diff --git a/.claude/skills/bump-tantivy/SKILL.md b/.claude/skills/bump-tantivy/SKILL.md new file mode 100644 index 00000000000..46e715b8551 --- /dev/null +++ b/.claude/skills/bump-tantivy/SKILL.md @@ -0,0 +1,83 @@ +--- +name: bump-tantivy +description: Bump tantivy to the latest commit on main branch, fix compilation issues, and open a PR +disable-model-invocation: true +--- + +# Bump Tantivy + +Follow these steps to bump tantivy to its latest version: + +## Step 1: Check that we are on the main branch + +Run: `git branch --show-current` + +If the current branch is not `main`, abort and ask the user to switch to the main branch first. + +## Step 2: Ensure main is up to date + +Run: `git pull origin main` + +This ensures we're working from the latest code. + +## Step 3: Get the latest tantivy SHA + +Run: `gh api repos/quickwit-oss/tantivy/commits/main --jq '.sha'` + +Extract the first 7 characters as the short SHA. + +## Step 4: Update Cargo.toml + +Edit `quickwit/Cargo.toml` and update the `rev` field in the tantivy dependency to the new short SHA. + +The line looks like: +```toml +tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "XXXXXXX", ... } +``` + +## Step 5: Run cargo check and fix compilation errors + +Run `cargo check` in the `quickwit` directory to verify compilation. + +If there are compilation errors: +- If the fix is straightforward (simple API changes, renames, etc.), fix them without asking +- If the fix is complex or unclear, ask the user before proceeding + +Repeat until cargo check passes. + +## Step 6: Format code + +Run `make fmt` from the `quickwit/` directory to format the code. + +## Step 7: Update licenses + +Run `make update-licenses` from the `quickwit/` directory, then move the generated file: +``` +mv quickwit/LICENSE-3rdparty.csv ./LICENSE-3rdparty.csv +``` + +## Step 8: Create a new branch + +Get the git username: `git config user.name | tr ' ' '-' | tr '[:upper:]' '[:lower:]'` + +Get today's date: `date +%Y-%m-%d` + +Create and checkout a new branch named: `{username}/bump-tantivy-{date}` + +Example: `paul/bump-tantivy-2024-03-15` + +## Step 9: Commit changes + +Stage all modified files and create a commit with message: +``` +Bump tantivy to {short-sha} +``` + +## Step 10: Push and open a PR + +Push the branch and open a PR using: +``` +gh pr create --title "Bump tantivy to {short-sha}" --body "Updates tantivy dependency to the latest commit on main." +``` + +Report the PR URL to the user when complete. diff --git a/.claude/skills/fix-clippy/SKILL.md b/.claude/skills/fix-clippy/SKILL.md new file mode 100644 index 00000000000..23e812c5523 --- /dev/null +++ b/.claude/skills/fix-clippy/SKILL.md @@ -0,0 +1,26 @@ +--- +name: fix-clippy +description: Fix all clippy lint warnings in the project +--- + +# Fix Clippy + +Clippy issues are **warnings**, not errors. Never grep for `error` when looking for clippy issues. + +## Step 1: Auto-fix + +Run `make fix` to automatically fix clippy warnings: + +``` +make fix +``` + +## Step 2: Fix remaining warnings manually + +Check for remaining warnings that couldn't be auto-fixed: + +``` +cargo clippy --tests 2>&1 | grep "^warning:" | sort -u +``` + +For each remaining warning, find the exact location and fix it manually. diff --git a/.claude/skills/fmt/SKILL.md b/.claude/skills/fmt/SKILL.md new file mode 100644 index 00000000000..30eabefd86e --- /dev/null +++ b/.claude/skills/fmt/SKILL.md @@ -0,0 +1,23 @@ +--- +name: fmt +description: Run `make fmt` to check the code format. +--- + +# Format Check + +Run `make fmt` from the `quickwit/` subdirectory to check code formatting: + +``` +cd /Users/paul.masurel/git/quickwit/quickwit && make fmt +``` + +This command checks: +1. Rust code formatting +2. License headers +3. Log format policy (no trailing punctuation, no uppercase first character) + +If there are log format issues, fix them by: +- Making the first character lowercase +- Removing trailing punctuation (periods, exclamation marks, etc.) + +Fix any issues found and re-run until clean. diff --git a/.claude/skills/rationalize-deps/SKILL.md b/.claude/skills/rationalize-deps/SKILL.md new file mode 100644 index 00000000000..829a70c67ec --- /dev/null +++ b/.claude/skills/rationalize-deps/SKILL.md @@ -0,0 +1,125 @@ +--- +name: rationalize-deps +description: Analyze Cargo.toml dependencies and attempt to remove unused features to reduce compile times and binary size +--- + +# Rationalize Dependencies + +This skill analyzes Cargo.toml dependencies to identify and remove unused features. + +## Overview + +Many crates enable features by default that may not be needed. This skill: +1. Identifies dependencies with default features enabled +2. Tests if `default-features = false` works +3. Identifies which specific features are actually needed +4. Verifies compilation after changes + +## Step 1: Identify the target + +Ask the user which crate(s) to analyze: +- A specific crate name (e.g., "tokio", "serde") +- A specific workspace member (e.g., "quickwit-search") +- "all" to scan the entire workspace + +## Step 2: Analyze current dependencies + +For the workspace Cargo.toml (`quickwit/Cargo.toml`), list dependencies that: +- Do NOT have `default-features = false` +- Have default features that might be unnecessary + +Run: `cargo tree -p -f "{p} {f}" --edges features` to see what features are actually used. + +## Step 3: For each candidate dependency + +### 3a: Check the crate's default features + +Look up the crate on crates.io or check its Cargo.toml to understand: +- What features are enabled by default +- What each feature provides + +Use: `cargo metadata --format-version=1 | jq '.packages[] | select(.name == "") | .features'` + +### 3b: Try disabling default features + +Modify the dependency in `quickwit/Cargo.toml`: + +From: +```toml +some-crate = { version = "1.0" } +``` + +To: +```toml +some-crate = { version = "1.0", default-features = false } +``` + +### 3c: Run cargo check + +Run: `cargo check --workspace` (or target specific packages for faster feedback) + +If compilation fails: +1. Read the error messages to identify which features are needed +2. Add only the required features explicitly: + ```toml + some-crate = { version = "1.0", default-features = false, features = ["needed-feature"] } + ``` +3. Re-run cargo check + +### 3d: Binary search for minimal features + +If there are many default features, use binary search: +1. Start with no features +2. If it fails, add half the default features +3. Continue until you find the minimal set + +## Step 4: Document findings + +For each dependency analyzed, report: +- Original configuration +- New configuration (if changed) +- Features that were removed +- Any features that are required + +## Step 5: Verify full build + +After all changes, run: +```bash +cargo check --workspace --all-targets +cargo test --workspace --no-run +``` + +## Common Patterns + +### Serde +Often only needs `derive`: +```toml +serde = { version = "1.0", default-features = false, features = ["derive", "std"] } +``` + +### Tokio +Identify which runtime features are actually used: +```toml +tokio = { version = "1.0", default-features = false, features = ["rt-multi-thread", "macros", "sync"] } +``` + +### Reqwest +Often doesn't need all TLS backends: +```toml +reqwest = { version = "0.11", default-features = false, features = ["rustls-tls", "json"] } +``` + +## Rollback + +If changes cause issues: +```bash +git checkout quickwit/Cargo.toml +cargo check --workspace +``` + +## Tips + +- Start with large crates that have many default features (tokio, reqwest, hyper) +- Use `cargo bloat --crates` to identify large dependencies +- Check `cargo tree -d` for duplicate dependencies that might indicate feature conflicts +- Some features are needed only for tests - consider using `[dev-dependencies]` features diff --git a/.claude/skills/simple-pr/SKILL.md b/.claude/skills/simple-pr/SKILL.md new file mode 100644 index 00000000000..00107e1c46b --- /dev/null +++ b/.claude/skills/simple-pr/SKILL.md @@ -0,0 +1,60 @@ +--- +name: simple-pr +description: Create a simple PR from staged changes with an auto-generated commit message +disable-model-invocation: true +--- + +# Simple PR + +Follow these steps to create a simple PR from staged changes: + +## Step 1: Check workspace state + +Run: `git status` + +Verify that all changes have been staged (no unstaged changes). If there are unstaged changes, abort and ask the user to stage their changes first with `git add`. + +Also verify that we are on the `main` branch. If not, abort and ask the user to switch to main first. + +## Step 2: Ensure main is up to date + +Run: `git pull origin main` + +This ensures we're working from the latest code. + +## Step 3: Review staged changes + +Run: `git diff --cached` + +Review the staged changes to understand what the PR will contain. + +## Step 4: Generate commit message + +Based on the staged changes, generate a concise commit message (1-2 sentences) that describes the "why" rather than the "what". + +Display the proposed commit message to the user and ask for confirmation before proceeding. + +## Step 5: Create a new branch + +Get the git username: `git config user.name | tr ' ' '-' | tr '[:upper:]' '[:lower:]'` + +Create a short, descriptive branch name based on the changes (e.g., `fix-typo-in-readme`, `add-retry-logic`, `update-deps`). + +Create and checkout the branch: `git checkout -b {username}/{short-descriptive-name}` + +## Step 6: Commit changes + +Commit with the message from step 3: +``` +git commit -m "{commit-message}" +``` + +## Step 7: Push and open a PR + +Push the branch and open a PR: +``` +git push -u origin {branch-name} +gh pr create --title "{commit-message-title}" --body "{longer-description-if-needed}" +``` + +Report the PR URL to the user when complete. diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index afeaa532c64..7782d81b43a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ { "name": "Quickwit", - "image": "mcr.microsoft.com/devcontainers/rust:latest", + "image": "mcr.microsoft.com/devcontainers/rust:bookworm", "customizations": { "codespaces": { "openFiles": [ @@ -33,7 +33,7 @@ "moby": true }, "ghcr.io/devcontainers/features/node:1": { - "version": "18" + "version": "24" }, "ghcr.io/devcontainers/features/aws-cli:1": {}, "ghcr.io/devcontainers-contrib/features/protoc:1": {} diff --git a/.github/actions/cargo-build-macos-binary/action.yml b/.github/actions/cargo-build-macos-binary/action.yml index 29da49e6ad6..4db9ce46198 100644 --- a/.github/actions/cargo-build-macos-binary/action.yml +++ b/.github/actions/cargo-build-macos-binary/action.yml @@ -17,7 +17,7 @@ runs: shell: bash - uses: actions/setup-node@v3 with: - node-version: 20 + node-version: 24 cache: "yarn" cache-dependency-path: quickwit/quickwit-ui/yarn.lock - run: yarn global add node-gyp @@ -36,9 +36,9 @@ runs: working-directory: ./quickwit - name: Retrieve and export commit date, hash, and tags run: | - echo "QW_COMMIT_DATE=$(TZ=UTC0 git log -1 --format=%cd --date=format-local:%Y-%m-%dT%H:%M:%SZ)" >> $GITHUB_ENV - echo "QW_COMMIT_HASH=$(git rev-parse HEAD)" >> $GITHUB_ENV - echo "QW_COMMIT_TAGS=$(git tag --points-at HEAD | tr '\n' ',')" >> $GITHUB_ENV + echo "QW_COMMIT_DATE=$(TZ=UTC0 git log -1 --format=%cd --date=format-local:%Y-%m-%dT%H:%M:%SZ)" >> $GITHUB_ENV + echo "QW_COMMIT_HASH=$(git rev-parse HEAD)" >> $GITHUB_ENV + echo "QW_COMMIT_TAGS=$(git tag --points-at HEAD | tr '\n' ',')" >> $GITHUB_ENV shell: bash - name: Build binary run: cargo build --release --features release-macos-feature-vendored-set --target ${{ matrix.target }} --bin quickwit @@ -60,7 +60,7 @@ runs: path: ./${{ env.ASSET_FULL_NAME }}.tar.gz retention-days: 3 - name: Deploy archive to GitHub release - uses: quickwit-inc/upload-to-github-release@v1 + uses: quickwit-inc/upload-to-github-release@9b2c40fba23bf8dea05b7d2eece24cbc95d4a190 env: GITHUB_TOKEN: ${{ inputs.token }} with: diff --git a/.github/actions/cross-build-binary/action.yml b/.github/actions/cross-build-binary/action.yml index d1916325d78..0f23b17aa22 100644 --- a/.github/actions/cross-build-binary/action.yml +++ b/.github/actions/cross-build-binary/action.yml @@ -17,7 +17,7 @@ runs: shell: bash - uses: actions/setup-node@v3 with: - node-version: 20 + node-version: 24 cache: "yarn" cache-dependency-path: quickwit/quickwit-ui/yarn.lock - run: yarn global add node-gyp @@ -32,9 +32,9 @@ runs: shell: bash - name: Retrieve and export commit date, hash, and tags run: | - echo "QW_COMMIT_DATE=$(TZ=UTC0 git log -1 --format=%cd --date=format-local:%Y-%m-%dT%H:%M:%SZ)" >> $GITHUB_ENV - echo "QW_COMMIT_HASH=$(git rev-parse HEAD)" >> $GITHUB_ENV - echo "QW_COMMIT_TAGS=$(git tag --points-at HEAD | tr '\n' ',')" >> $GITHUB_ENV + echo "QW_COMMIT_DATE=$(TZ=UTC0 git log -1 --format=%cd --date=format-local:%Y-%m-%dT%H:%M:%SZ)" >> $GITHUB_ENV + echo "QW_COMMIT_HASH=$(git rev-parse HEAD)" >> $GITHUB_ENV + echo "QW_COMMIT_TAGS=$(git tag --points-at HEAD | tr '\n' ',')" >> $GITHUB_ENV shell: bash - name: Build Quickwit run: cross build --release --features release-feature-vendored-set --target ${{ inputs.target }} --bin quickwit @@ -56,7 +56,7 @@ runs: path: ./${{ env.ASSET_FULL_NAME }}.tar.gz retention-days: 3 - name: Upload archive - uses: quickwit-inc/upload-to-github-release@v1 + uses: quickwit-inc/upload-to-github-release@9b2c40fba23bf8dea05b7d2eece24cbc95d4a190 env: GITHUB_TOKEN: ${{ inputs.token }} with: diff --git a/.github/dependabot.yml b/.github/dependabot.yml index adbe90f325c..ef256bac12e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,7 +4,7 @@ updates: - package-ecosystem: cargo directory: "/quickwit" schedule: - interval: "weekly" + interval: "monthly" groups: rust-dependencies: patterns: @@ -13,32 +13,40 @@ updates: ignore: - dependency-name: "*" update-types: ["version-update:semver-patch"] + cooldown: + default-days: 14 # Docker dependencies - package-ecosystem: docker directory: "/" schedule: - interval: "weekly" + interval: "monthly" open-pull-requests-limit: 10 + cooldown: + default-days: 14 # GitHub Actions - package-ecosystem: github-actions directory: "/" schedule: - interval: "weekly" + interval: "monthly" groups: github-actions: patterns: - "*" open-pull-requests-limit: 10 + cooldown: + default-days: 14 # NPM dependencies - package-ecosystem: npm directory: "/" schedule: - interval: "weekly" + interval: "monthly" groups: npm-dependencies: patterns: - "*" open-pull-requests-limit: 10 + cooldown: + default-days: 14 diff --git a/.github/workflows/cbench.yml b/.github/workflows/cbench.yml deleted file mode 100644 index 79fbed7ff8b..00000000000 --- a/.github/workflows/cbench.yml +++ /dev/null @@ -1,137 +0,0 @@ -name: CBENCH - -on: - workflow_dispatch: - push: - branches: - - main - paths: - - "quickwit/**" - - "!quickwit/quickwit-ui/**" - # For security reasons (to make sure the list of allowed users is - # trusted), make sure we run the workflow definition from the base - # commit of the pull request. - pull_request_target: - -# This is required for github.rest.issues.createComment. -permissions: - issues: write - pull-requests: write - -env: - RUSTFLAGS: --cfg tokio_unstable - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - tests: - name: Benchmark - # The self-hosted runner must have the system deps installed for QW and - # the benchmark, because we don't have root access. - runs-on: self-hosted - timeout-minutes: 60 - steps: - - name: Set authorized users - id: authorized-users - # List of users allowed to trigger this workflow. - # Because it executes code on a self-hosted runner, it must be restricted to trusted users. - run: | - echo 'users=["ddelemeny", "fmassot", "fulmicoton", "guilload", "PSeitz", "rdettai", "trinity-1686a"]' >> $GITHUB_OUTPUT - - - uses: actions/checkout@v5 - if: contains(fromJSON(steps.authorized-users.outputs.users), github.actor) && github.event_name == 'pull_request_target' - name: Checkout quickwit (pull request commit) - with: - repository: quickwit-oss/quickwit - ref: ${{ github.event.pull_request.head.sha }} - path: ./quickwit - - - uses: actions/checkout@v5 - if: contains(fromJSON(steps.authorized-users.outputs.users), github.actor) && github.event_name != 'pull_request_target' - name: Checkout quickwit - with: - repository: quickwit-oss/quickwit - ref: ${{ github.sha }} - path: ./quickwit - - - name: Checkout benchmarking code - uses: actions/checkout@v5 - if: contains(fromJSON(steps.authorized-users.outputs.users), github.actor) - with: - repository: quickwit-oss/benchmarks - ref: main - path: ./benchmarks - - - name: Install Rust - run: rustup update stable - - - name: Install protoc - uses: taiki-e/install-action@v2 - with: - tool: protoc - - # We don't use rust-cache as it requires root access on the self-hosted runner, which we don't have. - - name: cargo build - if: contains(fromJSON(steps.authorized-users.outputs.users), github.actor) - run: cargo build --release --bin quickwit - working-directory: ./quickwit/quickwit - - name: Compile qbench - if: contains(fromJSON(steps.authorized-users.outputs.users), github.actor) - run: cargo build --release - working-directory: ./benchmarks/qbench - - name: Run Benchmark on SSD - if: contains(fromJSON(steps.authorized-users.outputs.users), github.actor) - id: bench-run-ssd - run: python3 ./run.py --search-only --storage pd-ssd --engine quickwit --track generated-logs --tags "${{ github.event_name }}_${{ github.ref_name }}" --manage-engine --source github_workflow --binary-path ../quickwit/quickwit/target/release/quickwit --instance "{autodetect_gcp}" --export-to-endpoint=https://qw-benchmarks.104.155.161.122.nip.io --engine-data-dir "{qwdata_local}" --github-workflow-user "${{ github.actor }}" --github-workflow-run-id "${{ github.run_id }}" --comparison-reference-tag="push_main" --github-pr "${{ github.event_name == 'pull_request_target' && github.event.number || 0 }}" --comparison-reference-commit "${{ github.event_name == 'pull_request_target' && github.sha || github.event.before }}" --write-exported-run-url-to-file $GITHUB_OUTPUT - working-directory: ./benchmarks - - name: Run Benchmark on cloud storage - if: contains(fromJSON(steps.authorized-users.outputs.users), github.actor) - id: bench-run-cloud-storage - run: python3 ./run.py --search-only --storage gcs --engine quickwit --track generated-logs --tags "${{ github.event_name }}_${{ github.ref_name }}" --manage-engine --source github_workflow --binary-path ../quickwit/quickwit/target/release/quickwit --instance "{autodetect_gcp}" --export-to-endpoint=https://qw-benchmarks.104.155.161.122.nip.io --engine-data-dir "{qwdata_gcs}" --engine-config-file engines/quickwit/configs/cbench_quickwit_gcs.yaml --github-workflow-user "${{ github.actor }}" --github-workflow-run-id "${{ github.run_id }}" --comparison-reference-tag="push_main" --github-pr "${{ github.event_name == 'pull_request_target' && github.event.number || 0 }}" --comparison-reference-commit "${{ github.event_name == 'pull_request_target' && github.sha || github.event.before }}" --write-exported-run-url-to-file $GITHUB_OUTPUT - working-directory: ./benchmarks - - name: Show results links - if: contains(fromJSON(steps.authorized-users.outputs.users), github.actor) - run: | - echo "::notice title=Benchmark Results on SSD::${{ steps.bench-run-ssd.outputs.url }}" - echo "::notice title=Comparison of results on SSD::${{ steps.bench-run-ssd.outputs.comparison_text }}" - echo "::notice title=Benchmark Results on Cloud Storage::${{ steps.bench-run-cloud-storage.outputs.url }}" - echo "::notice title=Comparison of results on Cloud Storage::${{ steps.bench-run-cloud-storage.outputs.comparison_text }}" - - name: In case of auth error - if: ${{ ! contains(fromJSON(steps.authorized-users.outputs.users), github.actor) }} - run: | - echo "::error title=User not allowed to run the benchmark::User must be in list ${{ steps.authorized-users.outputs.users }}" - - name: Add a PR comment with comparison results - uses: actions/github-script@v7 - if: contains(fromJSON(steps.authorized-users.outputs.users), github.actor) && github.event_name == 'pull_request_target' - # Inspired from: https://github.com/actions/github-script/blob/60a0d83039c74a4aee543508d2ffcb1c3799cdea/.github/workflows/pull-request-test.yml - with: - script: | - // Get the existing comments. - const {data: comments} = await github.rest.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.number, - }) - - // Find any comment already made by the bot to update it. - const botComment = comments.find(comment => comment.user.id === 41898282) - const commentBody = "### On SSD:\n${{ steps.bench-run-ssd.outputs.comparison_text }}\n### On GCS:\n${{ steps.bench-run-cloud-storage.outputs.comparison_text }}\n" - if (botComment) { - // Update existing comment. - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: botComment.id, - body: commentBody - }) - } else { - // New comment. - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.number, - body: commentBody - }) - } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41de4779f43..4ce6f1ec568 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,10 @@ on: paths: - "quickwit/**" - "!quickwit/quickwit-ui/**" + +permissions: + contents: read + env: CARGO_INCREMENTAL: 0 QW_DISABLE_TELEMETRY: 1 @@ -27,7 +31,10 @@ jobs: tests: name: Unit tests runs-on: "ubuntu-latest" - timeout-minutes: 40 + timeout-minutes: 60 + permissions: + contents: read + actions: write services: # PostgreSQL service container postgres: @@ -45,10 +52,31 @@ jobs: --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@v5 + - name: Cleanup Disk Space + run: | + df -h + + if [ "$(df -BG / | awk 'NR==2 {gsub("G","",$4); print $4}')" -lt 30 ]; then + echo "Less than 30GiB available. Running cleanup..." + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /usr/share/swift + sudo rm -rf /usr/local/.ghcup + sudo rm -rf /opt/hostedtoolcache/CodeQL + df -h + else + echo "30GiB or more available. Skipping cleanup." + fi + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install Ubuntu packages - run: sudo apt-get -y install protobuf-compiler python3 - - uses: dorny/paths-filter@v3 + run: | + sudo apt-get update + sudo apt-get -y install protobuf-compiler + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v.6.2.0 + with: + python-version: '3.11' + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 id: modified with: filters: | @@ -58,49 +86,51 @@ jobs: - quickwit/**/*.proto - quickwit/rest-api-tests/** - .github/workflows/ci.yml - # The following step is just meant to install rustup actually. - # The next one installs the correct toolchain. - - name: Install rustup - if: steps.modified.outputs.rust_src == 'true' - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain none -y - name: Setup stable Rust Toolchain if: steps.modified.outputs.rust_src == 'true' - run: rustup show active-toolchain || rustup toolchain install - working-directory: ./quickwit + uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9 # master + with: + toolchain: stable - name: Setup cache - uses: Swatinem/rust-cache@v2 + uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 if: steps.modified.outputs.rust_src == 'true' with: workspaces: "./quickwit -> target" + shared-key: "quickwit-cargo" - name: Install nextest if: always() && steps.modified.outputs.rust_src == 'true' - uses: taiki-e/cache-cargo-install-action@v2 + uses: taiki-e/install-action@aba36d755ec7ca22d38b12111787c26115943952 with: tool: cargo-nextest - - name: cargo nextest + - name: cargo build if: always() && steps.modified.outputs.rust_src == 'true' - run: cargo nextest run --features=postgres --retries 1 + run: cargo build --features=postgres --tests --bin quickwit working-directory: ./quickwit - - name: cargo build + - name: cargo nextest if: always() && steps.modified.outputs.rust_src == 'true' - run: cargo build --features=postgres --bin quickwit + run: cargo nextest run --features=postgres --retries 1 working-directory: ./quickwit - name: Install python packages if: always() && steps.modified.outputs.rust_src == 'true' - run: python3 -m venv venv && source venv/bin/activate && pip install pyaml requests + run: | + pip install --user --require-hashes -r ${{ github.workspace }}/.github/workflows/requirements.txt + pipenv install --deploy --ignore-pipfile working-directory: ./quickwit/rest-api-tests - name: Run REST API tests if: always() && steps.modified.outputs.rust_src == 'true' - run: source venv/bin/activate && python3 ./run_tests.py --binary ../target/debug/quickwit + run: pipenv run python3 ./run_tests.py --binary ../target/debug/quickwit working-directory: ./quickwit/rest-api-tests lints: name: Lints runs-on: "ubuntu-latest" - timeout-minutes: 20 + timeout-minutes: 60 + permissions: + contents: read + actions: write steps: - - uses: actions/checkout@v5 - - uses: dorny/paths-filter@v3 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 id: modified with: filters: | @@ -111,28 +141,37 @@ jobs: - .github/workflows/ci.yml - name: Install Ubuntu packages if: always() && steps.modified.outputs.rust_src == 'true' - run: sudo apt-get -y install protobuf-compiler python3 python3-pip - - name: Install rustup - if: steps.modified.outputs.rust_src == 'true' - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain none -y + run: | + sudo apt-get update + sudo apt-get -y install protobuf-compiler - name: Setup nightly Rust Toolchain (for rustfmt) if: steps.modified.outputs.rust_src == 'true' - run: rustup toolchain install nightly + uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9 # master + with: + toolchain: nightly + components: rustfmt - name: Setup stable Rust Toolchain if: steps.modified.outputs.rust_src == 'true' - run: rustup show active-toolchain || rustup toolchain install - working-directory: ./quickwit + uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9 # master + with: + toolchain: stable - name: Setup cache if: steps.modified.outputs.rust_src == 'true' - uses: Swatinem/rust-cache@v2 + uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 with: workspaces: "./quickwit -> target" + shared-key: "quickwit-cargo" - name: Install cargo deny if: always() && steps.modified.outputs.rust_src == 'true' - uses: taiki-e/cache-cargo-install-action@v2 + uses: taiki-e/cache-cargo-install-action@59027ebf20a9617c4e819eb53ccd2673cb162b89 # v3.0.3 with: # 0.18 requires rustc 1.85 tool: cargo-deny@0.17.0 + - name: Install cargo machete + if: always() && steps.modified.outputs.rust_src == 'true' + uses: taiki-e/cache-cargo-install-action@59027ebf20a9617c4e819eb53ccd2673cb162b89 # v3.0.3 + with: + tool: cargo-machete - name: cargo clippy if: always() && steps.modified.outputs.rust_src == 'true' run: cargo clippy --workspace --tests --all-features @@ -141,9 +180,13 @@ jobs: if: always() && steps.modified.outputs.rust_src == 'true' run: cargo deny check licenses working-directory: ./quickwit + - name: cargo machete + if: always() && steps.modified.outputs.rust_src == 'true' + run: cargo machete + working-directory: ./quickwit - name: cargo doc if: always() && steps.modified.outputs.rust_src == 'true' - run: cargo doc + run: cargo doc --no-deps working-directory: ./quickwit - name: License headers check if: always() @@ -157,13 +200,18 @@ jobs: thirdparty-license: name: Check Datadog third-party license file runs-on: ubuntu-latest + permissions: + contents: read + actions: write steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9 # master + with: + toolchain: stable - name: Cache cargo tools - uses: actions/cache@v4 + uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 with: path: ~/.cargo/bin key: ${{ runner.os }}-cargo-tools-${{ hashFiles('**/Cargo.lock') }} diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 82609e14590..d8191e8a2d3 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -11,12 +11,16 @@ on: - quickwit/Cargo.lock - quickwit/quickwit-*/** +permissions: + contents: read + env: AWS_REGION: us-east-1 AWS_ACCESS_KEY_ID: "placeholder" AWS_SECRET_ACCESS_KEY: "placeholder" CARGO_INCREMENTAL: 0 - PUBSUB_EMULATOR_HOST: "localhost:9898" + LOCALSTACK_ACKNOWLEDGE_ACCOUNT_REQUIREMENT: 1 + PUBSUB_EMULATOR_HOST: "localhost:8681" QW_DISABLE_TELEMETRY: 1 QW_S3_ENDPOINT: "http://localhost:4566" # Services are exposed as localhost because we are not running coverage in a container. QW_S3_FORCE_PATH_STYLE_ACCESS: 1 @@ -28,6 +32,9 @@ jobs: name: Coverage runs-on: gh-ubuntu-arm64 timeout-minutes: 40 + permissions: + contents: read + actions: write # Setting a containing will require to fix the QW_S3_ENDPOINT to http://localstack:4566 services: localstack: @@ -59,51 +66,50 @@ jobs: --health-retries 5 kafka-broker: - image: confluentinc/cp-kafka:7.2.1 + image: confluentinc/confluent-local:7.4.11 ports: - "9092:9092" - "9101:9101" env: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-broker:29092,PLAINTEXT_HOST://localhost:9092 + # Mode KRaft (Single Node) + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: 'broker,controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@localhost:9093' + KAFKA_LOG4J_LOGGERS: "org.apache.kafka.image.loader.MetadataLoader=WARN" + + # Listeners + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT' + KAFKA_LISTENERS: 'EXTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093' + KAFKA_ADVERTISED_LISTENERS: 'EXTERNAL://localhost:9092' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_INTER_BROKER_LISTENER_NAME: 'EXTERNAL' + + # Configuration simplifiée KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_JMX_PORT: 9101 - KAFKA_JMX_HOSTNAME: localhost - KAFKA_HEAP_OPTS: -Xms256M -Xmx256M - options: >- - --health-cmd "cub kafka-ready -b localhost:9092 1 5" - --health-interval 10s - --health-timeout 5s - --health-retries 5 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + + # ID du Cluster (Nécessaire pour KRaft) + CLUSTER_ID: 'MkU3OEVBNTcwNTJENDM2Qk' - zookeeper: - image: confluentinc/cp-zookeeper:7.2.1 - ports: - - "2181:2181" - env: KAFKA_HEAP_OPTS: -Xms256M -Xmx256M - ZOOKEEPER_CLIENT_PORT: 2181 - ZOOKEEPER_TICK_TIME: 2000 + options: >- - --health-cmd "cub zk-ready localhost:2181 5" + --health-cmd "ub kafka-ready -b localhost:9092 1 5" --health-interval 10s --health-timeout 5s --health-retries 5 gcp-pubsub-emulator: - image: thekevjames/gcloud-pubsub-emulator:7555256f2c + image: thekevjames/gcloud-pubsub-emulator:550.0.0 ports: - - "9898:8681" + - "8681:8681" env: PUBSUB_PROJECT1: "quickwit-emulator,emulator_topic:emulator_subscription" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install lib libsasl2 run: | @@ -111,7 +117,11 @@ jobs: sudo apt install libsasl2-dev sudo apt install libsasl2-2 - - uses: actions/cache@v4 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v.6.2.0 + with: + python-version: '3.11' + + - uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 with: path: | ~/.cargo/git @@ -121,12 +131,15 @@ jobs: ${{ runner.os }}-cargo-test-${{ hashFiles('Cargo.lock') }} ${{ runner.os }}-cargo-test - - name: Install awslocal - run: pip install awscli-local + - name: Install python packages + run: | + pip install --user --require-hashes -r ${{ github.workspace }}/.github/workflows/requirements.txt + pipenv install --deploy --ignore-pipfile + working-directory: ./quickwit/quickwit-cli/tests - name: Prepare LocalStack S3 - run: ./quickwit-cli/tests/prepare_tests.sh - working-directory: ./quickwit + run: pipenv run ./prepare_tests.sh + working-directory: ./quickwit/quickwit-cli/tests # GitHub Actions does not allow services to be started with a custom command, # so we are running Azurite as a container manually. @@ -145,7 +158,7 @@ jobs: run: rustup update stable - name: Install cargo-llvm-cov, cargo-nextest, and protoc - uses: taiki-e/install-action@v2 + uses: taiki-e/install-action@90558ad1e179036f31467972b00dec6cb80701fa # v2.66.3 with: tool: cargo-llvm-cov,nextest,protoc @@ -160,7 +173,7 @@ jobs: working-directory: ./quickwit - name: Upload coverage to Codecov - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 with: token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos files: ./quickwit/lcov.info @@ -172,7 +185,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Send Message - uses: sarisia/actions-status-discord@v1 + uses: sarisia/actions-status-discord@eb045afee445dc055c18d3d90bd0f244fd062708 # v1.16.0 with: webhook: ${{ secrets.DISCORD_WEBHOOK }} nodetail: true diff --git a/.github/workflows/dependency.yml b/.github/workflows/dependency.yml index b6712648b9d..6b4db7a509d 100644 --- a/.github/workflows/dependency.yml +++ b/.github/workflows/dependency.yml @@ -14,9 +14,9 @@ jobs: runs-on: ubuntu-latest steps: - name: "Checkout Repository" - uses: actions/checkout@v5 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: "Dependency Review" - uses: actions/dependency-review-action@v4 + uses: actions/dependency-review-action@f5b971718edcbb31275a1db40004592335c0e031 # v4.8.1 with: # This is an minor vuln on the rsa crate, used for # google storage. diff --git a/.github/workflows/publish_cross_images.yml b/.github/workflows/publish_cross_images.yml index ca1817103ac..13f9f874f5d 100644 --- a/.github/workflows/publish_cross_images.yml +++ b/.github/workflows/publish_cross_images.yml @@ -8,6 +8,9 @@ on: paths: - "build/cross-images/**" +permissions: + contents: read + jobs: build-cross-images: name: Publish cross images @@ -16,9 +19,9 @@ jobs: name: production steps: - name: Check out the repo - uses: actions/checkout@v5 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Log in to Docker Hub - uses: docker/login-action@v3 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} diff --git a/.github/workflows/publish_docker_images.yml b/.github/workflows/publish_docker_images.yml index 927ed923696..e8b8f8a5851 100644 --- a/.github/workflows/publish_docker_images.yml +++ b/.github/workflows/publish_docker_images.yml @@ -5,6 +5,7 @@ on: push: branches: - main + - release-0.9 paths: - "quickwit/**" tags: @@ -12,6 +13,10 @@ on: - happy-plazza - qw* - v* + +permissions: + contents: read + env: REGISTRY_IMAGE: quickwit/quickwit @@ -27,27 +32,40 @@ jobs: platform: linux/arm64 platform_suffix: arm64 runs-on: ${{ matrix.os }} + permissions: + contents: read + actions: write environment: name: production steps: + - name: Cleanup Disk Space + run: | + df -h + sudo rm -rf /opt/hostedtoolcache/CodeQL + sudo rm -rf /usr/local/.ghcup + sudo rm -rf /usr/local/lib/android + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/share/swift + df -h + - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Login to Docker Hub - uses: docker/login-action@v3 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Docker meta id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 with: images: | ${{ env.REGISTRY_IMAGE }} @@ -69,7 +87,7 @@ jobs: fi - name: Build and push image - uses: docker/build-push-action@v6 + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 id: build with: context: . @@ -89,7 +107,7 @@ jobs: touch "/tmp/digests/${digest#sha256:}" - name: Upload digest - uses: actions/upload-artifact@v4.6.2 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: digest-${{ matrix.platform_suffix }} path: /tmp/digests/* @@ -99,21 +117,24 @@ jobs: merge: runs-on: ubuntu-latest needs: [docker] + permissions: + contents: read + actions: read environment: production steps: - name: Download digests - uses: actions/download-artifact@v5.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: digest-* path: /tmp/digests merge-multiple: true - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Docker meta id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 with: images: ${{ env.REGISTRY_IMAGE }} flavor: | @@ -125,8 +146,9 @@ jobs: type=semver,pattern={{version}},value=latest type=semver,pattern={{version}},suffix=-slim-bookworm type=ref,event=tag + type=raw,value=v0.9.0-rc,enable=${{ github.ref == 'refs/heads/release-0.9' }} - name: Login to Docker Hub - uses: docker/login-action@v3 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} diff --git a/.github/workflows/publish_lambda.yaml b/.github/workflows/publish_lambda.yaml new file mode 100644 index 00000000000..002b4f7236f --- /dev/null +++ b/.github/workflows/publish_lambda.yaml @@ -0,0 +1,78 @@ +# This workflow creates a new release for a quickwit search aws lambda. +# The artifact is a zip file containing a binary for ARM 64, +# ready to be deployed by the deployer. +# +# See quickwit-lambda-client/README.md +name: Release Lambda binary + +on: + push: + tags: + - 'lambda-*' + workflow_dispatch: + inputs: + version: + description: 'Version tag (e.g., v0.8.0)' + required: false + default: 'dev' + +permissions: + contents: read + +jobs: + build-lambda: + name: Build Lambda ARM64 + runs-on: ubuntu-latest + permissions: + contents: write + actions: write + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set version + run: | + if [ "${{ github.ref_type }}" = "tag" ]; then + # Extract version from tag (e.g., lambda-v0.8.0 -> v0.8.0) + echo "ASSET_VERSION=${GITHUB_REF_NAME#lambda-}" >> $GITHUB_ENV + elif [ -n "${{ github.event.inputs.version }}" ] && [ "${{ github.event.inputs.version }}" != "dev" ]; then + echo "ASSET_VERSION=${{ github.event.inputs.version }}" >> $GITHUB_ENV + else + echo "ASSET_VERSION=dev-$(git rev-parse --short HEAD)" >> $GITHUB_ENV + fi + + - name: Install rustup + run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain none -y + + - name: Install cross + run: cargo install cross + + - name: Retrieve and export commit date, hash, and tags + run: | + echo "QW_COMMIT_DATE=$(TZ=UTC0 git log -1 --format=%cd --date=format-local:%Y-%m-%dT%H:%M:%SZ)" >> $GITHUB_ENV + echo "QW_COMMIT_HASH=$(git rev-parse HEAD)" >> $GITHUB_ENV + echo "QW_COMMIT_TAGS=$(git tag --points-at HEAD | tr '\n' ',')" >> $GITHUB_ENV + + - name: Build Lambda binary + run: cross build --release --features lambda-release --target aarch64-unknown-linux-gnu -p quickwit-lambda-server --bin quickwit-aws-lambda-leaf-search + env: + QW_COMMIT_DATE: ${{ env.QW_COMMIT_DATE }} + QW_COMMIT_HASH: ${{ env.QW_COMMIT_HASH }} + QW_COMMIT_TAGS: ${{ env.QW_COMMIT_TAGS }} + working-directory: ./quickwit + + - name: Create Lambda zip + run: | + cd quickwit/target/aarch64-unknown-linux-gnu/release + cp quickwit-aws-lambda-leaf-search bootstrap + zip quickwit-aws-lambda-${{ env.ASSET_VERSION }}-aarch64.zip bootstrap + mv quickwit-aws-lambda-${{ env.ASSET_VERSION }}-aarch64.zip ../../../../ + + - name: Upload to GitHub release + uses: quickwit-inc/upload-to-github-release@9b2c40fba23bf8dea05b7d2eece24cbc95d4a190 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + file: quickwit-aws-lambda-${{ env.ASSET_VERSION }}-aarch64.zip + overwrite: true + draft: true + tag_name: ${{ env.ASSET_VERSION }} diff --git a/.github/workflows/publish_lambda_packages.yml b/.github/workflows/publish_lambda_packages.yml deleted file mode 100644 index 01ed5ea70b7..00000000000 --- a/.github/workflows/publish_lambda_packages.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Build and publish AWS Lambda packages - -on: - push: - tags: - - "aws-lambda-beta-*" - -jobs: - build-lambdas: - name: Build Quickwit Lambdas - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v5 - - name: Install Ubuntu packages - run: sudo apt-get -y install protobuf-compiler python3 python3-pip - - name: Install rustup - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain none -y - - name: Install python dependencies - run: | - pip install --user pipenv - pipenv install --system - working-directory: ./distribution/lambda - - name: Lint and format - run: | - mypy . - black . --check - working-directory: ./distribution/lambda - - name: Retrieve and export commit date, hash, and tags - run: | - echo "QW_COMMIT_DATE=$(TZ=UTC0 git log -1 --format=%cd --date=format-local:%Y-%m-%dT%H:%M:%SZ)" >> $GITHUB_ENV - echo "QW_COMMIT_HASH=$(git rev-parse HEAD)" >> $GITHUB_ENV - echo "QW_COMMIT_TAGS=$(git tag --points-at HEAD | tr '\n' ',')" >> $GITHUB_ENV - - name: Build Quickwit Lambdas - run: make package - env: - QW_COMMIT_DATE: ${{ env.QW_COMMIT_DATE }} - QW_COMMIT_HASH: ${{ env.QW_COMMIT_HASH }} - QW_COMMIT_TAGS: ${{ env.QW_COMMIT_TAGS }} - QW_LAMBDA_BUILD: 1 - working-directory: ./distribution/lambda - - name: Extract package locations - run: | - echo "SEARCHER_PACKAGE_LOCATION=./distribution/lambda/$(make searcher-package-path)" >> $GITHUB_ENV - echo "INDEXER_PACKAGE_LOCATION=./distribution/lambda/$(make indexer-package-path)" >> $GITHUB_ENV - working-directory: ./distribution/lambda - - name: Upload Lambda archives - uses: quickwit-inc/upload-to-github-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - file: ${{ env.SEARCHER_PACKAGE_LOCATION }};${{ env.INDEXER_PACKAGE_LOCATION }} - overwrite: true - draft: true - tag_name: ${{ github.ref_name }} diff --git a/.github/workflows/publish_nightly_packages.yml b/.github/workflows/publish_nightly_packages.yml index 5c47886d33a..7e428fef676 100644 --- a/.github/workflows/publish_nightly_packages.yml +++ b/.github/workflows/publish_nightly_packages.yml @@ -5,16 +5,22 @@ on: schedule: - cron: "0 5 * * *" +permissions: + contents: read + jobs: build-macos-binaries: name: Build ${{ matrix.target }} runs-on: macos-latest + permissions: + contents: write + actions: write strategy: fail-fast: false matrix: target: [x86_64-apple-darwin, aarch64-apple-darwin] steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/cargo-build-macos-binary with: target: ${{ matrix.target }} @@ -27,8 +33,11 @@ jobs: target: [x86_64-unknown-linux-gnu, aarch64-unknown-linux-gnu] name: Build ${{ matrix.target }} runs-on: ubuntu-latest + permissions: + contents: write + actions: write steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/cross-build-binary with: target: ${{ matrix.target }} diff --git a/.github/workflows/publish_release_packages.yml b/.github/workflows/publish_release_packages.yml index 0f6f8af3996..baaf3469e4c 100644 --- a/.github/workflows/publish_release_packages.yml +++ b/.github/workflows/publish_release_packages.yml @@ -5,16 +5,22 @@ on: tags: - "v*" +permissions: + contents: read + jobs: build-macos-binaries: name: Build ${{ matrix.target }} runs-on: macos-latest + permissions: + contents: write + actions: write strategy: matrix: target: [x86_64-apple-darwin, aarch64-apple-darwin] steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Extract asset version run: echo "ASSET_VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV - uses: ./.github/actions/cargo-build-macos-binary @@ -29,8 +35,11 @@ jobs: target: [x86_64-unknown-linux-gnu, aarch64-unknown-linux-gnu] name: Build ${{ matrix.target }} runs-on: ubuntu-latest + permissions: + contents: write + actions: write steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Extract asset version run: echo "ASSET_VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV - uses: ./.github/actions/cross-build-binary diff --git a/.github/workflows/requirements.txt b/.github/workflows/requirements.txt new file mode 100644 index 00000000000..3f0ea6f3309 --- /dev/null +++ b/.github/workflows/requirements.txt @@ -0,0 +1,22 @@ +# contains pinned dependencies for installing pipenv to ensure repeatable builds in CI/CD workflows +certifi==2025.10.5 \ + --hash=sha256:0f212c2744a9bb6de0c56639a6f68afe01ecd92d91f14ae897c4fe7bbeeef0de \ + --hash=sha256:47c09d31ccf2acf0be3f701ea53595ee7e0b8fa08801c6624be771df09ae7b43 +distlib==0.4.0 \ + --hash=sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16 \ + --hash=sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d +filelock==3.20.3 \ + --hash=sha256:18c57ee915c7ec61cff0ecf7f0f869936c7c30191bb0cf406f1341778d0834e1 \ + --hash=sha256:4b0dda527ee31078689fc205ec4f1c1bf7d56cf88b6dc9426c4f230e46c2dce1 +packaging==25.0 \ + --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 \ + --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f +pipenv==2025.0.4 \ + --hash=sha256:36fc2a7841ccdb2f58a9f787b296c2e15dea3b5b79b84d4071812f28b7e8d7a2 \ + --hash=sha256:e1fbe4cfd25ab179f123d1fbb1fa1cdc0b3ffcdb1f21c775dcaa12ccc356f2bb +platformdirs==4.5.0 \ + --hash=sha256:70ddccdd7c99fc5942e9fc25636a8b34d04c24b335100223152c2803e4063312 \ + --hash=sha256:e578a81bb873cbb89a41fcc904c7ef523cc18284b7e3b3ccf06aca1403b7ebd3 +virtualenv==20.36.1 \ + --hash=sha256:575a8d6b124ef88f6f51d56d656132389f961062a9177016a50e4f507bbcc19f \ + --hash=sha256:8befb5c81842c641f8ee658481e42641c68b5eab3521d8e092d18320902466ba diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 00000000000..faf87818d7a --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,50 @@ +name: OpenSSF Scorecard +on: + schedule: + - cron: '0 0 * * 0' + push: + branches: + - main + +permissions: + contents: read + +jobs: + analysis: + name: Scorecards analysis + runs-on: ubuntu-latest + permissions: + # Needed to upload the results to code-scanning dashboard. + security-events: write + # Needed to publish results + id-token: write + actions: read + contents: read + + steps: + - name: 'Checkout code' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: 'Run analysis' + uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3 + with: + results_file: results.sarif + results_format: sarif + repo_token: ${{ secrets.GITHUB_TOKEN }} + publish_results: true + + # Upload the results as artifacts. + - name: 'Upload artifact' + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + # Upload the results to GitHub's code scanning dashboard. + - name: 'Upload to code-scanning' + uses: github/codeql-action/upload-sarif@cdefb33c0f6224e58673d9004f47f7cb3e328b89 # v4.31.10 + with: + sarif_file: results.sarif diff --git a/.github/workflows/ui-ci.yml b/.github/workflows/ui-ci.yml index 5979517a96d..6eba480974b 100644 --- a/.github/workflows/ui-ci.yml +++ b/.github/workflows/ui-ci.yml @@ -14,29 +14,40 @@ on: - "quickwit/quickwit-ui/**" - ".github/workflows/ui-ci.yml" +permissions: + contents: read + jobs: - tests: - name: ${{ matrix.task.name }} + checks: + name: Lint, type check & unit tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: 24 + cache: "yarn" + cache-dependency-path: quickwit/quickwit-ui/yarn.lock + - name: Install JS dependencies + run: yarn --cwd quickwit-ui install + working-directory: ./quickwit + - name: Lint + run: yarn --cwd quickwit-ui lint + working-directory: ./quickwit + - name: Type check + run: yarn --cwd quickwit-ui type + working-directory: ./quickwit + - name: Unit tests + run: yarn --cwd quickwit-ui test + working-directory: ./quickwit + + e2e: + name: Playwright e2e runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - task: - - name: Cypress run - command: | - sudo apt-get -y install protobuf-compiler - rustup show active-toolchain || rustup toolchain install - CI=false yarn --cwd quickwit-ui build - RUSTFLAGS="--cfg tokio_unstable" cargo build --features=postgres - mkdir qwdata - RUSTFLAGS="--cfg tokio_unstable" cargo run --features=postgres -- run --service searcher --service metastore --config ../config/quickwit.yaml & - yarn --cwd quickwit-ui cypress run - - name: Lint - command: yarn --cwd quickwit-ui lint - - name: Unit Test - command: yarn --cwd quickwit-ui test + permissions: + contents: read + actions: write services: - # PostgreSQL service container postgres: image: postgres:latest ports: @@ -45,7 +56,6 @@ jobs: POSTGRES_USER: quickwit-dev POSTGRES_PASSWORD: quickwit-dev POSTGRES_DB: quickwit-metastore-dev - # Set health checks to wait until postgres has started options: >- --health-cmd pg_isready --health-interval 10s @@ -54,26 +64,42 @@ jobs: env: CARGO_INCREMENTAL: 0 RUST_BACKTRACE: 1 - RUSTFLAGS: -Dwarnings -C lto=off + RUSTFLAGS: -Dwarnings --cfg tokio_unstable RUSTDOCFLAGS: -Dwarnings -Arustdoc::private_intra_doc_links QW_TEST_DATABASE_URL: postgres://quickwit-dev:quickwit-dev@postgres:5432/quickwit-metastore-dev steps: - - uses: actions/checkout@v5 - - uses: actions/setup-node@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: - node-version: 20 + node-version: 24 cache: "yarn" cache-dependency-path: quickwit/quickwit-ui/yarn.lock - - name: Install rustup - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain none -y - - name: Install JS dependencies - run: yarn --cwd quickwit-ui install - working-directory: ./quickwit + - name: Setup stable Rust Toolchain + uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9 # master + with: + toolchain: stable - name: Setup Rust cache - if: matrix.task.name == 'Cypress run' - uses: Swatinem/rust-cache@v2 + uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 with: workspaces: "./quickwit -> target" - - name: ${{ matrix.task.name }} - run: ${{ matrix.task.command }} + shared-key: "quickwit-cargo" + - name: Install JS dependencies + run: yarn --cwd quickwit-ui install + working-directory: ./quickwit + - name: Install Playwright browsers + run: npx playwright install chromium --with-deps --only-shell + working-directory: ./quickwit/quickwit-ui + - name: Build UI + run: CI=false yarn --cwd quickwit-ui build + working-directory: ./quickwit + - name: Build Quickwit + run: | + sudo apt-get update && sudo apt-get -y install protobuf-compiler + cargo build --features=postgres + working-directory: ./quickwit + - name: Run e2e tests + run: | + mkdir -p qwdata + cargo run --features=postgres -- run --service searcher --service metastore --config ../config/quickwit.yaml & + yarn --cwd quickwit-ui e2e-test working-directory: ./quickwit diff --git a/CHANGELOG.md b/CHANGELOG.md index fe09833f35a..6cf3abcc4f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,7 +39,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Retry on S3 internal error (#5504) - Allow specifying OTEL index ID in header (#5503) - Add a metric to count storage errors and their error code (#5497) -- Add support for concatenated fields (#4773, #5369, #5331) +- Add support for concatenated fields (#4773, #5369, #5331) - Add number of splits per root/leaf search histograms (#5472) - Introduce a searcher config option to timeout get requests (#5467) - Add fingerprint to task in cluster state (#5464) @@ -52,7 +52,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add more env var config for Postgres (#5365) - Enable str fast field range queries (#5324) - Allow querying non-existing fields (#5308) -- Support updating doc mapper through api (#5253) +- Support updating doc mapper through api (#5253) - Add optional special handling for hex in code tokenizer (#5200) - Added a circuit breaker layer (#5134) - Various performance optimizations in Tantivy (https://github.com/quickwit-oss/tantivy/blob/main/CHANGELOG.md) @@ -83,7 +83,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fix grafana searchers dashboard (#5455) - Fix jaeger http endpoint (#5378) - Fix file re-ingestion after EOF (#5330) -- Fix source path in Lambda distrib (#5327) - Fix configuration interpolation (#5403) - Fix jaeger duration parse error (#5518) - Fix unit conversion in jaeger http search endpoint (#5519) @@ -91,7 +90,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed - Remove support for 2-digit years in java datetime parser (#5596) - Remove DocMapper trait (#5508) - +- Remove support for AWS Lambda (#5884) +- Remove search stream endpoint (#5886) # [0.8.1] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bbc2dd7be5a..ba4e68c831c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,7 +30,7 @@ You will be notified by email from the CI system if any issues are discovered, b ### Local Development 1. Install Rust, CMake, Docker (https://docs.docker.com/engine/install/) and Docker Compose (https://docs.docker.com/compose/install/) -2. Install node@20 and `npm install -g yarn` +2. Install node@24 and `npm install -g yarn` 3. Install awslocal https://github.com/localstack/awscli-local 4. Install protoc https://grpc.io/docs/protoc-installation/ (you may need to install the latest binaries rather than your distro's flavor) 5. Install nextest https://nexte.st/docs/installation/pre-built-binaries/ @@ -48,7 +48,7 @@ Run `make test-all` to run all tests. * `make test-all` - starts necessary Docker services and runs all tests. * `make -k test-all docker-compose-down` - the same as above, but tears down the Docker services after running all the tests. * `make fmt` - runs formatter, this command requires the nightly toolchain to be installed by running `rustup toolchain install nightly`. -* `make fix` - runs formatter and clippy checks. +* `make fix` - runs formatter and clippy checks as well as removing unused dependencies (requires `cargo install cargo-machete`). * `make typos` - runs the spellcheck tool over the codebase. (Install by running `cargo install typos-cli`) * `make doc` - builds docs. * `make docker-compose-up` - starts Docker services. diff --git a/Dockerfile b/Dockerfile index 1cf380bc86a..b3656edaec3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20 AS ui-builder +FROM node:24@sha256:b2b2184ba9b78c022e1d6a7924ec6fba577adf28f15c9d9c457730cc4ad3807a AS ui-builder COPY quickwit/quickwit-ui /quickwit/quickwit-ui @@ -8,7 +8,7 @@ RUN touch .gitignore_for_build_directory \ && NODE_ENV=production make install build -FROM rust:bookworm AS bin-builder +FROM rust:bookworm@sha256:b5efaabfd787a695d2e46b37d3d9c54040e11f4c10bc2e714bbadbfcc0cd6c39 AS bin-builder ARG CARGO_FEATURES=release-feature-set ARG CARGO_PROFILE=release @@ -49,7 +49,7 @@ RUN echo "Building workspace with feature(s) '$CARGO_FEATURES' and profile '$CAR && find target/$CARGO_PROFILE -maxdepth 1 -perm /a+x -type f -exec mv {} /quickwit/bin \; -FROM debian:bookworm-slim AS quickwit +FROM debian:bookworm-slim@sha256:e899040a73d36e2b36fa33216943539d9957cba8172b858097c2cabcdb20a3e2 AS quickwit LABEL org.opencontainers.image.title="Quickwit" LABEL maintainer="Quickwit, Inc. " diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index b19672aa0bc..5b917ab361c 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -1,13 +1,17 @@ Component,Origin,License,Copyright +RustyXML,https://github.com/Florob/RustyXML,MIT OR Apache-2.0,Florian Zeitz addr2line,https://github.com/gimli-rs/addr2line,Apache-2.0 OR MIT,The addr2line Authors adler2,https://github.com/oyvindln/adler2,0BSD OR MIT OR Apache-2.0,"Jonas Schievink , oyvindln " advapi32-sys,https://github.com/retep998/winapi-rs,MIT,Peter Atashian +aead,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers aes,https://github.com/RustCrypto/block-ciphers,MIT OR Apache-2.0,RustCrypto Developers +aes-siv,https://github.com/RustCrypto/AEADs,Apache-2.0 OR MIT,RustCrypto Developers +ahash,https://github.com/tkaitchuck/ahash,MIT OR Apache-2.0,Tom Kaitchuck aho-corasick,https://github.com/BurntSushi/aho-corasick,Unlicense OR MIT,Andrew Gallant aliasable,https://github.com/avitex/rust-aliasable,MIT,avitex +aligned-vec,https://github.com/sarah-ek/aligned-vec,MIT,sarah <> alloca,https://github.com/playXE/alloca-rs,MIT,"Adel Prokurov , StackOverflowExcept1on" allocator-api2,https://github.com/zakarumych/allocator-api2,MIT OR Apache-2.0,Zakarum -android-tzdata,https://github.com/RumovZ/android-tzdata,MIT OR Apache-2.0,RumovZ android_system_properties,https://github.com/nical/android_system_properties,MIT OR Apache-2.0,Nicolas Silva anes,https://github.com/zrzka/anes-rs,MIT OR Apache-2.0,Robert Vojta ansi-str,https://github.com/zhiburt/ansi-str,MIT,Maxim Zhiburt @@ -20,19 +24,42 @@ anstyle-wincon,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle anyhow,https://github.com/dtolnay/anyhow,MIT OR Apache-2.0,David Tolnay arc-swap,https://github.com/vorner/arc-swap,MIT OR Apache-2.0,Michal 'vorner' Vaner arrayvec,https://github.com/bluss/arrayvec,MIT OR Apache-2.0,bluss +arrow,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-arith,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-array,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-buffer,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-cast,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-data,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-ipc,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-ord,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-row,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-schema,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-select,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-string,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +ascii-canvas,https://github.com/lalrpop/ascii-canvas,Apache-2.0 OR MIT,Niko Matsakis assert-json-diff,https://github.com/davidpdrsn/assert-json-diff,MIT,David Pedersen +async-channel,https://github.com/smol-rs/async-channel,Apache-2.0 OR MIT,Stjepan Glavina async-compression,https://github.com/Nullus157/async-compression,MIT OR Apache-2.0,"Wim Looman , Allen Bui " +async-io,https://github.com/smol-rs/async-io,Apache-2.0 OR MIT,Stjepan Glavina +async-lock,https://github.com/smol-rs/async-lock,Apache-2.0 OR MIT,Stjepan Glavina +async-process,https://github.com/smol-rs/async-process,Apache-2.0 OR MIT,Stjepan Glavina +async-signal,https://github.com/smol-rs/async-signal,Apache-2.0 OR MIT,John Nunley async-speed-limit,https://github.com/tikv/async-speed-limit,MIT OR Apache-2.0,The TiKV Project Developers async-stream,https://github.com/tokio-rs/async-stream,MIT,Carl Lerche async-stream-impl,https://github.com/tokio-rs/async-stream,MIT,Carl Lerche +async-task,https://github.com/smol-rs/async-task,Apache-2.0 OR MIT,Stjepan Glavina async-trait,https://github.com/dtolnay/async-trait,MIT OR Apache-2.0,David Tolnay +atoi,https://github.com/pacman82/atoi-rs,MIT,Markus Klein atomic-waker,https://github.com/smol-rs/atomic-waker,Apache-2.0 OR MIT,"Stjepan Glavina , Contributors to futures-rs" aws-config,https://github.com/smithy-lang/smithy-rs,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-credential-types,https://github.com/smithy-lang/smithy-rs,Apache-2.0,AWS Rust SDK Team aws-lc-rs,https://github.com/aws/aws-lc-rs,ISC AND (Apache-2.0 OR ISC),AWS-LibCrypto aws-lc-sys,https://github.com/aws/aws-lc-rs,ISC AND (Apache-2.0 OR ISC) AND OpenSSL,AWS-LC aws-runtime,https://github.com/smithy-lang/smithy-rs,Apache-2.0,AWS Rust SDK Team +aws-sdk-kinesis,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " +aws-sdk-lambda,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-s3,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " +aws-sdk-sqs,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-sso,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-ssooidc,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-sts,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " @@ -51,123 +78,185 @@ aws-smithy-runtime-api,https://github.com/smithy-lang/smithy-rs,Apache-2.0,"AWS aws-smithy-types,https://github.com/smithy-lang/smithy-rs,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-smithy-xml,https://github.com/smithy-lang/smithy-rs,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-types,https://github.com/smithy-lang/smithy-rs,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " -aws_lambda_events,https://github.com/awslabs/aws-lambda-rust-runtime,MIT,"Christian Legnitto , Sam Rijs , David Calavera " axum,https://github.com/tokio-rs/axum,MIT,The axum Authors axum-core,https://github.com/tokio-rs/axum,MIT,The axum-core Authors +azure_core,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft Corp. +azure_identity,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft Corp. +azure_storage,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft Corp. +azure_storage_blobs,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft Corp. +azure_svc_blobstorage,https://github.com/azure/azure-sdk-for-rust,MIT,The azure_svc_blobstorage Authors +backon,https://github.com/Xuanwo/backon,Apache-2.0,The backon Authors backtrace,https://github.com/rust-lang/backtrace-rs,MIT OR Apache-2.0,The Rust Project Developers +base16,https://github.com/thomcc/rust-base16,CC0-1.0,Thom Chiovoloni base16ct,https://github.com/RustCrypto/formats/tree/master/base16ct,Apache-2.0 OR MIT,RustCrypto Developers +base62,https://github.com/fbernier/base62,MIT,"François Bernier , Chai T. Rex " base64,https://github.com/marshallpierce/rust-base64,MIT OR Apache-2.0,"Alice Maz , Marshall Pierce " base64,https://github.com/marshallpierce/rust-base64,MIT OR Apache-2.0,Marshall Pierce base64-simd,https://github.com/Nugine/simd,MIT,The base64-simd Authors base64ct,https://github.com/RustCrypto/formats,Apache-2.0 OR MIT,RustCrypto Developers -bincode,https://github.com/servo/bincode,MIT,"Ty Overby , Francesco Mazzoli , David Tolnay , Zoey Riordan " bit-set,https://github.com/contain-rs/bit-set,Apache-2.0 OR MIT,Alexis Beingessner bit-vec,https://github.com/contain-rs/bit-vec,Apache-2.0 OR MIT,Alexis Beingessner bitflags,https://github.com/bitflags/bitflags,MIT OR Apache-2.0,The Rust Project Developers bitpacking,https://github.com/quickwit-oss/bitpacking,MIT,Paul Masurel block-buffer,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers +block-padding,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers +blocking,https://github.com/smol-rs/blocking,Apache-2.0 OR MIT,Stjepan Glavina bon,https://github.com/elastio/bon,MIT OR Apache-2.0,The bon Authors bon-macros,https://github.com/elastio/bon,MIT OR Apache-2.0,The bon-macros Authors +borrow-or-share,https://github.com/yescallop/borrow-or-share,MIT-0,Scallop Ye bpu_trasher,https://github.com/pseitz/bpu_trasher,MIT,Pascal Seitz bs58,https://github.com/Nullus157/bs58-rs,MIT OR Apache-2.0,The bs58 Authors bumpalo,https://github.com/fitzgen/bumpalo,MIT OR Apache-2.0,Nick Fitzgerald bytecount,https://github.com/llogiq/bytecount,Apache-2.0 OR MIT,"Andre Bogus , Joshua Landau " +bytemuck,https://github.com/Lokathor/bytemuck,Zlib OR Apache-2.0 OR MIT,Lokathor byteorder,https://github.com/BurntSushi/byteorder,Unlicense OR MIT,Andrew Gallant bytes,https://github.com/tokio-rs/bytes,MIT,"Carl Lerche , Sean McArthur " bytes-utils,https://github.com/vorner/bytes-utils,Apache-2.0 OR MIT,Michal 'vorner' Vaner -bytesize,https://github.com/bytesize-rs/bytesize,Apache-2.0,Hyunsik Choi +bytesize,https://github.com/bytesize-rs/bytesize,Apache-2.0,"Hyunsik Choi , MrCroxx , Rob Ede " bytestring,https://github.com/actix/actix-net,MIT OR Apache-2.0,"Nikolay Kim , Rob Ede " -bzip2,https://github.com/alexcrichton/bzip2-rs,MIT OR Apache-2.0,Alex Crichton -bzip2-sys,https://github.com/alexcrichton/bzip2-rs,MIT OR Apache-2.0,Alex Crichton +camino,https://github.com/camino-rs/camino,MIT OR Apache-2.0,"Without Boats , Ashley Williams , Steve Klabnik , Rain " +cargo-platform,https://github.com/rust-lang/cargo,MIT OR Apache-2.0,The cargo-platform Authors +cargo_metadata,https://github.com/oli-obk/cargo_metadata,MIT,Oliver Schneider cast,https://github.com/japaric/cast.rs,MIT OR Apache-2.0,Jorge Aparicio +cbc,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers cbor-diag,https://github.com/Nullus157/cbor-diag-rs,MIT OR Apache-2.0,The cbor-diag Authors cc,https://github.com/rust-lang/cc-rs,MIT OR Apache-2.0,Alex Crichton census,https://github.com/quickwit-inc/census,MIT,Paul Masurel cexpr,https://github.com/jethrogb/rust-cexpr,Apache-2.0 OR MIT,Jethro Beekman +cfb-mode,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers cfg-if,https://github.com/rust-lang/cfg-if,MIT OR Apache-2.0,Alex Crichton +chacha20,https://github.com/RustCrypto/stream-ciphers,Apache-2.0 OR MIT,RustCrypto Developers +chacha20poly1305,https://github.com/RustCrypto/AEADs/tree/master/chacha20poly1305,Apache-2.0 OR MIT,RustCrypto Developers +charset,https://github.com/hsivonen/charset,Apache-2.0 OR MIT,Henri Sivonen chitchat,https://github.com/quickwit-oss/chitchat,MIT,"Quickwit, Inc. " chrono,https://github.com/chronotope/chrono,MIT OR Apache-2.0,The chrono Authors +chrono-tz,https://github.com/chronotope/chrono-tz,MIT OR Apache-2.0,The chrono-tz Authors ciborium,https://github.com/enarx/ciborium,Apache-2.0,Nathaniel McCallum ciborium-io,https://github.com/enarx/ciborium,Apache-2.0,Nathaniel McCallum ciborium-ll,https://github.com/enarx/ciborium,Apache-2.0,Nathaniel McCallum +cidr,https://github.com/stbuehler/rust-cidr,MIT,Stefan Bühler cipher,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers clang-sys,https://github.com/KyleMayes/clang-sys,Apache-2.0,Kyle Mayes clap,https://github.com/clap-rs/clap,MIT OR Apache-2.0,The clap Authors clap_builder,https://github.com/clap-rs/clap,MIT OR Apache-2.0,The clap_builder Authors clap_lex,https://github.com/clap-rs/clap,MIT OR Apache-2.0,The clap_lex Authors +cmac,https://github.com/RustCrypto/MACs,MIT OR Apache-2.0,RustCrypto Developers coarsetime,https://github.com/jedisct1/rust-coarsetime,ISC,Frank Denis cobs,https://github.com/jamesmunns/cobs.rs,MIT OR Apache-2.0,"Allen Welkie <>, James Munns " +codespan-reporting,https://github.com/brendanzab/codespan,Apache-2.0,Brendan Zabarauskas colorchoice,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The colorchoice Authors colored,https://github.com/mackwic/colored,MPL-2.0,Thomas Wickham -console,https://github.com/console-rs/console,MIT,Armin Ronacher +community-id,https://github.com/traceflight/rs-community-id,MIT OR Apache-2.0,Julian Wang +compression-codecs,https://github.com/Nullus157/async-compression,MIT OR Apache-2.0,"Wim Looman , Allen Bui " +compression-core,https://github.com/Nullus157/async-compression,MIT OR Apache-2.0,"Wim Looman , Allen Bui " +concurrent-queue,https://github.com/smol-rs/concurrent-queue,Apache-2.0 OR MIT,"Stjepan Glavina , Taiki Endo , John Nunley " +console,https://github.com/console-rs/console,MIT,The console Authors +console-api,https://github.com/tokio-rs/console,MIT,"Eliza Weisman , Tokio Contributors " +console-subscriber,https://github.com/tokio-rs/console,MIT,"Eliza Weisman , Tokio Contributors " const-oid,https://github.com/RustCrypto/formats/tree/master/const-oid,Apache-2.0 OR MIT,RustCrypto Developers -constant_time_eq,https://github.com/cesarb/constant_time_eq,CC0-1.0,Cesar Eduardo Barros +const-random,https://github.com/tkaitchuck/constrandom,MIT OR Apache-2.0,Tom Kaitchuck +const-random-macro,https://github.com/tkaitchuck/constrandom,MIT OR Apache-2.0,Tom Kaitchuck +const_fn,https://github.com/taiki-e/const_fn,Apache-2.0 OR MIT,The const_fn Authors +convert_case,https://github.com/rutrum/convert-case,MIT,rutrum core-foundation,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers core-foundation-sys,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers +cpp_demangle,https://github.com/gimli-rs/cpp_demangle,MIT OR Apache-2.0,"Nick Fitzgerald , Jim Blandy , Kyle Huey " cpufeatures,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers +crc,https://github.com/mrhooray/crc-rs,MIT OR Apache-2.0,"Rui Hu , Akhil Velagapudi <4@4khil.com>" +crc-catalog,https://github.com/akhilles/crc-catalog,MIT OR Apache-2.0,Akhil Velagapudi crc32c,https://github.com/zowens/crc32c,Apache-2.0 OR MIT,Zack Owens crc32fast,https://github.com/srijs/rust-crc32fast,MIT OR Apache-2.0,"Sam Rijs , Alex Crichton " -criterion-plot,https://github.com/bheisler/criterion.rs,MIT OR Apache-2.0,"Jorge Aparicio , Brook Heisler " +criterion-plot,https://github.com/criterion-rs/criterion.rs,Apache-2.0 OR MIT,"Jorge Aparicio , Brook Heisler " cron,https://github.com/zslayton/cron,MIT OR Apache-2.0,Zack Slayton crossbeam-channel,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-channel Authors crossbeam-deque,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-deque Authors crossbeam-epoch,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-epoch Authors +crossbeam-queue,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-queue Authors crossbeam-utils,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-utils Authors crunchy,https://github.com/eira-fransham/crunchy,MIT,Eira Fransham crypto-bigint,https://github.com/RustCrypto/crypto-bigint,Apache-2.0 OR MIT,RustCrypto Developers crypto-common,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers +crypto_secretbox,https://github.com/RustCrypto/nacl-compat/tree/master/crypto_secretbox,Apache-2.0 OR MIT,RustCrypto Developers csv,https://github.com/BurntSushi/rust-csv,Unlicense OR MIT,Andrew Gallant csv-core,https://github.com/BurntSushi/rust-csv,Unlicense OR MIT,Andrew Gallant +ctr,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers +curve25519-dalek,https://github.com/dalek-cryptography/curve25519-dalek/tree/main/curve25519-dalek,BSD-3-Clause,"Isis Lovecruft , Henry de Valence " +curve25519-dalek-derive,https://github.com/dalek-cryptography/curve25519-dalek,MIT OR Apache-2.0,The curve25519-dalek-derive Authors darling,https://github.com/TedDriggs/darling,MIT,Ted Driggs darling_core,https://github.com/TedDriggs/darling,MIT,Ted Driggs darling_macro,https://github.com/TedDriggs/darling,MIT,Ted Driggs +dashmap,https://github.com/xacrimon/dashmap,MIT,Acrimon data-encoding,https://github.com/ia0/data-encoding,MIT,Julien Cretin +data-url,https://github.com/servo/rust-url,MIT OR Apache-2.0,Simon Sapin +datasketches,https://github.com/apache/datasketches-rust,Apache-2.0,The datasketches Authors +dbl,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers deadpool,https://github.com/bikeshedder/deadpool,MIT OR Apache-2.0,Michael P. Jung deadpool-runtime,https://github.com/bikeshedder/deadpool,MIT OR Apache-2.0,Michael P. Jung +debugid,https://github.com/getsentry/rust-debugid,Apache-2.0,Sentry der,https://github.com/RustCrypto/formats/tree/master/der,Apache-2.0 OR MIT,RustCrypto Developers deranged,https://github.com/jhpratt/deranged,MIT OR Apache-2.0,Jacob Pratt -dialoguer,https://github.com/mitsuhiko/dialoguer,MIT,"Armin Ronacher , Pavan Kumar Sunkara " +derivative,https://github.com/mcarton/rust-derivative,MIT OR Apache-2.0,mcarton +dialoguer,https://github.com/console-rs/dialoguer,MIT,The dialoguer Authors diff,https://github.com/utkarshkukreti/diff.rs,MIT OR Apache-2.0,Utkarsh Kukreti difflib,https://github.com/DimaKudosh/difflib,MIT,Dima Kudosh digest,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers displaydoc,https://github.com/yaahc/displaydoc,MIT OR Apache-2.0,Jane Lusby +dns-lookup,https://github.com/keeperofdakeys/dns-lookup,MIT OR Apache-2.0,Josh Driver +doc-comment,https://github.com/GuillaumeGomez/doc-comment,MIT,Guillaume Gomez +domain,https://github.com/nlnetlabs/domain,BSD-3-Clause,NLnet Labs +domain-macros,https://github.com/nlnetlabs/domain,BSD-3-Clause,NLnet Labs +dotenvy,https://github.com/allan2/dotenvy,MIT,"Noemi Lapresta , Craig Hills , Mike Piccolo , Alice Maz , Sean Griffin , Adam Sharp , Arpad Borsos , Allan Zhang " downcast,https://github.com/fkoep/downcast-rs,MIT,Felix Köpge downcast-rs,https://github.com/marcianx/downcast-rs,MIT OR Apache-2.0,The downcast-rs Authors dtoa,https://github.com/dtolnay/dtoa,MIT OR Apache-2.0,David Tolnay dyn-clone,https://github.com/dtolnay/dyn-clone,MIT OR Apache-2.0,David Tolnay ecdsa,https://github.com/RustCrypto/signatures/tree/master/ecdsa,Apache-2.0 OR MIT,RustCrypto Developers +ed25519,https://github.com/RustCrypto/signatures/tree/master/ed25519,Apache-2.0 OR MIT,RustCrypto Developers +ed25519-dalek,https://github.com/dalek-cryptography/curve25519-dalek/tree/main/ed25519-dalek,BSD-3-Clause,"isis lovecruft , Tony Arcieri , Michael Rosenberg " either,https://github.com/rayon-rs/either,MIT OR Apache-2.0,bluss -elasticsearch-dsl,https://github.com/vinted/elasticsearch-dsl-rs,MIT OR Apache-2.0,"Evaldas Buinauskas , Boost " +elasticsearch-dsl,https://github.com/vinted/elasticsearch-dsl-rs,MIT OR Apache-2.0,"Evaldas Buinauskas , Search Platform " elliptic-curve,https://github.com/RustCrypto/traits/tree/master/elliptic-curve,Apache-2.0 OR MIT,RustCrypto Developers +email_address,https://github.com/johnstonskj/rust-email_address,MIT,Simon Johnston embedded-io,https://github.com/embassy-rs/embedded-io,MIT OR Apache-2.0,The embedded-io Authors embedded-io,https://github.com/rust-embedded/embedded-hal,MIT OR Apache-2.0,The embedded-io Authors +ena,https://github.com/rust-lang/ena,MIT OR Apache-2.0,Niko Matsakis encode_unicode,https://github.com/tormol/encode_unicode,Apache-2.0 OR MIT,Torbjørn Birch Moltu -encoding,https://github.com/lifthrasiir/rust-encoding,MIT,Kang Seonghoon -encoding-index-japanese,https://github.com/lifthrasiir/rust-encoding,CC0-1.0,Kang Seonghoon -encoding-index-korean,https://github.com/lifthrasiir/rust-encoding,CC0-1.0,Kang Seonghoon -encoding-index-simpchinese,https://github.com/lifthrasiir/rust-encoding,CC0-1.0,Kang Seonghoon -encoding-index-singlebyte,https://github.com/lifthrasiir/rust-encoding,CC0-1.0,Kang Seonghoon -encoding-index-tradchinese,https://github.com/lifthrasiir/rust-encoding,CC0-1.0,Kang Seonghoon -encoding_index_tests,https://github.com/lifthrasiir/rust-encoding,CC0-1.0,Kang Seonghoon encoding_rs,https://github.com/hsivonen/encoding_rs,(Apache-2.0 OR MIT) AND BSD-3-Clause,Henri Sivonen -encoding_rs_io,https://github.com/BurntSushi/encoding_rs_io,MIT OR Apache-2.0,Andrew Gallant -enum-iterator,https://github.com/stephaneyfx/enum-iterator,MIT,Stephane Raux +enum-iterator,https://github.com/stephaneyfx/enum-iterator,0BSD,Stephane Raux enum-iterator-derive,https://github.com/stephaneyfx/enum-iterator,0BSD,Stephane Raux +env_filter,https://github.com/rust-cli/env_logger,MIT OR Apache-2.0,The env_filter Authors env_logger,https://github.com/rust-cli/env_logger,MIT OR Apache-2.0,The env_logger Authors +equator,https://github.com/sarah-ek/equator,MIT,sarah <> +equator-macro,https://github.com/sarah-ek/equator,MIT,sarah <> equivalent,https://github.com/indexmap-rs/equivalent,Apache-2.0 OR MIT,The equivalent Authors +erased-serde,https://github.com/dtolnay/erased-serde,MIT OR Apache-2.0,David Tolnay errno,https://github.com/lambda-fairy/rust-errno,MIT OR Apache-2.0,"Chris Wong , Dan Gohman " +error-chain,https://github.com/rust-lang-nursery/error-chain,MIT OR Apache-2.0,"Brian Anderson , Paul Colomiets , Colin Kiegel , Yamakaky , Andrew Gauger " +etcetera,https://github.com/lunacookies/etcetera,MIT OR Apache-2.0,The etcetera Authors +event-listener,https://github.com/smol-rs/event-listener,Apache-2.0 OR MIT,Stjepan Glavina +event-listener,https://github.com/smol-rs/event-listener,Apache-2.0 OR MIT,"Stjepan Glavina , John Nunley " +event-listener-strategy,https://github.com/smol-rs/event-listener-strategy,Apache-2.0 OR MIT,John Nunley fail,https://github.com/tikv/fail-rs,Apache-2.0,The TiKV Project Developers +fancy-regex,https://github.com/fancy-regex/fancy-regex,MIT,"Raph Levien , Robin Stocker " +fancy-regex,https://github.com/fancy-regex/fancy-regex,MIT,"Raph Levien , Robin Stocker , Keith Hall " fastdivide,https://github.com/fulmicoton/fastdivide,zlib-acknowledgement OR MIT,Paul Masurel fastrand,https://github.com/smol-rs/fastrand,Apache-2.0 OR MIT,Stjepan Glavina ff,https://github.com/zkcrypto/ff,MIT OR Apache-2.0,"Sean Bowe , Jack Grigg " -filetime,https://github.com/alexcrichton/filetime,MIT OR Apache-2.0,Alex Crichton +fiat-crypto,https://github.com/mit-plv/fiat-crypto,MIT OR Apache-2.0 OR BSD-1-Clause,Fiat Crypto library authors +find-msvc-tools,https://github.com/rust-lang/cc-rs,MIT OR Apache-2.0,The find-msvc-tools Authors +findshlibs,https://github.com/gimli-rs/findshlibs,MIT OR Apache-2.0,The findshlibs Authors fixedbitset,https://github.com/petgraph/fixedbitset,MIT OR Apache-2.0,bluss +flatbuffers,https://github.com/google/flatbuffers,Apache-2.0,"Robert Winslow , FlatBuffers Maintainers" flate2,https://github.com/rust-lang/flate2-rs,MIT OR Apache-2.0,"Alex Crichton , Josh Triplett " float-cmp,https://github.com/mikedilger/float-cmp,MIT,Mike Dilger +fluent-uri,https://github.com/yescallop/fluent-uri-rs,MIT,Scallop Ye flume,https://github.com/zesterer/flume,Apache-2.0 OR MIT,Joshua Barretto fnv,https://github.com/servo/rust-fnv,Apache-2.0 OR MIT,Alex Crichton foldhash,https://github.com/orlp/foldhash,Zlib,Orson Peters +foreign-types,https://github.com/sfackler/foreign-types,MIT OR Apache-2.0,Steven Fackler +foreign-types-shared,https://github.com/sfackler/foreign-types,MIT OR Apache-2.0,Steven Fackler form_urlencoded,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers +fraction,https://github.com/dnsl48/fraction,MIT OR Apache-2.0,dnsl48 fragile,https://github.com/mitsuhiko/fragile,Apache-2.0,Armin Ronacher fs4,https://github.com/al8n/fs4-rs,MIT OR Apache-2.0,"Dan Burkert , Al Liu " fslock,https://github.com/brunoczim/fslock,MIT,The fslock Authors @@ -175,7 +264,9 @@ futures,https://github.com/rust-lang/futures-rs,MIT OR Apache-2.0,The futures Au futures-channel,https://github.com/rust-lang/futures-rs,MIT OR Apache-2.0,The futures-channel Authors futures-core,https://github.com/rust-lang/futures-rs,MIT OR Apache-2.0,The futures-core Authors futures-executor,https://github.com/rust-lang/futures-rs,MIT OR Apache-2.0,The futures-executor Authors +futures-intrusive,https://github.com/Matthias247/futures-intrusive,MIT OR Apache-2.0,Matthias Einwag futures-io,https://github.com/rust-lang/futures-rs,MIT OR Apache-2.0,The futures-io Authors +futures-lite,https://github.com/smol-rs/futures-lite,Apache-2.0 OR MIT,"Stjepan Glavina , Contributors to futures-rs" futures-macro,https://github.com/rust-lang/futures-rs,MIT OR Apache-2.0,The futures-macro Authors futures-sink,https://github.com/rust-lang/futures-rs,MIT OR Apache-2.0,The futures-sink Authors futures-task,https://github.com/rust-lang/futures-rs,MIT OR Apache-2.0,The futures-task Authors @@ -185,33 +276,45 @@ generic-array,https://github.com/fizyk20/generic-array,MIT,"Bartłomiej Kamińsk getrandom,https://github.com/rust-random/getrandom,MIT OR Apache-2.0,The Rand Project Developers gimli,https://github.com/gimli-rs/gimli,MIT OR Apache-2.0,The gimli Authors glob,https://github.com/rust-lang/glob,MIT OR Apache-2.0,The Rust Project Developers +gloo-timers,https://github.com/rustwasm/gloo/tree/master/crates/timers,MIT OR Apache-2.0,Rust and WebAssembly Working Group +google-cloud-auth,https://github.com/yoshidan/google-cloud-rust/tree/main/foundation/auth,MIT,yoshidan +google-cloud-gax,https://github.com/yoshidan/google-cloud-rust/tree/main/foundation/gax,MIT,yoshidan +google-cloud-googleapis,https://github.com/yoshidan/google-cloud-rust/tree/main/googleapis,MIT,yoshidan +google-cloud-metadata,https://github.com/yoshidan/google-cloud-rust/tree/main/foundation/metadata,MIT,yoshidan +google-cloud-pubsub,https://github.com/yoshidan/google-cloud-rust/tree/main/pubsub,MIT,yoshidan +google-cloud-token,https://github.com/yoshidan/google-cloud-rust/tree/main/foundation/token,MIT,yoshidan +governor,https://github.com/boinkor-net/governor,MIT,Andreas Fuchs +grok,https://github.com/mmastrac/grok,Apache-2.0,"Matt Mastracci , Michael Nitschinger " group,https://github.com/zkcrypto/group,MIT OR Apache-2.0,"Sean Bowe , Jack Grigg " h2,https://github.com/hyperium/h2,MIT,"Carl Lerche , Sean McArthur " half,https://github.com/VoidStarKat/half-rs,MIT OR Apache-2.0,Kathryn Long hashbrown,https://github.com/rust-lang/hashbrown,MIT OR Apache-2.0,Amanieu d'Antras +hashlink,https://github.com/kyren/hashlink,MIT OR Apache-2.0,kyren +hdrhistogram,https://github.com/HdrHistogram/HdrHistogram_rust,MIT OR Apache-2.0,"Jon Gjengset , Marshall Pierce " headers,https://github.com/hyperium/headers,MIT,Sean McArthur headers-core,https://github.com/hyperium/headers,MIT,Sean McArthur heck,https://github.com/withoutboats/heck,MIT OR Apache-2.0,The heck Authors heck,https://github.com/withoutboats/heck,MIT OR Apache-2.0,Without Boats hermit-abi,https://github.com/hermit-os/hermit-rs,MIT OR Apache-2.0,Stefan Lankes hex,https://github.com/KokaKiwi/rust-hex,MIT OR Apache-2.0,KokaKiwi +hkdf,https://github.com/RustCrypto/KDFs,MIT OR Apache-2.0,RustCrypto Developers hmac,https://github.com/RustCrypto/MACs,MIT OR Apache-2.0,RustCrypto Developers home,https://github.com/rust-lang/cargo,MIT OR Apache-2.0,Brian Anderson -hostname,https://github.com/svartalf/hostname,MIT,"fengcen , svartalf " +hostname,https://github.com/djc/hostname,MIT,The hostname Authors htmlescape,https://github.com/veddan/rust-htmlescape,Apache-2.0 OR MIT OR MPL-2.0,Viktor Dahl http,https://github.com/hyperium/http,MIT OR Apache-2.0,"Alex Crichton , Carl Lerche , Sean McArthur " http-body,https://github.com/hyperium/http-body,MIT,"Carl Lerche , Lucio Franco , Sean McArthur " http-body-util,https://github.com/hyperium/http-body,MIT,"Carl Lerche , Lucio Franco , Sean McArthur " -http-range-header,https://github.com/MarcusGrass/parse-range-headers,MIT,The http-range-header Authors http-serde,https://gitlab.com/kornelski/http-serde,Apache-2.0 OR MIT,Kornel +http-types,https://github.com/http-rs/http-types,MIT OR Apache-2.0,Yoshua Wuyts httparse,https://github.com/seanmonstar/httparse,MIT OR Apache-2.0,Sean McArthur httpdate,https://github.com/pyfisch/httpdate,MIT OR Apache-2.0,Pyfisch humantime,https://github.com/chronotope/humantime,MIT OR Apache-2.0,The humantime Authors hyper,https://github.com/hyperium/hyper,MIT,Sean McArthur hyper-rustls,https://github.com/rustls/hyper-rustls,Apache-2.0 OR ISC OR MIT,The hyper-rustls Authors hyper-timeout,https://github.com/hjr3/hyper-timeout,MIT OR Apache-2.0,Herman J. Radtke III +hyper-tls,https://github.com/hyperium/hyper-tls,MIT OR Apache-2.0,Sean McArthur hyper-util,https://github.com/hyperium/hyper-util,MIT,Sean McArthur -hyperloglogplus,https://github.com/tabac/hyperloglog.rs,MIT,Tasos Bakogiannis iana-time-zone,https://github.com/strawlab/iana-time-zone,MIT OR Apache-2.0,"Andrew Straw , René Kijewski , Ryan Lopopolo " iana-time-zone-haiku,https://github.com/strawlab/iana-time-zone,MIT OR Apache-2.0,René Kijewski icu_collections,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers @@ -227,9 +330,16 @@ idna_adapter,https://github.com/hsivonen/idna_adapter,Apache-2.0 OR MIT,The rust indexmap,https://github.com/bluss/indexmap,Apache-2.0 OR MIT,The indexmap Authors indexmap,https://github.com/indexmap-rs/indexmap,Apache-2.0 OR MIT,The indexmap Authors indicatif,https://github.com/console-rs/indicatif,MIT,The indicatif Authors +indoc,https://github.com/dtolnay/indoc,MIT OR Apache-2.0,David Tolnay +infer,https://github.com/bojand/infer,MIT,Bojan +inferno,https://github.com/jonhoo/inferno,CDDL-1.0,Jon Gjengset +influxdb-line-protocol,https://github.com/influxdata/influxdb_iox/tree/main/influxdb_line_protocol,MIT OR Apache-2.0,InfluxDB IOx Project Developers +inherent,https://github.com/dtolnay/inherent,MIT OR Apache-2.0,David Tolnay inout,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers instant,https://github.com/sebcrozet/instant,BSD-3-Clause,sebcrozet -io-uring,https://github.com/tokio-rs/io-uring,MIT OR Apache-2.0,quininer +integer-encoding,https://github.com/dermesser/integer-encoding-rs,MIT,Lewin Bormann +inventory,https://github.com/dtolnay/inventory,MIT OR Apache-2.0,David Tolnay +ipcrypt-rs,https://github.com/jedisct1/rust-ipcrypt2,ISC,Frank Denis ipnet,https://github.com/krisprice/ipnet,MIT OR Apache-2.0,Kris Price ipnetwork,https://github.com/achanda/ipnetwork,MIT OR Apache-2.0,"Abhishek Chanda , Linus Färnstrand " iri-string,https://github.com/lo48576/iri-string,MIT OR Apache-2.0,YOSHIOKA Takuma @@ -237,32 +347,35 @@ is-terminal,https://github.com/sunfishcode/is-terminal,MIT,"softprops +jiff,https://github.com/BurntSushi/jiff,Unlicense OR MIT,Andrew Gallant +jiff-static,https://github.com/BurntSushi/jiff,Unlicense OR MIT,Andrew Gallant +jiff-tzdb,https://github.com/BurntSushi/jiff,Unlicense OR MIT,Andrew Gallant +jiff-tzdb-platform,https://github.com/BurntSushi/jiff,Unlicense OR MIT,Andrew Gallant jobserver,https://github.com/rust-lang/jobserver-rs,MIT OR Apache-2.0,Alex Crichton -js-sys,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/js-sys,MIT OR Apache-2.0,The wasm-bindgen Developers +js-sys,https://github.com/wasm-bindgen/wasm-bindgen/tree/master/crates/js-sys,MIT OR Apache-2.0,The wasm-bindgen Developers json_comments,https://github.com/tmccombs/json-comments-rs,Apache-2.0,Thayne McCombs -lambda_http,https://github.com/awslabs/aws-lambda-rust-runtime,Apache-2.0,"David Calavera , Harold Sun " +jsonschema,https://github.com/Stranger6667/jsonschema,MIT,Dmitry Dygalo +jsonwebtoken,https://github.com/Keats/jsonwebtoken,MIT,Vincent Prouillet +keccak,https://github.com/RustCrypto/sponges/tree/master/keccak,Apache-2.0 OR MIT,RustCrypto Developers +krb5-src,https://github.com/MaterializeInc/rust-krb5-src,Apache-2.0,"Materialize, Inc." +lalrpop-util,https://github.com/lalrpop/lalrpop,Apache-2.0 OR MIT,Niko Matsakis lambda_runtime,https://github.com/awslabs/aws-lambda-rust-runtime,Apache-2.0,"David Calavera , Harold Sun " lambda_runtime_api_client,https://github.com/awslabs/aws-lambda-rust-runtime,Apache-2.0,"David Calavera , Harold Sun " lazy_static,https://github.com/rust-lang-nursery/lazy-static.rs,MIT OR Apache-2.0,Marvin Löbel -lazycell,https://github.com/indiv0/lazycell,MIT OR Apache-2.0,"Alex Crichton , Nikita Pekin " levenshtein_automata,https://github.com/tantivy-search/levenshtein-automata,MIT,Paul Masurel +lexical-core,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh +lexical-parse-float,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh +lexical-parse-integer,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh +lexical-util,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh +lexical-write-float,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh +lexical-write-integer,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh libc,https://github.com/rust-lang/libc,MIT OR Apache-2.0,The Rust Project Developers libloading,https://github.com/nagisa/rust_libloading,ISC,Simonas Kazlauskas libm,https://github.com/rust-lang/compiler-builtins,MIT,Jorge Aparicio libredox,https://gitlab.redox-os.org/redox-os/libredox,MIT,4lDO2 <4lDO2@protonmail.com> -lindera-cc-cedict,https://github.com/lindera-morphology/lindera,MIT,The lindera-cc-cedict Authors -lindera-cc-cedict-builder,https://github.com/lindera-morphology/lindera,MIT,The lindera-cc-cedict-builder Authors -lindera-compress,https://github.com/lindera-morphology/lindera,MIT,The lindera-compress Authors -lindera-core,https://github.com/lindera-morphology/lindera,MIT,The lindera-core Authors -lindera-decompress,https://github.com/lindera-morphology/lindera,MIT,The lindera-decompress Authors -lindera-dictionary,https://github.com/lindera-morphology/lindera,MIT,The lindera-dictionary Authors -lindera-ipadic,https://github.com/lindera-morphology/lindera,MIT,The lindera-ipadic Authors -lindera-ipadic-builder,https://github.com/lindera-morphology/lindera,MIT,The lindera-ipadic-builder Authors -lindera-ipadic-neologd-builder,https://github.com/lindera-morphology/lindera,MIT,The lindera-ipadic-neologd-builder Authors -lindera-ko-dic,https://github.com/lindera-morphology/lindera,MIT,The lindera-ko-dic Authors -lindera-ko-dic-builder,https://github.com/lindera-morphology/lindera,MIT,The lindera-ko-dic-builder Authors -lindera-tokenizer,https://github.com/lindera-morphology/lindera,MIT,The lindera-tokenizer Authors -lindera-unidic-builder,https://github.com/lindera-morphology/lindera,MIT,The lindera-unidic-builder Authors +libsqlite3-sys,https://github.com/rusqlite/rusqlite,MIT,The rusqlite developers +libz-rs-sys,https://github.com/trifectatechfoundation/zlib-rs,Zlib,The libz-rs-sys Authors +libz-sys,https://github.com/rust-lang/libz-sys,MIT OR Apache-2.0,"Alex Crichton , Josh Triplett , Sebastian Thiel " linked-hash-map,https://github.com/contain-rs/linked-hash-map,MIT OR Apache-2.0,"Stepan Koltsov , Andrew Paseltiner " linux-raw-sys,https://github.com/sunfishcode/linux-raw-sys,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Dan Gohman litemap,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers @@ -270,8 +383,9 @@ lock_api,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'Antr log,https://github.com/rust-lang/log,MIT OR Apache-2.0,The Rust Project Developers lru,https://github.com/jeromefroe/lru-rs,MIT,Jerome Froelich lru-slab,https://github.com/Ralith/lru-slab,MIT OR Apache-2.0 OR Zlib,Benjamin Saunders +lz4,https://github.com/10xGenomics/lz4-rs,MIT,"Jens Heyens , Artem V. Navrotskiy , Patrick Marks " +lz4-sys,https://github.com/10xGenomics/lz4-rs,MIT,"Jens Heyens , Artem V. Navrotskiy , Patrick Marks " lz4_flex,https://github.com/pseitz/lz4_flex,MIT,"Pascal Seitz , Arthur Silva , ticki " -match_cfg,https://github.com/gnzlbg/match_cfg,MIT OR Apache-2.0,gnzlbg matchers,https://github.com/hawkw/matchers,MIT,Eliza Weisman matchit,https://github.com/ibraheemdev/matchit,MIT AND BSD-3-Clause,Ibraheem Ahmed md-5,https://github.com/RustCrypto/hashes,MIT OR Apache-2.0,RustCrypto Developers @@ -282,64 +396,114 @@ memmap2,https://github.com/RazrFalcon/memmap2-rs,MIT OR Apache-2.0,"Dan Burkert mime,https://github.com/hyperium/mime,MIT OR Apache-2.0,Sean McArthur mime_guess,https://github.com/abonander/mime_guess,MIT,Austin Bonander mini-internal,https://github.com/dtolnay/miniserde,MIT OR Apache-2.0,David Tolnay +mini-moka,https://github.com/moka-rs/mini-moka,MIT OR Apache-2.0,The mini-moka Authors minimal-lexical,https://github.com/Alexhuszagh/minimal-lexical,MIT OR Apache-2.0,Alex Huszagh miniserde,https://github.com/dtolnay/miniserde,MIT OR Apache-2.0,David Tolnay miniz_oxide,https://github.com/Frommi/miniz_oxide/tree/master/miniz_oxide,MIT OR Zlib OR Apache-2.0,"Frommi , oyvindln , Rich Geldreich richgel99@gmail.com" mio,https://github.com/tokio-rs/mio,MIT,"Carl Lerche , Thomas de Zeeuw , Tokio Contributors " mockall,https://github.com/asomers/mockall,MIT OR Apache-2.0,Alan Somers mockall_derive,https://github.com/asomers/mockall,MIT OR Apache-2.0,Alan Somers +moka,https://github.com/moka-rs/moka,(MIT OR Apache-2.0) AND Apache-2.0,The moka Authors mrecordlog,https://github.com/quickwit-oss/mrecordlog,MIT,The mrecordlog Authors -multer,https://github.com/rousan/multer-rs,MIT,Rousan Ali multimap,https://github.com/havarnov/multimap,MIT OR Apache-2.0,Håvar Nøvik +murmur3,https://github.com/stusmall/murmur3,MIT OR Apache-2.0,Stu Small murmurhash32,https://github.com/quickwit-inc/murmurhash32,MIT,Paul Masurel -nanorand,https://github.com/Absolucy/nanorand-rs,Zlib,Lucy +native-tls,https://github.com/sfackler/rust-native-tls,MIT OR Apache-2.0,Steven Fackler +new_debug_unreachable,https://github.com/mbrubeck/rust-debug-unreachable,MIT,"Matt Brubeck , Jonathan Reem " new_string_template,https://github.com/hasezoey/new_string_template,MIT,hasezoey +nix,https://github.com/nix-rust/nix,MIT,The nix-rust Project Developers no-std-net,https://github.com/dunmatt/no-std-net,MIT,M@ Dunlap +nohash,https://github.com/tetcoin/nohash,Apache-2.0 OR MIT,Parity Technologies nom,https://github.com/Geal/nom,MIT,contact@geoffroycouprie.com +nom,https://github.com/rust-bakery/nom,MIT,contact@geoffroycouprie.com +nom-language,https://github.com/rust-bakery/nom,MIT,contact@geoffroycouprie.com +nonzero_ext,https://github.com/antifuchs/nonzero_ext,Apache-2.0,Andreas Fuchs normalize-line-endings,https://github.com/derekdreery/normalize-line-endings,Apache-2.0,Richard Dodd -ntapi,https://github.com/MSxDOS/ntapi,Apache-2.0 OR MIT,MSxDOS nu-ansi-term,https://github.com/nushell/nu-ansi-term,MIT,"ogham@bsago.me, Ryan Scheel (Havvy) , Josh Triplett , The Nushell Project Developers" +num,https://github.com/rust-num/num,MIT OR Apache-2.0,The Rust Project Developers num-bigint,https://github.com/rust-num/num-bigint,MIT OR Apache-2.0,The Rust Project Developers +num-bigint-dig,https://github.com/dignifiedquire/num-bigint,MIT OR Apache-2.0,"dignifiedquire , The Rust Project Developers" +num-cmp,https://github.com/lifthrasiir/num-cmp,MIT OR Apache-2.0,Kang Seonghoon +num-complex,https://github.com/rust-num/num-complex,MIT OR Apache-2.0,The Rust Project Developers num-conv,https://github.com/jhpratt/num-conv,MIT OR Apache-2.0,Jacob Pratt +num-format,https://github.com/bcmyers/num-format,MIT OR Apache-2.0,Brian Myers num-integer,https://github.com/rust-num/num-integer,MIT OR Apache-2.0,The Rust Project Developers +num-iter,https://github.com/rust-num/num-iter,MIT OR Apache-2.0,The Rust Project Developers num-rational,https://github.com/rust-num/num-rational,MIT OR Apache-2.0,The Rust Project Developers num-traits,https://github.com/rust-num/num-traits,MIT OR Apache-2.0,The Rust Project Developers num_cpus,https://github.com/seanmonstar/num_cpus,MIT OR Apache-2.0,Sean McArthur -number_prefix,https://github.com/ogham/rust-number-prefix,MIT,Benjamin Sago -numfmt,https://github.com/kdr-aus/numfmt,MIT,Kurt Lawrence +num_enum,https://github.com/illicitonion/num_enum,BSD-3-Clause OR MIT OR Apache-2.0,"Daniel Wagner-Hall , Daniel Henry-Mantilla , Vincent Esche " +num_enum_derive,https://github.com/illicitonion/num_enum,BSD-3-Clause OR MIT OR Apache-2.0,"Daniel Wagner-Hall , Daniel Henry-Mantilla , Vincent Esche " +num_threads,https://github.com/jhpratt/num_threads,MIT OR Apache-2.0,Jacob Pratt +numfmt,https://github.com/kurtlawrence/numfmt,MIT,Kurt Lawrence +oauth2,https://github.com/ramosbugs/oauth2-rs,MIT OR Apache-2.0,"Alex Crichton , Florin Lipan , David A. Ramos " +objc2-core-foundation,https://github.com/madsmtm/objc2,Zlib OR Apache-2.0 OR MIT,The objc2-core-foundation Authors +objc2-io-kit,https://github.com/madsmtm/objc2,Zlib OR Apache-2.0 OR MIT,The objc2-io-kit Authors object,https://github.com/gimli-rs/object,Apache-2.0 OR MIT,The object Authors +octseq,https://github.com/NLnetLabs/octets/,BSD-3-Clause,NLnet Labs +ofb,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers once_cell,https://github.com/matklad/once_cell,MIT OR Apache-2.0,Aleksey Kladov once_cell_polyfill,https://github.com/polyfill-rs/once_cell_polyfill,MIT OR Apache-2.0,The once_cell_polyfill Authors oneshot,https://github.com/faern/oneshot,MIT OR Apache-2.0,Linus Färnstrand +onig,https://github.com/iwillspeak/rust-onig,MIT,"Will Speak , Ivan Ivashchenko " +onig_sys,https://github.com/iwillspeak/rust-onig,MIT,"Will Speak , Ivan Ivashchenko " oorandom,https://hg.sr.ht/~icefox/oorandom,MIT,Simon Heath +opaque-debug,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers +opendal,https://github.com/apache/opendal,Apache-2.0,Apache OpenDAL +openidconnect,https://github.com/ramosbugs/openidconnect-rs,MIT,David A. Ramos +openssl,https://github.com/rust-openssl/rust-openssl,Apache-2.0,Steven Fackler +openssl-macros,https://github.com/sfackler/rust-openssl,MIT OR Apache-2.0,The openssl-macros Authors openssl-probe,https://github.com/alexcrichton/openssl-probe,MIT OR Apache-2.0,Alex Crichton -opentelemetry,https://github.com/open-telemetry/opentelemetry-rust,Apache-2.0,The opentelemetry Authors -opentelemetry-http,https://github.com/open-telemetry/opentelemetry-rust,Apache-2.0,The opentelemetry-http Authors +openssl-sys,https://github.com/rust-openssl/rust-openssl,MIT,"Alex Crichton , Steven Fackler " +opentelemetry,https://github.com/open-telemetry/opentelemetry-rust/tree/main/opentelemetry,Apache-2.0,The opentelemetry Authors +opentelemetry-appender-tracing,https://github.com/open-telemetry/opentelemetry-rust/tree/main/opentelemetry-appender-tracing,Apache-2.0,The opentelemetry-appender-tracing Authors +opentelemetry-http,https://github.com/open-telemetry/opentelemetry-rust/tree/main/opentelemetry-http,Apache-2.0,The opentelemetry-http Authors opentelemetry-otlp,https://github.com/open-telemetry/opentelemetry-rust/tree/main/opentelemetry-otlp,Apache-2.0,The opentelemetry-otlp Authors opentelemetry-proto,https://github.com/open-telemetry/opentelemetry-rust/tree/main/opentelemetry-proto,Apache-2.0,The opentelemetry-proto Authors -opentelemetry_sdk,https://github.com/open-telemetry/opentelemetry-rust,Apache-2.0,The opentelemetry_sdk Authors +opentelemetry_sdk,https://github.com/open-telemetry/opentelemetry-rust/tree/main/opentelemetry-sdk,Apache-2.0,The opentelemetry_sdk Authors +ordered-float,https://github.com/reem/rust-ordered-float,MIT,"Jonathan Reem , Matt Brubeck " +os_pipe,https://github.com/oconnor663/os_pipe.rs,MIT,Jack O'Connor ouroboros,https://github.com/someguynamedjosh/ouroboros,MIT OR Apache-2.0,Josh ouroboros_macro,https://github.com/someguynamedjosh/ouroboros,MIT OR Apache-2.0,Josh outref,https://github.com/Nugine/outref,MIT,The outref Authors -overload,https://github.com/danaugrs/overload,MIT,Daniel Salvadori ownedbytes,https://github.com/quickwit-oss/tantivy,MIT,"Paul Masurel , Pascal Seitz " p256,https://github.com/RustCrypto/elliptic-curves/tree/master/p256,Apache-2.0 OR MIT,RustCrypto Developers +p384,https://github.com/RustCrypto/elliptic-curves/tree/master/p384,Apache-2.0 OR MIT,"RustCrypto Developers, Frank Denis " +page_size,https://github.com/Elzair/page_size_rs,MIT OR Apache-2.0,Philip Woods papergrid,https://github.com/zhiburt/tabled,MIT,Maxim Zhiburt +parking,https://github.com/smol-rs/parking,Apache-2.0 OR MIT,"Stjepan Glavina , The Rust Project Developers" parking_lot,https://github.com/Amanieu/parking_lot,Apache-2.0 OR MIT,Amanieu d'Antras parking_lot,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'Antras parking_lot_core,https://github.com/Amanieu/parking_lot,Apache-2.0 OR MIT,Amanieu d'Antras parking_lot_core,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'Antras -password-hash,https://github.com/RustCrypto/traits/tree/master/password-hash,MIT OR Apache-2.0,RustCrypto Developers +parquet,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +parquet-variant,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +parquet-variant-compute,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +parquet-variant-json,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +parse-size,https://github.com/kennytm/parse-size,MIT,kennytm +paste,https://github.com/dtolnay/paste,MIT OR Apache-2.0,David Tolnay pbkdf2,https://github.com/RustCrypto/password-hashes/tree/master/pbkdf2,MIT OR Apache-2.0,RustCrypto Developers peakmem-alloc,https://github.com/PSeitz/peakmem-alloc,MIT,Pascal Seitz +peeking_take_while,https://github.com/fitzgen/peeking_take_while,MIT OR Apache-2.0,Nick Fitzgerald +pem,https://github.com/jcreekmore/pem-rs,MIT,Jonathan Creekmore +pem-rfc7468,https://github.com/RustCrypto/formats/tree/master/pem-rfc7468,Apache-2.0 OR MIT,RustCrypto Developers percent-encoding,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers perf-event,https://github.com/jimblandy/perf-event,MIT OR Apache-2.0,Jim Blandy perf-event-open-sys,https://github.com/jimblandy/perf-event-open-sys,MIT OR Apache-2.0,Jim Blandy +pest,https://github.com/pest-parser/pest,MIT OR Apache-2.0,Dragoș Tiselice +pest_derive,https://github.com/pest-parser/pest,MIT OR Apache-2.0,Dragoș Tiselice +pest_generator,https://github.com/pest-parser/pest,MIT OR Apache-2.0,Dragoș Tiselice +pest_meta,https://github.com/pest-parser/pest,MIT OR Apache-2.0,Dragoș Tiselice petgraph,https://github.com/petgraph/petgraph,MIT OR Apache-2.0,"bluss, mitchmindtree" +phf,https://github.com/rust-phf/rust-phf,MIT,Steven Fackler +phf_shared,https://github.com/rust-phf/rust-phf,MIT,Steven Fackler pin-project,https://github.com/taiki-e/pin-project,Apache-2.0 OR MIT,The pin-project Authors pin-project-internal,https://github.com/taiki-e/pin-project,Apache-2.0 OR MIT,The pin-project-internal Authors pin-project-lite,https://github.com/taiki-e/pin-project-lite,Apache-2.0 OR MIT,The pin-project-lite Authors pin-utils,https://github.com/rust-lang-nursery/pin-utils,MIT OR Apache-2.0,Josef Brandl +piper,https://github.com/smol-rs/piper,MIT OR Apache-2.0,"Stjepan Glavina , John Nunley " +pkcs1,https://github.com/RustCrypto/formats/tree/master/pkcs1,Apache-2.0 OR MIT,RustCrypto Developers +pkcs5,https://github.com/RustCrypto/formats/tree/master/pkcs5,Apache-2.0 OR MIT,RustCrypto Developers pkcs8,https://github.com/RustCrypto/formats/tree/master/pkcs8,Apache-2.0 OR MIT,RustCrypto Developers plotters,https://github.com/plotters-rs/plotters,MIT,Hao Hou plotters-backend,https://github.com/plotters-rs/plotters,MIT,Hao Hou @@ -352,18 +516,27 @@ pnet_macros_support,https://github.com/libpnet/libpnet,MIT OR Apache-2.0,Robert pnet_packet,https://github.com/libpnet/libpnet,MIT OR Apache-2.0,Robert Clipsham pnet_sys,https://github.com/libpnet/libpnet,MIT OR Apache-2.0,"Robert Clipsham , Linus Färnstrand " pnet_transport,https://github.com/libpnet/libpnet,MIT OR Apache-2.0,Robert Clipsham +polling,https://github.com/smol-rs/polling,Apache-2.0 OR MIT,"Stjepan Glavina , John Nunley " +poly1305,https://github.com/RustCrypto/universal-hashes,Apache-2.0 OR MIT,RustCrypto Developers portable-atomic,https://github.com/taiki-e/portable-atomic,Apache-2.0 OR MIT,The portable-atomic Authors +portable-atomic-util,https://github.com/taiki-e/portable-atomic,Apache-2.0 OR MIT,The portable-atomic-util Authors postcard,https://github.com/jamesmunns/postcard,MIT OR Apache-2.0,James Munns potential_utf,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers powerfmt,https://github.com/jhpratt/powerfmt,MIT OR Apache-2.0,Jacob Pratt +pprof,https://github.com/tikv/pprof-rs,Apache-2.0,Yang Keao ppv-lite86,https://github.com/cryptocorrosion/cryptocorrosion,MIT OR Apache-2.0,The CryptoCorrosion Contributors +precomputed-hash,https://github.com/emilio/precomputed-hash,MIT,Emilio Cobos Álvarez predicates,https://github.com/assert-rs/predicates-rs,MIT OR Apache-2.0,Nick Stevens predicates-core,https://github.com/assert-rs/predicates-rs/tree/master/crates/core,MIT OR Apache-2.0,Nick Stevens predicates-tree,https://github.com/assert-rs/predicates-rs/tree/master/crates/tree,MIT OR Apache-2.0,Nick Stevens pretty_assertions,https://github.com/rust-pretty-assertions/rust-pretty-assertions,MIT OR Apache-2.0,"Colin Kiegel , Florent Fayolle , Tom Milligan " prettyplease,https://github.com/dtolnay/prettyplease,MIT OR Apache-2.0,David Tolnay +primeorder,https://github.com/RustCrypto/elliptic-curves/tree/master/primeorder,Apache-2.0 OR MIT,RustCrypto Developers +proc-macro-crate,https://github.com/bkchr/proc-macro-crate,MIT OR Apache-2.0,Bastian Köcher proc-macro-error,https://gitlab.com/CreepySkeleton/proc-macro-error,MIT OR Apache-2.0,CreepySkeleton proc-macro-error-attr,https://gitlab.com/CreepySkeleton/proc-macro-error,MIT OR Apache-2.0,CreepySkeleton +proc-macro-error-attr2,https://github.com/GnomedDev/proc-macro-error-2,MIT OR Apache-2.0,"CreepySkeleton , GnomedDev " +proc-macro-error2,https://github.com/GnomedDev/proc-macro-error-2,MIT OR Apache-2.0,"CreepySkeleton , GnomedDev " proc-macro2,https://github.com/dtolnay/proc-macro2,MIT OR Apache-2.0,"David Tolnay , Alex Crichton " proc-macro2-diagnostics,https://github.com/SergioBenitez/proc-macro2-diagnostics,MIT OR Apache-2.0,Sergio Benitez procfs,https://github.com/eminence/procfs,MIT OR Apache-2.0,Andrew Chin @@ -372,48 +545,62 @@ prometheus,https://github.com/tikv/rust-prometheus,Apache-2.0,"overvenus@gmail.c prost,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Casper Meijn , Tokio Contributors " prost-build,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Casper Meijn , Tokio Contributors " prost-derive,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Casper Meijn , Tokio Contributors " +prost-reflect,https://github.com/andrewhickman/prost-reflect,MIT OR Apache-2.0,Andrew Hickman prost-types,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Casper Meijn , Tokio Contributors " -protobuf,https://github.com/stepancheg/rust-protobuf,MIT,Stepan Koltsov -query_map,https://github.com/calavera/query-map-rs,MIT,The query_map Authors +psl,https://github.com/addr-rs/psl,MIT OR Apache-2.0,rushmorem +psl-types,https://github.com/addr-rs/psl-types,MIT OR Apache-2.0,rushmorem +publicsuffix,https://github.com/rushmorem/publicsuffix,MIT OR Apache-2.0,rushmorem +pulldown-cmark,https://github.com/raphlinus/pulldown-cmark,MIT,"Raph Levien , Marcus Klaas de Vries " +pulldown-cmark-to-cmark,https://github.com/Byron/pulldown-cmark-to-cmark,Apache-2.0,"Sebastian Thiel , Dylan Owen , Alessandro Ogier , Zixian Cai <2891235+caizixian@users.noreply.github.com>, Andrew Lyjak " +pulsar,https://github.com/streamnative/pulsar-rs,MIT OR Apache-2.0,"Colin Stearns , Kevin Stenerson , Geoffroy Couprie " +quanta,https://github.com/metrics-rs/quanta,MIT,Toby Lawrence quick-error,http://github.com/tailhook/quick-error,MIT OR Apache-2.0,"Paul Colomiets , Colin Kiegel " +quick-xml,https://github.com/tafia/quick-xml,MIT,The quick-xml Authors +quick_cache,https://github.com/arthurprs/quick-cache,MIT,Arthur Silva quinn,https://github.com/quinn-rs/quinn,MIT OR Apache-2.0,The quinn Authors quinn-proto,https://github.com/quinn-rs/quinn,MIT OR Apache-2.0,The quinn-proto Authors quinn-udp,https://github.com/quinn-rs/quinn,MIT OR Apache-2.0,The quinn-udp Authors quote,https://github.com/dtolnay/quote,MIT OR Apache-2.0,David Tolnay +quoted_printable,https://github.com/staktrace/quoted-printable,0BSD,Kartikaya Gupta r-efi,https://github.com/r-efi/r-efi,MIT OR Apache-2.0 OR LGPL-2.1-or-later,The r-efi Authors rand,https://github.com/rust-random/rand,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers" rand_chacha,https://github.com/rust-random/rand,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers, The CryptoCorrosion Contributors" rand_core,https://github.com/rust-random/rand,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers" -rand_distr,https://github.com/rust-random/rand,MIT OR Apache-2.0,The Rand Project Developers +rand_hc,https://github.com/rust-random/rand,MIT OR Apache-2.0,The Rand Project Developers rand_xorshift,https://github.com/rust-random/rngs,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers" -rayon,https://github.com/rayon-rs/rayon,MIT OR Apache-2.0,"Niko Matsakis , Josh Stone " -rayon-core,https://github.com/rayon-rs/rayon,MIT OR Apache-2.0,"Niko Matsakis , Josh Stone " +raw-cpuid,https://github.com/gz/rust-cpuid,MIT,Gerd Zellweger +rayon,https://github.com/rayon-rs/rayon,MIT OR Apache-2.0,The rayon Authors +rayon-core,https://github.com/rayon-rs/rayon,MIT OR Apache-2.0,The rayon-core Authors +rdkafka,https://github.com/fede1024/rust-rdkafka,MIT,Federico Giraud +rdkafka-sys,https://github.com/fede1024/rust-rdkafka,MIT,Federico Giraud redox_syscall,https://gitlab.redox-os.org/redox-os/syscall,MIT,Jeremy Soller ref-cast,https://github.com/dtolnay/ref-cast,MIT OR Apache-2.0,David Tolnay ref-cast-impl,https://github.com/dtolnay/ref-cast,MIT OR Apache-2.0,David Tolnay +referencing,https://github.com/Stranger6667/jsonschema,MIT,Dmitry Dygalo regex,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " -regex-automata,https://github.com/BurntSushi/regex-automata,Unlicense OR MIT,Andrew Gallant -regex-automata,https://github.com/rust-lang/regex/tree/master/regex-automata,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " -regex-lite,https://github.com/rust-lang/regex/tree/master/regex-lite,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " -regex-syntax,https://github.com/rust-lang/regex,MIT OR Apache-2.0,The Rust Project Developers -regex-syntax,https://github.com/rust-lang/regex/tree/master/regex-syntax,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " +regex-automata,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " +regex-filtered,https://github.com/ua-parser/uap-rust,BSD-3-Clause,The regex-filtered Authors +regex-lite,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " +regex-syntax,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " +reqsign,https://github.com/Xuanwo/reqsign,Apache-2.0,Xuanwo reqwest,https://github.com/seanmonstar/reqwest,MIT OR Apache-2.0,Sean McArthur reqwest-middleware,https://github.com/TrueLayer/reqwest-middleware,MIT OR Apache-2.0,Rodrigo Gryzinski reqwest-retry,https://github.com/TrueLayer/reqwest-middleware,MIT OR Apache-2.0,Rodrigo Gryzinski retry-policies,https://github.com/TrueLayer/retry-policies,MIT OR Apache-2.0,Luca Palmieri rfc6979,https://github.com/RustCrypto/signatures/tree/master/rfc6979,Apache-2.0 OR MIT,RustCrypto Developers +rgb,https://github.com/kornelski/rust-rgb,MIT,"Kornel Lesiński , James Forster " ring,https://github.com/briansmith/ring,Apache-2.0 AND ISC,The ring Authors roxmltree,https://github.com/RazrFalcon/roxmltree,MIT OR Apache-2.0,Evgeniy Reizner -rust-embed,https://github.com/pyros2097/rust-embed,MIT,pyros2097 -rust-embed-impl,https://github.com/pyros2097/rust-embed,MIT,pyros2097 -rust-embed-utils,https://github.com/pyros2097/rust-embed,MIT,pyros2097 -rust-stemmers,https://github.com/CurrySoftware/rust-stemmers,MIT OR BSD-3-Clause,"Jakob Demler , CurrySoftware " +roxmltree,https://github.com/RazrFalcon/roxmltree,MIT OR Apache-2.0,Yevhenii Reizner +rsa,https://github.com/RustCrypto/RSA,MIT OR Apache-2.0,"RustCrypto Developers, dignifiedquire " +rust-embed,https://pyrossh.dev/repos/rust-embed,MIT,pyrossh +rust-embed-impl,https://pyrossh.dev/repos/rust-embed,MIT,pyrossh +rust-embed-utils,https://pyrossh.dev/repos/rust-embed,MIT,pyrossh +rust_decimal,https://github.com/paupino/rust-decimal,MIT,Paul Mason rustc-demangle,https://github.com/rust-lang/rustc-demangle,MIT OR Apache-2.0,Alex Crichton -rustc-hash,https://github.com/rust-lang-nursery/rustc-hash,Apache-2.0 OR MIT,The Rust Project Developers rustc-hash,https://github.com/rust-lang/rustc-hash,Apache-2.0 OR MIT,The Rust Project Developers rustix,https://github.com/bytecodealliance/rustix,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,"Dan Gohman , Jakub Konka " rustls,https://github.com/rustls/rustls,Apache-2.0 OR ISC OR MIT,The rustls Authors -rustls-native-certs,https://github.com/ctz/rustls-native-certs,Apache-2.0 OR ISC OR MIT,The rustls-native-certs Authors rustls-native-certs,https://github.com/rustls/rustls-native-certs,Apache-2.0 OR ISC OR MIT,The rustls-native-certs Authors rustls-pemfile,https://github.com/rustls/pemfile,Apache-2.0 OR ISC OR MIT,The rustls-pemfile Authors rustls-pki-types,https://github.com/rustls/pki-types,MIT OR Apache-2.0,The rustls-pki-types Authors @@ -422,25 +609,35 @@ rustop,https://chiselapp.com/user/fifr/repository/rustop,MIT,Frank Fischer rusty-fork,https://github.com/altsysrq/rusty-fork,MIT OR Apache-2.0,Jason Lingle ryu,https://github.com/dtolnay/ryu,Apache-2.0 OR BSL-1.0,David Tolnay +salsa20,https://github.com/RustCrypto/stream-ciphers,MIT OR Apache-2.0,RustCrypto Developers same-file,https://github.com/BurntSushi/same-file,Unlicense OR MIT,Andrew Gallant +sasl2-sys,https://github.com/MaterializeInc/rust-sasl,Apache-2.0,"Materialize, Inc." scc,https://github.com/wvwwvwwv/scalable-concurrent-containers,Apache-2.0,wvwwvwwv schannel,https://github.com/steffengy/schannel-rs,MIT,"Steven Fackler , Steffen Butzer " schemars,https://github.com/GREsau/schemars,MIT,Graham Esau scoped-tls,https://github.com/alexcrichton/scoped-tls,MIT OR Apache-2.0,Alex Crichton scopeguard,https://github.com/bluss/scopeguard,MIT OR Apache-2.0,bluss +scrypt,https://github.com/RustCrypto/password-hashes/tree/master/scrypt,MIT OR Apache-2.0,RustCrypto Developers sct,https://github.com/rustls/sct.rs,Apache-2.0 OR ISC OR MIT,Joseph Birr-Pixton sdd,https://github.com/wvwwvwwv/scalable-delayed-dealloc,Apache-2.0,wvwwvwwv +sea-query,https://github.com/SeaQL/sea-query,MIT OR Apache-2.0,"Chris Tsang , Billy Chan , Ivan Krivosheev " +sea-query-binder,https://github.com/SeaQL/sea-query,MIT OR Apache-2.0,"Valentin Tolmer , Ivan Krivosheev " +sea-query-derive,https://github.com/SeaQL/sea-query,MIT OR Apache-2.0,"Follpvosten , Rene Leveille " +seahash,https://gitlab.redox-os.org/redox-os/seahash,MIT,"ticki , Tom Almeida " sec1,https://github.com/RustCrypto/formats/tree/master/sec1,Apache-2.0 OR MIT,RustCrypto Developers security-framework,https://github.com/kornelski/rust-security-framework,MIT OR Apache-2.0,"Steven Fackler , Kornel " security-framework-sys,https://github.com/kornelski/rust-security-framework,MIT OR Apache-2.0,"Steven Fackler , Kornel " semver,https://github.com/dtolnay/semver,MIT OR Apache-2.0,David Tolnay separator,https://github.com/saghm/rust-separator,MIT,Saghm Rossi +seq-macro,https://github.com/dtolnay/seq-macro,MIT OR Apache-2.0,David Tolnay serde,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " +serde-value,https://github.com/arcnmx/serde-value,MIT,arcnmx +serde_core,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " serde_derive,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " -serde_dynamo,https://github.com/zenlist/serde_dynamo,MIT,Bryan Burgers serde_json,https://github.com/serde-rs/json,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " serde_json_borrow,https://github.com/PSeitz/serde_json_borrow,MIT,Pascal Seitz serde_path_to_error,https://github.com/dtolnay/path-to-error,MIT OR Apache-2.0,David Tolnay +serde_plain,https://github.com/mitsuhiko/serde-plain,MIT OR Apache-2.0,Armin Ronacher serde_qs,https://github.com/samscott89/serde_qs,MIT OR Apache-2.0,Sam Scott serde_spanned,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The serde_spanned Authors serde_urlencoded,https://github.com/nox/serde_urlencoded,MIT OR Apache-2.0,Anthony Ramine @@ -448,32 +645,57 @@ serde_with,https://github.com/jonasbb/serde_with,MIT OR Apache-2.0,"Jonas Bushar serde_with_macros,https://github.com/jonasbb/serde_with,MIT OR Apache-2.0,Jonas Bushart serde_yaml,https://github.com/dtolnay/serde-yaml,MIT OR Apache-2.0,David Tolnay serial_test_derive,https://github.com/palfrey/serial_test,MIT,Tom Parker-Shemilt +sha-1,https://github.com/RustCrypto/hashes,MIT OR Apache-2.0,RustCrypto Developers sha1,https://github.com/RustCrypto/hashes,MIT OR Apache-2.0,RustCrypto Developers sha2,https://github.com/RustCrypto/hashes,MIT OR Apache-2.0,RustCrypto Developers +sha3,https://github.com/RustCrypto/hashes,MIT OR Apache-2.0,RustCrypto Developers sharded-slab,https://github.com/hawkw/sharded-slab,MIT,Eliza Weisman +shared_child,https://github.com/oconnor663/shared_child.rs,MIT,jacko shell-words,https://github.com/tmiasko/shell-words,MIT OR Apache-2.0,Tomasz Miąsko shlex,https://github.com/comex/rust-shlex,MIT OR Apache-2.0,"comex , Fenhl , Adrian Taylor , Alex Touchet , Daniel Parks , Garrett Berg " -signal-hook-registry,https://github.com/vorner/signal-hook,Apache-2.0 OR MIT,"Michal 'vorner' Vaner , Masaki Hara " +sigchld,https://github.com/oconnor663/sigchld.rs,MIT,Jack O'Connor +signal-hook,https://github.com/vorner/signal-hook,Apache-2.0 OR MIT,"Michal 'vorner' Vaner , Thomas Himmelstoss " +signal-hook-registry,https://github.com/vorner/signal-hook,MIT OR Apache-2.0,"Michal 'vorner' Vaner , Masaki Hara " signature,https://github.com/RustCrypto/traits/tree/master/signature,Apache-2.0 OR MIT,RustCrypto Developers +simd-adler32,https://github.com/mcountryman/simd-adler32,MIT,Marvin Countryman +simdutf8,https://github.com/rusticstuff/simdutf8,MIT OR Apache-2.0,Hans Kratz +simple_asn1,https://github.com/acw/simple_asn1,ISC,Adam Wick siphasher,https://github.com/jedisct1/rust-siphash,MIT OR Apache-2.0,Frank Denis sketches-ddsketch,https://github.com/mheffner/rust-sketches-ddsketch,Apache-2.0,Mike Heffner slab,https://github.com/tokio-rs/slab,MIT,Carl Lerche smallvec,https://github.com/servo/rust-smallvec,MIT OR Apache-2.0,The Servo Project Developers +snafu,https://github.com/shepmaster/snafu,MIT OR Apache-2.0,Jake Goulding +snafu-derive,https://github.com/shepmaster/snafu,MIT OR Apache-2.0,Jake Goulding +snap,https://github.com/BurntSushi/rust-snappy,BSD-3-Clause,Andrew Gallant socket2,https://github.com/rust-lang/socket2,MIT OR Apache-2.0,"Alex Crichton , Thomas de Zeeuw " spin,https://github.com/mvdnes/spin-rs,MIT,"Mathijs van de Nes , John Ericson , Joshua Barretto " +spinning_top,https://github.com/rust-osdev/spinning_top,MIT OR Apache-2.0,Philipp Oppermann spki,https://github.com/RustCrypto/formats/tree/master/spki,Apache-2.0 OR MIT,RustCrypto Developers +sqlx,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " +sqlx-core,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " +sqlx-macros,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " +sqlx-macros-core,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " +sqlx-mysql,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " +sqlx-postgres,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " +sqlx-sqlite,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " stable_deref_trait,https://github.com/storyyeller/stable_deref_trait,MIT OR Apache-2.0,Robert Grosse static_assertions,https://github.com/nvzqz/static-assertions-rs,MIT OR Apache-2.0,Nikolai Vazquez +str_stack,https://github.com/Stebalien/str_stack,MIT OR Apache-2.0,Steven Allen +string_cache,https://github.com/servo/string-cache,MIT OR Apache-2.0,The Servo Project Developers +stringprep,https://github.com/sfackler/rust-stringprep,MIT OR Apache-2.0,Steven Fackler +strip-ansi-escapes,https://github.com/luser/strip-ansi-escapes,Apache-2.0 OR MIT,Ted Mielczarek strsim,https://github.com/rapidfuzz/strsim-rs,MIT,"Danny Guo , maxbachmann " subtle,https://github.com/dalek-cryptography/subtle,BSD-3-Clause,"Isis Lovecruft , Henry de Valence " +symbolic-common,https://github.com/getsentry/symbolic,MIT,"Armin Ronacher , Jan Michael Auer " +symbolic-demangle,https://github.com/getsentry/symbolic,MIT,"Armin Ronacher , Jan Michael Auer " syn,https://github.com/dtolnay/syn,MIT OR Apache-2.0,David Tolnay sync_wrapper,https://github.com/Actyx/sync_wrapper,Apache-2.0,Actyx AG synstructure,https://github.com/mystor/synstructure,MIT,Nika Layzell sysinfo,https://github.com/GuillaumeGomez/sysinfo,MIT,Guillaume Gomez -system-configuration,https://github.com/mullvad/system-configuration-rs,MIT OR Apache-2.0,Mullvad VPN -system-configuration-sys,https://github.com/mullvad/system-configuration-rs,MIT OR Apache-2.0,Mullvad VPN +syslog_loose,https://github.com/FungusHumungus/syslog-loose,MIT,Stephen Wakely tabled,https://github.com/zhiburt/tabled,MIT,Maxim Zhiburt tabled_derive,https://github.com/zhiburt/tabled,MIT,Maxim Zhiburt +tagptr,https://github.com/oliver-giersch/tagptr,MIT OR Apache-2.0,Oliver Giersch tantivy,https://github.com/quickwit-oss/tantivy,MIT,Paul Masurel tantivy-bitpacker,https://github.com/quickwit-oss/tantivy,MIT,Paul Masurel tantivy-columnar,https://github.com/quickwit-oss/tantivy,MIT,The tantivy-columnar Authors @@ -484,16 +706,23 @@ tantivy-sstable,https://github.com/quickwit-oss/tantivy,MIT,The tantivy-sstable tantivy-stacker,https://github.com/quickwit-oss/tantivy,MIT,The tantivy-stacker Authors tantivy-tokenizer-api,https://github.com/quickwit-oss/tantivy,MIT,The tantivy-tokenizer-api Authors tempfile,https://github.com/Stebalien/tempfile,MIT OR Apache-2.0,"Steven Allen , The Rust Project Developers, Ashley Mannix , Jason White " +term,https://github.com/Stebalien/term,MIT OR Apache-2.0,"The Rust Project Developers, Steven Allen" termcolor,https://github.com/BurntSushi/termcolor,Unlicense OR MIT,Andrew Gallant termtree,https://github.com/rust-cli/termtree,MIT,The termtree Authors +testing_table,https://github.com/zhiburt/tabled,MIT,Maxim Zhiburt thiserror,https://github.com/dtolnay/thiserror,MIT OR Apache-2.0,David Tolnay thiserror-impl,https://github.com/dtolnay/thiserror,MIT OR Apache-2.0,David Tolnay thousands,https://github.com/tov/thousands-rs,MIT OR Apache-2.0,Jesse A. Tov thread_local,https://github.com/Amanieu/thread_local-rs,MIT OR Apache-2.0,Amanieu d'Antras +thrift,https://github.com/apache/thrift/tree/master/lib/rs,Apache-2.0,Apache Thrift Developers +tikv-jemalloc-ctl,https://github.com/tikv/jemallocator,MIT OR Apache-2.0,"Steven Fackler , Gonzalo Brito Gadeschi , The TiKV Project Developers" +tikv-jemalloc-sys,https://github.com/tikv/jemallocator,MIT OR Apache-2.0,"Alex Crichton , Gonzalo Brito Gadeschi , The TiKV Project Developers" +tikv-jemallocator,https://github.com/tikv/jemallocator,MIT OR Apache-2.0,"Alex Crichton , Gonzalo Brito Gadeschi , Simon Sapin , Steven Fackler , The TiKV Project Developers" time,https://github.com/time-rs/time,MIT OR Apache-2.0,"Jacob Pratt , Time contributors" time-core,https://github.com/time-rs/time,MIT OR Apache-2.0,"Jacob Pratt , Time contributors" time-fmt,https://github.com/MiSawa/time-fmt,MIT OR Apache-2.0,mi_sawa time-macros,https://github.com/time-rs/time,MIT OR Apache-2.0,"Jacob Pratt , Time contributors" +tiny-keccak,https://github.com/debris/tiny-keccak,CC0-1.0,debris tinystr,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers tinytemplate,https://github.com/bheisler/TinyTemplate,Apache-2.0 OR MIT,Brook Heisler tinyvec,https://github.com/Lokathor/tinyvec,Zlib OR Apache-2.0 OR MIT,Lokathor @@ -501,16 +730,21 @@ tinyvec_macros,https://github.com/Soveu/tinyvec_macros,MIT OR Apache-2.0 OR Zlib tokio,https://github.com/tokio-rs/tokio,MIT,Tokio Contributors tokio-macros,https://github.com/tokio-rs/tokio,MIT,Tokio Contributors tokio-metrics,https://github.com/tokio-rs/tokio-metrics,MIT,Tokio Contributors +tokio-native-tls,https://github.com/tokio-rs/tls,MIT,Tokio Contributors +tokio-retry2,https://github.com/naomijub/tokio-retry,MIT,"Julia Naomi , Sam Rijs " tokio-rustls,https://github.com/rustls/tokio-rustls,MIT OR Apache-2.0,The tokio-rustls Authors tokio-stream,https://github.com/tokio-rs/tokio,MIT,Tokio Contributors -tokio-tungstenite,https://github.com/snapview/tokio-tungstenite,MIT,"Daniel Abramov , Alexey Galakhov " tokio-util,https://github.com/tokio-rs/tokio,MIT,Tokio Contributors -toml,https://github.com/toml-rs/toml,MIT OR Apache-2.0,Alex Crichton +toml,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml Authors toml_datetime,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_datetime Authors -toml_edit,https://github.com/toml-rs/toml,MIT OR Apache-2.0,"Andronik Ordian , Ed Page " +toml_edit,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_edit Authors +toml_parser,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_parser Authors +toml_writer,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_writer Authors tonic,https://github.com/hyperium/tonic,MIT,Lucio Franco tonic-build,https://github.com/hyperium/tonic,MIT,Lucio Franco tonic-health,https://github.com/hyperium/tonic,MIT,James Nugent +tonic-prost,https://github.com/hyperium/tonic,MIT,Lucio Franco +tonic-prost-build,https://github.com/hyperium/tonic,MIT,Lucio Franco tonic-reflection,https://github.com/hyperium/tonic,MIT,"James Nugent , Samani G. Gikandi " tower,https://github.com/tower-rs/tower,MIT,Tower Maintainers tower-http,https://github.com/tower-rs/tower-http,MIT,Tower Maintainers @@ -523,79 +757,116 @@ tracing-log,https://github.com/tokio-rs/tracing,MIT,Tokio Contributors tracing-subscriber,https://github.com/tokio-rs/tracing,MIT,"Eliza Weisman , David Barsky , Tokio Contributors " +triomphe,https://github.com/Manishearth/triomphe,MIT OR Apache-2.0,"Manish Goregaokar , The Servo Project Developers" try-lock,https://github.com/seanmonstar/try-lock,MIT,Sean McArthur ttl_cache,https://github.com/stusmall/ttl_cache,MIT OR Apache-2.0,Stu Small -tungstenite,https://github.com/snapview/tungstenite-rs,MIT OR Apache-2.0,"Alexey Galakhov, Daniel Abramov" +twox-hash,https://github.com/shepmaster/twox-hash,MIT,Jake Goulding +typeid,https://github.com/dtolnay/typeid,MIT OR Apache-2.0,David Tolnay typenum,https://github.com/paholg/typenum,MIT OR Apache-2.0,"Paho Lurie-Gregg , Andre Bogus " +typetag,https://github.com/dtolnay/typetag,MIT OR Apache-2.0,David Tolnay +typetag-impl,https://github.com/dtolnay/typetag,MIT OR Apache-2.0,David Tolnay +tz-rs,https://github.com/x-hgg-x/tz-rs,MIT OR Apache-2.0,x-hgg-x +ua-parser,https://github.com/ua-parser/uap-rust,Apache-2.0,The ua-parser Authors +ucd-trie,https://github.com/BurntSushi/ucd-generate,MIT OR Apache-2.0,Andrew Gallant ulid,https://github.com/dylanhart/ulid-rs,MIT,dylanhart unarray,https://github.com/cameron1024/unarray,MIT OR Apache-2.0,The unarray Authors unicase,https://github.com/seanmonstar/unicase,MIT OR Apache-2.0,Sean McArthur +unicode-bidi,https://github.com/servo/unicode-bidi,MIT OR Apache-2.0,The Servo Project Developers +unicode-general-category,https://github.com/yeslogic/unicode-general-category,Apache-2.0,YesLogic Pty. Ltd. unicode-ident,https://github.com/dtolnay/unicode-ident,(MIT OR Apache-2.0) AND Unicode-3.0,David Tolnay +unicode-normalization,https://github.com/unicode-rs/unicode-normalization,MIT OR Apache-2.0,"kwantam , Manish Goregaokar " +unicode-properties,https://github.com/unicode-rs/unicode-properties,MIT OR Apache-2.0,"Charles Lew , Manish Goregaokar " +unicode-segmentation,https://github.com/unicode-rs/unicode-segmentation,MIT OR Apache-2.0,"kwantam , Manish Goregaokar " unicode-width,https://github.com/unicode-rs/unicode-width,MIT OR Apache-2.0,"kwantam , Manish Goregaokar " +unicode-xid,https://github.com/unicode-rs/unicode-xid,MIT OR Apache-2.0,"erick.tryzelaar , kwantam , Manish Goregaokar " +unit-prefix,https://codeberg.org/commons-rs/unit-prefix,MIT,"Fabio Valentini , Benjamin Sago " +universal-hash,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers unsafe-libyaml,https://github.com/dtolnay/unsafe-libyaml,MIT,David Tolnay untrusted,https://github.com/briansmith/untrusted,ISC,Brian Smith +ureq-proto,https://github.com/algesten/ureq-proto,MIT OR Apache-2.0,Martin Algesten url,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers urlencoding,https://github.com/kornelski/rust_urlencoding,MIT,"Kornel , Bertram Truong " username,https://pijul.org/darcs/user,MIT OR Apache-2.0,Pierre-Étienne Meunier utf-8,https://github.com/SimonSapin/rust-utf8,MIT OR Apache-2.0,Simon Sapin utf8-ranges,https://github.com/BurntSushi/utf8-ranges,Unlicense OR MIT,Andrew Gallant +utf8-width,https://github.com/magiclen/utf8-width,MIT,Magic Len utf8_iter,https://github.com/hsivonen/utf8_iter,Apache-2.0 OR MIT,Henri Sivonen utf8parse,https://github.com/alacritty/vte,Apache-2.0 OR MIT,"Joe Wilm , Christian Duerr " utoipa,https://github.com/juhaku/utoipa,MIT OR Apache-2.0,Juha Kukkonen utoipa-gen,https://github.com/juhaku/utoipa,MIT OR Apache-2.0,Juha Kukkonen uuid,https://github.com/uuid-rs/uuid,Apache-2.0 OR MIT,"Ashley Mannix, Dylan DPC, Hunar Roop Kahlon" +uuid-simd,https://github.com/Nugine/simd,MIT,The uuid-simd Authors valuable,https://github.com/tokio-rs/valuable,MIT,The valuable Authors +vrl,https://github.com/vectordotdev/vrl,MPL-2.0,Vector Contributors vsimd,https://github.com/Nugine/simd,MIT,The vsimd Authors vte,https://github.com/alacritty/vte,Apache-2.0 OR MIT,"Joe Wilm , Christian Duerr " -vte_generate_state_changes,https://github.com/alacritty/vte,Apache-2.0 OR MIT,Christian Duerr wait-timeout,https://github.com/alexcrichton/wait-timeout,MIT OR Apache-2.0,Alex Crichton +waker-fn,https://github.com/smol-rs/waker-fn,Apache-2.0 OR MIT,Stjepan Glavina walkdir,https://github.com/BurntSushi/walkdir,Unlicense OR MIT,Andrew Gallant want,https://github.com/seanmonstar/want,MIT,Sean McArthur warp,https://github.com/seanmonstar/warp,MIT,Sean McArthur wasi,https://github.com/bytecodealliance/wasi,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The Cranelift Project Developers -wasi,https://github.com/bytecodealliance/wasi-rs,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The Cranelift Project Developers +wasip2,https://github.com/bytecodealliance/wasi-rs,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The wasip2 Authors +wasite,https://github.com/ardaku/wasite,Apache-2.0 OR BSL-1.0 OR MIT,The wasite Authors wasix,https://github.com/wasix-org/wasix-abi-rust,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,"The Cranelift Project Developers, john-sharratt" -wasm-bindgen,https://github.com/rustwasm/wasm-bindgen,MIT OR Apache-2.0,The wasm-bindgen Developers -wasm-bindgen-backend,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/backend,MIT OR Apache-2.0,The wasm-bindgen Developers -wasm-bindgen-futures,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/futures,MIT OR Apache-2.0,The wasm-bindgen Developers -wasm-bindgen-macro,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/macro,MIT OR Apache-2.0,The wasm-bindgen Developers -wasm-bindgen-macro-support,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/macro-support,MIT OR Apache-2.0,The wasm-bindgen Developers -wasm-bindgen-shared,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/shared,MIT OR Apache-2.0,The wasm-bindgen Developers +wasm-bindgen,https://github.com/wasm-bindgen/wasm-bindgen,MIT OR Apache-2.0,The wasm-bindgen Developers +wasm-bindgen-futures,https://github.com/wasm-bindgen/wasm-bindgen/tree/master/crates/futures,MIT OR Apache-2.0,The wasm-bindgen Developers +wasm-bindgen-macro,https://github.com/wasm-bindgen/wasm-bindgen/tree/master/crates/macro,MIT OR Apache-2.0,The wasm-bindgen Developers +wasm-bindgen-macro-support,https://github.com/wasm-bindgen/wasm-bindgen/tree/master/crates/macro-support,MIT OR Apache-2.0,The wasm-bindgen Developers +wasm-bindgen-shared,https://github.com/wasm-bindgen/wasm-bindgen/tree/master/crates/shared,MIT OR Apache-2.0,The wasm-bindgen Developers +wasm-streams,https://github.com/MattiasBuelens/wasm-streams,MIT OR Apache-2.0,Mattias Buelens wasm-timer,https://github.com/tomaka/wasm-timer,MIT,Pierre Krieger -web-sys,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/web-sys,MIT OR Apache-2.0,The wasm-bindgen Developers +wasmtimer,https://github.com/whizsid/wasmtimer-rs,MIT,"WhizSid , Pierre Krieger " +web-sys,https://github.com/wasm-bindgen/wasm-bindgen/tree/master/crates/web-sys,MIT OR Apache-2.0,The wasm-bindgen Developers web-time,https://github.com/daxpedda/web-time,MIT OR Apache-2.0,The web-time Authors webpki-roots,https://github.com/rustls/webpki-roots,CDLA-Permissive-2.0,The webpki-roots Authors -which,https://github.com/harryfei/which-rs,MIT,Harry Fei -whichlang,https://github.com/quickwit-oss/whichlang,MIT,"Quickwit, Inc. " +whoami,https://github.com/ardaku/whoami,Apache-2.0 OR BSL-1.0 OR MIT,The whoami Authors winapi,https://github.com/retep998/winapi-rs,MIT,Peter Atashian winapi,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian winapi-i686-pc-windows-gnu,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian winapi-util,https://github.com/BurntSushi/winapi-util,Unlicense OR MIT,Andrew Gallant winapi-x86_64-pc-windows-gnu,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian windows,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows-collections,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-collections Authors windows-core,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft -windows-implement,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft -windows-interface,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows-core,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-core Authors +windows-future,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-future Authors +windows-implement,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-implement Authors +windows-interface,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-interface Authors windows-link,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft -windows-registry,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows-link,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-link Authors +windows-numerics,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-numerics Authors windows-result,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows-result,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-result Authors windows-strings,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows-strings,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-strings Authors windows-sys,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows-sys,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-sys Authors windows-targets,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows-targets,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-targets Authors +windows-threading,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows_aarch64_gnullvm,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_aarch64_gnullvm,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows_aarch64_gnullvm Authors windows_aarch64_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_aarch64_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows_aarch64_msvc Authors windows_i686_gnu,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_i686_gnu,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows_i686_gnu Authors windows_i686_gnullvm,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_i686_gnullvm,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows_i686_gnullvm Authors windows_i686_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_i686_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows_i686_msvc Authors windows_x86_64_gnu,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_x86_64_gnu,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows_x86_64_gnu Authors windows_x86_64_gnullvm,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_x86_64_gnullvm,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows_x86_64_gnullvm Authors windows_x86_64_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_x86_64_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows_x86_64_msvc Authors winnow,https://github.com/winnow-rs/winnow,MIT,The winnow Authors -wit-bindgen-rt,https://github.com/bytecodealliance/wit-bindgen,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The wit-bindgen-rt Authors +wit-bindgen,https://github.com/bytecodealliance/wit-bindgen,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Alex Crichton +woothee,https://github.com/woothee/woothee-rust,Apache-2.0,hhatto writeable,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers -xattr,https://github.com/Stebalien/xattr,MIT OR Apache-2.0,Steven Allen xmlparser,https://github.com/RazrFalcon/xmlparser,MIT OR Apache-2.0,Yevhenii Reizner -yada,https://github.com/takuyaa/yada,MIT OR Apache-2.0,Takuya Asano +xxhash-rust,https://github.com/DoumanAsh/xxhash-rust,BSL-1.0,Douman yansi,https://github.com/SergioBenitez/yansi,MIT OR Apache-2.0,Sergio Benitez yoke,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaokar yoke-derive,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaokar @@ -603,10 +874,12 @@ zerocopy,https://github.com/google/zerocopy,BSD-2-Clause OR Apache-2.0 OR MIT,"J zerocopy-derive,https://github.com/google/zerocopy,BSD-2-Clause OR Apache-2.0 OR MIT,"Joshua Liebow-Feeser , Jack Wrenn " zerofrom,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaokar zerofrom-derive,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaokar -zeroize,https://github.com/RustCrypto/utils/tree/master/zeroize,Apache-2.0 OR MIT,The RustCrypto Project Developers +zeroize,https://github.com/RustCrypto/utils,Apache-2.0 OR MIT,The RustCrypto Project Developers zerotrie,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers zerovec,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers zerovec-derive,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaokar +zlib-rs,https://github.com/trifectatechfoundation/zlib-rs,Zlib,The zlib-rs Authors +zmij,https://github.com/dtolnay/zmij,MIT,David Tolnay zstd,https://github.com/gyscos/zstd-rs,MIT,Alexandre Bury zstd-safe,https://github.com/gyscos/zstd-rs,MIT OR Apache-2.0,Alexandre Bury zstd-sys,https://github.com/gyscos/zstd-rs,MIT OR Apache-2.0,Alexandre Bury diff --git a/Makefile b/Makefile index a3af1a4455c..4feb8035234 100644 --- a/Makefile +++ b/Makefile @@ -62,10 +62,6 @@ test-all: docker-compose-up test-failpoints: @$(MAKE) -C $(QUICKWIT_SRC) test-failpoints -test-lambda: DOCKER_SERVICES=localstack -test-lambda: docker-compose-up - @$(MAKE) -C $(QUICKWIT_SRC) test-lambda - # This will build and push all custom cross images for cross-compilation. # You will need to login into Docker Hub with the `quickwit` account. IMAGE_TAGS = x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu x86_64-unknown-linux-musl aarch64-unknown-linux-musl diff --git a/README.md b/README.md index dd546c9eb84..2c8c7315ed7 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ [![CI](https://github.com/quickwit-oss/quickwit/actions/workflows/ci.yml/badge.svg)](https://github.com/quickwit-oss/quickwit/actions?query=workflow%3ACI+branch%3Amain) [![codecov](https://codecov.io/gh/quickwit-oss/quickwit/branch/main/graph/badge.svg?token=06SRGAV5SS)](https://codecov.io/gh/quickwit-oss/quickwit) +[![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/quickwit-oss/quickwit/badge)](https://scorecard.dev/viewer/?uri=github.com/quickwit-oss/quickwit) [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.0-4baaaa.svg)](CODE_OF_CONDUCT.md) [![License: Apache 2.0](https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square)](LICENSE) [![Twitter Follow](https://img.shields.io/twitter/follow/Quickwit_Inc?color=%231DA1F2&logo=Twitter&style=plastic)](https://twitter.com/Quickwit_Inc) diff --git a/build/cross-images/aarch64-unknown-linux-gnu.dockerfile b/build/cross-images/aarch64-unknown-linux-gnu.dockerfile index e1fd145a6b9..82dc2d5cff2 100644 --- a/build/cross-images/aarch64-unknown-linux-gnu.dockerfile +++ b/build/cross-images/aarch64-unknown-linux-gnu.dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/cross-rs/aarch64-unknown-linux-gnu:0.2.4 +FROM ghcr.io/cross-rs/aarch64-unknown-linux-gnu:0.2.4@sha256:3356619b020614effd22e83cec41236e69f17ce581ffe35e252898b0c693b4e2 ARG PBC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v21.5/protoc-21.5-linux-x86_64.zip" diff --git a/build/cross-images/aarch64-unknown-linux-musl.dockerfile b/build/cross-images/aarch64-unknown-linux-musl.dockerfile index 29de948cc6a..3a2be3e81bf 100644 --- a/build/cross-images/aarch64-unknown-linux-musl.dockerfile +++ b/build/cross-images/aarch64-unknown-linux-musl.dockerfile @@ -1,4 +1,4 @@ -FROM rustembedded/cross:aarch64-unknown-linux-musl +FROM rustembedded/cross:aarch64-unknown-linux-musl@sha256:22627e0ba533781062127b13601c37216fdca27123390b07dfabd3f31f3c84a0 # The Rust toolchain to use when building our image. Set by `hooks/build`. diff --git a/build/cross-images/x86_64-unknown-linux-gnu.dockerfile b/build/cross-images/x86_64-unknown-linux-gnu.dockerfile index 62d0c7855b4..78701740713 100644 --- a/build/cross-images/x86_64-unknown-linux-gnu.dockerfile +++ b/build/cross-images/x86_64-unknown-linux-gnu.dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/cross-rs/x86_64-unknown-linux-gnu:0.2.4 +FROM ghcr.io/cross-rs/x86_64-unknown-linux-gnu:0.2.4@sha256:7c9067212c2283be2a1d5585af5ecebd4c4a2e18091e2a6aafd23f9b4b81d496 ARG PBC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v21.5/protoc-21.5-linux-x86_64.zip" diff --git a/build/cross-images/x86_64-unknown-linux-musl.dockerfile b/build/cross-images/x86_64-unknown-linux-musl.dockerfile index d98bcd7b68a..48b86280959 100644 --- a/build/cross-images/x86_64-unknown-linux-musl.dockerfile +++ b/build/cross-images/x86_64-unknown-linux-musl.dockerfile @@ -1,4 +1,4 @@ -FROM quickwit/cross-base:x86_64-unknown-linux-musl +FROM quickwit/cross-base:x86_64-unknown-linux-musl@sha256:5bcc7843aab64f89bf85c464fa2c5a00ecc634a8b1ac88c84a864f60054450cb # See https://github.com/quickwit-inc/rust-musl-builder RUN echo "Upgrading CMake" && \ diff --git a/distribution/docker/ubuntu/Dockerfile b/distribution/docker/ubuntu/Dockerfile index 1855e03cd2a..a3ddc75df00 100644 --- a/distribution/docker/ubuntu/Dockerfile +++ b/distribution/docker/ubuntu/Dockerfile @@ -1,10 +1,10 @@ -FROM ubuntu:noble AS builder +FROM ubuntu:noble@sha256:66460d557b25769b102175144d538d88219c077c678a49af4afca6fbfc1b5252 AS builder RUN apt-get update && apt-get install -y curl RUN curl -L https://install.quickwit.io | sh -FROM ubuntu:noble AS quickwit +FROM ubuntu:noble@sha256:66460d557b25769b102175144d538d88219c077c678a49af4afca6fbfc1b5252 AS quickwit LABEL org.opencontainers.image.title="Quickwit" LABEL maintainer="Quickwit, Inc. " diff --git a/distribution/lambda/.gitignore b/distribution/lambda/.gitignore deleted file mode 100644 index 85cfc7bfdef..00000000000 --- a/distribution/lambda/.gitignore +++ /dev/null @@ -1,18 +0,0 @@ -*.swp -package-lock.json -__pycache__ -.pytest_cache -.venv -*.egg-info -build/ -.mypy_cache - -# CDK asset staging directory -.cdk.staging -cdk.out - -# AWS SAM build directory -.aws-sam - -# Benchmark output files -*.log diff --git a/distribution/lambda/Makefile b/distribution/lambda/Makefile deleted file mode 100644 index 0c4decfce57..00000000000 --- a/distribution/lambda/Makefile +++ /dev/null @@ -1,149 +0,0 @@ -.SILENT: -.ONESHELL: -SHELL := bash -.SHELLFLAGS := -eu -o pipefail -c - -QW_LAMBDA_VERSION?=$(shell git tag --sort=committerdate | grep -E "aws-lambda-beta-" | tail -1 | cut -b 12-) -$(if $(QW_LAMBDA_VERSION),,$(error "Version tag not found, try 'git fetch --tags' or specify QW_LAMBDA_VERSION=beta-")) -PACKAGE_BASE_URL=https://github.com/quickwit-oss/quickwit/releases/download/aws-lambda-$(QW_LAMBDA_VERSION)/ -SEARCHER_PACKAGE_FILE=quickwit-lambda-searcher-$(QW_LAMBDA_VERSION)-x86_64.zip -INDEXER_PACKAGE_FILE=quickwit-lambda-indexer-$(QW_LAMBDA_VERSION)-x86_64.zip -export SEARCHER_PACKAGE_PATH=cdk.out/$(SEARCHER_PACKAGE_FILE) -export INDEXER_PACKAGE_PATH=cdk.out/$(INDEXER_PACKAGE_FILE) - -check-env: -ifndef CDK_ACCOUNT - $(error CDK_ACCOUNT is undefined) -endif -ifndef CDK_REGION - $(error CDK_REGION is undefined) -endif - -# Build or download the packages from the release page -# - Download by default, the version can be set with QW_LAMBDA_VERSION -# - To build locally, set QW_LAMBDA_BUILD=1 -package: - mkdir -p cdk.out - if [ "$${QW_LAMBDA_BUILD:-0}" = "1" ] - then - pushd ../../quickwit/ - rustc --version - # TODO: remove --disable-optimizations when upgrading to a release containing - # https://github.com/cargo-lambda/cargo-lambda/issues/649 (> 1.2.1) - cargo lambda build \ - -p quickwit-lambda \ - --disable-optimizations \ - --release \ - --output-format zip \ - --target x86_64-unknown-linux-gnu - popd - cp -u ../../quickwit/target/lambda/searcher/bootstrap.zip $(SEARCHER_PACKAGE_PATH) - cp -u ../../quickwit/target/lambda/indexer/bootstrap.zip $(INDEXER_PACKAGE_PATH) - else - if ! [ -f $(SEARCHER_PACKAGE_PATH) ]; then - echo "Downloading package $(PACKAGE_BASE_URL)$(SEARCHER_PACKAGE_FILE)" - curl --fail -C - -Ls -o $(SEARCHER_PACKAGE_PATH) $(PACKAGE_BASE_URL)$(SEARCHER_PACKAGE_FILE) - else - echo "Using cached package $(SEARCHER_PACKAGE_PATH)" - fi - if ! [ -f $(INDEXER_PACKAGE_PATH) ]; then - echo "Downloading package $(PACKAGE_BASE_URL)$(INDEXER_PACKAGE_FILE)" - curl --fail -C - -Ls -o $(INDEXER_PACKAGE_PATH) $(PACKAGE_BASE_URL)$(INDEXER_PACKAGE_FILE) - else - echo "Using cached package $(INDEXER_PACKAGE_PATH)" - fi - fi - -indexer-package-path: - echo -n $(INDEXER_PACKAGE_PATH) - -searcher-package-path: - echo -n $(SEARCHER_PACKAGE_PATH) - -bootstrap: - cdk bootstrap aws://$$CDK_ACCOUNT/$$CDK_REGION - -deploy-hdfs: package check-env - cdk deploy --require-approval never -a cdk/app.py HdfsStack - -deploy-mock-data: package check-env - cdk deploy --require-approval never -a cdk/app.py MockDataStack - -print-mock-data-metastore: check-env - python -c 'from cdk import cli; cli.print_mock_data_metastore()' - -# address https://github.com/aws/aws-cdk/issues/20060 -before-destroy: - mkdir -p cdk.out - touch $(INDEXER_PACKAGE_PATH) - touch $(SEARCHER_PACKAGE_PATH) - -destroy-hdfs: before-destroy check-env - python -c 'from cdk import cli; cli.empty_hdfs_bucket()' - cdk destroy --force -a cdk/app.py HdfsStack - -destroy-mock-data: before-destroy check-env - python -c 'from cdk import cli; cli.empty_mock_data_buckets()' - cdk destroy --force -a cdk/app.py MockDataStack - -clean: - rm -rf cdk.out - -## Invocation examples - -invoke-mock-data-searcher: check-env - python -c 'from cdk import cli; cli.invoke_mock_data_searcher()' - -invoke-hdfs-indexer: check-env - python -c 'from cdk import cli; cli.upload_hdfs_src_file()' - python -c 'from cdk import cli; cli.invoke_hdfs_indexer()' - -invoke-hdfs-searcher-term: check-env - python -c 'from cdk import cli; cli.invoke_hdfs_searcher("""{"query": "severity_text:ERROR", "max_hits": 10}""")' - -invoke-hdfs-searcher-histogram: check-env - python -c 'from cdk import cli; cli.invoke_hdfs_searcher("""{ "query": "*", "max_hits": 0, "aggs": { "events": { "date_histogram": { "field": "timestamp", "fixed_interval": "1d" }, "aggs": { "log_level": { "terms": { "size": 10, "field": "severity_text", "order": { "_count": "desc" } } } } } } }""")' - -bench-index: - mem_sizes=( 10240 8192 6144 4096 3072 2048 ) - export QW_LAMBDA_DISABLE_MERGE=true - for mem_size in "$${mem_sizes[@]}" - do - export INDEXER_MEMORY_SIZE=$${mem_size} - $(MAKE) deploy-hdfs - python -c 'from cdk import cli; cli.benchmark_hdfs_indexing()' - done - -bench-search-term: - export QW_LAMBDA_ENABLE_VERBOSE_JSON_LOGS=true - mem_sizes=( 1024 2048 4096 8192 ) - for mem_size in "$${mem_sizes[@]}" - do - export SEARCHER_MEMORY_SIZE=$${mem_size} - $(MAKE) deploy-hdfs - python -c 'from cdk import cli; cli.benchmark_hdfs_search("""{"query": "severity_text:ERROR", "max_hits": 10}""")' - done - -bench-search-histogram: - export QW_LAMBDA_ENABLE_VERBOSE_JSON_LOGS=true - mem_sizes=( 1024 2048 4096 8192 ) - for mem_size in "$${mem_sizes[@]}" - do - export SEARCHER_MEMORY_SIZE=$${mem_size} - $(MAKE) deploy-hdfs - python -c 'from cdk import cli; cli.benchmark_hdfs_search("""{ "query": "*", "max_hits": 0, "aggs": { "events": { "date_histogram": { "field": "timestamp", "fixed_interval": "1d" }, "aggs": { "log_level": { "terms": { "size": 10, "field": "severity_text", "order": { "_count": "desc" } } } } } } }""")' - done - -bench-search: - for run in {1..30} - do - export QW_LAMBDA_PARTIAL_REQUEST_CACHE_CAPACITY=0 - $(MAKE) bench-search-term - $(MAKE) bench-search-histogram - export QW_LAMBDA_PARTIAL_REQUEST_CACHE_CAPACITY=64MB - $(MAKE) bench-search-term - $(MAKE) bench-search-histogram - done - -test-mock-data-endpoints: - python -c 'from cdk import cli; cli.test_mock_data_endpoints()' diff --git a/distribution/lambda/Pipfile b/distribution/lambda/Pipfile deleted file mode 100644 index a82f12135a2..00000000000 --- a/distribution/lambda/Pipfile +++ /dev/null @@ -1,23 +0,0 @@ -[[source]] -url = "https://pypi.org/simple" -verify_ssl = true -name = "pypi" - -[packages] -cdk = {file = "cdk", editable = true} -aws-cdk-lib = "2.95.1" -cargo-lambda = "1.1.0" -constructs = "10.3.0" -pyyaml = "6.0.1" -black = "24.3.0" -boto3 = "1.28.59" -mypy = "1.7.0" -ziglang = "0.11.0" - -# types -boto3-stubs = "1.28.59" -types-requests = "2.31.0.2" -types-pyyaml = "6.0.12.11" - -[requires] -python_version = "3.10" diff --git a/distribution/lambda/Pipfile.lock b/distribution/lambda/Pipfile.lock deleted file mode 100644 index 1f7189cf900..00000000000 --- a/distribution/lambda/Pipfile.lock +++ /dev/null @@ -1,441 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "1ff8dd1cf56643a683dac32ad8cc42e8d2d24474dc2fd98b2c1b78c26203b829" - }, - "pipfile-spec": 6, - "requires": { - "python_version": "3.10" - }, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "attrs": { - "hashes": [ - "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30", - "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1" - ], - "markers": "python_version >= '3.7'", - "version": "==23.2.0" - }, - "aws-cdk-lib": { - "hashes": [ - "sha256:3d16664234db35cfe8731390f67f13fd155ba9482c1c0207e2c04c123a73e0a1", - "sha256:f99fc196fb233c09f73eb5699bd6ff9d6aac75496cc01dc1edb589e25a2d69c1" - ], - "index": "pypi", - "markers": "python_version ~= '3.7'", - "version": "==2.95.1" - }, - "aws-cdk.asset-awscli-v1": { - "hashes": [ - "sha256:3ef87d6530736b3a7b0f777fe3b4297994dd40c3ce9306d95f80f48fb18036e8", - "sha256:96205ea2e5e132ec52fabfff37ea25b9b859498f167d05b32564c949822cd331" - ], - "markers": "python_version ~= '3.8'", - "version": "==2.2.202" - }, - "aws-cdk.asset-kubectl-v20": { - "hashes": [ - "sha256:346283e43018a43e3b3ca571de3f44e85d49c038dc20851894cb8f9b2052b164", - "sha256:7f0617ab6cb942b066bd7174bf3e1f377e57878c3e1cddc21d6b2d13c92d0cc1" - ], - "markers": "python_version ~= '3.7'", - "version": "==2.1.2" - }, - "aws-cdk.asset-node-proxy-agent-v6": { - "hashes": [ - "sha256:b62cb10c69a42cab135e6bc670e3d2d3121fd4f53a0f61e53449da4b12738a6f", - "sha256:ef2ff0634ab037e2ebddbe69d7c92515a847c6c8bb2abdfc85b089f5e87761cb" - ], - "markers": "python_version ~= '3.8'", - "version": "==2.0.3" - }, - "black": { - "hashes": [ - "sha256:2818cf72dfd5d289e48f37ccfa08b460bf469e67fb7c4abb07edc2e9f16fb63f", - "sha256:41622020d7120e01d377f74249e677039d20e6344ff5851de8a10f11f513bf93", - "sha256:4acf672def7eb1725f41f38bf6bf425c8237248bb0804faa3965c036f7672d11", - "sha256:4be5bb28e090456adfc1255e03967fb67ca846a03be7aadf6249096100ee32d0", - "sha256:4f1373a7808a8f135b774039f61d59e4be7eb56b2513d3d2f02a8b9365b8a8a9", - "sha256:56f52cfbd3dabe2798d76dbdd299faa046a901041faf2cf33288bc4e6dae57b5", - "sha256:65b76c275e4c1c5ce6e9870911384bff5ca31ab63d19c76811cb1fb162678213", - "sha256:65c02e4ea2ae09d16314d30912a58ada9a5c4fdfedf9512d23326128ac08ac3d", - "sha256:6905238a754ceb7788a73f02b45637d820b2f5478b20fec82ea865e4f5d4d9f7", - "sha256:79dcf34b33e38ed1b17434693763301d7ccbd1c5860674a8f871bd15139e7837", - "sha256:7bb041dca0d784697af4646d3b62ba4a6b028276ae878e53f6b4f74ddd6db99f", - "sha256:7d5e026f8da0322b5662fa7a8e752b3fa2dac1c1cbc213c3d7ff9bdd0ab12395", - "sha256:9f50ea1132e2189d8dff0115ab75b65590a3e97de1e143795adb4ce317934995", - "sha256:a0c9c4a0771afc6919578cec71ce82a3e31e054904e7197deacbc9382671c41f", - "sha256:aadf7a02d947936ee418777e0247ea114f78aff0d0959461057cae8a04f20597", - "sha256:b5991d523eee14756f3c8d5df5231550ae8993e2286b8014e2fdea7156ed0959", - "sha256:bf21b7b230718a5f08bd32d5e4f1db7fc8788345c8aea1d155fc17852b3410f5", - "sha256:c45f8dff244b3c431b36e3224b6be4a127c6aca780853574c00faf99258041eb", - "sha256:c7ed6668cbbfcd231fa0dc1b137d3e40c04c7f786e626b405c62bcd5db5857e4", - "sha256:d7de8d330763c66663661a1ffd432274a2f92f07feeddd89ffd085b5744f85e7", - "sha256:e19cb1c6365fd6dc38a6eae2dcb691d7d83935c10215aef8e6c38edee3f77abd", - "sha256:e2af80566f43c85f5797365077fb64a393861a3730bd110971ab7a0c94e873e7" - ], - "index": "pypi", - "markers": "python_version >= '3.8'", - "version": "==24.3.0" - }, - "boto3": { - "hashes": [ - "sha256:7623b52c135becf145f762a9cc4203a1fb30055bb1cc7a254f82e5f7954d44a1", - "sha256:aa861e5568a564a5ce2fff5413d6ae2cda0eed7399b3a949bc861a20915e2046" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==1.28.59" - }, - "boto3-stubs": { - "hashes": [ - "sha256:4d8f4387e128561413a54944396a3d316b3113aa9c74ccf34ff850c2e0e4bb9a", - "sha256:921e5ef6acf22736191087d7fef08d095353310b752d4f341d975f0965fe176b" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==1.28.59" - }, - "botocore": { - "hashes": [ - "sha256:b8f35d65f2b45af50c36fc25cc1844d6bd61d38d2148b2ef133b8f10e198555d", - "sha256:ce58e688222df73ec5691f934be1a2122a52c9d11d3037b586b3fff16ed6d25f" - ], - "markers": "python_version >= '3.7'", - "version": "==1.31.85" - }, - "botocore-stubs": { - "hashes": [ - "sha256:64d80a3467e3b19939e9c2750af33328b3087f8f524998dbdf7ed168227f507d", - "sha256:b0345f55babd8b901c53804fc5c326a4a0bd2e23e3b71f9ea5d9f7663466e6ba" - ], - "markers": "python_version >= '3.8' and python_version < '4.0'", - "version": "==1.34.94" - }, - "cargo-lambda": { - "hashes": [ - "sha256:43d2d5b7d175bb4ad1fa92e520ca381213ab6725b41aff87f5bf9515c3912c43", - "sha256:4877adea6c25b5db94cf14dab286370de6fc69822d242527076ebf2c733f5dd6", - "sha256:504c549b07e0801b56d99c52224c8ec89df21e728c5290542512b61099190f12", - "sha256:50d375944be1051009c4ce24cd927302c3d8b260217ad41aaf643a014a02e696", - "sha256:51c29d9873a608f01f25192e0d0dd4c6e2eb364658cc9cc0d028d0b1186d1c44", - "sha256:55a30f4fe3c6c1d86ff7d9936605ea847d3b98f1f34c87577d58c646b69eed81", - "sha256:a6a5463821ed5e113feec15837ab9201fc75671344a442dd19e407648d085550", - "sha256:b3e402961550aee058f830dc41b53294fd00aeb3620c08f1af4f65468dbe6d13" - ], - "index": "pypi", - "version": "==1.1.0" - }, - "cattrs": { - "hashes": [ - "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108", - "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f" - ], - "markers": "python_version >= '3.8'", - "version": "==23.2.3" - }, - "cdk": { - "editable": true, - "file": "cdk" - }, - "click": { - "hashes": [ - "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", - "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de" - ], - "markers": "python_version >= '3.7'", - "version": "==8.1.7" - }, - "constructs": { - "hashes": [ - "sha256:2972f514837565ff5b09171cfba50c0159dfa75ee86a42921ea8c86f2941b3d2", - "sha256:518551135ec236f9cc6b86500f4fbbe83b803ccdc6c2cb7684e0b7c4d234e7b1" - ], - "index": "pypi", - "markers": "python_version ~= '3.7'", - "version": "==10.3.0" - }, - "exceptiongroup": { - "hashes": [ - "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad", - "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16" - ], - "markers": "python_version < '3.11'", - "version": "==1.2.1" - }, - "importlib-resources": { - "hashes": [ - "sha256:50d10f043df931902d4194ea07ec57960f66a80449ff867bfe782b4c486ba78c", - "sha256:cdb2b453b8046ca4e3798eb1d84f3cce1446a0e8e7b5ef4efb600f19fc398145" - ], - "markers": "python_version >= '3.8'", - "version": "==6.4.0" - }, - "jmespath": { - "hashes": [ - "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", - "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" - ], - "markers": "python_version >= '3.7'", - "version": "==1.0.1" - }, - "jsii": { - "hashes": [ - "sha256:3067d523126ce8178374dd958c60350efc831fc2ef3eb94a0a755d64fa4cc22d", - "sha256:64bbaf9c494626bc0afd1b95834f0dba66a2f2ecbb0da97fa3000c4b01d67857" - ], - "markers": "python_version ~= '3.8'", - "version": "==1.98.0" - }, - "mypy": { - "hashes": [ - "sha256:0e81ffd120ee24959b449b647c4b2fbfcf8acf3465e082b8d58fd6c4c2b27e46", - "sha256:185cff9b9a7fec1f9f7d8352dff8a4c713b2e3eea9c6c4b5ff7f0edf46b91e41", - "sha256:1e280b5697202efa698372d2f39e9a6713a0395a756b1c6bd48995f8d72690dc", - "sha256:1fe46e96ae319df21359c8db77e1aecac8e5949da4773c0274c0ef3d8d1268a9", - "sha256:2b53655a295c1ed1af9e96b462a736bf083adba7b314ae775563e3fb4e6795f5", - "sha256:551d4a0cdcbd1d2cccdcc7cb516bb4ae888794929f5b040bb51aae1846062901", - "sha256:55d28d7963bef00c330cb6461db80b0b72afe2f3c4e2963c99517cf06454e665", - "sha256:5da84d7bf257fd8f66b4f759a904fd2c5a765f70d8b52dde62b521972a0a2357", - "sha256:6cb8d5f6d0fcd9e708bb190b224089e45902cacef6f6915481806b0c77f7786d", - "sha256:7a7b1e399c47b18feb6f8ad4a3eef3813e28c1e871ea7d4ea5d444b2ac03c418", - "sha256:870bd1ffc8a5862e593185a4c169804f2744112b4a7c55b93eb50f48e7a77010", - "sha256:87c076c174e2c7ef8ab416c4e252d94c08cd4980a10967754f91571070bf5fbe", - "sha256:96650d9a4c651bc2a4991cf46f100973f656d69edc7faf91844e87fe627f7e96", - "sha256:a3637c03f4025f6405737570d6cbfa4f1400eb3c649317634d273687a09ffc2f", - "sha256:a79cdc12a02eb526d808a32a934c6fe6df07b05f3573d210e41808020aed8b5d", - "sha256:b633f188fc5ae1b6edca39dae566974d7ef4e9aaaae00bc36efe1f855e5173ac", - "sha256:bf7a2f0a6907f231d5e41adba1a82d7d88cf1f61a70335889412dec99feeb0f8", - "sha256:c1b06b4b109e342f7dccc9efda965fc3970a604db70f8560ddfdee7ef19afb05", - "sha256:cddee95dea7990e2215576fae95f6b78a8c12f4c089d7e4367564704e99118d3", - "sha256:d01921dbd691c4061a3e2ecdbfbfad029410c5c2b1ee88946bf45c62c6c91210", - "sha256:d0fa29919d2e720c8dbaf07d5578f93d7b313c3e9954c8ec05b6d83da592e5d9", - "sha256:d6ed9a3997b90c6f891138e3f83fb8f475c74db4ccaa942a1c7bf99e83a989a1", - "sha256:d93e76c2256aa50d9c82a88e2f569232e9862c9982095f6d54e13509f01222fc", - "sha256:df67fbeb666ee8828f675fee724cc2cbd2e4828cc3df56703e02fe6a421b7401", - "sha256:f29386804c3577c83d76520abf18cfcd7d68264c7e431c5907d250ab502658ee", - "sha256:f65f385a6f43211effe8c682e8ec3f55d79391f70a201575def73d08db68ead1", - "sha256:fc9fe455ad58a20ec68599139ed1113b21f977b536a91b42bef3ffed5cce7391" - ], - "index": "pypi", - "markers": "python_version >= '3.8'", - "version": "==1.7.0" - }, - "mypy-extensions": { - "hashes": [ - "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d", - "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782" - ], - "markers": "python_version >= '3.5'", - "version": "==1.0.0" - }, - "packaging": { - "hashes": [ - "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", - "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9" - ], - "markers": "python_version >= '3.7'", - "version": "==24.0" - }, - "pathspec": { - "hashes": [ - "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", - "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712" - ], - "markers": "python_version >= '3.8'", - "version": "==0.12.1" - }, - "platformdirs": { - "hashes": [ - "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee", - "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3" - ], - "markers": "python_version >= '3.8'", - "version": "==4.2.2" - }, - "publication": { - "hashes": [ - "sha256:0248885351febc11d8a1098d5c8e3ab2dabcf3e8c0c96db1e17ecd12b53afbe6", - "sha256:68416a0de76dddcdd2930d1c8ef853a743cc96c82416c4e4d3b5d901c6276dc4" - ], - "version": "==0.0.3" - }, - "python-dateutil": { - "hashes": [ - "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", - "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.9.0.post0" - }, - "pyyaml": { - "hashes": [ - "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5", - "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc", - "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df", - "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741", - "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206", - "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27", - "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595", - "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62", - "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98", - "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696", - "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290", - "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9", - "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d", - "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6", - "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867", - "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47", - "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486", - "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6", - "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3", - "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007", - "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938", - "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0", - "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c", - "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735", - "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d", - "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28", - "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4", - "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba", - "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8", - "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef", - "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5", - "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd", - "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3", - "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0", - "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515", - "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c", - "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c", - "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924", - "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34", - "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43", - "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859", - "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673", - "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54", - "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a", - "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b", - "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab", - "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa", - "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c", - "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585", - "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d", - "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f" - ], - "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==6.0.1" - }, - "s3transfer": { - "hashes": [ - "sha256:10d6923c6359175f264811ef4bf6161a3156ce8e350e705396a7557d6293c33a", - "sha256:fd3889a66f5fe17299fe75b82eae6cf722554edca744ca5d5fe308b104883d2e" - ], - "markers": "python_version >= '3.7'", - "version": "==0.7.0" - }, - "six": { - "hashes": [ - "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", - "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.16.0" - }, - "tomli": { - "hashes": [ - "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", - "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" - ], - "markers": "python_version < '3.11'", - "version": "==2.0.1" - }, - "typeguard": { - "hashes": [ - "sha256:00edaa8da3a133674796cf5ea87d9f4b4c367d77476e185e80251cc13dfbb8c4", - "sha256:5e3e3be01e887e7eafae5af63d1f36c849aaa94e3a0112097312aabfa16284f1" - ], - "markers": "python_full_version >= '3.5.3'", - "version": "==2.13.3" - }, - "types-awscrt": { - "hashes": [ - "sha256:3ae374b553e7228ba41a528cf42bd0b2ad7303d806c73eff4aaaac1515e3ea4e", - "sha256:64898a2f4a2468f66233cb8c29c5f66de907cf80ba1ef5bb1359aef2f81bb521" - ], - "markers": "python_version >= '3.7' and python_version < '4.0'", - "version": "==0.20.9" - }, - "types-pyyaml": { - "hashes": [ - "sha256:7d340b19ca28cddfdba438ee638cd4084bde213e501a3978738543e27094775b", - "sha256:a461508f3096d1d5810ec5ab95d7eeecb651f3a15b71959999988942063bf01d" - ], - "index": "pypi", - "version": "==6.0.12.11" - }, - "types-requests": { - "hashes": [ - "sha256:56d181c85b5925cbc59f4489a57e72a8b2166f18273fd8ba7b6fe0c0b986f12a", - "sha256:6aa3f7faf0ea52d728bb18c0a0d1522d9bfd8c72d26ff6f61bfc3d06a411cf40" - ], - "index": "pypi", - "version": "==2.31.0.2" - }, - "types-s3transfer": { - "hashes": [ - "sha256:02154cce46528287ad76ad1a0153840e0492239a0887e8833466eccf84b98da0", - "sha256:49a7c81fa609ac1532f8de3756e64b58afcecad8767933310228002ec7adff74" - ], - "markers": "python_version >= '3.8' and python_version < '4.0'", - "version": "==0.10.1" - }, - "types-urllib3": { - "hashes": [ - "sha256:229b7f577c951b8c1b92c1bc2b2fdb0b49847bd2af6d1cc2a2e3dd340f3bda8f", - "sha256:9683bbb7fb72e32bfe9d2be6e04875fbe1b3eeec3cbb4ea231435aa7fd6b4f0e" - ], - "version": "==1.26.25.14" - }, - "typing-extensions": { - "hashes": [ - "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0", - "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a" - ], - "markers": "python_version < '3.12'", - "version": "==4.11.0" - }, - "urllib3": { - "hashes": [ - "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84", - "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e" - ], - "markers": "python_version >= '3.10'", - "version": "==2.0.7" - }, - "ziglang": { - "hashes": [ - "sha256:038b95cac9adef0c6dce9b72bdad895a0e4e0654c77c4a8f84fe79d2909a366e", - "sha256:45e8116428267e20529b9ee43a7e7364791c1a092845d2143b248a1dbf6760b0", - "sha256:4f848c8cca520cb12357cfa3d303bf1149a30566f4c1e5999284dbdf921cc2b8", - "sha256:5fe81f91fd872fc32ed0f82807df6c680a82cbea56a9f24f818e9da299049022", - "sha256:97ac4312a358d2a4ba2c153fdb1827caf6bc158501a468ebd6a554b50edee42e", - "sha256:a7edc7020e7ffbbb3af3a40c17a9bda65d5a65132ff933e153ffa80d8f5ad731", - "sha256:bd046eeab97ad51048575768f6dae10468b3a4449f4467ed61dae621faf6ee55", - "sha256:d6372bada34714a5395539cc4d76e9cc6062739cee5ce9949a250f7c525ceb94" - ], - "index": "pypi", - "markers": "python_version ~= '3.5'", - "version": "==0.11.0" - } - }, - "develop": {} -} diff --git a/distribution/lambda/README.md b/distribution/lambda/README.md deleted file mode 100644 index 5aee77e6643..00000000000 --- a/distribution/lambda/README.md +++ /dev/null @@ -1,174 +0,0 @@ - -# Quickwit on AWS Lambda - -- Get the latest [Lambda binaries](https://github.com/quickwit-oss/quickwit/releases/tag/aws-lambda-beta-01) -- Read the [beta release announcement](https://quickwit.io/blog/quickwit-lambda-beta) - -## Quickstart - -- [Search on 20 million log dataset on S3 with lambda](https://quickwit.io/docs/get-started/tutorials/tutorial-aws-lambda-simple) -- [E2E use case with HTTP API](https://quickwit.io/docs/guides/e2e-serverless-aws-lambda) - -## Build and run yourself - -### Prerequisites - -- Install AWS CDK Toolkit (cdk command) - - `npm install -g aws-cdk` -- Ensure `curl` and `make` are installed -- To run the invocation example `make` commands, you will also need Python 3.10 - or later and `pip` installed (see [Python venv](#python-venv) below). - -### AWS Lambda service quotas - -For newly created AWS accounts, a conservative quota of 10 concurrent executions -is applied to Lambda in each individual region. If that's the case, CDK won't be -able to apply the reserved concurrency of the indexing Quickwit lambda. You can -increase the quota without charge using the [Service Quotas -console](https://console.aws.amazon.com/servicequotas/home/services/lambda/quotas). - -> **Note:** The request can take hours or even days to be processed. - -### Python venv - -The Python environment is configured using pipenv: - -``` -# Install pipenv if needed. -pip install --user pipenv -pipenv shell -pipenv install -``` - -### Example stacks - -Provided demonstration setups: -- HDFS example data: index the [HDFS - dataset](https://quickwit-datasets-public.s3.amazonaws.com/hdfs-logs-multitenants-10000.json) - by triggering the Quickwit lambda manually. -- Mock Data generator: start a mock data generator lambda that pushes mock JSON - data every X minutes to S3. Those files trigger the Quickwit indexer lambda - automatically. - -### Deploy and run - -The Makefile is a useful entrypoint to show how the Lambda deployment can used. - -Configure your shell and AWS account: -```bash -# replace with your AWS account ID and preferred region -export CDK_ACCOUNT=123456789 -export CDK_REGION=us-east-1 -make bootstrap -``` - -Deploy, index and query the HDFS dataset: -```bash -make deploy-hdfs -make invoke-hdfs-indexer -make invoke-hdfs-searcher -``` - -Deploy the mock data generator and query the indexed data: -```bash -make deploy-mock-data -# wait a few minutes... -make invoke-mock-data-searcher -``` - -### Configurations - -The following environment variables can be configured on the Lambda functions. -Note that only a small subset of all Quickwit configurations are exposed to -simplify the setup and avoid unstable deployments. - -| Variable | Description | Default | -|---|---|---| -| QW_LAMBDA_INDEX_ID | the index this Lambda interacts with (one and only one) | required | -| QW_LAMBDA_METASTORE_BUCKET | bucket name for metastore files | required | -| QW_LAMBDA_INDEX_BUCKET | bucket name for split files | required | -| QW_LAMBDA_OPENTELEMETRY_URL | HTTP OTEL tracing collector endpoint | none, OTEL disabled | -| QW_LAMBDA_OPENTELEMETRY_AUTHORIZATION | Authorization header value for HTTP OTEL calls | none, OTEL disabled | -| QW_LAMBDA_ENABLE_VERBOSE_JSON_LOGS | true to enable JSON logging of spans and logs in Cloudwatch | false | -| RUST_LOG | [Rust logging config][1] | info | - -[1]: https://rust-lang-nursery.github.io/rust-cookbook/development_tools/debugging/config_log.html - - -> [!TIP] -> The Indexer Lambda's logging is quite verbose. To reduce the associated -> CloudWatch costs, you can disable some lower level logs by setting the -> `RUST_LOG` environment variable to `info,quickwit_actors=warn`, or disable -> INFO logs altogether by setting `RUST_LOG=warn`. - -Indexer only: -| Variable | Description | Default | -|---|---|---| -| QW_LAMBDA_INDEX_CONFIG_URI | location of the index configuration file, e.g `s3://mybucket/index-config.yaml` | required | -| QW_LAMBDA_DISABLE_MERGE | true to disable compaction merges | false | -| QW_LAMBDA_DISABLE_JANITOR | true to disable retention enforcement and garbage collection | false | -| QW_LAMBDA_MAX_CHECKPOINTS | maximum number of ingested file names to keep in source history | 100 | - -Searcher only: -| Variable | Description | Default | -|---|---|---| -| QW_LAMBDA_SEARCHER_METASTORE_POLLING_INTERVAL_SECONDS | refresh interval of the metastore | 60 | -| QW_LAMBDA_PARTIAL_REQUEST_CACHE_CAPACITY | `searcher.partial_request_cache_capacity` node config | 64M | - - -### Set up a search API - -You can configure an HTTP API endpoint around the Quickwit Searcher Lambda. The -mock data example stack shows such a configuration. The API Gateway is enabled -when the `SEARCHER_API_KEY` environment variable is set: - -```bash -SEARCHER_API_KEY=my-at-least-20-char-long-key make deploy-mock-data -``` - -> [!WARNING] -> The API key is stored in plain text in the CDK stack. For a real world -> deployment, the key should be fetched from something like [AWS Secrets -> Manager](https://docs.aws.amazon.com/cdk/v2/guide/get_secrets_manager_value.html). - -Note that the response is always gzipped compressed, regardless of the -`Accept-Encoding` request header: - -```bash -curl -d '{"query":"quantity:>5", "max_hits": 10}' -H "Content-Type: application/json" -H "x-api-key: my-at-least-20-char-long-key" -X POST https://{api_id}.execute-api.{region}.amazonaws.com/api/v1/mock-sales/search --compressed -``` - -### Useful CDK commands - - * `cdk ls` list all stacks in the app - * `cdk synth` emits the synthesized CloudFormation template - * `cdk deploy` deploy this stack to your default AWS account/region - * `cdk diff` compare deployed stack with current state - * `cdk docs` open CDK documentation - -### Grafana data source setup - -You can query and visualize the Quickwit Searcher Lambda from Grafana by using the [Quickwit data source for Grafana](https://grafana.com/grafana/plugins/quickwit-quickwit-datasource/). - -#### Prerequisites - -- [Set up HTTP API endpoint for Quickwit Searcher Lambda](#set-up-a-search-api) -- [Install Quickwit data source plugin on Grafana](https://github.com/quickwit-oss/quickwit-datasource#installation) - -#### Configure Grafana data source - -If you don't have a Grafana instance running yet, you can start one with the Quickwit plugin installed using Docker: - -```bash -docker run -e GF_INSTALL_PLUGINS="quickwit-quickwit-datasource" -p 3000:3000 grafana/grafana -``` - -In the `Connections > Data sources` page, add a new Quickwit data source and configure the following settings: - -|Variable|Description|Example| -|--|--|--| -|HTTP URL| HTTP search endpoint for Quickwit Searcher Lambda | https://*******.execute-api.us-east-1.amazonaws.com/api/v1 | -|Custom HTTP Headers| If you configure API Gateway to require an API key, set `x-api-key` HTTP Header | Header: `x-api-key`
Value: API key value| -|Index ID| Same as `QW_LAMBDA_INDEX_ID` | hdfs-logs | - -After entering these values, click "Save & test". You can now query your Quickwit Lambda from Grafana! diff --git a/distribution/lambda/cdk/__init__.py b/distribution/lambda/cdk/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/distribution/lambda/cdk/app.py b/distribution/lambda/cdk/app.py deleted file mode 100755 index d9a4fe9d52e..00000000000 --- a/distribution/lambda/cdk/app.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 -import os -from typing import Literal - -import aws_cdk as cdk - -from stacks.services.quickwit_service import DEFAULT_LAMBDA_MEMORY_SIZE -from stacks.examples.hdfs_stack import HdfsStack -from stacks.examples.mock_data_stack import MockDataStack - -HDFS_STACK_NAME = "HdfsStack" -MOCK_DATA_STACK_NAME = "MockDataStack" - - -def package_location_from_env(type: Literal["searcher"] | Literal["indexer"]) -> str: - path_var = f"{type.upper()}_PACKAGE_PATH" - if path_var in os.environ: - return os.environ[path_var] - else: - print( - f"Could not infer the {type} package location. Configure it using the {path_var} environment variable" - ) - exit(1) - - -app = cdk.App() - -HdfsStack( - app, - HDFS_STACK_NAME, - env=cdk.Environment( - account=os.getenv("CDK_ACCOUNT"), region=os.getenv("CDK_REGION") - ), - indexer_memory_size=int( - os.environ.get("INDEXER_MEMORY_SIZE", DEFAULT_LAMBDA_MEMORY_SIZE) - ), - searcher_memory_size=int( - os.environ.get("SEARCHER_MEMORY_SIZE", DEFAULT_LAMBDA_MEMORY_SIZE) - ), - indexer_package_location=package_location_from_env("indexer"), - searcher_package_location=package_location_from_env("searcher"), -) - -MockDataStack( - app, - MOCK_DATA_STACK_NAME, - env=cdk.Environment( - account=os.getenv("CDK_ACCOUNT"), region=os.getenv("CDK_REGION") - ), - indexer_package_location=package_location_from_env("indexer"), - searcher_package_location=package_location_from_env("searcher"), - search_api_key=os.getenv("SEARCHER_API_KEY", None), - data_generation_interval_sec=int(os.getenv("DATA_GENERATION_INTERVAL_SEC", 300)), -) - -app.synth() diff --git a/distribution/lambda/cdk/cdk.json b/distribution/lambda/cdk/cdk.json deleted file mode 100644 index ed7ea2f776b..00000000000 --- a/distribution/lambda/cdk/cdk.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "app": "python3 app.py", - "watch": { - "include": [ - "**" - ], - "exclude": [ - "README.md", - "cdk*.json", - "requirements*.txt", - "**/__init__.py", - "python/__pycache__", - "tests" - ] - }, - "context": { - "@aws-cdk/aws-lambda:recognizeLayerVersion": true, - "@aws-cdk/core:checkSecretUsage": true, - "@aws-cdk/core:target-partitions": [ - "aws", - "aws-cn" - ], - "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, - "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, - "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, - "@aws-cdk/aws-iam:minimizePolicies": true, - "@aws-cdk/core:validateSnapshotRemovalPolicy": true, - "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, - "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, - "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, - "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, - "@aws-cdk/core:enablePartitionLiterals": true, - "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true, - "@aws-cdk/aws-iam:standardizedServicePrincipals": true, - "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true, - "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true, - "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true, - "@aws-cdk/aws-route53-patters:useCertificate": true, - "@aws-cdk/customresources:installLatestAwsSdkDefault": false, - "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true, - "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true, - "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true, - "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true, - "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true, - "@aws-cdk/aws-redshift:columnId": true, - "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true, - "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true, - "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true, - "@aws-cdk/aws-kms:aliasNameRef": true, - "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true, - "@aws-cdk/core:includePrefixInUniqueNameGeneration": true, - "@aws-cdk/aws-efs:denyAnonymousAccess": true, - "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true, - "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true, - "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true - } -} diff --git a/distribution/lambda/cdk/cli.py b/distribution/lambda/cdk/cli.py deleted file mode 100644 index ecb3ffdb155..00000000000 --- a/distribution/lambda/cdk/cli.py +++ /dev/null @@ -1,465 +0,0 @@ -"""Helper scripts to test and explore the deployed infrastructure. - -These functions are wrapped by the Makefile for convenience.""" - -import base64 -import gzip -import http.client -import json -import os -import re -import subprocess -import tempfile -import time -from dataclasses import dataclass -from functools import cache -from io import BytesIO -from urllib.parse import urlparse - -import boto3 -import botocore.config -import botocore.exceptions -from . import app -from stacks.examples import hdfs_stack, mock_data_stack - -region = os.environ["CDK_REGION"] - -example_host = "quickwit-datasets-public.s3.amazonaws.com" -example_hdfs_file = "hdfs-logs-multitenants.json.gz" -INDEXING_BOTO_CONFIG = botocore.config.Config( - retries={"max_attempts": 0}, read_timeout=60 * 15 -) -session = boto3.Session(region_name=region) -mock_sales_index_id = "mock-sales" -hdfs_logs_index_id = "hdfs-logs" - - -@cache -def _get_cloudformation_output_value(stack_name: str, export_name: str) -> str: - client = session.client("cloudformation") - stacks = client.describe_stacks(StackName=stack_name)["Stacks"] - if len(stacks) != 1: - print(f"Stack {stack_name} not identified uniquely, found {stacks}") - outputs = stacks[0]["Outputs"] - for output in outputs: - if "ExportName" in output and output["ExportName"] == export_name: - return output["OutputValue"] - else: - print(f"Export name {export_name} not found in stack {stack_name}") - exit(1) - - -def _decompress_if_gzip(payload: bytes, headers: dict) -> str: - if headers.get("content-encoding", "") == "gzip": - return gzip.GzipFile(mode="rb", fileobj=BytesIO(payload)).read().decode() - else: - return payload.decode() - - -@dataclass -class LambdaResult: - function_error: str - log_tail: str - payload: str - raw_size_bytes: int - status_code: int - - @staticmethod - def from_lambda_response(lambda_resp: dict) -> "LambdaResult": - payload = lambda_resp["Payload"].read().decode() - return LambdaResult( - function_error=lambda_resp.get("FunctionError", ""), - log_tail=base64.b64decode(lambda_resp["LogResult"]).decode(), - payload=payload, - raw_size_bytes=len(payload), - status_code=0, - ) - - @staticmethod - def from_lambda_gateway_response(lambda_resp: dict) -> "LambdaResult": - gw_str = lambda_resp["Payload"].read().decode() - gw_obj = json.loads(gw_str) - if "body" in gw_obj: - payload = gw_obj["body"] - status_code = gw_obj["statusCode"] - else: - payload = gw_str - status_code = -1 - if gw_obj.get("isBase64Encoded", False): - dec_payload = base64.b64decode(payload) - payload = _decompress_if_gzip(dec_payload, gw_obj.get("headers", {})) - return LambdaResult( - function_error=lambda_resp.get("FunctionError", ""), - log_tail=base64.b64decode(lambda_resp["LogResult"]).decode(), - payload=payload, - raw_size_bytes=len(gw_str), - status_code=status_code, - ) - - def extract_report(self) -> str: - """Expect "REPORT RequestId: xxx Duration: yyy..." to be in log tail""" - for line in reversed(self.log_tail.strip().splitlines()): - if line.startswith("REPORT"): - return line - else: - raise ValueError(f"Could not find report in log tail") - - def request_id(self) -> str: - report = self.extract_report() - match = re.search(r"RequestId: ([0-9a-z\-]+)", report) - if match: - return match.group(1) - else: - raise ValueError(f"Could not find RequestId in report: {report}") - - -def _format_lambda_output( - lambda_result: LambdaResult, duration=None, max_resp_size=10 * 1000 -): - if lambda_result.function_error != "": - print("\n## FUNCTION ERROR:") - print(lambda_result.function_error) - print("\n## RAW RESPONSE SIZE (BYTES):") - if len(lambda_result.payload) == 0: - ratio = "empty payload" - else: - ratio = f"{(lambda_result.raw_size_bytes / len(lambda_result.payload)):.1f}x the final payload" - print(f"{lambda_result.raw_size_bytes} ({ratio})") - print(f"\n## RESPONSE [{lambda_result.status_code}]:") - payload_size = len(lambda_result.payload) - print(lambda_result.payload[:max_resp_size]) - if payload_size > max_resp_size: - print(f"Response too long ({payload_size}), truncated to {max_resp_size} bytes") - - if duration is not None: - print("\n## TOTAL INVOCATION DURATION:") - print(duration) - - -def upload_hdfs_src_file(): - bucket_name = _get_cloudformation_output_value( - app.HDFS_STACK_NAME, hdfs_stack.INDEX_STORE_BUCKET_NAME_EXPORT_NAME - ) - uri = f"s3://{bucket_name}/{example_hdfs_file}" - try: - resp = session.client("s3").head_object( - Bucket=bucket_name, Key=example_hdfs_file - ) - print(f"{uri} already exists ({resp['ContentLength']} bytes), skipping upload") - return - except botocore.exceptions.ClientError as e: - if e.response["Error"]["Code"] != "404": - raise e - print(f"download dataset https://{example_host}/{example_hdfs_file}") - conn = http.client.HTTPSConnection(example_host) - conn.request("GET", f"/{example_hdfs_file}") - response = conn.getresponse() - if response.status != 200: - print(f"Failed to fetch dataset") - exit(1) - with tempfile.NamedTemporaryFile() as tmp: - while True: - chunk = response.read(1024 * 1024) - if len(chunk) == 0: - break - tmp.write(chunk) - tmp.flush() - print(f"downloaded {tmp.tell()} bytes") - print(f"upload dataset to {uri}") - session.client("s3").upload_file( - Bucket=bucket_name, Filename=tmp.name, Key=example_hdfs_file - ) - - -def invoke_hdfs_indexer() -> LambdaResult: - function_name = _get_cloudformation_output_value( - app.HDFS_STACK_NAME, hdfs_stack.INDEXER_FUNCTION_NAME_EXPORT_NAME - ) - print(f"indexer function name: {function_name}") - bucket_name = _get_cloudformation_output_value( - app.HDFS_STACK_NAME, hdfs_stack.INDEX_STORE_BUCKET_NAME_EXPORT_NAME - ) - source_uri = f"s3://{bucket_name}/{example_hdfs_file}" - print(f"src_file: {source_uri}") - invoke_start = time.time() - resp = session.client("lambda", config=INDEXING_BOTO_CONFIG).invoke( - FunctionName=function_name, - InvocationType="RequestResponse", - LogType="Tail", - Payload=f"""{{ "source_uri": "{source_uri}" }}""", - ) - invoke_duration = time.time() - invoke_start - lambda_result = LambdaResult.from_lambda_response(resp) - _format_lambda_output(lambda_result, invoke_duration) - return lambda_result - - -def _invoke_searcher( - stack_name: str, - index_id: str, - function_export_name: str, - payload: str, - download_logs: bool, -) -> LambdaResult: - function_name = _get_cloudformation_output_value(stack_name, function_export_name) - client = session.client("lambda") - print(f"searcher function name: {function_name}") - invoke_start = time.time() - resp = client.invoke( - FunctionName=function_name, - InvocationType="RequestResponse", - LogType="Tail", - Payload=json.dumps( - { - "resource": f"/api/v1/{index_id}/search", - "path": f"/api/v1/{index_id}/search", - "httpMethod": "POST", - "headers": { - "Content-Type": "application/json", - }, - "requestContext": { - "httpMethod": "POST", - }, - "body": payload, - "isBase64Encoded": False, - } - ), - ) - invoke_duration = time.time() - invoke_start - lambda_result = LambdaResult.from_lambda_gateway_response(resp) - _format_lambda_output(lambda_result, invoke_duration) - if download_logs: - download_logs_to_file(lambda_result.request_id(), function_name, invoke_start) - return lambda_result - - -def invoke_hdfs_searcher(payload: str, download_logs: bool = True) -> LambdaResult: - return _invoke_searcher( - app.HDFS_STACK_NAME, - hdfs_logs_index_id, - hdfs_stack.SEARCHER_FUNCTION_NAME_EXPORT_NAME, - payload, - download_logs, - ) - - -def get_logs( - function_name: str, request_id: str, timestamp_unix_ms: int, timeout: float = 60 -): - print(f"Getting logs for requestId: {request_id}...") - client = session.client("logs") - log_group_name = f"/aws/lambda/{function_name}" - paginator = client.get_paginator("filter_log_events") - lower_time_bound = timestamp_unix_ms - 1000 * 3600 - upper_time_bound = timestamp_unix_ms + 1000 * 3600 - last_event_id = "" - last_event_found = True - start_time = time.time() - while time.time() - start_time < timeout: - describe_resp = client.describe_log_groups(logGroupNamePrefix=log_group_name) - group_names = [group["logGroupName"] for group in describe_resp["logGroups"]] - if log_group_name in group_names: - break - print(f"log group not found, retrying...") - time.sleep(3) - while time.time() - start_time < timeout: - for page in paginator.paginate( - logGroupName=log_group_name, - filterPattern=f"%{request_id}%", - startTime=lower_time_bound, - endTime=upper_time_bound, - ): - for event in page["events"]: - if last_event_found or event["eventId"] == last_event_id: - last_event_found = True - last_event_id = event["eventId"] - yield event["message"] - if event["message"].startswith("REPORT"): - last_event_id = "REPORT" - break - if last_event_id == "REPORT": - break - if last_event_id == "REPORT": - break - elif last_event_id == "": - print(f"no event found, retrying...") - else: - print(f"last event not found, retrying...") - last_event_found = False - time.sleep(3) - - else: - raise TimeoutError(f"Log collection timed out after {timeout}s") - - -def download_logs_to_file(request_id: str, function_name: str, invoke_start: float): - try: - with open(f"lambda.{request_id}.log", "w") as f: - for log in get_logs( - function_name, - request_id, - int(invoke_start * 1000), - ): - f.write(log) - print(f"Logs written to lambda.{request_id}.log") - except Exception as e: - print(f"Failed to download logs: {e}") - - -def invoke_mock_data_searcher(): - _invoke_searcher( - app.MOCK_DATA_STACK_NAME, - mock_sales_index_id, - mock_data_stack.SEARCHER_FUNCTION_NAME_EXPORT_NAME, - """{"query": "id:1", "sort_by": "ts", "max_hits": 10}""", - True, - ) - - -def _clean_s3_bucket(bucket_name: str, prefix: str = ""): - print(f"Cleaning up bucket {bucket_name}/{prefix}...") - s3 = session.resource("s3") - bucket = s3.Bucket(bucket_name) - try: - bucket.objects.filter(Prefix=prefix).delete() - except s3.meta.client.exceptions.NoSuchBucket: - print(f"Bucket {bucket_name} not found, skipping cleanup") - - -def empty_hdfs_bucket(): - bucket_name = _get_cloudformation_output_value( - app.HDFS_STACK_NAME, hdfs_stack.INDEX_STORE_BUCKET_NAME_EXPORT_NAME - ) - _clean_s3_bucket(bucket_name) - - -def empty_mock_data_buckets(): - bucket_name = _get_cloudformation_output_value( - app.MOCK_DATA_STACK_NAME, mock_data_stack.INDEX_STORE_BUCKET_NAME_EXPORT_NAME - ) - _clean_s3_bucket(bucket_name) - bucket_name = _get_cloudformation_output_value( - app.MOCK_DATA_STACK_NAME, mock_data_stack.SOURCE_BUCKET_NAME_EXPORT_NAME - ) - _clean_s3_bucket(bucket_name) - - -def print_mock_data_metastore(): - bucket_name = _get_cloudformation_output_value( - app.MOCK_DATA_STACK_NAME, mock_data_stack.INDEX_STORE_BUCKET_NAME_EXPORT_NAME - ) - s3 = session.client("s3") - response = s3.get_object( - Bucket=bucket_name, Key=f"index/{mock_sales_index_id}/metastore.json" - ) - print(response["Body"].read().decode()) - - -@cache -def _git_commit(): - return subprocess.run( - ["git", "describe", "--dirty"], check=True, capture_output=True, text=True - ).stdout.strip() - - -def benchmark_hdfs_indexing(): - memory_size = os.environ["INDEXER_MEMORY_SIZE"] - bucket_name = _get_cloudformation_output_value( - app.HDFS_STACK_NAME, hdfs_stack.INDEX_STORE_BUCKET_NAME_EXPORT_NAME - ) - _clean_s3_bucket(bucket_name, "index/") - bench_result = { - "run": "benchmark_hdfs_indexing", - "ts": time.time(), - "commit": _git_commit(), - "memory_size": memory_size, - "env": { - k: os.environ[k] - for k in os.environ.keys() - if k.startswith("QW_LAMBDA_") - and k != "QW_LAMBDA_OPENTELEMETRY_AUTHORIZATION" - }, - } - try: - indexer_result = invoke_hdfs_indexer() - bench_result["lambda_report"] = indexer_result.extract_report() - except Exception as e: - bench_result["invocation_error"] = repr(e) - print(f"Failed to invoke indexer") - - with open(f"lambda-bench.log", "a+") as f: - f.write(json.dumps(bench_result)) - f.write("\n") - - -def benchmark_hdfs_search(payload: str): - memory_size = os.environ["SEARCHER_MEMORY_SIZE"] - for _ in range(2): - bench_result = { - "run": "benchmark_hdfs_search", - "ts": time.time(), - "commit": _git_commit(), - "memory_size": memory_size, - "payload": json.loads(payload), - "env": { - k: os.environ[k] - for k in os.environ.keys() - if k != "QW_LAMBDA_OPENTELEMETRY_AUTHORIZATION" - }, - } - try: - indexer_result = invoke_hdfs_searcher(payload, download_logs=False) - bench_result["lambda_report"] = indexer_result.extract_report() - except Exception as e: - bench_result["invocation_error"] = repr(e) - print(f"Failed to invoke searcher") - - with open(f"lambda-bench.log", "a+") as f: - f.write(json.dumps(bench_result)) - f.write("\n") - - -def test_mock_data_endpoints(): - apigw_url = _get_cloudformation_output_value( - app.MOCK_DATA_STACK_NAME, mock_data_stack.API_GATEWAY_EXPORT_NAME - ) - - def req(method, path, body=None, expected_status=200): - conn = http.client.HTTPSConnection(urlparse(apigw_url).netloc) - conn.request( - method, - path, - body, - headers={"x-api-key": os.getenv("SEARCHER_API_KEY")}, - ) - response = conn.getresponse() - print(f"{method} {path}") - headers = {k: v for (k, v) in response.getheaders()} - body = _decompress_if_gzip(response.read(), headers) - if response.status != expected_status: - print(f"[{response.status}] => {body}") - exit(1) - else: - print(f"[{response.status}] => {json.dumps(json.loads(body))[0:100]}") - - req("GET", f"/api/v1/{mock_sales_index_id}/search?query=animal") - req( - "POST", - f"/api/v1/{mock_sales_index_id}/search", - '{"query":"quantity:>5", "max_hits": 10}', - ) - req("GET", f"/api/v1/_elastic/{mock_sales_index_id}/_search?q=animal") - req( - "POST", - f"/api/v1/_elastic/{mock_sales_index_id}/_search", - '{"query":{"bool":{"must":[{"range":{"quantity":{"gt":5}}}]}},"size":10}', - ) - req("GET", f"/api/v1/_elastic/{mock_sales_index_id}/_field_caps?fields=quantity") - # expected errors - req( - "GET", - f"/api/v1/_elastic/{mock_sales_index_id}/_search?query=animal", - expected_status=400, - ) - req("GET", f"/api/v1/_elastic/_search?q=animal", expected_status=501) - req("GET", f"/api/v1/indexes/{mock_sales_index_id}") diff --git a/distribution/lambda/cdk/setup.py b/distribution/lambda/cdk/setup.py deleted file mode 100644 index 2f0f3ebd7e5..00000000000 --- a/distribution/lambda/cdk/setup.py +++ /dev/null @@ -1,7 +0,0 @@ -from setuptools import setup, find_packages - -setup( - name="cdk", - version="0.1.0", - packages=find_packages(), -) diff --git a/distribution/lambda/cdk/stacks/__init__.py b/distribution/lambda/cdk/stacks/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/distribution/lambda/cdk/stacks/examples/__init__.py b/distribution/lambda/cdk/stacks/examples/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/distribution/lambda/cdk/stacks/examples/hdfs_stack.py b/distribution/lambda/cdk/stacks/examples/hdfs_stack.py deleted file mode 100644 index 5e1d7350a99..00000000000 --- a/distribution/lambda/cdk/stacks/examples/hdfs_stack.py +++ /dev/null @@ -1,74 +0,0 @@ -import aws_cdk -from aws_cdk import Stack, aws_s3_assets -from constructs import Construct -import yaml - -from ..services import quickwit_service - - -INDEX_STORE_BUCKET_NAME_EXPORT_NAME = "hdfs-index-store-bucket-name" -INDEXER_FUNCTION_NAME_EXPORT_NAME = "hdfs-indexer-function-name" -SEARCHER_FUNCTION_NAME_EXPORT_NAME = "hdfs-searcher-function-name" - - -class HdfsStack(Stack): - def __init__( - self, - scope: Construct, - construct_id: str, - indexer_memory_size: int, - searcher_memory_size: int, - indexer_package_location: str, - searcher_package_location: str, - **kwargs - ) -> None: - super().__init__(scope, construct_id, **kwargs) - - index_config_local_path = "./resources/hdfs-logs.yaml" - - with open(index_config_local_path) as f: - index_config_dict = yaml.safe_load(f) - index_id = index_config_dict["index_id"] - - index_config = aws_s3_assets.Asset( - self, - "mock-data-index-config", - path=index_config_local_path, - ) - lambda_env = { - **quickwit_service.extract_local_env(), - "RUST_LOG": "quickwit=debug", - } - qw_svc = quickwit_service.QuickwitService( - self, - "Quickwit", - index_id=index_id, - index_config_bucket=index_config.s3_bucket_name, - index_config_key=index_config.s3_object_key, - indexer_environment=lambda_env, - searcher_environment=lambda_env, - indexer_memory_size=indexer_memory_size, - searcher_memory_size=searcher_memory_size, - indexer_package_location=indexer_package_location, - searcher_package_location=searcher_package_location, - indexer_timeout=aws_cdk.Duration.minutes(10), - ) - - aws_cdk.CfnOutput( - self, - "index-store-bucket-name", - value=qw_svc.bucket.bucket_name, - export_name=INDEX_STORE_BUCKET_NAME_EXPORT_NAME, - ) - aws_cdk.CfnOutput( - self, - "indexer-function-name", - value=qw_svc.indexer.lambda_function.function_name, - export_name=INDEXER_FUNCTION_NAME_EXPORT_NAME, - ) - aws_cdk.CfnOutput( - self, - "searcher-function-name", - value=qw_svc.searcher.lambda_function.function_name, - export_name=SEARCHER_FUNCTION_NAME_EXPORT_NAME, - ) diff --git a/distribution/lambda/cdk/stacks/examples/mock_data_stack.py b/distribution/lambda/cdk/stacks/examples/mock_data_stack.py deleted file mode 100644 index 8a4a5c9290b..00000000000 --- a/distribution/lambda/cdk/stacks/examples/mock_data_stack.py +++ /dev/null @@ -1,207 +0,0 @@ -import aws_cdk -from aws_cdk import ( - Stack, - aws_apigateway, - aws_lambda, - aws_s3, - aws_s3_assets, - aws_s3_notifications, - aws_events, - aws_events_targets, -) -from constructs import Construct -import yaml - -from ..services import quickwit_service - -SEARCHER_FUNCTION_NAME_EXPORT_NAME = "mock-data-searcher-function-name" -INDEX_STORE_BUCKET_NAME_EXPORT_NAME = "mock-data-index-store-bucket-name" -SOURCE_BUCKET_NAME_EXPORT_NAME = "mock-data-source-bucket-name" -API_GATEWAY_EXPORT_NAME = "mock-data-api-gateway-url" - - -class Source(Construct): - """An synthetic data source that generates mock data and pushes it to the - indexer through a staging S3 bucket""" - - def __init__( - self, - scope: Construct, - construct_id: str, - index_id: str, - qw_svc: quickwit_service.QuickwitService, - data_generation_interval_sec: int, - **kwargs, - ): - super().__init__(scope, construct_id, **kwargs) - mock_data_bucket = aws_s3.Bucket( - self, - "mock-data", - removal_policy=aws_cdk.RemovalPolicy.DESTROY, - lifecycle_rules=[ - aws_s3.LifecycleRule(enabled=True, expiration=aws_cdk.Duration.days(1)) - ], - ) - - generator_lambda = aws_lambda.Function( - self, - id="MockDataGenerator", - code=aws_lambda.Code.from_asset("resources/data-generator/"), - runtime=aws_lambda.Runtime.PYTHON_3_10, - handler="handler.lambda_handler", - environment={ - "BUCKET_NAME": mock_data_bucket.bucket_name, - "PREFIX": index_id, - }, - timeout=aws_cdk.Duration.seconds(30), - memory_size=1024, - ) - mock_data_bucket.grant_read_write(generator_lambda) - rule = aws_events.Rule( - self, - "ScheduledRule", - schedule=aws_events.Schedule.rate( - aws_cdk.Duration.seconds(data_generation_interval_sec) - ), - ) - rule.add_target(aws_events_targets.LambdaFunction(generator_lambda)) - - mock_data_bucket.grant_read(qw_svc.indexer.lambda_function) - mock_data_bucket.add_object_created_notification( - aws_s3_notifications.LambdaDestination(qw_svc.indexer.lambda_function) - ) - aws_cdk.CfnOutput( - self, - "source-bucket-name", - value=mock_data_bucket.bucket_name, - export_name=SOURCE_BUCKET_NAME_EXPORT_NAME, - ) - - -class SearchAPI(Construct): - """An API Gateway example configuration to expose the Searcher Lambda - function as a Quickwit search endpoint.""" - - def __init__( - self, - scope: Construct, - construct_id: str, - index_id: str, - qw_svc: quickwit_service.QuickwitService, - api_key: str, - **kwargs, - ) -> None: - super().__init__(scope, construct_id, **kwargs) - - api = aws_apigateway.RestApi( - self, - "quickwit-search-api", - rest_api_name=f"Quickwit {index_id} search API", - deploy=False, - ) - searcher_integration = aws_apigateway.LambdaIntegration( - qw_svc.searcher.lambda_function - ) - search_resource = api.root.add_resource("v1").add_resource("{proxy+}") - search_resource.add_method("POST", searcher_integration, api_key_required=True) - search_resource.add_method("GET", searcher_integration, api_key_required=True) - # Change the deployment id (api-deployment-x) each time the API changes, - # otherwise changes are not deployed. - api_deployment = aws_apigateway.Deployment(self, "api-deployment-1", api=api) - api_stage = aws_apigateway.Stage( - self, "api", deployment=api_deployment, stage_name="api" - ) - plan = aws_apigateway.UsagePlan( - self, - "default-usage-plan", - api_stages=[aws_apigateway.UsagePlanPerApiStage(api=api, stage=api_stage)], - description="Usage plan for the Quickwit search API", - ) - key = aws_apigateway.ApiKey( - self, - "default-api-key", - value=api_key, - description="Default API key for the Quickwit search API", - ) - plan.add_api_key(key) - api.deployment_stage = api_stage - - aws_cdk.CfnOutput( - self, - "search-api-url", - value=api.url.rstrip("/") + search_resource.path, - export_name=API_GATEWAY_EXPORT_NAME, - ) - - -class MockDataStack(Stack): - def __init__( - self, - scope: Construct, - construct_id: str, - indexer_package_location: str, - searcher_package_location: str, - search_api_key: str | None = None, - data_generation_interval_sec: int = 300, - **kwargs, - ) -> None: - """If `search_api_key` is not set, the search API is not deployed.""" - super().__init__(scope, construct_id, **kwargs) - - index_config_local_path = "resources/mock-sales.yaml" - with open(index_config_local_path) as f: - index_config_dict = yaml.safe_load(f) - index_id = index_config_dict["index_id"] - - index_config = aws_s3_assets.Asset( - self, - "mock-data-index-config", - path=index_config_local_path, - ) - lambda_env = quickwit_service.extract_local_env() - qw_svc = quickwit_service.QuickwitService( - self, - "Quickwit", - index_id=index_id, - index_config_bucket=index_config.s3_bucket_name, - index_config_key=index_config.s3_object_key, - indexer_environment={ - # the actor system is very verbose when the source is shutting - # down (each Lambda invocation) - "RUST_LOG": "info,quickwit_actors=warn", - **lambda_env, - }, - searcher_environment=lambda_env, - indexer_package_location=indexer_package_location, - searcher_package_location=searcher_package_location, - ) - - Source( - self, - "Source", - index_id=index_id, - qw_svc=qw_svc, - data_generation_interval_sec=data_generation_interval_sec, - ) - - if search_api_key is not None: - SearchAPI( - self, - "SearchAPI", - index_id=index_id, - qw_svc=qw_svc, - api_key=search_api_key, - ) - - aws_cdk.CfnOutput( - self, - "index-store-bucket-name", - value=qw_svc.bucket.bucket_name, - export_name=INDEX_STORE_BUCKET_NAME_EXPORT_NAME, - ) - aws_cdk.CfnOutput( - self, - "searcher-function-name", - value=qw_svc.searcher.lambda_function.function_name, - export_name=SEARCHER_FUNCTION_NAME_EXPORT_NAME, - ) diff --git a/distribution/lambda/cdk/stacks/services/__init__.py b/distribution/lambda/cdk/stacks/services/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/distribution/lambda/cdk/stacks/services/indexer_service.py b/distribution/lambda/cdk/stacks/services/indexer_service.py deleted file mode 100644 index 56ce7be5def..00000000000 --- a/distribution/lambda/cdk/stacks/services/indexer_service.py +++ /dev/null @@ -1,48 +0,0 @@ -import aws_cdk -from aws_cdk import aws_lambda, aws_s3, aws_iam -from constructs import Construct - - -class IndexerService(Construct): - def __init__( - self, - scope: Construct, - construct_id: str, - store_bucket: aws_s3.Bucket, - index_id: str, - index_config_bucket: str, - index_config_key: str, - memory_size: int, - timeout: aws_cdk.Duration, - environment: dict[str, str], - asset_path: str, - **kwargs, - ) -> None: - super().__init__(scope, construct_id, **kwargs) - - self.lambda_function = aws_lambda.Function( - self, - id="Lambda", - code=aws_lambda.Code.from_asset(asset_path), - runtime=aws_lambda.Runtime.PROVIDED_AL2, - handler="N/A", - environment={ - "QW_LAMBDA_INDEX_BUCKET": store_bucket.bucket_name, - "QW_LAMBDA_METASTORE_BUCKET": store_bucket.bucket_name, - "QW_LAMBDA_INDEX_ID": index_id, - "QW_LAMBDA_INDEX_CONFIG_URI": f"s3://{index_config_bucket}/{index_config_key}", - **environment, - }, - timeout=timeout, - retry_attempts=0, - reserved_concurrent_executions=1, - memory_size=memory_size, - ephemeral_storage_size=aws_cdk.Size.gibibytes(10), - ) - self.lambda_function.add_to_role_policy( - aws_iam.PolicyStatement( - actions=["s3:GetObject"], - resources=[f"arn:aws:s3:::{index_config_bucket}/{index_config_key}"], - ) - ) - store_bucket.grant_read_write(self.lambda_function) diff --git a/distribution/lambda/cdk/stacks/services/quickwit_service.py b/distribution/lambda/cdk/stacks/services/quickwit_service.py deleted file mode 100644 index 66294ec239b..00000000000 --- a/distribution/lambda/cdk/stacks/services/quickwit_service.py +++ /dev/null @@ -1,72 +0,0 @@ -import os - -import aws_cdk -from aws_cdk import aws_s3 -from constructs import Construct - -from . import indexer_service, searcher_service - -# Using 3008MB as default because higher memory configurations need to be -# enabled for each AWS account through the support. -DEFAULT_LAMBDA_MEMORY_SIZE = 3008 - - -def extract_local_env() -> dict[str, str]: - """Extracts local environment variables QW_LAMBDA_* and QW_DISABLE_TELEMETRY""" - return { - k: os.environ[k] - for k in os.environ.keys() - if (k.startswith("QW_LAMBDA_") or k == "QW_DISABLE_TELEMETRY") - } - - -class QuickwitService(Construct): - def __init__( - self, - scope: Construct, - construct_id: str, - index_config_bucket: str, - index_config_key: str, - index_id: str, - searcher_package_location: str, - indexer_package_location: str, - indexer_memory_size: int = DEFAULT_LAMBDA_MEMORY_SIZE, - indexer_environment: dict[str, str] = {}, - # small default timeout to avoid unexpected costs and hanging indexers - indexer_timeout: aws_cdk.Duration = aws_cdk.Duration.minutes(1), - searcher_memory_size: int = DEFAULT_LAMBDA_MEMORY_SIZE, - searcher_environment: dict[str, str] = {}, - **kwargs, - ) -> None: - """Create a new Quickwit Lambda service construct node. - - `{indexer|searcher}_package_location` is the path of the `zip` asset for - the Lambda function. - """ - super().__init__(scope, construct_id, **kwargs) - self.bucket = aws_s3.Bucket( - self, - "IndexStore", - removal_policy=aws_cdk.RemovalPolicy.DESTROY, - ) - self.indexer = indexer_service.IndexerService( - self, - "Indexer", - store_bucket=self.bucket, - index_id=index_id, - index_config_bucket=index_config_bucket, - index_config_key=index_config_key, - memory_size=indexer_memory_size, - timeout=indexer_timeout, - environment=indexer_environment, - asset_path=indexer_package_location, - ) - self.searcher = searcher_service.SearcherService( - self, - "Searcher", - store_bucket=self.bucket, - index_id=index_id, - memory_size=searcher_memory_size, - environment=searcher_environment, - asset_path=searcher_package_location, - ) diff --git a/distribution/lambda/cdk/stacks/services/searcher_service.py b/distribution/lambda/cdk/stacks/services/searcher_service.py deleted file mode 100644 index 5950a2de73e..00000000000 --- a/distribution/lambda/cdk/stacks/services/searcher_service.py +++ /dev/null @@ -1,37 +0,0 @@ -import aws_cdk -from aws_cdk import aws_lambda, aws_s3, PhysicalName -from constructs import Construct - - -class SearcherService(Construct): - def __init__( - self, - scope: Construct, - construct_id: str, - store_bucket: aws_s3.Bucket, - index_id: str, - memory_size: int, - environment: dict[str, str], - asset_path: str, - **kwargs - ) -> None: - super().__init__(scope, construct_id, **kwargs) - - self.lambda_function = aws_lambda.Function( - self, - id="Lambda", - code=aws_lambda.Code.from_asset(asset_path), - runtime=aws_lambda.Runtime.PROVIDED_AL2, - handler="N/A", - environment={ - "QW_LAMBDA_INDEX_BUCKET": store_bucket.bucket_name, - "QW_LAMBDA_METASTORE_BUCKET": store_bucket.bucket_name, - "QW_LAMBDA_INDEX_ID": index_id, - **environment, - }, - timeout=aws_cdk.Duration.seconds(30), - memory_size=memory_size, - ephemeral_storage_size=aws_cdk.Size.gibibytes(10), - ) - - store_bucket.grant_read_write(self.lambda_function) diff --git a/distribution/lambda/resources/data-generator/handler.py b/distribution/lambda/resources/data-generator/handler.py deleted file mode 100644 index 7382246ddb6..00000000000 --- a/distribution/lambda/resources/data-generator/handler.py +++ /dev/null @@ -1,37 +0,0 @@ -import json -import os -import random -import time -import gzip -import random - -import boto3 - -s3 = boto3.client("s3") - -with gzip.open("wordlist.json.gz", "r") as f: - words: list[str] = json.loads(f.read()) - - -def lambda_handler(event, context): - data = [] - base_time = time.time() - for i in range(100000): - item = { - "ts": int(base_time * 1000) + i, - "id": i, - "name": f"Item {i}", - "price": round(random.uniform(1, 100), 2), - "quantity": random.randint(1, 10), - "description": " ".join(random.choices(words, k=8)), - } - data.append(item) - - json_str = "\n".join(json.dumps(d) for d in data) - json_gz = gzip.compress(json_str.encode(), compresslevel=6) - - s3.put_object( - Bucket=os.environ["BUCKET_NAME"], - Key=f"{os.environ['PREFIX']}/{int(base_time)}.gz", - Body=json_gz, - ) diff --git a/distribution/lambda/resources/data-generator/wordlist.json.gz b/distribution/lambda/resources/data-generator/wordlist.json.gz deleted file mode 100644 index d24eadcd6a4..00000000000 Binary files a/distribution/lambda/resources/data-generator/wordlist.json.gz and /dev/null differ diff --git a/distribution/lambda/resources/hdfs-logs.yaml b/distribution/lambda/resources/hdfs-logs.yaml deleted file mode 100644 index 1fe923d2233..00000000000 --- a/distribution/lambda/resources/hdfs-logs.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# -# Index config file for hdfs-logs dataset. -# - -version: 0.8 - -index_id: hdfs-logs - -doc_mapping: - field_mappings: - - name: timestamp - type: datetime - input_formats: - - unix_timestamp - output_format: unix_timestamp_secs - fast_precision: seconds - fast: true - - name: tenant_id - type: u64 - - name: severity_text - type: text - tokenizer: raw - fast: true - - name: body - type: text - tokenizer: default - record: position - - name: resource - type: json - tokenizer: raw - tag_fields: [tenant_id] - timestamp_field: timestamp - -search_settings: - default_search_fields: [severity_text, body] - -indexing_settings: - split_num_docs_target: 2000000 diff --git a/distribution/lambda/resources/mock-sales.yaml b/distribution/lambda/resources/mock-sales.yaml deleted file mode 100644 index c8a367ad540..00000000000 --- a/distribution/lambda/resources/mock-sales.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# -# Index config file for mock-sales data generator. -# - -version: 0.8 - -index_id: mock-sales - -doc_mapping: - field_mappings: - - name: ts - type: datetime - input_formats: - - unix_timestamp - output_format: unix_timestamp_millis - precision: milliseconds - fast: true - - name: id - type: u64 - - name: name - type: text - tokenizer: raw - - name: price - type: f64 - fast: true - - name: quantity - type: u64 - fast: true - - name: description - type: text - tokenizer: default - timestamp_field: ts - -search_settings: - default_search_fields: [name, description] - -indexing_settings: - split_num_docs_target: 2000000 diff --git a/docker-compose.yml b/docker-compose.yml index 149e44890f0..24f5ed29b47 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,8 +6,7 @@ # Notes on image versions: # - For the key services such as postgres and pulsar we are trying to run # against the oldest supported version -# - For the zookeeper and kafka we are trying to use the oldest supported -# version that has arm64 images +# - For kafka we use the oldest version that supports KRaft. # - For everything else we are trying to run against the latest version. # # To run against the latest image versions update .env file. See .env.example @@ -74,67 +73,59 @@ services: # The oldest version with arm64 docker images. EOL May 2 2025 image: apachepulsar/pulsar:${PULSAR_VERSION:-3.0.0} container_name: pulsar-broker - command: bin/pulsar standalone + command: bin/pulsar standalone --no-functions-worker ports: - "${MAP_HOST_PULSAR:-127.0.0.1}:6650:6650" - "${MAP_HOST_PULSAR:-127.0.0.1}:8081:8080" environment: PULSAR_MEM: "-Xms384M -Xmx384M" + # Disable functions worker to save memory/time + PULSAR_PREFIX_functionsWorkerEnabled: "false" profiles: - all - pulsar kafka-broker: - # The oldest supported version with arm64 docker images. EOL October 27, 2023 - image: confluentinc/cp-kafka:${CP_VERSION:-7.0.9} + image: confluentinc/confluent-local:${CP_VERSION:-7.4.11} container_name: kafka-broker - depends_on: - - zookeeper ports: - "${MAP_HOST_KAFKA:-127.0.0.1}:9092:9092" - "${MAP_HOST_KAFKA:-127.0.0.1}:9101:9101" - - "${MAP_HOST_KAFKA:-127.0.0.1}:29092:29092" profiles: - all - kafka - environment: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT_HOST://localhost:9092,PLAINTEXT://kafka-broker:29092 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_JMX_PORT: 9101 - KAFKA_JMX_HOSTNAME: localhost - KAFKA_HEAP_OPTS: -Xms256M -Xmx256M - healthcheck: - test: ["CMD", "cub", "kafka-ready", "-b", "localhost:9092", "1", "5"] - start_period: 5s - interval: 5s - timeout: 10s - retries: 100 - zookeeper: - # The oldest supported version with arm64 images. EOL October 27, 2023 - image: confluentinc/cp-zookeeper:${CP_VERSION:-7.0.9} - container_name: zookeeper - ports: - - "${MAP_HOST_ZOOKEEPER:-127.0.0.1}:2181:2181" - profiles: - - all - - kafka environment: - KAFKA_HEAP_OPTS: -Xms256M -Xmx256M - ZOOKEEPER_CLIENT_PORT: 2181 - ZOOKEEPER_TICK_TIME: 2000 + # Mode KRaft (Single Node) + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: 'broker,controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@localhost:9093' + KAFKA_LOG4J_LOGGERS: "org.apache.kafka.image.loader.MetadataLoader=WARN" + + # Listeners + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT' + KAFKA_LISTENERS: 'EXTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093' + KAFKA_ADVERTISED_LISTENERS: 'EXTERNAL://localhost:9092' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_INTER_BROKER_LISTENER_NAME: 'EXTERNAL' + + # Configuration simplifiée + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + + # ID du Cluster (Nécessaire pour KRaft) + CLUSTER_ID: 'MkU3OEVBNTcwNTJENDM2Qk' + + KAFKA_HEAP_OPTS: -Xms256M -Xmx256M healthcheck: - test: ["CMD", "cub", "zk-ready", "localhost:2181", "5"] - start_period: 5s - interval: 5s - timeout: 10s - retries: 100 + # test: ["CMD-SHELL", "nc -z localhost 9092 || exit 1"] + test: ["CMD", "ub", "kafka-ready", "-b", "localhost:9092", "1", "5"] + start_period: 5s + interval: 5s + timeout: 10s + retries: 100 azurite: image: mcr.microsoft.com/azure-storage/azurite:${AZURITE_VERSION:-3.24.0} @@ -222,7 +213,7 @@ services: # It is not an official docker image # if we prefer we can build a docker from the official docker image (gcloud cli) # and install the pubsub emulator https://cloud.google.com/pubsub/docs/emulator - image: thekevjames/gcloud-pubsub-emulator:${GCLOUD_EMULATOR:-455.0.0} + image: thekevjames/gcloud-pubsub-emulator:${GCLOUD_EMULATOR:-550.0.0} container_name: gcp-pubsub-emulator ports: - "${MAP_HOST_GCLOUD_EMULATOR:-127.0.0.1}:8681:8681" diff --git a/docs/configuration/index-config.md b/docs/configuration/index-config.md index 8c09186c874..24ce8677902 100644 --- a/docs/configuration/index-config.md +++ b/docs/configuration/index-config.md @@ -358,6 +358,14 @@ fast: normalizer: lowercase ``` +Stored primitive types are inferred from the JSON value types using the following rules: +- a boolean value `true` or `false` is stored as `bool` +- numeric values are cast to the first compatible format between `i64`, `u64` or + `f64` (in this order) +- for string values (surrounded with quotes), Tantivy attempts to parse a date + in `rfc3339` format. If the parsing fails, the value is stored as `text` using + the configured tokenization rules + **Parameters for JSON field** | Variable | Description | Default value | @@ -435,8 +443,13 @@ record: basic Concatenate fields don't support fast fields, and are never stored. They uses their own tokenizer, independently of the tokenizer configured on the individual fields. At query time, concatenate fields don't support range queries. -Only the following types are supported inside a concatenate field: text, bool, i64, u64, f64, json. Other types are rejected -at index creation, or silently discarded during indexation if they are found inside a json field. +Only the following types are supported inside a concatenate field: text, bool, +i64, u64, f64, json. Other types are rejected at index creation, or silently +discarded during indexation if they are found inside a json field. Unlike +regular JSON fields, JSON fields in a concatenate field don't store RFC3339 +dates as Tantivy dates. This means you can still perform prefix queries, +e.g `my_default_field:"2025-12-12"*` to work around the lack of support for range +queries. Adding an object field to a concatenate field doesn't automatically add its subfields (yet). It isn't possible to add subfields from a json field to a concatenate field. For instance if `attributes` is a json field, it's not possible to add only `attributes.color` to a concatenate field. @@ -552,6 +565,8 @@ src.port:53 src.port:53 AND query_params.ctk:e42bb897d ``` +The stored primitive type inference is the [same as for JSON fields](#json-type). + ### Field name validation rules Currently Quickwit only accepts field name that matches the following regular expression: diff --git a/docs/configuration/lambda-config.md b/docs/configuration/lambda-config.md new file mode 100644 index 00000000000..4f522da4b35 --- /dev/null +++ b/docs/configuration/lambda-config.md @@ -0,0 +1,240 @@ +--- +title: Lambda configuration +sidebar_position: 6 +--- + +Quickwit supports offloading leaf search operations to AWS Lambda for horizontal scaling. When the local search queue becomes saturated, overflow splits are automatically sent to Lambda functions for processing. + +:::note +Lambda offloading is currently only supported on AWS. +::: + +## How it works + +Lambda offloading is **only active when a `lambda` configuration section is present** under `searcher` in your node configuration. When configured: + +1. Quickwit monitors the local search queue depth +2. When pending searches exceed the `offload_threshold`, new splits are sent to Lambda instead of being queued locally +3. Lambda returns per-split search results that are cached and merged with local results + +This allows Quickwit to handle traffic spikes without provisioning additional searcher nodes. + +## Startup validation + +When a `lambda` configuration is defined, Quickwit performs a **dry run invocation** at startup to verify that: +- The Lambda function exists +- The function version matches the embedded binary +- The invoker has permission to call the function + +If this validation fails, **Quickwit will fail to start**. This ensures that Lambda offloading works correctly before the node begins serving traffic. + +## Configuration + +Add a `lambda` section under `searcher` in your node configuration: + +```yaml +searcher: + lambda: + offload_threshold: 100 + auto_deploy: + execution_role_arn: arn:aws:iam::123456789012:role/quickwit-lambda-role + memory_size: 5 GiB + invocation_timeout_secs: 15 +``` + +### Lambda configuration options + +| Property | Description | Default value | +| --- | --- | --- | +| `function_name` | Name of the AWS Lambda function to invoke. | `quickwit-lambda-search` | +| `max_splits_per_invocation` | Maximum number of splits to send in a single Lambda invocation. Must be at least 1. | `10` | +| `offload_threshold` | Number of pending local searches before offloading to Lambda. A value of `0` offloads everything to Lambda. | `100` | +| `auto_deploy` | Auto-deployment configuration. If set, Quickwit automatically deploys or updates the Lambda function at startup. | (none) | + +### Auto-deploy configuration options + +| Property | Description | Default value | +| --- | --- | --- | +| `execution_role_arn` | **Required.** IAM role ARN for the Lambda function's execution role. | | +| `memory_size` | Memory allocated to the Lambda function. More memory provides more CPU. | `5 GiB` | +| `invocation_timeout_secs` | Timeout for Lambda invocations in seconds. | `15` | + +## Deployment options + +### Automatic deployment (recommended) + +With `auto_deploy` configured, Quickwit automatically: +1. Creates the Lambda function if it doesn't exist +2. Updates the function code if the embedded binary has changed +3. Publishes a new version with a unique identifier +4. Garbage collects old versions (keeps current + 5 most recent) + +This is the recommended approach as it ensures the Lambda function always matches the Quickwit binary version. + +### Manual deployment + +You can deploy the Lambda function manually without `auto_deploy`: +1. Download the Lambda zip from [GitHub releases](https://github.com/quickwit-oss/quickwit/releases) +2. Create or update the Lambda function using AWS CLI, Terraform, or the AWS Console +3. Publish a version with description format `quickwit_{version}_{sha256}_{timeout}_{deploy_config}"` (e.g., `quickwit_0_8_0_fa940f44_5120_60s_6c3b2`) + +The description must match the format Quickwit expects, or it won't find the function version. + +## IAM permissions + +### Permissions for the Quickwit node + +The IAM role or user running Quickwit needs the following permissions to invoke Lambda: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "lambda:InvokeFunction" + ], + "Resource": "arn:aws:lambda:*:*:function:quickwit-lambda-search:*" + } + ] +} +``` + +If using `auto_deploy`, additional permissions are required for deployment: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "lambda:CreateFunction", + "lambda:GetFunction", + "lambda:UpdateFunctionCode", + "lambda:PublishVersion", + "lambda:ListVersionsByFunction", + "lambda:DeleteFunction" + ], + "Resource": "arn:aws:lambda:*:*:function:quickwit-lambda-search" + }, + { + "Effect": "Allow", + "Action": "iam:PassRole", + "Resource": "arn:aws:iam::*:role/quickwit-lambda-role", + "Condition": { + "StringEquals": { + "iam:PassedToService": "lambda.amazonaws.com" + } + } + } + ] +} +``` + +### Lambda execution role + +The Lambda function requires an execution role with S3 read access to your index data. + +Example policy: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::your-index-bucket/*" + } + ] +} +``` + +The execution role must also have a trust policy allowing Lambda to assume it: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] +} +``` + +## CloudWatch logging + +The Lambda function emits structured logs (JSON) to stdout. To have these logs captured by CloudWatch, add the following iam permissions to the Lambda execution role: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Resource": "arn:aws:logs:*:*:*" + } + ] +} +``` + +No additional configuration is needed on the Quickwit side. + +## Versioning + +Quickwit uses content-based versioning for Lambda: +- A SHA256 hash of the Lambda binary is computed at build time +- This hash is embedded in the Lambda function description as `quickwit:{version}-{sha256_short}` +- When Quickwit starts, it searches for a version matching this description +- Different Quickwit builds with the same Lambda binary share the same Lambda version +- Updating the Lambda binary automatically triggers a new deployment + +## Example configuration + + +Minimal configuration (with auto-deployment): + +```yaml +searcher: + lambda: + auto_deploy: + execution_role_arn: arn:aws:iam::123456789012:role/quickwit-lambda-role +``` + + +Full configuration (auto-deployment): + +```yaml +searcher: + lambda: + function_name: quickwit-lambda-search + max_splits_per_invocation: 10 + offload_threshold: 10 + auto_deploy: + execution_role_arn: arn:aws:iam::123456789012:role/quickwit-lambda-role + memory_size: 5 GiB + invocation_timeout_secs: 15 +``` + +Aggressive offloading (send everything to Lambda): + +```yaml +searcher: + lambda: + function_name: quickwit-lambda-search + offload_threshold: 0 + auto_deploy: + execution_role_arn: arn:aws:iam::123456789012:role/quickwit-lambda-role +``` diff --git a/docs/configuration/node-config.md b/docs/configuration/node-config.md index 03bdb9164d6..ed4bca51517 100644 --- a/docs/configuration/node-config.md +++ b/docs/configuration/node-config.md @@ -198,11 +198,10 @@ This section contains the configuration options for a Searcher. | --- | --- | --- | | `aggregation_memory_limit` | Controls the maximum amount of memory that can be used for aggregations before aborting. This limit is per searcher node. A node may run concurrent queries, which share the limit. The first query that will hit the limit will be aborted and frees its memory. It is used to prevent excessive memory usage during the aggregation phase, which can lead to performance degradation or crashes. | `500M`| | `aggregation_bucket_limit` | Determines the maximum number of buckets returned to the client. | `65000` | -| `fast_field_cache_capacity` | Fast field in memory cache capacity on a Searcher. If your filter by dates, run aggregations, range queries, or if you use the search stream API, or even for tracing, it might worth increasing this parameter. The [metrics](../reference/metrics.md) starting by `quickwit_cache_fastfields_cache` can help you make an informed choice when setting this value. | `1G` | +| `fast_field_cache_capacity` | Fast field in memory cache capacity on a Searcher. If your filter by dates, run aggregations, range queries, or even for tracing, it might worth increasing this parameter. The [metrics](../reference/metrics.md) starting by `quickwit_cache_fastfields_cache` can help you make an informed choice when setting this value. | `1G` | | `split_footer_cache_capacity` | Split footer in memory cache (it is essentially the hotcache) capacity on a Searcher.| `500M` | | `partial_request_cache_capacity` | Partial request in memory cache capacity on a Searcher. Cache intermediate state for a request, possibly making subsequent requests faster. It can be disabled by setting the size to `0`. | `64M` | | `max_num_concurrent_split_searches` | Maximum number of concurrent split search requests running on a Searcher. | `100` | -| `max_num_concurrent_split_streams` | Maximum number of concurrent split stream requests running on a Searcher. | `100` | | `split_cache` | Searcher split cache configuration options defined in the section below. Cache disabled if unspecified. | | | `request_timeout_secs` | The time before a search request is cancelled. This should match the timeout of the stack calling into quickwit if there is one set. | `30` | diff --git a/docs/deployment/cluster-sizing.md b/docs/deployment/cluster-sizing.md index 59e1aa6566c..d69b5235310 100644 --- a/docs/deployment/cluster-sizing.md +++ b/docs/deployment/cluster-sizing.md @@ -74,17 +74,22 @@ Searcher nodes based on: ### Other services The Control Plane, Metastore and, Janitor are lightweight components. -Each of these services requires 1 replica. -The Control Plane needs a single core and 2GB of RAM. It doesn't require any disk. - -The Metastore also requires a single core and 2GB of RAM. For clusters handling -hundreds of indexes, you may increase the size to 2 cores and 4GB of RAM. It -doesn't write to disk. - -In general, the Janitor requires 1 core and 2GB of RAM and doesn't use the disk. -If you use the [delete API](https://quickwit.io/docs/overview/concepts/deletes), -the Janitor should be dimensioned like an indexer. +- **Control Plane**: A cluster must have only one Control Plane. It needs a + single core and 2GB of RAM. It doesn't require any disk. + +- **Metastore**: A cluster must have exactly one Metastore when using the + [file-backed metastore](../configuration/metastore-config.md#file-backed-metastore). + When using the [PostgreSQL metastore](#postgres-metastore-backend), you can + run one or several Metastore pods for high availability (HA). The Metastore + requires a single core and 2GB of RAM. For clusters handling hundreds of + indexes, you may increase the size to 2 cores and 4GB of RAM. It doesn't + write to disk (when using PostgreSQL, the database handles persistence). + +- **Janitor**: A cluster must have only one Janitor. In general, it requires 1 + core and 2GB of RAM and doesn't use the disk. If you use the [delete + API](https://quickwit.io/docs/overview/concepts/deletes), the Janitor should + be dimensioned like an indexer. ### Single node deployments diff --git a/docs/get-started/tutorials/tutorial-aws-lambda-simple.md b/docs/get-started/tutorials/tutorial-aws-lambda-simple.md deleted file mode 100644 index 58e59a7e4c4..00000000000 --- a/docs/get-started/tutorials/tutorial-aws-lambda-simple.md +++ /dev/null @@ -1,143 +0,0 @@ ---- -title: Search with AWS Lambda -description: Index and search using AWS Lambda on 20 million log entries -tags: [aws, integration] -icon_url: /img/tutorials/aws-logo.png -sidebar_position: 4 ---- - -In this tutorial, we will index and search about 20 million log entries (7 GB decompressed) located on AWS S3 with Quickwit Lambda. - -Concretely, we will deploy an AWS CloudFormation stack with the Quickwit Lambdas, and two buckets: one staging for hosting gzipped newline-delimited JSON files to be indexed and one for hosting the index data. The staging bucket is optional as Quickwit indexer can read data from any S3 files it has access to. - -![Tutorial stack overview](../../assets/images/quickwit-lambda-service.svg) - -## Install - -### Install AWS CDK - -We will use [AWS CDK](https://aws.amazon.com/cdk/) for our infrastructure automation script. Install it using [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm): -```bash -npm install -g aws-cdk - -You also need AWS credentials to be properly configured in your shell. One way is using the [credentials file](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). - -Finally, clone the Quickwit repository: -```bash -git clone https://github.com/quickwit-oss/tutorials.git -cd tutorials/simple-lambda-stack -``` - -### Setup python environment - -We use python 3.10 to define the AWS CloudFormation stack we need to deploy, and a python CLI to invoke Lambdas. -Let's install those few packages (boto3, aws-cdk-lib, click, pyyaml). - -```bash -# Install pipenv if needed. -pip install --user pipenv -pipenv shell -pipenv install -``` - -### Download Quickwit Lambdas - -```bash -mkdir -p cdk.out -wget -P cdk.out https://github.com/quickwit-oss/quickwit/releases/download/aws-lambda-beta-01/quickwit-lambda-indexer-beta-01-x86_64.zip -wget -P cdk.out https://github.com/quickwit-oss/quickwit/releases/download/aws-lambda-beta-01/quickwit-lambda-searcher-beta-01-x86_64.zip -``` - -### Bootstrap and deploy - -Configure the AWS region and [account id](https://docs.aws.amazon.com/IAM/latest/UserGuide/console_account-alias.html) where you want to deploy the stack: - -```bash -export CDK_ACCOUNT=123456789 -# us-east-1 is where the Quickwit public dataset bucket is located -export CDK_REGION=us-east-1 -``` - -If this region/account pair was not bootstrapped by CDK yet, run: -```bash -cdk bootstrap aws://$CDK_ACCOUNT/$CDK_REGION -``` - -This initializes some basic resources to host artifacts such as Lambda packages. We can now deploy the stack: -```bash -cdk deploy -a cdk/app.py -``` - -## Index the HDFS logs dataset - -Here is an example of a log entry of the dataset: -```json -{ - "timestamp": 1460530013, - "severity_text": "INFO", - "body": "PacketResponder: BP-108841162-10.10.34.11-1440074360971:blk_1074072698_331874, type=HAS_DOWNSTREAM_IN_PIPELINE terminating", - "resource": { - "service": "datanode/01" - }, - "attributes": { - "class": "org.apache.hadoop.hdfs.server.datanode.DataNode" - }, - "tenant_id": 58 -} -``` - -If you have 5 minutes ahead of you, you can index the whole dataset which is available on our public S3 bucket. - -```bash -python cli.py index s3://quickwit-datasets-public/hdfs-logs-multitenants.json.gz -``` - -:::note - -Once the indexing Lambda is started, you cannot stop it manually. A keyboard interrupt will stop the client but not the function. If you try running it again, you might get a `TooManyRequestsException` error. That's because the concurrency of the indexer is set to 1 to avoid race conditions on the metastore. - -::: - -If you don't want to wait, just index the 10,000 documents dataset: - -```bash -python cli.py index s3://quickwit-datasets-public/hdfs-logs-multitenants-10000.json -``` - -## Execute search queries - -Let's start with a query on the field `severity_text` and look for errors: `severity_text:ERROR`: - -```bash -python cli.py search '{"query":"severity_text:ERROR"}' -``` - -It should respond in under 1 second and return 10 hits out of 345 if you indexed the whole dataset. If you index the first 10,000 documents, you won't have any hits, try to query `INFO` logs instead. - - -Let's now run a more advanced query: a date histogram with a term aggregation on the `severity_text`` field: - -```bash -python cli.py search '{ "query": "*", "max_hits": 0, "aggs": { "events": { "date_histogram": { "field": "timestamp", "fixed_interval": "30d" }, "aggs": { "log_level": { "terms": { "size": 10, "field": "severity_text", "order": { "_count": "desc" } } } } } } }' -``` - -It should respond under 2 seconds and return the top log levels per 30 days. - - -### Cleaning up - -First, you have to delete the files created on your S3 buckets. -Once done, you can delete the stack. - -```bash -cdk destroy -a cdk/app.py -rm -rf cdk.out -``` - -Congratz! You finished this tutorial! You can level up with the following tutorials to discover all Quickwit features. - -## Next steps - -- [Search log service on AWS](/blog/log-search-service-for-under-7-dollars) which covers an end-to-end use case. -- [Search REST API](../../reference/rest-api) -- [Query language](../../reference/query-language) diff --git a/docs/guides/add-full-text-search-to-your-olap-db.md b/docs/guides/add-full-text-search-to-your-olap-db.md deleted file mode 100644 index 999a831cda5..00000000000 --- a/docs/guides/add-full-text-search-to-your-olap-db.md +++ /dev/null @@ -1,254 +0,0 @@ ---- -title: Full-text search on ClickHouse -description: Add full-text search to ClickHouse, using the Quickwit search streaming feature. -tags: [clickhouse, integration] -icon_url: /img/tutorials/clickhouse.svg -sidebar_position: 10 ---- - - -This guide will help you add full-text search to a well-known OLAP database, ClickHouse, using the Quickwit search streaming feature. Indeed Quickwit exposes a REST endpoint that streams ids or whatever attributes matching a search query **extremely fast** (up to 50 million in 1 second), and ClickHouse can easily use them with joins queries. - -We will take the [GitHub archive dataset](https://www.gharchive.org/), which gathers more than 3 billion GitHub events: `PullRequestEvent`, `IssuesEvent`... You can dive into this [great analysis](https://ghe.clickhouse.tech/) made by ClickHouse to have a good understanding of the dataset. We also took strong inspiration from this work, and we are very grateful to them for sharing this. - -## Install - -```bash -curl -L https://install.quickwit.io | sh -cd quickwit-v*/ -``` - -## Start a Quickwit server - -```bash -./quickwit run -``` - -## Create a Quickwit index - -After [starting Quickwit], we need to create an index configured to receive these events. Let's first look at the data to ingest. Here is an event example: - -```JSON -{ - "id": 11410577343, - "event_type": "PullRequestEvent", - "actor_login": "renovate[bot]", - "repo_name": "dmtrKovalenko/reason-date-fns", - "created_at": 1580515200000, - "action": "closed", - "number": 44, - "title": "Update dependency rollup to ^1.31.0", - "labels": [], - "ref": null, - "additions": 5, - "deletions": 5, - "commit_id": null, - "body":"This PR contains the following updates..." -} -``` - -We don't need to index all fields described above as `title` and `body` are the fields of interest for our full-text search tutorial. -The `id` will be helpful for making the JOINs in ClickHouse, `created_at` and `event_type` may also be beneficial for timestamp pruning and filtering. - -```yaml title="gh-archive-index-config.yaml" -version: 0.7 -index_id: gh-archive -# By default, the index will be stored in your data directory, -# but you can store it on s3 or on a custom path as follows: -# index_uri: s3://my-bucket/gh-archive -# index_uri: file://my-big-ssd-harddrive/ -doc_mapping: - store_source: false - field_mappings: - - name: id - type: u64 - fast: true - - name: created_at - type: datetime - input_formats: - - unix_timestamp - output_format: unix_timestamp_secs - fast_precision: seconds - fast: true - - name: event_type - type: text - tokenizer: raw - - name: title - type: text - tokenizer: default - record: position - - name: body - type: text - tokenizer: default - record: position - timestamp_field: created_at - -search_settings: - default_search_fields: [title, body] -``` - -```bash -curl -o gh-archive-index-config.yaml https://raw.githubusercontent.com/quickwit-oss/quickwit/main/config/tutorials/gh-archive/index-config-for-clickhouse.yaml -./quickwit index create --index-config gh-archive-index-config.yaml -``` - -## Indexing events - -The dataset is a compressed [NDJSON file](https://quickwit-datasets-public.s3.amazonaws.com/gh-archive/gh-archive-2021-12.json.gz). -Let's index it. - -```bash -wget https://quickwit-datasets-public.s3.amazonaws.com/gh-archive/gh-archive-2021-12-text-only.json.gz -gunzip -c gh-archive-2021-12-text-only.json.gz | ./quickwit index ingest --index gh-archive -``` - -You can check it's working by using the `search` command and looking for `tantivy` word: -```bash -./quickwit index search --index gh-archive --query "tantivy" -``` - - -## Streaming IDs - -We are now ready to fetch some ids with the search stream endpoint. Let's start by streaming them on a simple -query and with a `csv` output format. - -```bash -curl "http://127.0.0.1:7280/api/v1/gh-archive/search/stream?query=tantivy&output_format=csv&fast_field=id" -``` - -We will use the `click_house` binary output format in the following sections to speed up queries. - - -## ClickHouse - -Let's leave Quickwit for now and [install ClickHouse](https://clickhouse.com/docs/en/install). Start a ClickHouse server. - -### Create database and table - -Once installed, just start a client and execute the following sql statements: -```SQL -CREATE DATABASE "gh-archive"; -USE "gh-archive"; - - -CREATE TABLE github_events -( - id UInt64, - event_type Enum('CommitCommentEvent' = 1, 'CreateEvent' = 2, 'DeleteEvent' = 3, 'ForkEvent' = 4, - 'GollumEvent' = 5, 'IssueCommentEvent' = 6, 'IssuesEvent' = 7, 'MemberEvent' = 8, - 'PublicEvent' = 9, 'PullRequestEvent' = 10, 'PullRequestReviewCommentEvent' = 11, - 'PushEvent' = 12, 'ReleaseEvent' = 13, 'SponsorshipEvent' = 14, 'WatchEvent' = 15, - 'GistEvent' = 16, 'FollowEvent' = 17, 'DownloadEvent' = 18, 'PullRequestReviewEvent' = 19, - 'ForkApplyEvent' = 20, 'Event' = 21, 'TeamAddEvent' = 22), - actor_login LowCardinality(String), - repo_name LowCardinality(String), - created_at Int64, - action Enum('none' = 0, 'created' = 1, 'added' = 2, 'edited' = 3, 'deleted' = 4, 'opened' = 5, 'closed' = 6, 'reopened' = 7, 'assigned' = 8, 'unassigned' = 9, - 'labeled' = 10, 'unlabeled' = 11, 'review_requested' = 12, 'review_request_removed' = 13, 'synchronize' = 14, 'started' = 15, 'published' = 16, 'update' = 17, 'create' = 18, 'fork' = 19, 'merged' = 20), - comment_id UInt64, - body String, - ref LowCardinality(String), - number UInt32, - title String, - labels Array(LowCardinality(String)), - additions UInt32, - deletions UInt32, - commit_id String -) ENGINE = MergeTree ORDER BY (event_type, repo_name, created_at); -``` - -### Import events - -We have created a second dataset, `gh-archive-2021-12.json.gz`, which gathers all events, even ones with no -text. So it's better to insert it into ClickHouse, but if you don't have the time, you can use the dataset -`gh-archive-2021-12-text-only.json.gz` used for Quickwit. - -```bash -wget https://quickwit-datasets-public.s3.amazonaws.com/gh-archive/gh-archive-2021-12.json.gz -gunzip -c gh-archive-2021-12.json.gz | clickhouse-client -d gh-archive --query="INSERT INTO github_events FORMAT JSONEachRow" -``` - -Let's check it's working: -```SQL -# Top repositories by stars -SELECT repo_name, count() AS stars -FROM github_events -GROUP BY repo_name -ORDER BY stars DESC LIMIT 5 - -┌─repo_name─────────────────────────────────┬─stars─┐ -│ test-organization-kkjeer/app-test-2 │ 16697 │ -│ test-organization-kkjeer/bot-validation-2 │ 15326 │ -│ microsoft/winget-pkgs │ 14099 │ -│ conda-forge/releases │ 13332 │ -│ NixOS/nixpkgs │ 12860 │ -└───────────────────────────────────────────┴───────┘ -``` - -### Use Quickwit search inside ClickHouse - -ClickHouse has an exciting feature called [URL Table Engine](https://clickhouse.com/docs/en/engines/table-engines/special/url/) that queries data from a remote HTTP/HTTPS server. -This is precisely what we need: by creating a table pointing to Quickwit search stream endpoint, we will fetch ids that match a query from ClickHouse. - -```SQL -SELECT count(*) FROM url('http://127.0.0.1:7280/api/v1/gh-archive/search/stream?query=log4j+OR+log4shell&fast_field=id&output_format=click_house_row_binary', RowBinary, 'id UInt64') - -┌─count()─┐ -│ 217469 │ -└─────────┘ - -1 row in set. Elapsed: 0.068 sec. Processed 217.47 thousand rows, 1.74 MB (3.19 million rows/s., 25.55 MB/s.) -``` - -We are fetching 217 469 u64 ids in 0.068 seconds. That's 3.19 million rows per second, not bad. And it's possible to increase the throughput if fast field are already cached. - - -Let's do another example with a more exciting query that will match `log4j` or `log4shell` and count events per day: - -```SQL -SELECT - count(*), - toDate(fromUnixTimestamp64Milli(created_at)) AS date -FROM github_events -WHERE id IN ( - SELECT id - FROM url('http://127.0.0.1:7280/api/v1/gh-archive/search/stream?query=log4j+OR+log4shell&fast_field=id&output_format=click_house_row_binary', RowBinary, 'id UInt64') -) -GROUP BY date - -Query id: 10cb0d5a-7817-424e-8248-820fa2c425b8 - -┌─count()─┬───────date─┐ -│ 96 │ 2021-12-01 │ -│ 66 │ 2021-12-02 │ -│ 70 │ 2021-12-03 │ -│ 62 │ 2021-12-04 │ -│ 67 │ 2021-12-05 │ -│ 167 │ 2021-12-06 │ -│ 140 │ 2021-12-07 │ -│ 104 │ 2021-12-08 │ -│ 157 │ 2021-12-09 │ -│ 88110 │ 2021-12-10 │ -│ 2937 │ 2021-12-11 │ -│ 1533 │ 2021-12-12 │ -│ 5935 │ 2021-12-13 │ -│ 118025 │ 2021-12-14 │ -└─────────┴────────────┘ - -14 rows in set. Elapsed: 0.124 sec. Processed 8.35 million rows, 123.10 MB (67.42 million rows/s., 993.55 MB/s.) - -``` - -We can see two spikes on the 2021-12-10 and 2021-12-14. - -## Wrapping up - -We have just scratched the surface of full-text search from ClickHouse with this small subset of GitHub archive. -You can play with the complete dataset that you can download from our public S3 bucket. -We have made available monthly gzipped ndjson files from 2015 until 2021. Here are `2015-01` links: -- full JSON dataset https://quickwit-datasets-public.s3.amazonaws.com/gh-archive/gh-archive-2015-01.json.gz -- text-only JSON dataset https://quickwit-datasets-public.s3.amazonaws.com/gh-archive/gh-archive-2015-01-text-only.json.gz - -The search stream endpoint is powerful enough to stream 100 million ids to ClickHouse in less than 2 seconds on a multi TB dataset. -And you should be comfortable playing with search stream on even bigger datasets. diff --git a/docs/overview/concepts/querying.md b/docs/overview/concepts/querying.md index 04e699e8781..a883ea21dce 100644 --- a/docs/overview/concepts/querying.md +++ b/docs/overview/concepts/querying.md @@ -3,11 +3,6 @@ title: Querying sidebar_position: 2 --- -Quickwit provides two endpoints with full-text search queries identified by the `query` parameter: - -- A search endpoint that returns a [JSON](../../reference/rest-api.md) -- A search stream endpoint that returns a stream of the requested [field values](../../reference/rest-api.md) - A search query received by a searcher will be executed using a map-reduce approach following these steps: 1. The Searcher identifies relevant splits based on the request’s [timestamp interval](#time-sharding) and [tags](#tag-pruning). @@ -90,10 +85,6 @@ For instance, if tenant\_1,app\_1 and tenant\_2,app\_2 are both sent to partitio still search inside the 1st partition as it will be tagged with tenant\_1,tenant\_2,app\_1 and app\_2. You should therefore prefer a partition key such as `hash_mod(tenant_id, 10),hash_mod(app_id, 5)` which will generate as many splits, but with better tags. -### Search stream query limits - -Search stream queries can take a huge amount of RAM. Quickwit limits the number of concurrent search streams per split to 100 by default. You can adjust this limit by setting the value of the searcher configuration property called `max_num_concurrent_split_streams` in the configuration file. - ### Caching Quickwit does caching in many places to deliver a highly performing query engine. diff --git a/docs/overview/introduction.md b/docs/overview/introduction.md index db84adfda6e..a219a8e82aa 100644 --- a/docs/overview/introduction.md +++ b/docs/overview/introduction.md @@ -27,7 +27,6 @@ Check out our guides to see how you can use Quickwit: - [Log management](../log-management/overview.md) - [Distributed Tracing](../distributed-tracing/overview.md) -- Adding full-text search capabilities to [OLAP databases such as ClickHouse](../guides/add-full-text-search-to-your-olap-db). ## Key features diff --git a/docs/reference/es_compatible_api.md b/docs/reference/es_compatible_api.md index 15006970f44..28ba1aa7eb2 100644 --- a/docs/reference/es_compatible_api.md +++ b/docs/reference/es_compatible_api.md @@ -365,6 +365,79 @@ Example response: [HTTP accept header]: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html + +### `_field_caps`   Field capabilities API + +``` +GET api/v1/_elastic//_field_caps +``` +``` +POST api/v1/_elastic//_field_caps +``` +``` +GET api/v1/_elastic/_field_caps +``` +``` +POST api/v1/_elastic/_field_caps +``` + +The [field capabilities API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-field-caps.html) returns information about the capabilities of fields among multiple indices. + +#### Supported Query string parameters + +| Variable | Type | Description | Default value | +| --------------------- | ---------- | ------------------------------------------------------------------------------ | ------------- | +| `fields` | `String` | Comma-separated list of fields to retrieve capabilities for. Supports wildcards (`*`). | (Optional) | +| `allow_no_indices` | `Boolean` | If `true`, missing or closed indices are not an error. | (Optional) | +| `expand_wildcards` | `String` | Controls what kind of indices that wildcard patterns can match. | (Optional) | +| `ignore_unavailable` | `Boolean` | If `true`, unavailable indices are ignored. | (Optional) | +| `start_timestamp` | `Integer` | *(Quickwit-specific)* If set, restricts splits to documents with a timestamp range start >= `start_timestamp` (seconds since epoch). | (Optional) | +| `end_timestamp` | `Integer` | *(Quickwit-specific)* If set, restricts splits to documents with a timestamp range end < `end_timestamp` (seconds since epoch). | (Optional) | + +#### Supported Request Body parameters + +| Variable | Type | Description | Default value | +| ------------------ | ------------- | --------------------------------------------------------------------------- | ------------- | +| `index_filter` | `Json object` | A query to filter indices. If provided, only fields from indices that can potentially match the filter are returned. See [index_filter](#index_filter). | (Optional) | +| `runtime_mappings` | `Json object` | Accepted but not supported. | (Optional) | + +#### `index_filter` + +The `index_filter` parameter allows you to filter which indices contribute to the field capabilities response. When provided, Quickwit uses the filter query to prune indices (splits) that cannot match the filter, and only returns field capabilities for the remaining ones. + +Like Elasticsearch, this is a **best-effort** approach: Quickwit may return field capabilities from indices that do not actually contain any matching documents. In Quickwit, the filtering is limited to the existing split-pruning based on metadata: + +- **Time pruning**: Range queries on the timestamp field can eliminate splits whose time range does not overlap with the filter. +- **Tag pruning**: Term queries on [tag fields](../configuration/index-config.md#tag-fields) can eliminate splits that do not contain the requested tag value. + +Other filter types (e.g. full-text queries or term queries on non-tag fields) are accepted but will not prune any splits — all indices will be returned as if no filter was specified. In particular, Quickwit does not check whether terms are present in the term dictionary. + +#### Request Body example + +```json +{ + "index_filter": { + "range": { + "timestamp": { + "gte": "2024-01-01T00:00:00Z", + "lt": "2024-02-01T00:00:00Z" + } + } + } +} +``` + +```json +{ + "index_filter": { + "term": { + "status": "active" + } + } +} +``` + + ## Query DSL [Elasticsearch Query DSL reference](https://www.elastic.co/guide/en/elasticsearch/reference/8.8/query-dsl.html). @@ -695,10 +768,11 @@ When working on text, it is recommended to only use `term` queries on fields con #### Supported Parameters -| Variable | Type | Description | Default | -| -------- | -------- | ---------------------------------------------------------------------------- | ------- | -| `value` | String | Term value. This is the string representation of a token after tokenization. | - | -| `boost` | `Number` | Multiplier boost for score computation | 1.0 | +| Variable | Type | Description | Default | +| ------------------ | ------- | ---------------------------------------------------------------------------- | ------- | +| `value` | String | Term value. This is the string representation of a token after tokenization. | - | +| `boost` | Number | Multiplier boost for score computation | 1.0 | +| `case_insensitive` | Boolean | Allows ASCII case insensitive matching of the value. | false | @@ -741,6 +815,91 @@ Query matching only documents containing a non-null value for a given field. | -------- | ------ | ------------------------------------------------------- | ------- | | `field` | String | Only documents with a value for field will be returned. | - | +### `prefix` + +[Elasticsearch reference documentation](https://www.elastic.co/guide/en/elasticsearch/reference/8.8/query-dsl-prefix-query.html) + +Returns documents that contain a specific prefix in a provided field. + +#### Example + +```json +{ + "query": { + "prefix": { + "author.login" { + "value": "adm", + } + } + } +} +``` + +#### Supported Parameters + +| Variable | Type | Description | Default | +| ------------------ | ------- | ---------------------------------------------------- | ------- | +| `value` | String | Beginning characters of terms you wish to find. | - | +| `case_insensitive` | Boolean | Allows ASCII case insensitive matching of the value. | false | + +### `wildcard` + +[Elasticsearch reference documentation](https://www.elastic.co/guide/en/elasticsearch/reference/8.8/query-dsl-wildcard-query.html) + +Returns documents that contain terms matching a wildcard pattern: +* `?` replaces one and only one term character +* `*` replaces any number of term characters or an empty string + +#### Example + +```json +{ + "query": { + "wildcard": { + "author.login" { + "value": "adm?n*", + } + } + } +} +``` + +#### Supported Parameters + +| Variable | Type | Description | Default | +| ------------------ | ------- | ---------------------------------------------------- | ------- | +| `value` | String | Wildcard pattern for terms you wish to find. | - | +| `boost` | Number | Multiplier boost for score computation. | 1.0 | +| `case_insensitive` | Boolean | Allows ASCII case insensitive matching of the value. | false | + + +### `regexp` + +[Elasticsearch reference documentation](https://www.elastic.co/guide/en/elasticsearch/reference/8.8/query-dsl-regexp-query.html) + +Returns documents that contain terms matching a regular expression. + +#### Example + +```json +{ + "query": { + "regexp": { + "author.login" { + "value": "adm.*n", + } + } + } +} +``` + +#### Supported Parameters + +| Variable | Type | Description | Default | +| ------------------ | ------- | ---------------------------------------------------- | ------- | +| `value` | String | Wildcard pattern for terms you wish to find. | - | +| `case_insensitive` | Boolean | Allows ASCII case insensitive matching of the value. | false | + ### About the `lenient` argument diff --git a/docs/reference/metrics.md b/docs/reference/metrics.md index 9df72335a05..9e854f8f1f6 100644 --- a/docs/reference/metrics.md +++ b/docs/reference/metrics.md @@ -42,8 +42,8 @@ Currently Quickwit exposes metrics for three caches: `fastfields`, `shortlived`, | Namespace | Metric Name | Description | Type | | --------- | ----------- | ----------- | ---- | -| `quickwit_ingest` | `ingested_num_bytes` | Total size of the docs ingested in bytes | `counter` | -| `quickwit_ingest` | `ingested_num_docs` | Number of docs received to be ingested | `counter` | +| `quickwit_ingest` | `docs_bytes_total` | Total size of the docs ingested, measured in ingester's leader, after validation and before persistence/replication | `counter` | +| `quickwit_ingest` | `docs_total` | Total number of the docs ingested, measured in ingester's leader, after validation and before persistence/replication | `counter` | | `quickwit_ingest` | `queue_count` | Number of queues currently active | `counter` | ## Metastore Metrics diff --git a/docs/reference/rest-api.md b/docs/reference/rest-api.md index 5047afd6308..54ac05c040b 100644 --- a/docs/reference/rest-api.md +++ b/docs/reference/rest-api.md @@ -113,61 +113,6 @@ GET api/v1/stackoverflow*/search } ``` -### Search stream in an index - -``` -GET api/v1//search/stream?query=searchterm&fast_field=my_id -``` - -Streams field values from ALL documents matching a search query in the target index ``, in a specified output format among the following: - -- [CSV](https://datatracker.ietf.org/doc/html/rfc4180) -- [ClickHouse RowBinary](https://clickhouse.tech/docs/en/interfaces/formats/#rowbinary). If `partition_by_field` is set, Quickwit returns chunks of data for each partition field value. Each chunk starts with 16 bytes being partition value and content length and then the `fast_field` values in `RowBinary` format. - -`fast_field` and `partition_by_field` must be fast fields of type `i64` or `u64`. - -This endpoint is available as long as you have at least one node running a searcher service in the cluster. - - - -:::note - -The endpoint will return 10 million values if 10 million documents match the query. This is expected, this endpoint is made to support queries matching millions of documents and return field values in a reasonable response time. - -::: - -#### Path variable - -| Variable | Description | -| ------------- | ------------- | -| `index id` | The index id | - -#### Get parameters - -| Variable | Type | Description | Default value | -|---------------------|------------|----------------------------------------------------------------------------------------------------------|----------------------------------------------------| -| `query` | `String` | Query text. See the [query language doc](query-language.md) | _required_ | -| `fast_field` | `String` | Name of a field to retrieve from documents. This field must be a fast field of type `i64` or `u64`. | _required_ | -| `search_field` | `[String]` | Fields to search on. Comma-separated list, e.g. "field1,field2" | index_config.search_settings.default_search_fields | -| `start_timestamp` | `i64` | If set, restrict search to documents with a `timestamp >= start_timestamp`. The value must be in seconds. | | -| `end_timestamp` | `i64` | If set, restrict search to documents with a `timestamp < end_timestamp`. The value must be in seconds. | | -| `partition_by_field` | `String` | If set, the endpoint returns chunks of data for each partition field value. This field must be a fast field of type `i64` or `u64`. | | -| `output_format` | `String` | Response output format. `csv` or `clickHouseRowBinary` | `csv` | - -:::info -The `start_timestamp` and `end_timestamp` should be specified in seconds regardless of the timestamp field precision. -::: - -#### Response - -The response is an HTTP stream. Depending on the client's capability, it is an HTTP1.1 [chunked transfer encoded stream](https://en.wikipedia.org/wiki/Chunked_transfer_encoding) or an HTTP2 stream. - -It returns a list of all the field values from documents matching the query. The field must be marked as "fast" in the index config for this to work. -The formatting is based on the specified output format. - -On error, an "X-Stream-Error" header will be sent via the trailers channel with information about the error, and the stream will be closed via [`sender.abort()`](https://docs.rs/hyper/0.14.16/hyper/body/struct.Sender.html#method.abort). -Depending on the client, the trailer header with error details may not be shown. The error will also be logged in quickwit ("Error when streaming search results"). - ## Ingest API ### Ingest data into an index diff --git a/monitoring/grafana/dashboards/searchers.json b/monitoring/grafana/dashboards/searchers.json index b6f7ade938b..a5e00656050 100644 --- a/monitoring/grafana/dashboards/searchers.json +++ b/monitoring/grafana/dashboards/searchers.json @@ -606,7 +606,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "rate(quickwit_cache_cache_hits_total{instance=~\"$instance\"}[$__rate_interval])", + "expr": "rate(quickwit_cache_cache_misses_total{instance=~\"$instance\"}[$__rate_interval])", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, diff --git a/quickwit/CLAUDE.md b/quickwit/CLAUDE.md new file mode 100644 index 00000000000..43c2bb18367 --- /dev/null +++ b/quickwit/CLAUDE.md @@ -0,0 +1,120 @@ +## Build & Test Commands + +### Formatting & Linting +- **`make fmt`** — Format and validate code (requires nightly toolchain: `rustup toolchain install nightly`): + 1. Runs `cargo +nightly fmt` + 2. Checks license headers on `.rs`, `.ts`, `.proto` files + 3. Enforces log format policy: no trailing punctuation, no uppercase first character in log and error messages +- **`make fix`** — Runs clippy with `--fix`, then `make fmt`, then `make unused-deps` +- **`make unused-deps`** — Detects unused dependencies via `cargo-machete` + +Log messages (`info!`, `warn!`, `error!`, `debug!`) must: +- Start with a **lowercase** letter +- Have **no trailing punctuation** + +### Testing +- **Single crate test**: `cargo nextest run -p quickwit-search my_test_name` +- **Single test**: `cargo test -p quickwit-common my_test_name` +- **`make test-all`** — Starts Docker services (LocalStack S3, PostgreSQL, Pub/Sub emulator) and runs the full test suite with `cargo nextest run --all-features --retries 5` +- **`make test-failpoints`** — Runs failpoint tests only: `cargo nextest run --test failpoints --features fail/failpoints` +- Docker services: `make docker-compose-up` / `make docker-compose-down` (subset: `DOCKER_SERVICES=kafka,postgres`) + +### Building +- **`make doc`** — Generates docs with `cargo doc --all-features` (warnings as errors) +- Rust toolchain: **1.93** + +## Code Conventions + +### Clippy Disallowed Methods +These methods are banned (see `clippy.toml`): +- `Path::exists` — (use try_exists) +- `Option::is_some_and`, `Option::is_none_or`, `Option::xor` +- `Option::map_or`, `Option::map_or_else` — use `.map(..).unwrap_or(..)` or `let Some(..) else {..}` instead + +### Formatting Shortcut +Use `/fmt` to automatically run format checks. + +## Architecture Overview + +Quickwit is a cloud-native distributed search engine for observability data (logs, traces). It's organized as a ~38-crate Rust workspace. + +### Key Layers + +**Protocol & Types** — `quickwit-proto` defines all gRPC service contracts and message types via protobuf. Service traits are auto-generated. + +**Actor System** — `quickwit-actors` is a custom lightweight actor framework. The indexing pipeline is fully actor-based: +``` +Source → DocProcessor → Indexer → IndexSerializer → Packager → Uploader → Sequencer → Publisher +``` +A parallel merge pipeline runs alongside. + +**Search** — `quickwit-search` implements a root-leaf pattern: root servers parse queries and coordinate, leaf servers search their assigned splits in parallel, leaf results are merged at root. + +**Storage** — `quickwit-storage` abstracts cloud storage (S3, Azure, GCS, local file, RAM) behind a `Storage` trait. + +**Metastore** — `quickwit-metastore` manages index metadata with file-backed (dev) and PostgreSQL (production) backends. + +**Cluster** — `quickwit-cluster` uses Chitchat gossip protocol for membership. `quickwit-control-plane` handles indexing task scheduling and placement. + +**API Surface** — `quickwit-serve` hosts both REST and gRPC endpoints over the same service traits, plus serves the embedded React UI. + +### Core Crates +| Crate | Purpose | +|-------|---------| +| `quickwit-cli` | CLI entry point and binary | +| `quickwit-serve` | REST/gRPC server | +| `quickwit-search` | Distributed search orchestration | +| `quickwit-indexing` | Actor-based indexing pipeline | +| `quickwit-ingest` | Distributed ingestion with replication | +| `quickwit-metastore` | Index metadata storage | +| `quickwit-storage` | Multi-cloud storage abstraction | +| `quickwit-config` | Configuration parsing/validation | +| `quickwit-doc-mapper` | Index schema and document mapping | +| `quickwit-query` | Query DSL parsing (ES-compatible) | +| `quickwit-cluster` | Cluster membership (Chitchat) | +| `quickwit-control-plane` | Indexing task scheduling | +| `quickwit-actors` | Actor framework | +| `quickwit-proto` | Protobuf definitions and gRPC traits | +| `quickwit-common` | Shared utilities and metrics | +| `quickwit-lambda-server` | AWS Lambda leaf search handler | +| `quickwit-lambda-client` | Lambda invocation with auto-deployment | + +quickwit-common contains shared utilities about metrics, rate limited logging, reading from environment variables, etc. +It also contains the `run_cpu_intensive` that should be use to run CPU-intensive tasks from tokio tasks. + +When the client is unlikely to match on an error, you can rely on the crate level Error or anyhow::Error. If you need to introduce a new Error type, use thiserror. + +### Design Patterns +- **Trait-based services**: `SearchService`, `MetastoreService`, etc. — enables mocking and multiple implementations +- **Feature gates**: Cloud backends (`azure`, `gcs`), message sources (`kafka`, `kinesis`, `pulsar`, `sqs`, `gcp-pubsub`), `postgres` metastore, `multilang` tokenizers +- **Metrics**: `once_cell::sync::Lazy` statics with `quickwit_common::metrics::*` factories + +### Key Dependencies +- **Tantivy**: Search engine library (custom fork) +- **Tonic/Prost**: gRPC framework and protobuf +- **Tokio**: Async runtime +- **SQLx**: PostgreSQL metastore + +# Quickwit Claude Guidelines + +When adding a new dependency, update license by running `make update-licenses`. +Prefer referring to the crate in workspace. +Make sure to keep features minimal. + +In other words, prefer +zip = { workspace = true, default-features = false, features=["deflate"] } +to +zip = "2" + +## Code Formatting +### Quick Fix + +Use `/fmt` to automatically run format checks and see issues. + +## Coding Style +- Avoid single-letter variable names except for indices (i, j, k) +- Document all "hidden contracts" (implicit assumptions, invariants, preconditions) +- Try to avoid deep nesting. In particular, prefer early return style +- Avoid abusing iterator chaining with complex constructs like `.transpose()` +- Write type names explicitly when it aids readability +- Use `with_capacity` to hint container capacity when size is known diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index df499e2a2ed..d83f33320a8 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -51,7 +51,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -70,17 +70,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "ahash" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" -dependencies = [ - "getrandom 0.2.16", - "once_cell", - "version_check", -] - [[package]] name = "ahash" version = "0.8.12" @@ -88,8 +77,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", - "getrandom 0.3.3", + "const-random", + "getrandom 0.3.4", "once_cell", + "serde", "version_check", "zerocopy", ] @@ -109,6 +100,15 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" +[[package]] +name = "aligned-vec" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" +dependencies = [ + "equator", +] + [[package]] name = "alloca" version = "0.4.0" @@ -147,21 +147,21 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "ansi-str" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cf4578926a981ab0ca955dc023541d19de37112bc24c1a197bd806d3d86ad1d" +checksum = "060de1453b69f46304b28274f382132f4e72c55637cf362920926a70d090890d" dependencies = [ "ansitok", ] [[package]] name = "ansitok" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "220044e6a1bb31ddee4e3db724d29767f352de47445a6cd75e1a173142136c83" +checksum = "c0a8acea8c2f1c60f0a92a8cd26bf96ca97db56f10bbcab238bbe0cceba659ee" dependencies = [ - "nom", - "vte 0.10.1", + "nom 7.1.3", + "vte", ] [[package]] @@ -216,27 +216,202 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.99" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "arc-swap" -version = "1.7.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +checksum = "a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6" +dependencies = [ + "rustversion", +] [[package]] name = "arrayvec" -version = "0.5.2" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] -name = "arrayvec" -version = "0.7.6" +name = "arrow" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +checksum = "e4754a624e5ae42081f464514be454b39711daae0458906dacde5f4c632f33a8" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7b3141e0ec5145a22d8694ea8b6d6f69305971c4fa1c1a13ef0195aef2d678b" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8955af33b25f3b175ee10af580577280b4bd01f7e823d94c7cdef7cf8c9aef" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.16.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c697ddca96183182f35b3a18e50b9110b11e916d7b7799cbfd4d34662f2c56c2" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "646bbb821e86fd57189c10b4fcdaa941deaf4181924917b0daa92735baa6ada5" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64 0.22.1", + "chrono", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-data" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fdd994a9d28e6365aa78e15da3f3950c0fdcea6b963a12fa1c391afb637b304" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ipc" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf7df950701ab528bf7c0cf7eeadc0445d03ef5d6ffc151eaae6b38a58feff1" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", +] + +[[package]] +name = "arrow-ord" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d8f1870e03d4cbed632959498bcc84083b5a24bded52905ae1695bd29da45b" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18228633bad92bff92a95746bbeb16e5fc318e8382b75619dec26db79e4de4c0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c872d36b7bf2a6a6a2b40de9156265f0242910791db366a2c17476ba8330d68" +dependencies = [ + "serde_core", + "serde_json", +] + +[[package]] +name = "arrow-select" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68bf3e3efbd1278f770d67e5dc410257300b161b93baedb3aae836144edcaf4b" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85e968097061b3c0e9fe3079cf2e703e487890700546b5b0647f60fca1b5a8d8" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] [[package]] name = "ascii-canvas" @@ -451,8 +626,8 @@ dependencies = [ "bytes", "fastrand 2.3.0", "hex", - "http 1.3.1", - "ring 0.17.14", + "http 1.4.0", + "ring", "time", "tokio", "tracing", @@ -462,9 +637,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.6" +version = "1.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d025db5d9f52cbc413b167136afb3d8aeea708c0d8884783cf6253be5e22f6f2" +checksum = "e26bbf46abc608f2dc61fd6cb3b7b0665497cc259a21520151ed98f8b37d2c79" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -497,23 +672,26 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.10" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c034a1bc1d70e16e7f4e4caf7e9f7693e4c9c24cd91cf17c2a0b21abaebc7c8b" +checksum = "b0f92058d22a46adf53ec57a6a96f34447daf02bff52e8fb956c66bcd5c6ac12" dependencies = [ "aws-credential-types", "aws-sigv4", "aws-smithy-async", "aws-smithy-eventstream", - "aws-smithy-http 0.62.3", + "aws-smithy-http 0.63.4", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", + "bytes-utils", "fastrand 2.3.0", "http 0.2.12", + "http 1.4.0", "http-body 0.4.6", + "http-body 1.0.1", "percent-encoding", "pin-project-lite", "tracing", @@ -522,16 +700,42 @@ dependencies = [ [[package]] name = "aws-sdk-kinesis" -version = "1.88.0" +version = "1.100.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d59431117b456eabf7e27ef9cc0137af9cf880937ce459e3ce9dfdf983328b5" +checksum = "5769458ed398a643d6f0a6307077311fe253655d9f3ecc3e53069dc61cbcc98c" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-eventstream", - "aws-smithy-http 0.62.3", - "aws-smithy-json 0.61.5", + "aws-smithy-http 0.63.4", + "aws-smithy-json 0.62.4", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand 2.3.0", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-lambda" +version = "1.116.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c16765a900546bf5bc6b60d317f78692d56da3829f3603e766ba326a4727fb3" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http 0.63.4", + "aws-smithy-json 0.62.4", + "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -539,6 +743,7 @@ dependencies = [ "bytes", "fastrand 2.3.0", "http 0.2.12", + "http 1.4.0", "regex-lite", "tracing", ] @@ -579,15 +784,16 @@ dependencies = [ [[package]] name = "aws-sdk-sqs" -version = "1.83.0" +version = "1.94.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b3e9f9a923e6c20272c40de609faa8624b687d203bc77df99ccb86a5de5944b" +checksum = "0894909314920ab2c907110e7f91b9f7a6c826bdc830ccf36079c69af2b82335" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.62.3", - "aws-smithy-json 0.61.5", + "aws-smithy-http 0.63.4", + "aws-smithy-json 0.62.4", + "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -595,6 +801,7 @@ dependencies = [ "bytes", "fastrand 2.3.0", "http 0.2.12", + "http 1.4.0", "regex-lite", "tracing", ] @@ -668,13 +875,13 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.3.4" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084c34162187d39e3740cb635acd73c4e3a551a36146ad6fe8883c929c9f876c" +checksum = "68f6ae9b71597dc5fd115d52849d7a5556ad9265885ad3492ea8d73b93bbc46e" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", - "aws-smithy-http 0.62.3", + "aws-smithy-http 0.63.4", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", @@ -683,10 +890,10 @@ dependencies = [ "hex", "hmac", "http 0.2.12", - "http 1.3.1", + "http 1.4.0", "p256 0.11.1", "percent-encoding", - "ring 0.17.14", + "ring", "sha2", "subtle", "time", @@ -696,9 +903,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.5" +version = "1.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e190749ea56f8c42bf15dd76c65e14f8f765233e6df9b0506d9d934ebef867c" +checksum = "2ffcaf626bdda484571968400c326a244598634dc75fd451325a54ad1a59acfc" dependencies = [ "futures-util", "pin-project-lite", @@ -728,9 +935,9 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.60.11" +version = "0.60.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "182b03393e8c677347fb5705a04a9392695d47d20ef0a2f8cfe28c8e6b9b9778" +checksum = "1c0b3e587fbaa5d7f7e870544508af8ce82ea47cd30376e69e1e37c4ac746f79" dependencies = [ "aws-smithy-types", "bytes", @@ -764,14 +971,13 @@ version = "0.62.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c4dacf2d38996cf729f55e7a762b30918229917eca115de45dfa8dfb97796c9" dependencies = [ - "aws-smithy-eventstream", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", "bytes-utils", "futures-core", "http 0.2.12", - "http 1.3.1", + "http 1.4.0", "http-body 0.4.6", "percent-encoding", "pin-project-lite", @@ -779,11 +985,33 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-smithy-http" +version = "0.63.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af4a8a5fe3e4ac7ee871237c340bbce13e982d37543b65700f4419e039f5d78e" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + [[package]] name = "aws-smithy-http-client" -version = "1.1.1" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147e8eea63a40315d704b97bf9bc9b8c1402ae94f89d5ad6f7550d963309da1b" +checksum = "0709f0083aa19b704132684bc26d3c868e06bd428ccc4373b0b55c3e8748a58b" dependencies = [ "aws-smithy-async", "aws-smithy-protocol-test", @@ -793,19 +1021,19 @@ dependencies = [ "h2 0.3.27", "h2 0.4.12", "http 0.2.12", - "http 1.3.1", + "http 1.4.0", "http-body 0.4.6", "http-body 1.0.1", "hyper 0.14.32", - "hyper 1.7.0", + "hyper 1.8.1", "hyper-rustls 0.24.2", "hyper-rustls 0.27.7", "hyper-util", - "indexmap 2.11.0", + "indexmap 2.13.0", "pin-project-lite", "rustls 0.21.12", "rustls 0.23.31", - "rustls-native-certs 0.8.1", + "rustls-native-certs", "rustls-pki-types", "serde", "serde_json", @@ -833,20 +1061,41 @@ dependencies = [ "aws-smithy-types", ] +[[package]] +name = "aws-smithy-json" +version = "0.62.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b3a779093e18cad88bbae08dc4261e1d95018c4c5b9356a52bcae7c0b6e9bb" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-mocks" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "943deb750d91da9f788024482ff1f3393cefb017d0dc30fc5cecb11e1be15770" +dependencies = [ + "aws-smithy-http-client", + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.4.0", +] + [[package]] name = "aws-smithy-observability" -version = "0.1.3" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9364d5989ac4dd918e5cc4c4bdcc61c9be17dcd2586ea7f69e348fc7c6cab393" +checksum = "4d3f39d5bb871aaf461d59144557f16d5927a5248a983a40654d9cf3b9ba183b" dependencies = [ "aws-smithy-runtime-api", ] [[package]] name = "aws-smithy-protocol-test" -version = "0.63.4" +version = "0.63.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2116be0e3a8e18771b126f8eb69901b2780fbe12b1890370cb1c462337d13b" +checksum = "b59f9305f7863a70f4a0c048fa6d81fb9dd9373a751358791faaad8881c1377f" dependencies = [ "assert-json-diff", "aws-smithy-runtime-api", @@ -873,12 +1122,12 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.9.1" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3946acbe1ead1301ba6862e712c7903ca9bb230bdf1fbd1b5ac54158ef2ab1f" +checksum = "8fd3dfc18c1ce097cf81fced7192731e63809829c6cbf933c1ec47452d08e1aa" dependencies = [ "aws-smithy-async", - "aws-smithy-http 0.62.3", + "aws-smithy-http 0.63.4", "aws-smithy-http-client", "aws-smithy-observability", "aws-smithy-runtime-api", @@ -886,9 +1135,10 @@ dependencies = [ "bytes", "fastrand 2.3.0", "http 0.2.12", - "http 1.3.1", + "http 1.4.0", "http-body 0.4.6", "http-body 1.0.1", + "http-body-util", "pin-project-lite", "pin-utils", "tokio", @@ -898,15 +1148,15 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.9.0" +version = "1.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07f5e0fc8a6b3f2303f331b94504bbf754d85488f402d6f1dd7a6080f99afe56" +checksum = "8c55e0837e9b8526f49e0b9bfa9ee18ddee70e853f5bc09c5d11ebceddcb0fec" dependencies = [ "aws-smithy-async", "aws-smithy-types", "bytes", "http 0.2.12", - "http 1.3.1", + "http 1.4.0", "pin-project-lite", "tokio", "tracing", @@ -915,16 +1165,16 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.3.2" +version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d498595448e43de7f4296b7b7a18a8a02c61ec9349128c80a368f7c3b4ab11a8" +checksum = "576b0d6991c9c32bc14fc340582ef148311f924d41815f641a308b5d11e8e7cd" dependencies = [ "base64-simd", "bytes", "bytes-utils", "futures-core", "http 0.2.12", - "http 1.3.1", + "http 1.4.0", "http-body 0.4.6", "http-body 1.0.1", "http-body-util", @@ -950,9 +1200,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.8" +version = "1.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b069d19bf01e46298eaedd7c6f283fe565a59263e53eebec945f3e6398f42390" +checksum = "6c50f3cdf47caa8d01f2be4a6663ea02418e892f9bbfd82c7b9a3a37eaccdd3a" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -962,171 +1212,59 @@ dependencies = [ "tracing", ] -[[package]] -name = "aws_lambda_events" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "144ec7565561115498a288850cc6a42b279e09b6c4b88f623eecb9c8ca96c08c" -dependencies = [ - "base64 0.22.1", - "bytes", - "chrono", - "flate2", - "http 1.3.1", - "http-body 1.0.1", - "http-serde 2.1.1", - "query_map", - "serde", - "serde_dynamo", - "serde_json", - "serde_with", -] - -[[package]] -name = "axum" -version = "0.6.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" -dependencies = [ - "async-trait", - "axum-core 0.3.4", - "bitflags 1.3.2", - "bytes", - "futures-util", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.32", - "itoa", - "matchit 0.7.3", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper 0.1.2", - "tower 0.4.13", - "tower-layer", - "tower-service", -] - [[package]] name = "axum" -version = "0.7.9" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +checksum = "021e862c184ae977658b36c4500f7feac3221ca5da43e3f25bd04ab6c79a29b5" dependencies = [ - "async-trait", - "axum-core 0.4.5", + "axum-core", "bytes", "futures-util", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "http-body-util", "itoa", - "matchit 0.7.3", + "matchit", "memchr", "mime", "percent-encoding", "pin-project-lite", "rustversion", "serde", - "sync_wrapper 1.0.2", + "sync_wrapper", "tower 0.5.2", "tower-layer", "tower-service", ] [[package]] -name = "axum" -version = "0.8.4" +name = "axum-core" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "021e862c184ae977658b36c4500f7feac3221ca5da43e3f25bd04ab6c79a29b5" +checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6" dependencies = [ - "axum-core 0.5.2", "bytes", - "futures-util", - "http 1.3.1", + "futures-core", + "http 1.4.0", "http-body 1.0.1", "http-body-util", - "itoa", - "matchit 0.8.4", - "memchr", "mime", - "percent-encoding", "pin-project-lite", "rustversion", - "serde", - "sync_wrapper 1.0.2", - "tower 0.5.2", + "sync_wrapper", "tower-layer", "tower-service", ] [[package]] -name = "axum-core" -version = "0.3.4" +name = "azure_core" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +checksum = "7b552ad43a45a746461ec3d3a51dfb6466b4759209414b439c165eb6a6b7729e" dependencies = [ "async-trait", - "bytes", - "futures-util", - "http 0.2.12", - "http-body 0.4.6", - "mime", - "rustversion", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http 1.3.1", - "http-body 1.0.1", - "http-body-util", - "mime", - "pin-project-lite", - "rustversion", - "sync_wrapper 1.0.2", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6" -dependencies = [ - "bytes", - "futures-core", - "http 1.3.1", - "http-body 1.0.1", - "http-body-util", - "mime", - "pin-project-lite", - "rustversion", - "sync_wrapper 1.0.2", - "tower-layer", - "tower-service", -] - -[[package]] -name = "azure_core" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b552ad43a45a746461ec3d3a51dfb6466b4759209414b439c165eb6a6b7729e" -dependencies = [ - "async-trait", - "base64 0.22.1", + "base64 0.22.1", "bytes", "dyn-clone", "futures", @@ -1138,7 +1276,7 @@ dependencies = [ "pin-project", "quick-xml 0.31.0", "rand 0.8.5", - "reqwest 0.12.23", + "reqwest", "rustc_version", "serde", "serde_json", @@ -1160,7 +1298,7 @@ dependencies = [ "async-trait", "azure_core", "futures", - "oauth2", + "oauth2 4.4.2", "pin-project", "serde", "time", @@ -1228,9 +1366,9 @@ dependencies = [ [[package]] name = "backon" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "592277618714fbcecda9a02ba7a8781f319d26532a88553bbacc77ba5d2b3a8d" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" dependencies = [ "fastrand 2.3.0", "gloo-timers", @@ -1313,15 +1451,6 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - [[package]] name = "bindgen" version = "0.69.5" @@ -1365,15 +1494,16 @@ dependencies = [ [[package]] name = "binggan" -version = "0.14.2" +version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da19f87f00cab3f37ed64c5bc7cdfe2bdbb9fef2b9406f764c7a0eed57918ebd" +checksum = "d104d03a0d9d525501ac0e0339b3c73d862492cdaab1ad5efb32bc820acf0379" dependencies = [ "alloca", "bpu_trasher", "miniserde", "peakmem-alloc", "perf-event", + "quanta", "rustc-hash 2.1.1", "rustop", "unicode-width 0.1.14", @@ -1412,25 +1542,13 @@ dependencies = [ [[package]] name = "bitpacking" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c1d3e2bfd8d06048a179f7b17afc3188effa10385e7b00dc65af6aae732ea92" +checksum = "96a7139abd3d9cebf8cd6f920a389cf3dc9576172e32f4563f188cae3c3eb019" dependencies = [ "crunchy", ] -[[package]] -name = "bitvec" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" -dependencies = [ - "funty", - "radium", - "tap", - "wyz", -] - [[package]] name = "block-buffer" version = "0.10.4" @@ -1488,27 +1606,10 @@ dependencies = [ ] [[package]] -name = "borsh" -version = "1.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8646f98db542e39fc66e68a20b2144f6a732636df7c2354e74645faaa433ce" -dependencies = [ - "borsh-derive", - "cfg_aliases", -] - -[[package]] -name = "borsh-derive" -version = "1.5.7" +name = "borrow-or-share" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdd1d3c0c2f5833f22386f252fe8ed005c7f59fdcddeef025c01b4c3b9fd9ac3" -dependencies = [ - "once_cell", - "proc-macro-crate 3.3.0", - "proc-macro2", - "quote", - "syn 2.0.106", -] +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" [[package]] name = "bpu_trasher" @@ -1534,28 +1635,6 @@ version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" -[[package]] -name = "bytecheck" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" -dependencies = [ - "bytecheck_derive", - "ptr_meta", - "simdutf8", -] - -[[package]] -name = "bytecheck_derive" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "bytecount" version = "0.6.9" @@ -1595,40 +1674,51 @@ dependencies = [ [[package]] name = "bytesize" -version = "1.3.3" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e93abca9e28e0a1b9877922aacb20576e05d4679ffa78c3d6dc22a26a216659" +checksum = "6bd91ee7b2422bcb158d90ef4d14f75ef67f340943fc4149891dcce8f8b972a3" dependencies = [ - "serde", + "serde_core", ] [[package]] name = "bytestring" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e465647ae23b2823b0753f50decb2d5a86d2bb2cac04788fafd1f80e45378e5f" +checksum = "113b4343b5f6617e7ad401ced8de3cc8b012e73a594347c307b90db3e9271289" dependencies = [ "bytes", ] [[package]] -name = "bzip2" -version = "0.4.4" +name = "camino" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +checksum = "e629a66d692cb9ff1a1c664e41771b3dcaf961985a9774c0eb0bd1b51cf60a48" dependencies = [ - "bzip2-sys", - "libc", + "serde_core", ] [[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" +name = "cargo-platform" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +checksum = "e35af189006b9c0f00a064685c727031e3ed2d8020f7ba284d78cc2671bd36ea" dependencies = [ - "cc", - "pkg-config", + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", ] [[package]] @@ -1656,7 +1746,7 @@ dependencies = [ "chrono", "data-encoding", "half", - "nom", + "nom 7.1.3", "num-bigint", "num-rational", "num-traits", @@ -1689,7 +1779,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ - "nom", + "nom 7.1.3", ] [[package]] @@ -1721,7 +1811,18 @@ checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", +] + +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.0", ] [[package]] @@ -1731,7 +1832,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" dependencies = [ "aead", - "chacha20", + "chacha20 0.9.1", "cipher", "poly1305", "zeroize", @@ -1749,8 +1850,9 @@ dependencies = [ [[package]] name = "chitchat" -version = "0.9.0" -source = "git+https://github.com/quickwit-oss/chitchat.git?rev=bd54c81#bd54c810700814f83599a31a7e29f2a5eb8324b3" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "735f8a51f68b353b17e351b38317433d6afcaa9cc04f4d0f6c9e9125c49c1efe" dependencies = [ "anyhow", "async-trait", @@ -1762,7 +1864,7 @@ dependencies = [ "tokio", "tokio-stream", "tracing", - "zstd 0.13.3", + "zstd", ] [[package]] @@ -1777,7 +1879,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -1819,20 +1921,9 @@ dependencies = [ [[package]] name = "cidr" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdf600c45bd958cf2945c445264471cca8b6c8e67bc87b71affd6d7e5682621" - -[[package]] -name = "cidr-utils" -version = "0.6.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25c0a9fb70c2c2cc2a520aa259b1d1345650046a07df1b6da1d3cefcd327f43e" -dependencies = [ - "cidr", - "num-bigint", - "num-traits", -] +checksum = "579504560394e388085d0c080ea587dfa5c15f7e251b4d5247d1e1a61d1d6928" [[package]] name = "cipher" @@ -1925,12 +2016,13 @@ dependencies = [ [[package]] name = "codespan-reporting" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +checksum = "fe6d2e5af09e8c8ad56c969f2157a3d4238cebc7c55f0a517728c38f7b200f81" dependencies = [ + "serde", "termcolor", - "unicode-width 0.1.14", + "unicode-width 0.2.1", ] [[package]] @@ -1941,12 +2033,11 @@ checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "colored" -version = "2.2.0" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" +checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "lazy_static", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -1972,8 +2063,8 @@ dependencies = [ "compression-core", "flate2", "memchr", - "zstd 0.13.3", - "zstd-safe 7.2.4", + "zstd", + "zstd-safe", ] [[package]] @@ -1993,48 +2084,51 @@ dependencies = [ [[package]] name = "console" -version = "0.15.11" +version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87" dependencies = [ "encode_unicode", "libc", - "once_cell", "unicode-width 0.2.1", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "console-api" -version = "0.5.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2895653b4d9f1538a83970077cb01dfc77a4810524e51a110944688e916b18e" +checksum = "e8599749b6667e2f0c910c1d0dff6901163ff698a52d5a39720f61b5be4b20d3" dependencies = [ - "prost 0.11.9", - "prost-types 0.11.9", - "tonic 0.9.2", + "futures-core", + "prost 0.14.3", + "prost-types 0.14.3", + "tonic 0.14.5", + "tonic-prost", "tracing-core", ] [[package]] name = "console-subscriber" -version = "0.1.10" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4cf42660ac07fcebed809cfe561dd8730bcd35b075215e6479c516bcd0d11cb" +checksum = "fb4915b7d8dd960457a1b6c380114c2944f728e7c65294ab247ae6b6f1f37592" dependencies = [ "console-api", "crossbeam-channel", "crossbeam-utils", - "futures", + "futures-task", "hdrhistogram", "humantime", - "prost-types 0.11.9", + "hyper-util", + "prost 0.14.3", + "prost-types 0.14.3", "serde", "serde_json", "thread_local", "tokio", "tokio-stream", - "tonic 0.9.2", + "tonic 0.14.5", "tracing", "tracing-core", "tracing-subscriber", @@ -2047,22 +2141,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] -name = "const_fn" -version = "0.4.11" +name = "const-random" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f8a2ca5ac02d09563609681103aada9e1777d54fc57a5acd7a41404f9c93b6e" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] [[package]] -name = "constant_time_eq" -version = "0.1.5" +name = "const-random-macro" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.16", + "once_cell", + "tiny-keccak", +] [[package]] -name = "convert_case" -version = "0.4.0" +name = "const_fn" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +checksum = "2f8a2ca5ac02d09563609681103aada9e1777d54fc57a5acd7a41404f9c93b6e" [[package]] name = "convert_case" @@ -2117,6 +2219,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.3.0" @@ -2152,26 +2263,24 @@ dependencies = [ [[package]] name = "criterion" -version = "0.5.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" dependencies = [ + "alloca", "anes", "cast", "ciborium", "clap", "criterion-plot", - "futures", - "is-terminal", - "itertools 0.10.5", + "itertools 0.13.0", "num-traits", - "once_cell", "oorandom", + "page_size", "plotters", "rayon", "regex", "serde", - "serde_derive", "serde_json", "tinytemplate", "tokio", @@ -2180,23 +2289,23 @@ dependencies = [ [[package]] name = "criterion-plot" -version = "0.5.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" dependencies = [ "cast", - "itertools 0.10.5", + "itertools 0.13.0", ] [[package]] name = "cron" -version = "0.12.1" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f8c3e73077b4b4a6ab1ea5047c37c57aee77657bc8ecd6f29b0af082d0b0c07" +checksum = "5877d3fbf742507b66bc2a1945106bd30dd8504019d596901ddd012a4dd01740" dependencies = [ "chrono", - "nom", "once_cell", + "winnow 0.6.26", ] [[package]] @@ -2335,7 +2444,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest", "fiat-crypto", @@ -2375,6 +2484,16 @@ dependencies = [ "darling_macro 0.21.3", ] +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core 0.23.0", + "darling_macro 0.23.0", +] + [[package]] name = "darling_core" version = "0.20.11" @@ -2385,7 +2504,6 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", "syn 2.0.106", ] @@ -2403,6 +2521,19 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.106", +] + [[package]] name = "darling_macro" version = "0.20.11" @@ -2425,6 +2556,44 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core 0.23.0", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "dashmap" +version = "5.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +dependencies = [ + "cfg-if", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core 0.9.11", +] + +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core 0.9.11", +] + [[package]] name = "data-encoding" version = "2.9.0" @@ -2437,6 +2606,12 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be1e0bca6c3637f992fc1cc7cbc52a78c1ef6db076dbf1059c4323d6a2048376" +[[package]] +name = "datasketches" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" + [[package]] name = "dbl" version = "0.3.2" @@ -2505,28 +2680,24 @@ dependencies = [ ] [[package]] -name = "derive_more" -version = "0.99.20" +name = "derivative" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" dependencies = [ - "convert_case 0.4.0", "proc-macro2", "quote", - "rustc_version", - "syn 2.0.106", + "syn 1.0.109", ] [[package]] name = "dialoguer" -version = "0.10.4" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59c6f2989294b9a498d3ad5491a79c6deb604617378e1cdc4bfc1c1361fe2f87" +checksum = "25f104b501bf2364e78d0d3974cbc774f738f5865306ed128e1e0d7499c0ad96" dependencies = [ "console", "shell-words", - "tempfile", - "zeroize", ] [[package]] @@ -2584,13 +2755,17 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "domain" -version = "0.10.4" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c84070523f8ba0f9127ff156920f27eb27b302b425efe60bf5f41ec244d1c60" +checksum = "7f7ff15f82df7d5086fb15dfc1c1e96598a6ded9829840a9bcfa1fa3ccd8d01d" dependencies = [ + "bumpalo", "bytes", + "domain-macros", "futures-util", - "moka", + "hashbrown 0.14.5", + "log", + "moka", "octseq", "rand 0.8.5", "serde", @@ -2600,6 +2775,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "domain-macros" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d1a6796ad411f6812d691955066ad27450196bfb181bb91b66a643cc3e8f5b7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + [[package]] name = "dotenvy" version = "0.15.7" @@ -2760,6 +2946,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "email_address" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" +dependencies = [ + "serde", +] + [[package]] name = "embedded-io" version = "0.4.0" @@ -2788,126 +2983,92 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" [[package]] -name = "encoding" -version = "0.2.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec" -dependencies = [ - "encoding-index-japanese", - "encoding-index-korean", - "encoding-index-simpchinese", - "encoding-index-singlebyte", - "encoding-index-tradchinese", -] - -[[package]] -name = "encoding-index-japanese" -version = "1.20141219.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04e8b2ff42e9a05335dbf8b5c6f7567e5591d0d916ccef4e0b1710d32a0d0c91" -dependencies = [ - "encoding_index_tests", -] - -[[package]] -name = "encoding-index-korean" -version = "1.20141219.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dc33fb8e6bcba213fe2f14275f0963fd16f0a02c878e3095ecfdf5bee529d81" -dependencies = [ - "encoding_index_tests", -] - -[[package]] -name = "encoding-index-simpchinese" -version = "1.20141219.5" +name = "encoding_rs" +version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d87a7194909b9118fc707194baa434a4e3b0fb6a5a757c73c3adb07aa25031f7" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ - "encoding_index_tests", + "cfg-if", ] [[package]] -name = "encoding-index-singlebyte" -version = "1.20141219.5" +name = "enum-iterator" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3351d5acffb224af9ca265f435b859c7c01537c0849754d3db3fdf2bfe2ae84a" +checksum = "a4549325971814bda7a44061bf3fe7e487d447cba01e4220a4b454d630d7a016" dependencies = [ - "encoding_index_tests", + "enum-iterator-derive", ] [[package]] -name = "encoding-index-tradchinese" -version = "1.20141219.5" +name = "enum-iterator-derive" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18" +checksum = "685adfa4d6f3d765a26bc5dbc936577de9abf756c1feeb3089b01dd395034842" dependencies = [ - "encoding_index_tests", + "proc-macro2", + "quote", + "syn 2.0.106", ] [[package]] -name = "encoding_index_tests" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" - -[[package]] -name = "encoding_rs" -version = "0.8.35" +name = "env_filter" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +checksum = "7a1c3cc8e57274ec99de65301228b537f1e4eedc1b8e0f9411c6caac8ae7308f" dependencies = [ - "cfg-if", + "log", ] [[package]] -name = "encoding_rs_io" -version = "0.1.7" +name = "env_logger" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cc3c5651fb62ab8aa3103998dade57efdd028544bd300516baa31840c252a83" +checksum = "b2daee4ea451f429a58296525ddf28b45a3b64f1acf6587e2067437bb11e218d" dependencies = [ - "encoding_rs", + "anstream", + "anstyle", + "env_filter", + "log", ] [[package]] -name = "enum-iterator" -version = "1.5.0" +name = "equator" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fd242f399be1da0a5354aa462d57b4ab2b4ee0683cc552f7c007d2d12d36e94" +checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" dependencies = [ - "enum-iterator-derive", + "equator-macro", ] [[package]] -name = "enum-iterator-derive" -version = "1.4.0" +name = "equator-macro" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1ab991c1362ac86c61ab6f556cff143daa22e5a15e4e189df818b2fd19fe65b" +checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" dependencies = [ "proc-macro2", "quote", "syn 2.0.106", ] -[[package]] -name = "env_logger" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" -dependencies = [ - "humantime", - "is-terminal", - "log", - "regex", - "termcolor", -] - [[package]] name = "equivalent" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + [[package]] name = "errno" version = "0.3.13" @@ -2918,6 +3079,15 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "error-chain" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" +dependencies = [ + "version_check", +] + [[package]] name = "etcetera" version = "0.8.0" @@ -2969,9 +3139,21 @@ dependencies = [ [[package]] name = "fancy-regex" -version = "0.14.0" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6215aee357f8c7c989ebb4b8466ca4d7dc93b3957039f2fc3ea2ade8ea5f279" +dependencies = [ + "bit-set", + "derivative", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "fancy-regex" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298" +checksum = "998b056554fbe42e03ae0e152895cd1a7e1002aec800fdc6635d20270260c46f" dependencies = [ "bit-set", "regex-automata", @@ -2998,6 +3180,9 @@ name = "fastrand" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +dependencies = [ + "getrandom 0.2.16", +] [[package]] name = "ff" @@ -3025,18 +3210,6 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" -[[package]] -name = "filetime" -version = "0.2.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc0505cd1b6fa6580283f6bdf70a73fcf4aba1184038c90902b92b3dd0df63ed" -dependencies = [ - "cfg-if", - "libc", - "libredox", - "windows-sys 0.60.2", -] - [[package]] name = "find-msvc-tools" version = "0.1.1" @@ -3061,6 +3234,16 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags 2.9.4", + "rustc_version", +] + [[package]] name = "flate2" version = "1.1.2" @@ -3068,25 +3251,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" dependencies = [ "crc32fast", + "libz-rs-sys", "miniz_oxide", ] [[package]] name = "float-cmp" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" dependencies = [ "num-traits", ] [[package]] -name = "float-cmp" -version = "0.10.0" +name = "fluent-uri" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" dependencies = [ - "num-traits", + "borrow-or-share", + "ref-cast", + "serde", ] [[package]] @@ -3097,7 +3283,18 @@ checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ "futures-core", "futures-sink", - "nanorand", + "spin 0.9.8", +] + +[[package]] +name = "flume" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" +dependencies = [ + "fastrand 2.3.0", + "futures-core", + "futures-sink", "spin 0.9.8", ] @@ -3113,6 +3310,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -3137,6 +3340,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fraction" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f158e3ff0a1b334408dc9fb811cd99b446986f4d8b741bb08f9df1604085ae7" +dependencies = [ + "lazy_static", + "num", +] + [[package]] name = "fragile" version = "2.0.1" @@ -3145,12 +3358,12 @@ checksum = "28dd6caf6059519a65843af8fe2a3ae298b14b80179855aeb4adc2c1934ee619" [[package]] name = "fs4" -version = "0.8.4" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8" +checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" dependencies = [ - "rustix 0.38.44", - "windows-sys 0.52.0", + "rustix 1.0.8", + "windows-sys 0.59.0", ] [[package]] @@ -3169,12 +3382,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "funty" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" - [[package]] name = "futures" version = "0.3.31" @@ -3320,7 +3527,7 @@ dependencies = [ "libc", "log", "rustversion", - "windows 0.61.3", + "windows", ] [[package]] @@ -3360,18 +3567,32 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", - "wasi 0.14.3+wasi-0.2.4", + "r-efi 5.3.0", + "wasip2", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.0", + "wasip2", + "wasip3", +] + [[package]] name = "gimli" version = "0.31.1" @@ -3398,17 +3619,17 @@ dependencies = [ [[package]] name = "google-cloud-auth" -version = "0.12.0" +version = "0.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "931bedb2264cb00f914b0a6a5c304e34865c34306632d3932e0951a073e4a67d" +checksum = "e57a13fbacc5e9c41ded3ad8d0373175a6b7a6ad430d99e89d314ac121b7ab06" dependencies = [ "async-trait", "base64 0.21.7", "google-cloud-metadata", "google-cloud-token", "home", - "jsonwebtoken 8.3.0", - "reqwest 0.11.27", + "jsonwebtoken", + "reqwest", "serde", "serde_json", "thiserror 1.0.69", @@ -3420,47 +3641,47 @@ dependencies = [ [[package]] name = "google-cloud-gax" -version = "0.15.0" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8bdaaa4bc036e8318274d1b25f0f2265b3e95418b765fd1ea1c7ef938fd69bd" +checksum = "de13e62d7e0ffc3eb40a0113ddf753cf6ec741be739164442b08893db4f9bfca" dependencies = [ "google-cloud-token", - "http 0.2.12", + "http 1.4.0", "thiserror 1.0.69", "tokio", - "tokio-retry", - "tonic 0.9.2", + "tokio-retry2", + "tonic 0.12.3", "tower 0.4.13", "tracing", ] [[package]] name = "google-cloud-googleapis" -version = "0.10.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a3b24a3f57be08afc02344e693afb55e48172c9c2ab86ff3fdb8efff550e4b9" +checksum = "886aa8ec755382a1fdf4651f6e6ec01f2f3bf49f2cb0f068b9a74cafd574a715" dependencies = [ - "prost 0.11.9", - "prost-types 0.11.9", - "tonic 0.9.2", + "prost 0.13.5", + "prost-types 0.13.5", + "tonic 0.12.3", ] [[package]] name = "google-cloud-metadata" -version = "0.3.2" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96e4ad0802d3f416f62e7ce01ac1460898ee0efc98f8b45cd4aab7611607012f" +checksum = "d901aeb453fd80e51d64df4ee005014f6cf39f2d736dd64f7239c132d9d39a6a" dependencies = [ - "reqwest 0.11.27", + "reqwest", "thiserror 1.0.69", "tokio", ] [[package]] name = "google-cloud-pubsub" -version = "0.18.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "095b104502b6e1abbad9b9768af944b9202e032dbc7f0947d3c30d4191761071" +checksum = "bebc6e7327e49a66ffb40508c673b7643191bd6b509530193bda97f09272cdcf" dependencies = [ "async-channel 1.9.0", "async-stream", @@ -3468,7 +3689,7 @@ dependencies = [ "google-cloud-gax", "google-cloud-googleapis", "google-cloud-token", - "prost-types 0.11.9", + "prost-types 0.13.5", "thiserror 1.0.69", "tokio", "tokio-util", @@ -3484,6 +3705,29 @@ dependencies = [ "async-trait", ] +[[package]] +name = "governor" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9efcab3c1958580ff1f25a2a41be1668f7603d849bb63af523b208a3cc1223b8" +dependencies = [ + "cfg-if", + "dashmap 6.1.0", + "futures-sink", + "futures-timer", + "futures-util", + "getrandom 0.3.4", + "hashbrown 0.16.1", + "nonzero_ext", + "parking_lot 0.12.4", + "portable-atomic", + "quanta", + "rand 0.9.2", + "smallvec", + "spinning_top", + "web-time", +] + [[package]] name = "grok" version = "2.4.0" @@ -3528,7 +3772,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.11.0", + "indexmap 2.13.0", "slab", "tokio", "tokio-util", @@ -3546,8 +3790,8 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.3.1", - "indexmap 2.11.0", + "http 1.4.0", + "indexmap 2.13.0", "slab", "tokio", "tokio-util", @@ -3562,6 +3806,7 @@ checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" dependencies = [ "cfg-if", "crunchy", + "num-traits", ] [[package]] @@ -3569,9 +3814,6 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -dependencies = [ - "ahash 0.7.8", -] [[package]] name = "hashbrown" @@ -3579,7 +3821,6 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ - "ahash 0.8.12", "allocator-api2", ] @@ -3591,16 +3832,27 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", ] [[package]] name = "hashlink" -version = "0.8.4" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" dependencies = [ - "hashbrown 0.14.5", + "hashbrown 0.15.5", ] [[package]] @@ -3612,20 +3864,20 @@ dependencies = [ "base64 0.21.7", "byteorder", "flate2", - "nom", + "nom 7.1.3", "num-traits", ] [[package]] name = "headers" -version = "0.3.9" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270" +checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "bytes", "headers-core", - "http 0.2.12", + "http 1.4.0", "httpdate", "mime", "sha1", @@ -3633,11 +3885,11 @@ dependencies = [ [[package]] name = "headers-core" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" +checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" dependencies = [ - "http 0.2.12", + "http 1.4.0", ] [[package]] @@ -3645,9 +3897,6 @@ name = "heck" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -dependencies = [ - "unicode-segmentation", -] [[package]] name = "heck" @@ -3694,17 +3943,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "hostname" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" -dependencies = [ - "libc", - "match_cfg", - "winapi 0.3.9", -] - [[package]] name = "hostname" version = "0.4.1" @@ -3713,7 +3951,7 @@ checksum = "a56f203cd1c76362b69e3863fd987520ac36cf70a8c92627449b2f64a8cf7d65" dependencies = [ "cfg-if", "libc", - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -3735,12 +3973,11 @@ dependencies = [ [[package]] name = "http" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" dependencies = [ "bytes", - "fnv", "itoa", ] @@ -3762,7 +3999,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.3.1", + "http 1.4.0", ] [[package]] @@ -3773,34 +4010,18 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "pin-project-lite", ] -[[package]] -name = "http-range-header" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" - [[package]] name = "http-serde" -version = "1.1.3" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f560b665ad9f1572cfcaf034f7fb84338a7ce945216d64a90fd81f046a3caee" +checksum = "0f056c8559e3757392c8d091e796416e4649d8e49e88b8d76df6c002f05027fd" dependencies = [ - "http 0.2.12", - "serde", -] - -[[package]] -name = "http-serde" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f056c8559e3757392c8d091e796416e4649d8e49e88b8d76df6c002f05027fd" -dependencies = [ - "http 1.3.1", + "http 1.4.0", "serde", ] @@ -3838,9 +4059,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hyper" @@ -3868,16 +4089,16 @@ dependencies = [ [[package]] name = "hyper" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", "h2 0.4.12", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "httparse", "httpdate", @@ -3900,7 +4121,6 @@ dependencies = [ "hyper 0.14.32", "log", "rustls 0.21.12", - "rustls-native-certs 0.6.3", "tokio", "tokio-rustls 0.24.1", ] @@ -3911,12 +4131,11 @@ version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "http 1.3.1", - "hyper 1.7.0", + "http 1.4.0", + "hyper 1.8.1", "hyper-util", - "log", "rustls 0.23.31", - "rustls-native-certs 0.8.1", + "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls 0.26.2", @@ -3924,44 +4143,19 @@ dependencies = [ "webpki-roots 1.0.2", ] -[[package]] -name = "hyper-timeout" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" -dependencies = [ - "hyper 0.14.32", - "pin-project-lite", - "tokio", - "tokio-io-timeout", -] - [[package]] name = "hyper-timeout" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.7.0", + "hyper 1.8.1", "hyper-util", "pin-project-lite", "tokio", "tower-service", ] -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper 0.14.32", - "native-tls", - "tokio", - "tokio-native-tls", -] - [[package]] name = "hyper-tls" version = "0.6.0" @@ -3970,7 +4164,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.1", "hyper-util", "native-tls", "tokio", @@ -3989,28 +4183,17 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", - "hyper 1.7.0", + "hyper 1.8.1", "ipnet", "libc", "percent-encoding", "pin-project-lite", "socket2 0.6.0", - "system-configuration 0.6.1", "tokio", "tower-service", "tracing", - "windows-registry", -] - -[[package]] -name = "hyperloglogplus" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "621debdf94dcac33e50475fdd76d34d5ea9c0362a834b9db08c3024696c1fbe3" -dependencies = [ - "serde", ] [[package]] @@ -4025,7 +4208,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.61.2", + "windows-core", ] [[package]] @@ -4123,6 +4306,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + [[package]] name = "ident_case" version = "1.0.1" @@ -4163,25 +4352,26 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.11.0" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2481980430f9f78649238835720ddccc57e52df14ffce1c6f37391d61b563e9" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ "equivalent", - "hashbrown 0.15.5", + "hashbrown 0.16.1", "serde", + "serde_core", ] [[package]] name = "indicatif" -version = "0.17.11" +version = "0.18.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" +checksum = "25470f23803092da7d239834776d653104d551bc4d7eacaf31e6837854b8e9eb" dependencies = [ "console", - "number_prefix", "portable-atomic", "unicode-width 0.2.1", + "unit-prefix", "web-time", ] @@ -4203,8 +4393,8 @@ version = "0.11.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" dependencies = [ - "ahash 0.8.12", - "indexmap 2.11.0", + "ahash", + "indexmap 2.13.0", "is-terminal", "itoa", "log", @@ -4223,7 +4413,7 @@ checksum = "22fa7ee6be451ea0b1912b962c91c8380835e97cf1584a77e18264e908448dcb" dependencies = [ "bytes", "log", - "nom", + "nom 7.1.3", "smallvec", "snafu 0.7.5", ] @@ -4262,14 +4452,27 @@ dependencies = [ ] [[package]] -name = "io-uring" -version = "0.7.10" +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + +[[package]] +name = "inventory" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" +checksum = "8062b737e5389949f477d4760a2ebbff0c366f97798f2419b8d8f366363d3342" dependencies = [ - "bitflags 2.9.4", - "cfg-if", - "libc", + "rustversion", +] + +[[package]] +name = "ipcrypt-rs" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96e4f67dbfc0f75d7b65953ecf0be3fd84ee0cb1ae72a00a4aa9a2f5518a2c80" +dependencies = [ + "aes", ] [[package]] @@ -4356,13 +4559,54 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "jiff" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" +dependencies = [ + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-sys 0.60.2", +] + +[[package]] +name = "jiff-static" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "jobserver" version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", "libc", ] @@ -4383,17 +4627,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9dbbfed4e59ba9750e15ba154fdfd9329cee16ff3df539c2666b70f58cc32105" [[package]] -name = "jsonwebtoken" -version = "8.3.0" +name = "jsonschema" +version = "0.37.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" +checksum = "73c9ffb2b5c56d58030e1b532d8e8389da94590515f118cf35b5cb68e4764a7e" dependencies = [ - "base64 0.21.7", - "pem 1.1.1", - "ring 0.16.20", + "ahash", + "bytecount", + "data-encoding", + "email_address", + "fancy-regex 0.16.2", + "fraction", + "getrandom 0.3.4", + "idna", + "itoa", + "num-cmp", + "num-traits", + "percent-encoding", + "referencing", + "regex", + "regex-syntax", "serde", "serde_json", - "simple_asn1", + "unicode-general-category", + "uuid-simd", ] [[package]] @@ -4404,8 +4661,8 @@ checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" dependencies = [ "base64 0.22.1", "js-sys", - "pem 3.0.5", - "ring 0.17.14", + "pem", + "ring", "serde", "serde_json", "simple_asn1", @@ -4417,7 +4674,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -4440,7 +4697,7 @@ dependencies = [ "ena", "itertools 0.14.0", "lalrpop-util", - "petgraph", + "petgraph 0.7.1", "regex", "regex-syntax", "sha3", @@ -4460,33 +4717,6 @@ dependencies = [ "rustversion", ] -[[package]] -name = "lambda_http" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b470cea1ec37b96e9543870a33c1f4b9e243754fe5892668efa3125ea12784" -dependencies = [ - "aws_lambda_events", - "base64 0.22.1", - "bytes", - "encoding_rs", - "futures", - "futures-util", - "http 1.3.1", - "http-body 1.0.1", - "http-body-util", - "hyper 1.7.0", - "lambda_runtime", - "mime", - "percent-encoding", - "pin-project-lite", - "serde", - "serde_json", - "serde_urlencoded", - "tokio-stream", - "url", -] - [[package]] name = "lambda_runtime" version = "0.13.0" @@ -4497,11 +4727,11 @@ dependencies = [ "base64 0.22.1", "bytes", "futures", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "http-body-util", - "http-serde 2.1.1", - "hyper 1.7.0", + "http-serde", + "hyper 1.8.1", "hyper-util", "lambda_runtime_api_client", "pin-project", @@ -4524,10 +4754,10 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.1", "hyper-util", "tokio", "tower 0.4.13", @@ -4552,292 +4782,136 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] -name = "levenshtein_automata" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" - -[[package]] -name = "libc" -version = "0.2.175" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" - -[[package]] -name = "libloading" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" -dependencies = [ - "cfg-if", - "windows-targets 0.53.3", -] - -[[package]] -name = "libm" -version = "0.2.15" +name = "leb128fmt" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] -name = "libredox" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3" -dependencies = [ - "bitflags 2.9.4", - "libc", - "redox_syscall 0.5.17", -] - -[[package]] -name = "libsqlite3-sys" -version = "0.27.0" +name = "levenshtein_automata" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", -] +checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" [[package]] -name = "libz-sys" -version = "1.1.22" +name = "lexical-core" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", ] [[package]] -name = "lindera-cc-cedict" -version = "0.27.2" +name = "lexical-parse-float" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b6bf27b9e3d76effb160531cda3d11c965616600459a6e17adc61aada8a3df2" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" dependencies = [ - "bincode", - "byteorder", - "encoding", - "lindera-cc-cedict-builder", - "lindera-core", - "lindera-decompress", - "once_cell", - "zip", + "lexical-parse-integer", + "lexical-util", ] [[package]] -name = "lindera-cc-cedict-builder" -version = "0.27.2" +name = "lexical-parse-integer" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a90d23f7cef31c6ab7ac0d4f3b23940754207f7b5a80b080c39193caffe99ac2" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" dependencies = [ - "anyhow", - "bincode", - "byteorder", - "csv", - "encoding", - "env_logger", - "glob", - "lindera-compress", - "lindera-core", - "lindera-decompress", - "log", - "yada", + "lexical-util", ] [[package]] -name = "lindera-compress" -version = "0.27.2" +name = "lexical-util" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1927b7d2bd4ffc19e07691bf8609722663c341f80260a1c636cee8f1ec420dce" -dependencies = [ - "anyhow", - "flate2", - "lindera-decompress", -] +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" [[package]] -name = "lindera-core" -version = "0.27.2" +name = "lexical-write-float" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3299caa2b81c9a076535a4651a83bf7d624c15f2349f243187fffc64b5a78251" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" dependencies = [ - "anyhow", - "bincode", - "byteorder", - "encoding_rs", - "log", - "once_cell", - "serde", - "thiserror 1.0.69", - "yada", + "lexical-util", + "lexical-write-integer", ] [[package]] -name = "lindera-decompress" -version = "0.27.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b82b8d2323a67dc8ff0c40751d199b7ba94cd5e3c13a5b31622d318acc79e5b" -dependencies = [ - "anyhow", - "flate2", - "serde", -] - -[[package]] -name = "lindera-dictionary" -version = "0.27.2" +name = "lexical-write-integer" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cddf783b459d54b130d956889bec052c25fcb478a304e03fa9b2289387572bc5" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" dependencies = [ - "anyhow", - "bincode", - "byteorder", - "lindera-cc-cedict", - "lindera-cc-cedict-builder", - "lindera-core", - "lindera-ipadic", - "lindera-ipadic-builder", - "lindera-ipadic-neologd-builder", - "lindera-ko-dic", - "lindera-ko-dic-builder", - "lindera-unidic-builder", - "serde", + "lexical-util", ] [[package]] -name = "lindera-ipadic" -version = "0.27.2" +name = "libc" +version = "0.2.175" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b21c060c9309d29ac7e5c8fb7309c85dbf31b60f2fc4236f9dcda17854131276" -dependencies = [ - "bincode", - "byteorder", - "encoding", - "flate2", - "lindera-core", - "lindera-decompress", - "lindera-ipadic-builder", - "once_cell", - "tar", -] +checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" [[package]] -name = "lindera-ipadic-builder" -version = "0.27.2" +name = "libloading" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c708f08f14b0806f6c4cce5324b4bcba27209463026b78c31f399f8be9d30d" +checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ - "anyhow", - "bincode", - "byteorder", - "csv", - "encoding_rs", - "encoding_rs_io", - "env_logger", - "glob", - "lindera-compress", - "lindera-core", - "lindera-decompress", - "log", - "serde", - "yada", + "cfg-if", + "windows-targets 0.53.3", ] [[package]] -name = "lindera-ipadic-neologd-builder" -version = "0.27.2" +name = "libm" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5e67eb91652203d202f7d27ead220d1d8c9099552709b8429eae9c70f2312fb" -dependencies = [ - "anyhow", - "bincode", - "byteorder", - "csv", - "encoding_rs", - "encoding_rs_io", - "env_logger", - "glob", - "lindera-core", - "lindera-decompress", - "log", - "serde", - "yada", -] +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" [[package]] -name = "lindera-ko-dic" -version = "0.27.2" +name = "libredox" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d45da8d9a5888f4d4e78bb29fc82ff9ae519962efb0d2d92343b6cf8e373952f" +checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3" dependencies = [ - "bincode", - "byteorder", - "encoding", - "flate2", - "lindera-core", - "lindera-decompress", - "lindera-ko-dic-builder", - "once_cell", - "tar", + "bitflags 2.9.4", + "libc", + "redox_syscall 0.5.17", ] [[package]] -name = "lindera-ko-dic-builder" -version = "0.27.2" +name = "libsqlite3-sys" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41c0933295dc945178bbc08f34111dc3ef22bfee38820f78453c8f8d4f3463d1" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" dependencies = [ - "anyhow", - "bincode", - "byteorder", - "csv", - "encoding", - "env_logger", - "glob", - "lindera-compress", - "lindera-core", - "lindera-decompress", - "log", - "yada", + "pkg-config", + "vcpkg", ] [[package]] -name = "lindera-tokenizer" -version = "0.27.2" +name = "libz-rs-sys" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "348ce9bb3f2e5edc577420b98cca05b2177f3af50ef5ae278a1d8a1351d56197" +checksum = "c10501e7805cee23da17c7790e59df2870c0d4043ec6d03f67d31e2b53e77415" dependencies = [ - "bincode", - "byteorder", - "lindera-core", - "lindera-dictionary", - "once_cell", - "serde", - "serde_json", + "zlib-rs", ] [[package]] -name = "lindera-unidic-builder" -version = "0.27.2" +name = "libz-sys" +version = "1.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34e5564ee81af82603cd6a03c3abe6e17cc0ae598bfa5078809f06e59e96e08" +checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" dependencies = [ - "anyhow", - "bincode", - "byteorder", - "csv", - "encoding", - "env_logger", - "glob", - "lindera-core", - "lindera-decompress", - "log", - "yada", + "cc", + "libc", + "pkg-config", + "vcpkg", ] [[package]] @@ -4911,6 +4985,15 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "lru" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" +dependencies = [ + "hashbrown 0.16.1", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -4941,12 +5024,15 @@ name = "lz4_flex" version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" +dependencies = [ + "twox-hash", +] [[package]] -name = "match_cfg" -version = "0.1.0" +name = "lz4_flex" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" +checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" [[package]] name = "matchers" @@ -4963,12 +5049,6 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" -[[package]] -name = "matchit" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" - [[package]] name = "matchit" version = "0.8.4" @@ -4987,9 +5067,9 @@ dependencies = [ [[package]] name = "md5" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" +checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0" [[package]] name = "measure_time" @@ -5042,6 +5122,21 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "mini-moka" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c325dfab65f261f386debee8b0969da215b3fa0037e74c8a1234db7ba986d803" +dependencies = [ + "crossbeam-channel", + "crossbeam-utils", + "dashmap 5.5.3", + "skeptic", + "smallvec", + "tagptr", + "triomphe", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -5081,29 +5176,28 @@ dependencies = [ [[package]] name = "mockall" -version = "0.11.4" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c84490118f2ee2d74570d114f3d0493cbf02790df303d2707606c3e14e07c96" +checksum = "f58d964098a5f9c6b63d0798e5372fd04708193510a7af313c22e9f29b7b620b" dependencies = [ "cfg-if", "downcast", "fragile", - "lazy_static", "mockall_derive", - "predicates 2.1.5", + "predicates", "predicates-tree", ] [[package]] name = "mockall_derive" -version = "0.11.4" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb" +checksum = "ca41ce716dda6a9be188b385aa78ee5260fc25cd3802cb2a8afdc6afbe6b6dbf" dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.106", ] [[package]] @@ -5141,24 +5235,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "multer" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01acbdc23469fd8fe07ab135923371d5f5a422fbf9c522158677c8eb15bc51c2" -dependencies = [ - "bytes", - "encoding_rs", - "futures-util", - "http 0.2.12", - "httparse", - "log", - "memchr", - "mime", - "spin 0.9.8", - "version_check", -] - [[package]] name = "multimap" version = "0.10.1" @@ -5166,19 +5242,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" [[package]] -name = "murmurhash32" -version = "0.3.1" +name = "murmur3" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" +checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" [[package]] -name = "nanorand" -version = "0.7.0" +name = "murmurhash32" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" -dependencies = [ - "getrandom 0.2.16", -] +checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" [[package]] name = "native-tls" @@ -5241,20 +5314,35 @@ dependencies = [ ] [[package]] -name = "normalize-line-endings" -version = "0.3.0" +name = "nom" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] [[package]] -name = "ntapi" -version = "0.4.1" +name = "nom-language" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +checksum = "2de2bc5b451bfedaef92c90b8939a8fff5770bdcc1fafd6239d086aab8fa6b29" dependencies = [ - "winapi 0.3.9", + "nom 8.0.0", ] +[[package]] +name = "nonzero_ext" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + [[package]] name = "nu-ansi-term" version = "0.50.1" @@ -5264,6 +5352,20 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -5292,10 +5394,25 @@ dependencies = [ ] [[package]] -name = "num-conv" +name = "num-cmp" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa" + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" [[package]] name = "num-format" @@ -5303,7 +5420,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" dependencies = [ - "arrayvec 0.7.6", + "arrayvec", "itoa", ] @@ -5410,12 +5527,6 @@ dependencies = [ "libc", ] -[[package]] -name = "number_prefix" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" - [[package]] name = "numfmt" version = "1.2.0" @@ -5435,9 +5546,28 @@ dependencies = [ "base64 0.13.1", "chrono", "getrandom 0.2.16", - "http 0.2.12", + "http 0.2.12", + "rand 0.8.5", + "serde", + "serde_json", + "serde_path_to_error", + "sha2", + "thiserror 1.0.69", + "url", +] + +[[package]] +name = "oauth2" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" +dependencies = [ + "base64 0.22.1", + "chrono", + "getrandom 0.2.16", + "http 1.4.0", "rand 0.8.5", - "reqwest 0.11.27", + "reqwest", "serde", "serde_json", "serde_path_to_error", @@ -5446,6 +5576,25 @@ dependencies = [ "url", ] +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.9.4", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + [[package]] name = "object" version = "0.36.7" @@ -5489,9 +5638,9 @@ checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" [[package]] name = "oneshot" -version = "0.1.11" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ce411919553d3f9fa53a0880544cda985a112117a0444d5ff1e870a893d6ea" +checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" [[package]] name = "onig" @@ -5529,54 +5678,53 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "opendal" -version = "0.53.3" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f947c4efbca344c1a125753366033c8107f552b2e3f8251815ed1908f116ca3e" +checksum = "d075ab8a203a6ab4bc1bce0a4b9fe486a72bf8b939037f4b78d95386384bc80a" dependencies = [ "anyhow", - "async-trait", "backon", "base64 0.22.1", "bytes", - "chrono", "futures", "getrandom 0.2.16", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", + "jiff", "log", "md-5", "percent-encoding", - "quick-xml 0.37.5", + "quick-xml 0.38.4", "reqsign", - "reqwest 0.12.23", + "reqwest", "serde", "serde_json", "tokio", + "url", "uuid", ] [[package]] name = "openidconnect" -version = "3.5.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f47e80a9cfae4462dd29c41e987edd228971d6565553fbc14b8a11e666d91590" +checksum = "0d8c6709ba2ea764bbed26bce1adf3c10517113ddea6f2d4196e4851757ef2b2" dependencies = [ - "base64 0.13.1", + "base64 0.21.7", "chrono", "dyn-clone", "ed25519-dalek", "hmac", - "http 0.2.12", + "http 1.4.0", "itertools 0.10.5", "log", - "oauth2", + "oauth2 5.0.0", "p256 0.13.2", "p384", "rand 0.8.5", "rsa", "serde", "serde-value", - "serde_derive", "serde_json", "serde_path_to_error", "serde_plain", @@ -5643,83 +5791,90 @@ dependencies = [ [[package]] name = "opentelemetry" -version = "0.27.1" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab70038c28ed37b97d8ed414b6429d343a8bbf44c9f79ec854f3a643029ba6d7" +checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" dependencies = [ "futures-core", "futures-sink", "js-sys", "pin-project-lite", - "thiserror 1.0.69", + "thiserror 2.0.16", + "tracing", +] + +[[package]] +name = "opentelemetry-appender-tracing" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef6a1ac5ca3accf562b8c306fa8483c85f4390f768185ab775f242f7fe8fdcc2" +dependencies = [ + "opentelemetry", "tracing", + "tracing-core", + "tracing-subscriber", ] [[package]] name = "opentelemetry-http" -version = "0.27.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a8a7f5f6ba7c1b286c2fbca0454eaba116f63bbe69ed250b642d36fbb04d80" +checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" dependencies = [ "async-trait", "bytes", - "http 1.3.1", + "http 1.4.0", "opentelemetry", - "reqwest 0.12.23", + "reqwest", ] [[package]] name = "opentelemetry-otlp" -version = "0.27.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91cf61a1868dacc576bf2b2a1c3e9ab150af7272909e80085c3173384fe11f76" +checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" dependencies = [ - "async-trait", - "futures-core", - "http 1.3.1", + "http 1.4.0", "opentelemetry", "opentelemetry-http", "opentelemetry-proto", "opentelemetry_sdk", - "prost 0.13.5", - "reqwest 0.12.23", - "thiserror 1.0.69", + "prost 0.14.3", + "reqwest", + "thiserror 2.0.16", "tokio", - "tonic 0.12.3", + "tonic 0.14.5", "tracing", ] [[package]] name = "opentelemetry-proto" -version = "0.27.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6e05acbfada5ec79023c85368af14abd0b307c015e9064d249b2a950ef459a6" +checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" dependencies = [ "opentelemetry", "opentelemetry_sdk", - "prost 0.13.5", - "tonic 0.12.3", + "prost 0.14.3", + "tonic 0.14.5", + "tonic-prost", ] [[package]] name = "opentelemetry_sdk" -version = "0.27.1" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "231e9d6ceef9b0b2546ddf52335785ce41252bc7474ee8ba05bfad277be13ab8" +checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" dependencies = [ - "async-trait", "futures-channel", "futures-executor", "futures-util", - "glob", "opentelemetry", "percent-encoding", - "rand 0.8.5", - "serde_json", - "thiserror 1.0.69", + "rand 0.9.2", + "thiserror 2.0.16", "tokio", "tokio-stream", - "tracing", ] [[package]] @@ -5740,6 +5895,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" +dependencies = [ + "num-traits", +] + [[package]] name = "os_pipe" version = "1.2.2" @@ -5782,8 +5946,8 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "ownedbytes" -version = "0.7.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=80f5f1e#80f5f1ecd4d5d30069b44e5acd3801f610d36056" +version = "0.9.0" +source = "git+https://github.com/indextables/tantivy/?rev=ffe408f49bb1f605d8eb28de9d18edd29c5245d5#ffe408f49bb1f605d8eb28de9d18edd29c5245d5" dependencies = [ "stable_deref_trait", ] @@ -5823,17 +5987,27 @@ dependencies = [ "sha2", ] +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi 0.3.9", +] + [[package]] name = "papergrid" -version = "0.10.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2ccbe15f2b6db62f9a9871642746427e297b0ceb85f9a7f1ee5ff47d184d0c8" +checksum = "6978128c8b51d8f4080631ceb2302ab51e32cc6e8615f735ee2f83fd269ae3f1" dependencies = [ "ansi-str", "ansitok", "bytecount", "fnv", - "unicode-width 0.1.14", + "unicode-width 0.2.1", ] [[package]] @@ -5891,40 +6065,94 @@ dependencies = [ ] [[package]] -name = "parse-size" -version = "1.1.0" +name = "parquet" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "487f2ccd1e17ce8c1bfab3a65c89525af41cfad4c8659021a1e9a2aacd73b89b" +checksum = "6ee96b29972a257b855ff2341b37e61af5f12d6af1158b6dcdb5b31ea07bb3cb" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64 0.22.1", + "bytes", + "chrono", + "half", + "hashbrown 0.16.1", + "num-bigint", + "num-integer", + "num-traits", + "parquet-variant", + "parquet-variant-compute", + "parquet-variant-json", + "paste", + "seq-macro", + "snap", + "thrift", + "twox-hash", + "zstd", +] [[package]] -name = "password-hash" -version = "0.4.2" +name = "parquet-variant" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" +checksum = "a6c31f8f9bfefb9dbf67b0807e00fd918676954a7477c889be971ac904103184" dependencies = [ - "base64ct", - "rand_core 0.6.4", - "subtle", + "arrow-schema", + "chrono", + "half", + "indexmap 2.13.0", + "simdutf8", + "uuid", ] [[package]] -name = "paste" -version = "1.0.15" +name = "parquet-variant-compute" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +checksum = "196cd9f7178fed3ac8d5e6d2b51193818e896bbc3640aea3fde3440114a8f39c" +dependencies = [ + "arrow", + "arrow-schema", + "chrono", + "half", + "indexmap 2.13.0", + "parquet-variant", + "parquet-variant-json", + "uuid", +] [[package]] -name = "pbkdf2" -version = "0.11.0" +name = "parquet-variant-json" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +checksum = "ed23d7acc90ef60f7fdbcc473fa2fdaefa33542ed15b84388959346d52c839be" dependencies = [ - "digest", - "hmac", - "password-hash", - "sha2", + "arrow-schema", + "base64 0.22.1", + "chrono", + "parquet-variant", + "serde_json", + "uuid", ] +[[package]] +name = "parse-size" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487f2ccd1e17ce8c1bfab3a65c89525af41cfad4c8659021a1e9a2aacd73b89b" + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "pbkdf2" version = "0.12.2" @@ -5947,15 +6175,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e9ed2178b0575fff8e1b83b58ba6f75e727aafac2e1b6c795169ad3b17eb518" -[[package]] -name = "pem" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" -dependencies = [ - "base64 0.13.1", -] - [[package]] name = "pem" version = "3.0.5" @@ -6051,7 +6270,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset", - "indexmap 2.11.0", + "indexmap 2.13.0", +] + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap 2.13.0", ] [[package]] @@ -6144,7 +6374,7 @@ dependencies = [ "aes", "cbc", "der 0.7.10", - "pbkdf2 0.12.2", + "pbkdf2", "scrypt", "sha2", "spki 0.7.3", @@ -6208,9 +6438,9 @@ dependencies = [ [[package]] name = "pnet" -version = "0.33.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd959a8268165518e2bf5546ba84c7b3222744435616381df3c456fe8d983576" +checksum = "682396b533413cc2e009fbb48aadf93619a149d3e57defba19ff50ce0201bd0d" dependencies = [ "ipnetwork", "pnet_base", @@ -6222,18 +6452,18 @@ dependencies = [ [[package]] name = "pnet_base" -version = "0.33.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872e46346144ebf35219ccaa64b1dffacd9c6f188cd7d012bd6977a2a838f42e" +checksum = "ffc190d4067df16af3aba49b3b74c469e611cad6314676eaf1157f31aa0fb2f7" dependencies = [ "no-std-net", ] [[package]] name = "pnet_datalink" -version = "0.33.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c302da22118d2793c312a35fb3da6846cb0fab6c3ad53fd67e37809b06cdafce" +checksum = "e79e70ec0be163102a332e1d2d5586d362ad76b01cec86f830241f2b6452a7b7" dependencies = [ "ipnetwork", "libc", @@ -6244,30 +6474,30 @@ dependencies = [ [[package]] name = "pnet_macros" -version = "0.33.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a780e80005c2e463ec25a6e9f928630049a10b43945fea83207207d4a7606f4" +checksum = "13325ac86ee1a80a480b0bc8e3d30c25d133616112bb16e86f712dcf8a71c863" dependencies = [ "proc-macro2", "quote", "regex", - "syn 1.0.109", + "syn 2.0.106", ] [[package]] name = "pnet_macros_support" -version = "0.33.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d932134f32efd7834eb8b16d42418dac87086347d1bc7d142370ef078582bc" +checksum = "eed67a952585d509dd0003049b1fc56b982ac665c8299b124b90ea2bdb3134ab" dependencies = [ "pnet_base", ] [[package]] name = "pnet_packet" -version = "0.33.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bde678bbd85cb1c2d99dc9fc596e57f03aa725f84f3168b0eaf33eeccb41706" +checksum = "4c96ebadfab635fcc23036ba30a7d33a80c39e8461b8bd7dc7bb186acb96560f" dependencies = [ "glob", "pnet_base", @@ -6277,9 +6507,9 @@ dependencies = [ [[package]] name = "pnet_sys" -version = "0.33.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf7a58b2803d818a374be9278a1fe8f88fce14b936afbe225000cfcd9c73f16" +checksum = "7d4643d3d4db6b08741050c2f3afa9a892c4244c085a72fcda93c9c2c9a00f4b" dependencies = [ "libc", "winapi 0.3.9", @@ -6287,9 +6517,9 @@ dependencies = [ [[package]] name = "pnet_transport" -version = "0.33.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "813d1c0e4defbe7ee22f6fe1755f122b77bfb5abe77145b1b5baaf463cab9249" +checksum = "5f604d98bc2a6591cf719b58d3203fd882bdd6bf1db696c4ac97978e9f4776bf" dependencies = [ "libc", "pnet_base", @@ -6317,16 +6547,25 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", "universal-hash", ] [[package]] name = "portable-atomic" -version = "1.11.1" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" +dependencies = [ + "portable-atomic", +] [[package]] name = "postcard" @@ -6357,10 +6596,11 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "pprof" -version = "0.13.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef5c97c51bd34c7e742402e216abdeb44d415fbe6ae41d56b114723e953711cb" +checksum = "38a01da47675efa7673b032bf8efd8214f1917d89685e07e395ab125ea42b187" dependencies = [ + "aligned-vec", "backtrace", "cfg-if", "findshlibs", @@ -6369,11 +6609,11 @@ dependencies = [ "log", "nix", "once_cell", - "parking_lot 0.12.4", "smallvec", + "spin 0.10.0", "symbolic-demangle", "tempfile", - "thiserror 1.0.69", + "thiserror 2.0.16", ] [[package]] @@ -6391,20 +6631,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" -[[package]] -name = "predicates" -version = "2.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" -dependencies = [ - "difflib", - "float-cmp 0.9.0", - "itertools 0.10.5", - "normalize-line-endings", - "predicates-core", - "regex", -] - [[package]] name = "predicates" version = "3.1.3" @@ -6413,7 +6639,7 @@ checksum = "a5d19ee57562043d37e82899fade9a22ebab7be9cef5026b07fda9cdd4293573" dependencies = [ "anstyle", "difflib", - "float-cmp 0.10.0", + "float-cmp", "normalize-line-endings", "predicates-core", "regex", @@ -6507,6 +6733,28 @@ dependencies = [ "version_check", ] +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn 2.0.106", +] + [[package]] name = "proc-macro2" version = "1.0.101" @@ -6531,22 +6779,21 @@ dependencies = [ [[package]] name = "procfs" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4" +checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" dependencies = [ "bitflags 2.9.4", "hex", - "lazy_static", "procfs-core", "rustix 0.38.44", ] [[package]] name = "procfs-core" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29" +checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" dependencies = [ "bitflags 2.9.4", "hex", @@ -6554,9 +6801,9 @@ dependencies = [ [[package]] name = "prometheus" -version = "0.13.4" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" +checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a" dependencies = [ "cfg-if", "fnv", @@ -6565,8 +6812,7 @@ dependencies = [ "memchr", "parking_lot 0.12.4", "procfs", - "protobuf", - "thiserror 1.0.69", + "thiserror 2.0.16", ] [[package]] @@ -6591,22 +6837,22 @@ dependencies = [ [[package]] name = "prost" -version = "0.11.9" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes", - "prost-derive 0.11.9", + "prost-derive 0.13.5", ] [[package]] name = "prost" -version = "0.13.5" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" dependencies = [ "bytes", - "prost-derive 0.13.5", + "prost-derive 0.14.3", ] [[package]] @@ -6620,7 +6866,7 @@ dependencies = [ "log", "multimap", "once_cell", - "petgraph", + "petgraph 0.7.1", "prettyplease", "prost 0.13.5", "prost-types 0.13.5", @@ -6629,24 +6875,45 @@ dependencies = [ "tempfile", ] +[[package]] +name = "prost-build" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" +dependencies = [ + "heck 0.5.0", + "itertools 0.14.0", + "log", + "multimap", + "petgraph 0.8.3", + "prettyplease", + "prost 0.14.3", + "prost-types 0.14.3", + "pulldown-cmark 0.13.3", + "pulldown-cmark-to-cmark", + "regex", + "syn 2.0.106", + "tempfile", +] + [[package]] name = "prost-derive" -version = "0.11.9" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.106", ] [[package]] name = "prost-derive" -version = "0.13.5" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6666,15 +6933,6 @@ dependencies = [ "prost-types 0.13.5", ] -[[package]] -name = "prost-types" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" -dependencies = [ - "prost 0.11.9", -] - [[package]] name = "prost-types" version = "0.13.5" @@ -6685,10 +6943,13 @@ dependencies = [ ] [[package]] -name = "protobuf" -version = "2.28.0" +name = "prost-types" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost 0.14.3", +] [[package]] name = "psl" @@ -6706,40 +6967,51 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac" [[package]] -name = "ptr_meta" -version = "0.1.4" +name = "publicsuffix" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42ea446cab60335f76979ec15e12619a2165b5ae2c12166bef27d283a9fadf" +dependencies = [ + "idna", + "psl-types", +] + +[[package]] +name = "pulldown-cmark" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b" dependencies = [ - "ptr_meta_derive", + "bitflags 2.9.4", + "memchr", + "unicase", ] [[package]] -name = "ptr_meta_derive" -version = "0.1.4" +name = "pulldown-cmark" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +checksum = "7c3a14896dfa883796f1cb410461aef38810ea05f2b2c33c5aded3649095fdad" dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", + "bitflags 2.9.4", + "memchr", + "unicase", ] [[package]] -name = "publicsuffix" -version = "2.3.0" +name = "pulldown-cmark-to-cmark" +version = "22.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f42ea446cab60335f76979ec15e12619a2165b5ae2c12166bef27d283a9fadf" +checksum = "50793def1b900256624a709439404384204a5dc3a6ec580281bfaac35e882e90" dependencies = [ - "idna", - "psl-types", + "pulldown-cmark 0.13.3", ] [[package]] name = "pulsar" -version = "6.3.1" +version = "6.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cee616af00383c461f9ceb0067d15dee68e7d313ae47dbd7f8543236aed7ee9" +checksum = "e2367cb38f1b65857bc11dd13b2adf13b7a1d991ef1cd43572f1420958c56cc2" dependencies = [ "async-channel 2.5.0", "async-trait", @@ -6751,13 +7023,14 @@ dependencies = [ "futures", "log", "lz4", + "murmur3", "native-tls", - "nom", - "oauth2", + "nom 7.1.3", + "oauth2 5.0.0", "openidconnect", - "pem 3.0.5", + "pem", "prost 0.13.5", - "prost-build", + "prost-build 0.13.5", "prost-derive 0.13.5", "rand 0.8.5", "regex", @@ -6769,18 +7042,22 @@ dependencies = [ "tokio-util", "url", "uuid", - "zstd 0.13.3", + "zstd", ] [[package]] -name = "query_map" -version = "0.7.0" +name = "quanta" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eab6b8b1074ef3359a863758dae650c7c0c6027927a085b7af911c8e0bf3a15" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" dependencies = [ - "form_urlencoded", - "serde", - "serde_derive", + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi 0.11.1+wasi-snapshot-preview1", + "web-sys", + "winapi 0.3.9", ] [[package]] @@ -6810,14 +7087,26 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.37.5" +version = "0.38.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" dependencies = [ "memchr", "serde", ] +[[package]] +name = "quick_cache" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a70b1b8b47e31d0498ecbc3c5470bb931399a8bfed1fd79d1717a61ce7f96e3" +dependencies = [ + "ahash", + "equivalent", + "hashbrown 0.16.1", + "parking_lot 0.12.4", +] + [[package]] name = "quickwit-actors" version = "0.8.0" @@ -6825,14 +7114,14 @@ dependencies = [ "anyhow", "async-trait", "criterion", - "flume", + "flume 0.12.0", "futures", "once_cell", "quickwit-common", - "rand 0.8.5", + "rand 0.9.2", "serde", "serde_json", - "sync_wrapper 1.0.2", + "sync_wrapper", "thiserror 2.0.16", "tokio", "tracing", @@ -6848,13 +7137,7 @@ dependencies = [ "aws-sdk-s3", "aws-sdk-sqs", "aws-smithy-async", - "aws-smithy-http-client", - "aws-smithy-runtime", - "aws-types", "futures", - "http-body-util", - "hyper 1.7.0", - "hyper-rustls 0.27.7", "quickwit-common", "tokio", ] @@ -6879,14 +7162,14 @@ dependencies = [ "once_cell", "openssl-probe", "opentelemetry", + "opentelemetry-appender-tracing", "opentelemetry-otlp", "opentelemetry_sdk", - "predicates 3.1.3", + "predicates", "quickwit-actors", "quickwit-cluster", "quickwit-common", "quickwit-config", - "quickwit-doc-mapper", "quickwit-index-management", "quickwit-indexing", "quickwit-ingest", @@ -6897,7 +7180,8 @@ dependencies = [ "quickwit-serve", "quickwit-storage", "quickwit-telemetry", - "reqwest 0.12.23", + "reqwest", + "rustls 0.23.31", "serde_json", "tabled", "tempfile", @@ -6928,13 +7212,13 @@ dependencies = [ "quickwit-common", "quickwit-config", "quickwit-proto", - "rand 0.8.5", + "rand 0.9.2", "serde", "serde_json", "time", "tokio", "tokio-stream", - "tonic 0.13.1", + "tonic 0.14.5", "tracing", "utoipa", ] @@ -6948,11 +7232,11 @@ dependencies = [ "heck 0.5.0", "prettyplease", "proc-macro2", - "prost-build", + "prost-build 0.14.3", "quote", "serde", "syn 2.0.106", - "tonic-build", + "tonic-prost-build", ] [[package]] @@ -6963,9 +7247,9 @@ dependencies = [ "async-trait", "bytesize", "futures", - "http 1.3.1", + "http 1.4.0", "mockall", - "prost 0.13.5", + "prost 0.14.3", "quickwit-actors", "quickwit-codegen", "quickwit-common", @@ -6974,7 +7258,8 @@ dependencies = [ "thiserror 2.0.16", "tokio", "tokio-stream", - "tonic 0.13.1", + "tonic 0.14.5", + "tonic-prost", "tower 0.5.2", "utoipa", ] @@ -6994,10 +7279,11 @@ dependencies = [ "env_logger", "fnv", "futures", + "governor", "home", - "hostname 0.3.1", - "http 1.3.1", - "hyper 1.7.0", + "hostname", + "http 1.4.0", + "hyper 1.8.1", "hyper-util", "itertools 0.14.0", "once_cell", @@ -7005,7 +7291,7 @@ dependencies = [ "pnet", "prometheus", "proptest", - "rand 0.8.5", + "rand 0.9.2", "rayon", "regex", "serde", @@ -7015,12 +7301,11 @@ dependencies = [ "sysinfo", "tempfile", "thiserror 2.0.16", - "tikv-jemalloc-ctl", "tikv-jemallocator", "tokio", "tokio-metrics", "tokio-stream", - "tonic 0.13.1", + "tonic 0.14.5", "tower 0.5.2", "tracing", ] @@ -7035,8 +7320,8 @@ dependencies = [ "chrono", "cron", "enum-iterator", - "http 0.2.12", - "http-serde 1.1.3", + "http 1.4.0", + "http-serde", "humantime", "itertools 0.14.0", "json_comments", @@ -7068,7 +7353,7 @@ dependencies = [ "fnv", "futures", "itertools 0.14.0", - "lru 0.13.0", + "lru 0.16.3", "mockall", "once_cell", "proptest", @@ -7080,13 +7365,11 @@ dependencies = [ "quickwit-ingest", "quickwit-metastore", "quickwit-proto", - "rand 0.8.5", + "rand 0.9.2", "serde", "serde_json", "smallvec", - "time", "tokio", - "tower 0.5.2", "tracing", "ulid", ] @@ -7095,7 +7378,6 @@ dependencies = [ name = "quickwit-datetime" version = "0.8.0" dependencies = [ - "anyhow", "itertools 0.14.0", "serde", "serde_json", @@ -7130,10 +7412,10 @@ dependencies = [ "binggan", "fnv", "hex", - "indexmap 2.11.0", + "indexmap 2.13.0", "itertools 0.14.0", "matches", - "nom", + "nom 8.0.0", "once_cell", "quickwit-common", "quickwit-datetime", @@ -7179,6 +7461,7 @@ version = "0.8.0" dependencies = [ "anyhow", "arc-swap", + "arrow", "async-compression", "async-trait", "aws-sdk-kinesis", @@ -7187,7 +7470,7 @@ dependencies = [ "bytesize", "criterion", "fail", - "flume", + "flume 0.12.0", "fnv", "futures", "google-cloud-auth", @@ -7200,9 +7483,10 @@ dependencies = [ "once_cell", "oneshot", "openssl", + "parquet", "percent-encoding", "proptest", - "prost 0.13.5", + "prost 0.14.3", "pulsar", "quickwit-actors", "quickwit-aws", @@ -7215,15 +7499,17 @@ dependencies = [ "quickwit-ingest", "quickwit-metastore", "quickwit-opentelemetry", + "quickwit-parquet-engine", "quickwit-proto", "quickwit-query", "quickwit-storage", - "rand 0.8.5", + "rand 0.9.2", "rdkafka", "regex", - "reqwest 0.12.23", + "reqwest", "serde", "serde_json", + "sqlx", "tantivy", "tempfile", "thiserror 2.0.16", @@ -7246,12 +7532,12 @@ dependencies = [ "bytesize", "fail", "futures", - "http 1.3.1", + "http 1.4.0", "itertools 0.14.0", "mockall", "mrecordlog", "once_cell", - "prost 0.13.5", + "prost 0.14.3", "quickwit-actors", "quickwit-cluster", "quickwit-codegen", @@ -7259,7 +7545,7 @@ dependencies = [ "quickwit-config", "quickwit-doc-mapper", "quickwit-proto", - "rand 0.8.5", + "rand 0.9.2", "rand_distr", "serde", "serde_json", @@ -7267,7 +7553,8 @@ dependencies = [ "tempfile", "thiserror 2.0.16", "tokio", - "tonic 0.13.1", + "tonic 0.14.5", + "tonic-prost", "tower 0.5.2", "tracing", "ulid", @@ -7281,7 +7568,7 @@ dependencies = [ "anyhow", "aws-sdk-sqs", "futures-util", - "hyper 1.7.0", + "hyper 1.8.1", "hyper-util", "itertools 0.14.0", "quickwit-actors", @@ -7289,18 +7576,20 @@ dependencies = [ "quickwit-common", "quickwit-config", "quickwit-indexing", + "quickwit-ingest", "quickwit-metastore", "quickwit-opentelemetry", "quickwit-proto", "quickwit-rest-client", "quickwit-serve", "quickwit-storage", - "rand 0.8.5", - "reqwest 0.12.23", + "rand 0.9.2", + "reqwest", + "rustls 0.23.31", "serde_json", "tempfile", "tokio", - "tonic 0.13.1", + "tonic 0.14.5", "tracing", "tracing-subscriber", ] @@ -7313,8 +7602,8 @@ dependencies = [ "itertools 0.14.0", "once_cell", "postcard", - "prost 0.13.5", - "prost-types 0.13.5", + "prost 0.14.3", + "prost-types 0.14.3", "quickwit-actors", "quickwit-cluster", "quickwit-common", @@ -7334,7 +7623,7 @@ dependencies = [ "time", "tokio", "tokio-stream", - "tonic 0.13.1", + "tonic 0.14.5", "tracing", ] @@ -7371,49 +7660,54 @@ dependencies = [ ] [[package]] -name = "quickwit-lambda" +name = "quickwit-lambda-client" version = "0.8.0" dependencies = [ "anyhow", - "aws_lambda_events", + "async-trait", + "aws-config", + "aws-sdk-lambda", + "aws-smithy-mocks", + "base64 0.22.1", "bytesize", - "chitchat", - "chrono", - "flate2", - "http 1.3.1", - "lambda_http", - "lambda_runtime", - "mime_guess", + "md5", "once_cell", - "opentelemetry", - "opentelemetry-otlp", - "opentelemetry_sdk", - "quickwit-actors", - "quickwit-cli", - "quickwit-cluster", + "prost 0.14.3", "quickwit-common", "quickwit-config", - "quickwit-index-management", - "quickwit-indexing", - "quickwit-ingest", - "quickwit-janitor", - "quickwit-metastore", + "quickwit-lambda-server", + "quickwit-proto", + "quickwit-search", + "quickwit-storage", + "serde_json", + "sha2", + "tokio", + "tracing", + "ureq", +] + +[[package]] +name = "quickwit-lambda-server" +version = "0.8.0" +dependencies = [ + "anyhow", + "base64 0.22.1", + "bytesize", + "lambda_runtime", + "openssl", + "prost 0.14.3", + "quickwit-common", + "quickwit-config", + "quickwit-doc-mapper", "quickwit-proto", "quickwit-search", - "quickwit-serve", "quickwit-storage", - "quickwit-telemetry", - "rand 0.8.5", - "reqwest 0.12.23", "serde", "serde_json", - "serial_test", - "time", + "thiserror 2.0.16", "tokio", "tracing", - "tracing-opentelemetry", "tracing-subscriber", - "warp", ] [[package]] @@ -7435,7 +7729,7 @@ dependencies = [ "bytesize", "dotenvy", "futures", - "http 1.3.1", + "http 1.4.0", "hyper-util", "itertools 0.14.0", "md5", @@ -7445,10 +7739,11 @@ dependencies = [ "quickwit-common", "quickwit-config", "quickwit-doc-mapper", + "quickwit-parquet-engine", "quickwit-proto", "quickwit-query", "quickwit-storage", - "rand 0.8.5", + "rand 0.9.2", "regex", "regex-syntax", "sea-query", @@ -7468,6 +7763,7 @@ dependencies = [ "tracing-subscriber", "ulid", "utoipa", + "uuid", ] [[package]] @@ -7475,10 +7771,11 @@ name = "quickwit-opentelemetry" version = "0.8.0" dependencies = [ "anyhow", + "arrow", "async-trait", - "hex", "once_cell", - "prost 0.13.5", + "parquet", + "prost 0.14.3", "quickwit-common", "quickwit-config", "quickwit-ingest", @@ -7489,8 +7786,26 @@ dependencies = [ "thiserror 2.0.16", "time", "tokio", - "tonic 0.13.1", + "tonic 0.14.5", + "tracing", +] + +[[package]] +name = "quickwit-parquet-engine" +version = "0.8.0" +dependencies = [ + "arrow", + "parquet", + "proptest", + "quickwit-common", + "sea-query", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.16", + "tokio", "tracing", + "ulid", ] [[package]] @@ -7504,12 +7819,13 @@ dependencies = [ "bytestring", "futures", "glob", - "http 1.3.1", + "hex", + "http 1.4.0", "mockall", "opentelemetry", - "prost 0.13.5", - "prost-build", - "prost-types 0.13.5", + "prost 0.14.3", + "prost-build 0.14.3", + "prost-types 0.14.3", "quickwit-actors", "quickwit-codegen", "quickwit-common", @@ -7519,14 +7835,16 @@ dependencies = [ "sqlx", "thiserror 2.0.16", "tokio", - "tonic 0.13.1", + "tonic 0.14.5", "tonic-build", + "tonic-prost", + "tonic-prost-build", "tower 0.5.2", "tracing", "tracing-opentelemetry", "ulid", "utoipa", - "zstd 0.13.3", + "zstd", ] [[package]] @@ -7535,15 +7853,14 @@ version = "0.8.0" dependencies = [ "anyhow", "base64 0.22.1", + "bitpacking", "criterion", "hex", - "lindera-core", - "lindera-dictionary", - "lindera-tokenizer", "once_cell", "proptest", "quickwit-common", "quickwit-datetime", + "quickwit-proto", "regex", "rustc-hash 2.1.1", "serde", @@ -7553,7 +7870,7 @@ dependencies = [ "tantivy-fst", "thiserror 2.0.16", "time", - "whichlang", + "tracing", ] [[package]] @@ -7562,7 +7879,7 @@ version = "0.8.0" dependencies = [ "anyhow", "bytes", - "http 1.3.1", + "http 1.4.0", "quickwit-cluster", "quickwit-common", "quickwit-config", @@ -7570,11 +7887,10 @@ dependencies = [ "quickwit-ingest", "quickwit-metastore", "quickwit-proto", - "quickwit-search", "quickwit-serve", - "reqwest 0.12.23", + "reqwest", "reqwest-middleware", - "reqwest-retry", + "reqwest-retry 0.8.0", "serde", "serde_json", "thiserror 2.0.16", @@ -7596,34 +7912,30 @@ dependencies = [ "chrono", "fnv", "futures", - "http 1.3.1", + "http 1.4.0", "itertools 0.14.0", "mockall", "once_cell", "pin-project", "postcard", "proptest", - "prost 0.13.5", + "prost 0.14.3", "quickwit-common", "quickwit-config", "quickwit-directories", "quickwit-doc-mapper", "quickwit-indexing", "quickwit-metastore", - "quickwit-opentelemetry", "quickwit-proto", "quickwit-query", "quickwit-storage", - "rand 0.8.5", - "rayon", + "rand 0.9.2", "serde", "serde_json", - "serde_json_borrow", "tantivy", "tantivy-fst", "thiserror 2.0.16", "tokio", - "tokio-stream", "tower 0.5.2", "tracing", "ttl_cache", @@ -7647,21 +7959,19 @@ dependencies = [ "futures-util", "glob", "hex", - "http 1.3.1", - "http-serde 1.1.3", + "http 1.4.0", + "http-body 1.0.1", + "http-serde", "humantime", - "hyper 1.7.0", - "hyper-rustls 0.27.7", "hyper-util", "itertools 0.14.0", "mime_guess", "mockall", "once_cell", - "opentelemetry", "percent-encoding", "pprof", - "prost 0.13.5", - "prost-types 0.13.5", + "prost 0.14.3", + "prost-types 0.14.3", "quickwit-actors", "quickwit-cluster", "quickwit-common", @@ -7673,6 +7983,7 @@ dependencies = [ "quickwit-ingest", "quickwit-jaeger", "quickwit-janitor", + "quickwit-lambda-client", "quickwit-metastore", "quickwit-opentelemetry", "quickwit-proto", @@ -7682,28 +7993,28 @@ dependencies = [ "quickwit-telemetry", "regex", "rust-embed", - "rustls 0.21.12", + "rustls 0.23.31", "rustls-pemfile", "serde", "serde_json", - "serde_qs 0.12.0", + "serde_qs 0.15.0", "serde_with", "tempfile", "thiserror 2.0.16", "time", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls 0.26.2", "tokio-stream", "tokio-util", - "tonic 0.13.1", + "tonic 0.14.5", "tonic-health", "tonic-reflection", "tower 0.5.2", - "tower-http 0.4.4", + "tower-http", "tracing", "utoipa", "warp", - "zstd 0.13.3", + "zstd", ] [[package]] @@ -7726,23 +8037,24 @@ dependencies = [ "bytesize", "fnv", "futures", + "http 1.4.0", "http-body-util", - "hyper 1.7.0", - "lru 0.13.0", + "hyper 1.8.1", + "lru 0.16.3", "md5", + "mini-moka", "mockall", "once_cell", "opendal", "pin-project", "proptest", + "quick_cache", "quickwit-aws", "quickwit-common", "quickwit-config", "quickwit-proto", - "rand 0.8.5", "regex", - "reqsign", - "reqwest 0.12.23", + "reqwest", "serde", "serde_json", "tantivy", @@ -7762,11 +8074,11 @@ version = "0.8.0" dependencies = [ "async-trait", "encoding_rs", - "hostname 0.3.1", + "hostname", "md5", "once_cell", "quickwit-common", - "reqwest 0.12.23", + "reqwest", "serde", "serde_json", "tokio", @@ -7802,10 +8114,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" dependencies = [ "bytes", - "getrandom 0.3.3", + "getrandom 0.3.4", "lru-slab", "rand 0.9.2", - "ring 0.17.14", + "ring", "rustc-hash 2.1.1", "rustls 0.23.31", "rustls-pki-types", @@ -7852,10 +8164,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] -name = "radium" -version = "0.7.0" +name = "r-efi" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" @@ -7891,6 +8203,17 @@ dependencies = [ "rand_core 0.9.3", ] +[[package]] +name = "rand" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" +dependencies = [ + "chacha20 0.10.0", + "getrandom 0.4.2", + "rand_core 0.10.0", +] + [[package]] name = "rand_chacha" version = "0.2.2" @@ -7945,17 +8268,23 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" + [[package]] name = "rand_distr" -version = "0.4.3" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.8.5", + "rand 0.9.2", ] [[package]] @@ -7976,6 +8305,15 @@ dependencies = [ "rand_core 0.9.3", ] +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.9.4", +] + [[package]] name = "rayon" version = "1.11.0" @@ -8068,11 +8406,26 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "referencing" +version = "0.37.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4283168a506f0dcbdce31c9f9cce3129c924da4c6bca46e46707fcb746d2d70c" +dependencies = [ + "ahash", + "fluent-uri", + "getrandom 0.3.4", + "hashbrown 0.16.1", + "parking_lot 0.12.4", + "percent-encoding", + "serde_json", +] + [[package]] name = "regex" -version = "1.11.2" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" dependencies = [ "aho-corasick", "memchr", @@ -8082,15 +8435,27 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.10" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", "regex-syntax", ] +[[package]] +name = "regex-filtered" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac5f7b31fbef748cc46643c1f9ba17f6d5c7c6f0ba5e372fc9c48d31ad1c8612" +dependencies = [ + "aho-corasick", + "itertools 0.14.0", + "regex", + "regex-syntax", +] + [[package]] name = "regex-lite" version = "0.1.7" @@ -8103,15 +8468,6 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" -[[package]] -name = "rend" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" -dependencies = [ - "bytecheck", -] - [[package]] name = "reqsign" version = "0.16.5" @@ -8127,12 +8483,12 @@ dependencies = [ "hex", "hmac", "home", - "http 1.3.1", - "jsonwebtoken 9.3.1", + "http 1.4.0", + "jsonwebtoken", "log", "percent-encoding", "rand 0.8.5", - "reqwest 0.12.23", + "reqwest", "rsa", "serde", "serde_json", @@ -8142,85 +8498,44 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" -dependencies = [ - "base64 0.21.7", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2 0.3.27", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.32", - "hyper-rustls 0.24.2", - "hyper-tls 0.5.0", - "ipnet", - "js-sys", - "log", - "mime", - "native-tls", - "once_cell", - "percent-encoding", - "pin-project-lite", - "rustls 0.21.12", - "rustls-pemfile", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper 0.1.2", - "system-configuration 0.5.1", - "tokio", - "tokio-native-tls", - "tokio-rustls 0.24.1", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "webpki-roots 0.25.4", - "winreg", -] - -[[package]] -name = "reqwest" -version = "0.12.23" +version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64 0.22.1", "bytes", + "encoding_rs", "futures-channel", "futures-core", "futures-util", - "http 1.3.1", + "h2 0.4.12", + "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.1", "hyper-rustls 0.27.7", - "hyper-tls 0.6.0", + "hyper-tls", "hyper-util", "js-sys", "log", + "mime", "native-tls", "percent-encoding", "pin-project-lite", "quinn", "rustls 0.23.31", - "rustls-native-certs 0.8.1", + "rustls-native-certs", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 1.0.2", + "sync_wrapper", "tokio", "tokio-native-tls", "tokio-rustls 0.26.2", "tokio-util", "tower 0.5.2", - "tower-http 0.6.6", + "tower-http", "tower-service", "url", "wasm-bindgen", @@ -8238,33 +8553,53 @@ checksum = "57f17d28a6e6acfe1733fe24bcd30774d13bffa4b8a22535b4c8c98423088d4e" dependencies = [ "anyhow", "async-trait", - "http 1.3.1", - "reqwest 0.12.23", + "http 1.4.0", + "reqwest", "serde", "thiserror 1.0.69", - "tower-service", + "tower-service", +] + +[[package]] +name = "reqwest-retry" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c73e4195a6bfbcb174b790d9b3407ab90646976c55de58a6515da25d851178" +dependencies = [ + "anyhow", + "async-trait", + "futures", + "getrandom 0.2.16", + "http 1.4.0", + "hyper 1.8.1", + "parking_lot 0.11.2", + "reqwest", + "reqwest-middleware", + "retry-policies 0.4.0", + "thiserror 1.0.69", + "tokio", + "wasm-timer", ] [[package]] name = "reqwest-retry" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c73e4195a6bfbcb174b790d9b3407ab90646976c55de58a6515da25d851178" +checksum = "105747e3a037fe5bf17458d794de91149e575b6183fc72c85623a44abb9683f5" dependencies = [ "anyhow", "async-trait", "futures", "getrandom 0.2.16", - "http 1.3.1", - "hyper 1.7.0", - "parking_lot 0.11.2", - "reqwest 0.12.23", + "http 1.4.0", + "hyper 1.8.1", + "reqwest", "reqwest-middleware", - "retry-policies", - "thiserror 1.0.69", + "retry-policies 0.5.1", + "thiserror 2.0.16", "tokio", "tracing", - "wasm-timer", + "wasmtimer", ] [[package]] @@ -8276,6 +8611,15 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "retry-policies" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46a4bd6027df676bcb752d3724db0ea3c0c5fc1dd0376fec51ac7dcaf9cc69be" +dependencies = [ + "rand 0.9.2", +] + [[package]] name = "rfc6979" version = "0.3.1" @@ -8306,21 +8650,6 @@ dependencies = [ "bytemuck", ] -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin 0.5.2", - "untrusted 0.7.1", - "web-sys", - "winapi 0.3.9", -] - [[package]] name = "ring" version = "0.17.14" @@ -8331,39 +8660,10 @@ dependencies = [ "cfg-if", "getrandom 0.2.16", "libc", - "untrusted 0.9.0", + "untrusted", "windows-sys 0.52.0", ] -[[package]] -name = "rkyv" -version = "0.7.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" -dependencies = [ - "bitvec", - "bytecheck", - "bytes", - "hashbrown 0.12.3", - "ptr_meta", - "rend", - "rkyv_derive", - "seahash", - "tinyvec", - "uuid", -] - -[[package]] -name = "rkyv_derive" -version = "0.7.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "roxmltree" version = "0.14.1" @@ -8402,9 +8702,9 @@ dependencies = [ [[package]] name = "rust-embed" -version = "6.8.1" +version = "8.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a36224c3276f8c4ebc8c20f158eca7ca4359c8db89991c4925132aaaf6702661" +checksum = "04113cb9355a377d83f06ef1f0a45b8ab8cd7d8b1288160717d66df5c7988d27" dependencies = [ "rust-embed-impl", "rust-embed-utils", @@ -8413,9 +8713,9 @@ dependencies = [ [[package]] name = "rust-embed-impl" -version = "6.8.1" +version = "8.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49b94b81e5b2c284684141a2fb9e2a31be90638caf040bf9afbc5a0416afe1ac" +checksum = "da0902e4c7c8e997159ab384e6d0fc91c221375f6894346ae107f47dd0f3ccaa" dependencies = [ "proc-macro2", "quote", @@ -8426,38 +8726,22 @@ dependencies = [ [[package]] name = "rust-embed-utils" -version = "7.8.1" +version = "8.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d38ff6bf570dc3bb7100fce9f7b60c33fa71d80e88da3f2580df4ff2bdded74" +checksum = "5bcdef0be6fe7f6fa333b1073c949729274b05f123a0ad7efcb8efd878e5c3b1" dependencies = [ "sha2", "walkdir", ] -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - [[package]] name = "rust_decimal" version = "1.37.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b203a6425500a03e0919c42d3c47caca51e79f1132046626d2c8871c5092035d" dependencies = [ - "arrayvec 0.7.6", - "borsh", - "bytes", + "arrayvec", "num-traits", - "rand 0.8.5", - "rkyv", - "serde", - "serde_json", ] [[package]] @@ -8520,7 +8804,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", - "ring 0.17.14", + "ring", "rustls-webpki 0.101.7", "sct", ] @@ -8534,25 +8818,13 @@ dependencies = [ "aws-lc-rs", "log", "once_cell", - "ring 0.17.14", + "ring", "rustls-pki-types", "rustls-webpki 0.103.4", "subtle", "zeroize", ] -[[package]] -name = "rustls-native-certs" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" -dependencies = [ - "openssl-probe", - "rustls-pemfile", - "schannel", - "security-framework 2.11.1", -] - [[package]] name = "rustls-native-certs" version = "0.8.1" @@ -8567,11 +8839,11 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.4" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "base64 0.21.7", + "rustls-pki-types", ] [[package]] @@ -8584,24 +8856,14 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rustls-webpki" -version = "0.100.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6a5fc258f1c1276dfe3016516945546e2d5383911efc0fc4f1cdc5df3a4ae3" -dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", -] - [[package]] name = "rustls-webpki" version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.17.14", - "untrusted 0.9.0", + "ring", + "untrusted", ] [[package]] @@ -8611,9 +8873,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a17884ae0c1b773f1ccd2bd4a8c72f16da897310a98b0e84bf349ad5ead92fc" dependencies = [ "aws-lc-rs", - "ring 0.17.14", + "ring", "rustls-pki-types", - "untrusted 0.9.0", + "untrusted", ] [[package]] @@ -8736,7 +8998,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" dependencies = [ - "pbkdf2 0.12.2", + "pbkdf2", "salsa20", "sha2", ] @@ -8747,8 +9009,8 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.17.14", - "untrusted 0.9.0", + "ring", + "untrusted", ] [[package]] @@ -8759,9 +9021,9 @@ checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca" [[package]] name = "sea-query" -version = "0.30.7" +version = "0.32.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4166a1e072292d46dc91f31617c2a1cdaf55a8be4b5c9f4bf2ba248e3ac4999b" +checksum = "8a5d1c518eaf5eda38e5773f902b26ab6d5e9e9e2bb2349ca6c64cf96f80448c" dependencies = [ "inherent", "sea-query-derive", @@ -8769,9 +9031,9 @@ dependencies = [ [[package]] name = "sea-query-binder" -version = "0.5.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36bbb68df92e820e4d5aeb17b4acd5cc8b5d18b2c36a4dd6f4626aabfa7ab1b9" +checksum = "b0019f47430f7995af63deda77e238c17323359af241233ec768aba1faea7608" dependencies = [ "sea-query", "sqlx", @@ -8866,6 +9128,9 @@ name = "semver" version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" +dependencies = [ + "serde", +] [[package]] name = "separator" @@ -8873,12 +9138,19 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5" +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + [[package]] name = "serde" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ + "serde_core", "serde_derive", ] @@ -8893,44 +9165,44 @@ dependencies = [ ] [[package]] -name = "serde_derive" -version = "1.0.219" +name = "serde_core" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", + "serde_derive", ] [[package]] -name = "serde_dynamo" -version = "4.2.14" +name = "serde_derive" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36c1b1792cfd9de29eb373ee6a4b74650369c096f55db7198ceb7b8921d1f7f" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ - "base64 0.21.7", - "serde", + "proc-macro2", + "quote", + "syn 2.0.106", ] [[package]] name = "serde_json" -version = "1.0.143" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ - "indexmap 2.11.0", + "indexmap 2.13.0", "itoa", "memchr", - "ryu", "serde", + "serde_core", + "zmij", ] [[package]] name = "serde_json_borrow" -version = "0.5.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a60291362be3646d15fb0b5a5bddfd8003ebf013b2186a3c60a534fd35d6a26" +checksum = "2780994ebdf8677793192229715c7f5d9dd4c37cefc7fb760b9418903ffb5c34" dependencies = [ "serde", "serde_json", @@ -8968,25 +9240,22 @@ dependencies = [ [[package]] name = "serde_qs" -version = "0.12.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0431a35568651e363364210c91983c1da5eb29404d9f0928b67d4ebcfa7d330c" +checksum = "f3faaf9e727533a19351a43cc5a8de957372163c7d35cc48c90b75cdda13c352" dependencies = [ - "futures", "percent-encoding", "serde", - "thiserror 1.0.69", - "tracing", - "warp", + "thiserror 2.0.16", ] [[package]] name = "serde_spanned" -version = "0.6.9" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +checksum = "876ac351060d4f882bb1032b6369eb0aef79ad9df1ea8bc404874d8cc3d0cd98" dependencies = [ - "serde", + "serde_core", ] [[package]] @@ -9003,19 +9272,18 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.14.0" +version = "3.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2c45cd61fefa9db6f254525d46e392b852e0e61d9a1fd36e5bd183450a556d5" +checksum = "dd5414fad8e6907dbdd5bc441a50ae8d6e26151a03b1de04d89a5576de61d01f" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.11.0", + "indexmap 2.13.0", "schemars 0.9.0", "schemars 1.0.4", - "serde", - "serde_derive", + "serde_core", "serde_json", "serde_with_macros", "time", @@ -9023,11 +9291,11 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.14.0" +version = "3.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f" +checksum = "d3db8978e608f1fe7357e211969fd9abdcae80bac1ba7a3369bb7eb6b404eb65" dependencies = [ - "darling 0.20.11", + "darling 0.23.0", "proc-macro2", "quote", "syn 2.0.106", @@ -9039,7 +9307,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.11.0", + "indexmap 2.13.0", "itoa", "ryu", "serde", @@ -9079,7 +9347,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -9090,7 +9358,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -9101,7 +9369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -9221,11 +9489,26 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" +[[package]] +name = "skeptic" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16d23b015676c90a0f01c197bfdc786c20342c73a0afdda9025adb0bc42940a8" +dependencies = [ + "bytecount", + "cargo_metadata", + "error-chain", + "glob", + "pulldown-cmark 0.9.6", + "tempfile", + "walkdir", +] + [[package]] name = "sketches-ddsketch" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1e9a774a6c28142ac54bb25d25562e6bcf957493a184f15ad4eebccb23e410a" +checksum = "05e40b6cf54d988dc1a2223531b969c9a9e30906ad90ef64890c27b4bfbb46ea" dependencies = [ "serde", ] @@ -9241,6 +9524,9 @@ name = "smallvec" version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +dependencies = [ + "serde", +] [[package]] name = "snafu" @@ -9313,15 +9599,27 @@ dependencies = [ [[package]] name = "spin" -version = "0.5.2" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] [[package]] name = "spin" -version = "0.9.8" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spinning_top" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96d2d1d716fb500937168cc09353ffdc7a012be8475ac7308e1bdf0e3923300" dependencies = [ "lock_api", ] @@ -9346,21 +9644,11 @@ dependencies = [ "der 0.7.10", ] -[[package]] -name = "sqlformat" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bba3a93db0cc4f7bdece8bb09e77e2e785c20bfebf79eb8340ed80708048790" -dependencies = [ - "nom", - "unicode_categories", -] - [[package]] name = "sqlx" -version = "0.7.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9a2ccff1a000a5a59cd33da541d9f2fdcd9e6e8229cc200565942bff36d0aaa" +checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" dependencies = [ "sqlx-core", "sqlx-macros", @@ -9371,69 +9659,63 @@ dependencies = [ [[package]] name = "sqlx-core" -version = "0.7.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24ba59a9342a3d9bab6c56c118be528b27c9b60e490080e9711a04dccac83ef6" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" dependencies = [ - "ahash 0.8.12", - "atoi", - "byteorder", + "base64 0.22.1", "bytes", "crc", "crossbeam-queue", "either", - "event-listener 2.5.3", - "futures-channel", + "event-listener 5.4.1", "futures-core", "futures-intrusive", "futures-io", "futures-util", + "hashbrown 0.15.5", "hashlink", - "hex", - "indexmap 2.11.0", + "indexmap 2.13.0", "log", "memchr", "once_cell", - "paste", "percent-encoding", - "rustls 0.21.12", - "rustls-pemfile", + "rustls 0.23.31", "serde", "serde_json", "sha2", "smallvec", - "sqlformat", - "thiserror 1.0.69", + "thiserror 2.0.16", "time", "tokio", "tokio-stream", "tracing", "url", - "webpki-roots 0.25.4", + "webpki-roots 0.26.11", ] [[package]] name = "sqlx-macros" -version = "0.7.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea40e2345eb2faa9e1e5e326db8c34711317d2b5e08d0d5741619048a803127" +checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" dependencies = [ "proc-macro2", "quote", "sqlx-core", "sqlx-macros-core", - "syn 1.0.109", + "syn 2.0.106", ] [[package]] name = "sqlx-macros-core" -version = "0.7.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5833ef53aaa16d860e92123292f1f6a3d53c34ba8b1969f152ef1a7bb803f3c8" +checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" dependencies = [ "dotenvy", "either", - "heck 0.4.1", + "heck 0.5.0", "hex", "once_cell", "proc-macro2", @@ -9445,20 +9727,19 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 1.0.109", - "tempfile", + "syn 2.0.106", "tokio", "url", ] [[package]] name = "sqlx-mysql" -version = "0.7.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ed31390216d20e538e447a7a9b959e06ed9fc51c37b514b46eb758016ecd418" +checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" dependencies = [ "atoi", - "base64 0.21.7", + "base64 0.22.1", "bitflags 2.9.4", "byteorder", "bytes", @@ -9488,7 +9769,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 1.0.69", + "thiserror 2.0.16", "time", "tracing", "whoami", @@ -9496,12 +9777,12 @@ dependencies = [ [[package]] name = "sqlx-postgres" -version = "0.7.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c824eb80b894f926f89a0b9da0c7f435d27cdd35b8c655b114e58223918577e" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ "atoi", - "base64 0.21.7", + "base64 0.22.1", "bitflags 2.9.4", "byteorder", "crc", @@ -9509,7 +9790,6 @@ dependencies = [ "etcetera", "futures-channel", "futures-core", - "futures-io", "futures-util", "hex", "hkdf", @@ -9527,7 +9807,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 1.0.69", + "thiserror 2.0.16", "time", "tracing", "whoami", @@ -9535,12 +9815,12 @@ dependencies = [ [[package]] name = "sqlx-sqlite" -version = "0.7.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b244ef0a8414da0bed4bb1910426e890b19e5e9bccc27ada6b797d05c55ae0aa" +checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" dependencies = [ "atoi", - "flume", + "flume 0.11.1", "futures-channel", "futures-core", "futures-executor", @@ -9550,11 +9830,12 @@ dependencies = [ "log", "percent-encoding", "serde", + "serde_urlencoded", "sqlx-core", + "thiserror 2.0.16", "time", "tracing", "url", - "urlencoding", ] [[package]] @@ -9604,7 +9885,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" dependencies = [ - "vte 0.14.1", + "vte", ] [[package]] @@ -9664,12 +9945,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - [[package]] name = "sync_wrapper" version = "1.0.2" @@ -9692,94 +9967,50 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.33.1" +version = "0.37.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fc858248ea01b66f19d8e8a6d55f41deaf91e9d495246fd01368d99935c6c01" +checksum = "16607d5caffd1c07ce073528f9ed972d88db15dd44023fa57142963be3feb11f" dependencies = [ - "core-foundation-sys", "libc", - "memchr", - "ntapi", - "rayon", - "windows 0.57.0", + "objc2-core-foundation", + "objc2-io-kit", + "windows", ] [[package]] name = "syslog_loose" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "161028c00842709450114c39db3b29f44c898055ed8833bb9b535aba7facf30e" +checksum = "d6ec4df26907adce53e94eac201a9ba38744baea3bc97f34ffd591d5646231a6" dependencies = [ "chrono", - "nom", -] - -[[package]] -name = "system-configuration" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" -dependencies = [ - "bitflags 1.3.2", - "core-foundation 0.9.4", - "system-configuration-sys 0.5.0", -] - -[[package]] -name = "system-configuration" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" -dependencies = [ - "bitflags 2.9.4", - "core-foundation 0.9.4", - "system-configuration-sys 0.6.0", -] - -[[package]] -name = "system-configuration-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "system-configuration-sys" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" -dependencies = [ - "core-foundation-sys", - "libc", + "nom 8.0.0", ] [[package]] name = "tabled" -version = "0.14.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfe9c3632da101aba5131ed63f9eed38665f8b3c68703a6bb18124835c1a5d22" +checksum = "e39a2ee1fbcd360805a771e1b300f78cc88fec7b8d3e2f71cd37bbf23e725c7d" dependencies = [ "ansi-str", "ansitok", "papergrid", "tabled_derive", - "unicode-width 0.1.14", + "testing_table", ] [[package]] name = "tabled_derive" -version = "0.6.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99f688a08b54f4f02f0a3c382aefdb7884d3d69609f785bd253dc033243e3fe4" +checksum = "0ea5d1b13ca6cff1f9231ffd62f15eefd72543dab5e468735f1a456728a02846" dependencies = [ - "heck 0.4.1", - "proc-macro-error", + "heck 0.5.0", + "proc-macro-error2", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.106", ] [[package]] @@ -9790,8 +10021,8 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] name = "tantivy" -version = "0.23.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=80f5f1e#80f5f1ecd4d5d30069b44e5acd3801f610d36056" +version = "0.26.0" +source = "git+https://github.com/indextables/tantivy/?rev=ffe408f49bb1f605d8eb28de9d18edd29c5245d5#ffe408f49bb1f605d8eb28de9d18edd29c5245d5" dependencies = [ "aho-corasick", "arc-swap", @@ -9802,6 +10033,7 @@ dependencies = [ "census", "crc32fast", "crossbeam-channel", + "datasketches", "downcast-rs", "fastdivide", "fnv", @@ -9809,19 +10041,17 @@ dependencies = [ "futures-channel", "futures-util", "htmlescape", - "hyperloglogplus", "itertools 0.14.0", "levenshtein_automata", "log", - "lru 0.12.5", - "lz4_flex", + "lru 0.16.3", + "lz4_flex 0.13.0", "measure_time", "memmap2", "once_cell", "oneshot", "rayon", "regex", - "rust-stemmers", "rustc-hash 2.1.1", "serde", "serde_json", @@ -9838,23 +10068,24 @@ dependencies = [ "tempfile", "thiserror 2.0.16", "time", + "typetag", "uuid", "winapi 0.3.9", - "zstd 0.13.3", + "zstd", ] [[package]] name = "tantivy-bitpacker" -version = "0.6.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=80f5f1e#80f5f1ecd4d5d30069b44e5acd3801f610d36056" +version = "0.9.0" +source = "git+https://github.com/indextables/tantivy/?rev=ffe408f49bb1f605d8eb28de9d18edd29c5245d5#ffe408f49bb1f605d8eb28de9d18edd29c5245d5" dependencies = [ "bitpacking", ] [[package]] name = "tantivy-columnar" -version = "0.3.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=80f5f1e#80f5f1ecd4d5d30069b44e5acd3801f610d36056" +version = "0.6.0" +source = "git+https://github.com/indextables/tantivy/?rev=ffe408f49bb1f605d8eb28de9d18edd29c5245d5#ffe408f49bb1f605d8eb28de9d18edd29c5245d5" dependencies = [ "downcast-rs", "fastdivide", @@ -9868,8 +10099,8 @@ dependencies = [ [[package]] name = "tantivy-common" -version = "0.7.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=80f5f1e#80f5f1ecd4d5d30069b44e5acd3801f610d36056" +version = "0.10.0" +source = "git+https://github.com/indextables/tantivy/?rev=ffe408f49bb1f605d8eb28de9d18edd29c5245d5#ffe408f49bb1f605d8eb28de9d18edd29c5245d5" dependencies = [ "async-trait", "byteorder", @@ -9891,60 +10122,46 @@ dependencies = [ [[package]] name = "tantivy-query-grammar" -version = "0.22.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=80f5f1e#80f5f1ecd4d5d30069b44e5acd3801f610d36056" +version = "0.25.0" +source = "git+https://github.com/indextables/tantivy/?rev=ffe408f49bb1f605d8eb28de9d18edd29c5245d5#ffe408f49bb1f605d8eb28de9d18edd29c5245d5" dependencies = [ - "nom", + "fnv", + "nom 7.1.3", + "ordered-float 5.3.0", + "serde", + "serde_json", ] [[package]] name = "tantivy-sstable" -version = "0.3.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=80f5f1e#80f5f1ecd4d5d30069b44e5acd3801f610d36056" +version = "0.6.0" +source = "git+https://github.com/indextables/tantivy/?rev=ffe408f49bb1f605d8eb28de9d18edd29c5245d5#ffe408f49bb1f605d8eb28de9d18edd29c5245d5" dependencies = [ "futures-util", "itertools 0.14.0", "tantivy-bitpacker", "tantivy-common", "tantivy-fst", - "zstd 0.13.3", + "zstd", ] [[package]] name = "tantivy-stacker" -version = "0.3.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=80f5f1e#80f5f1ecd4d5d30069b44e5acd3801f610d36056" +version = "0.6.0" +source = "git+https://github.com/indextables/tantivy/?rev=ffe408f49bb1f605d8eb28de9d18edd29c5245d5#ffe408f49bb1f605d8eb28de9d18edd29c5245d5" dependencies = [ "murmurhash32", - "rand_distr", "tantivy-common", ] [[package]] name = "tantivy-tokenizer-api" -version = "0.3.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=80f5f1e#80f5f1ecd4d5d30069b44e5acd3801f610d36056" +version = "0.6.0" +source = "git+https://github.com/indextables/tantivy/?rev=ffe408f49bb1f605d8eb28de9d18edd29c5245d5#ffe408f49bb1f605d8eb28de9d18edd29c5245d5" dependencies = [ "serde", ] -[[package]] -name = "tap" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" - -[[package]] -name = "tar" -version = "0.4.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" -dependencies = [ - "filetime", - "libc", - "xattr", -] - [[package]] name = "tempfile" version = "3.21.0" @@ -9952,7 +10169,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15b61f8f20e3a6f7e0649d825294eaf317edce30f82cf6026e7e4cb9222a7d1e" dependencies = [ "fastrand 2.3.0", - "getrandom 0.3.3", + "getrandom 0.3.4", "once_cell", "rustix 1.0.8", "windows-sys 0.60.2", @@ -9982,6 +10199,16 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" +[[package]] +name = "testing_table" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f8daae29995a24f65619e19d8d31dea5b389f3d853d8bf297bbf607cd0014cc" +dependencies = [ + "ansitok", + "unicode-width 0.2.1", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -10037,11 +10264,22 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "ordered-float 2.10.1", +] + [[package]] name = "tikv-jemalloc-ctl" -version = "0.5.4" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "619bfed27d807b54f7f776b9430d4f8060e66ee138a28632ca898584d462c31c" +checksum = "661f1f6a57b3a36dc9174a2c10f19513b4866816e13425d3e418b11cc37bc24c" dependencies = [ "libc", "paste", @@ -10050,9 +10288,9 @@ dependencies = [ [[package]] name = "tikv-jemalloc-sys" -version = "0.5.4+5.3.0-patched" +version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9402443cb8fd499b6f327e40565234ff34dbda27460c5b47db0db77443dd85d1" +checksum = "cd8aa5b2ab86a2cefa406d889139c162cbb230092f7d1d7cbc1716405d852a3b" dependencies = [ "cc", "libc", @@ -10060,9 +10298,9 @@ dependencies = [ [[package]] name = "tikv-jemallocator" -version = "0.5.4" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "965fe0c26be5c56c94e38ba547249074803efd52adfb66de62107d95aab3eaca" +checksum = "0359b4327f954e0567e69fb191cf1436617748813819c94b8cd4a431422d053a" dependencies = [ "libc", "tikv-jemalloc-sys", @@ -10070,26 +10308,27 @@ dependencies = [ [[package]] name = "time" -version = "0.3.43" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83bde6f1ec10e72d583d91623c939f623002284ef622b87de38cfd546cbf2031" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", + "itoa", "js-sys", "libc", "num-conv", "num_threads", "powerfmt", - "serde", + "serde_core", "time-core", "time-macros", ] [[package]] name = "time-core" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] name = "time-fmt" @@ -10103,14 +10342,23 @@ dependencies = [ [[package]] name = "time-macros" -version = "0.2.24" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" dependencies = [ "num-conv", "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinystr" version = "0.8.1" @@ -10148,40 +10396,27 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.47.1" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" dependencies = [ - "backtrace", "bytes", - "io-uring", "libc", "mio", "parking_lot 0.12.4", "pin-project-lite", "signal-hook-registry", - "slab", "socket2 0.6.0", "tokio-macros", "tracing", - "windows-sys 0.59.0", -] - -[[package]] -name = "tokio-io-timeout" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bd86198d9ee903fedd2f9a2e72014287c0d9167e4ae43b5853007205dda1b76" -dependencies = [ - "pin-project-lite", - "tokio", + "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" dependencies = [ "proc-macro2", "quote", @@ -10190,9 +10425,9 @@ dependencies = [ [[package]] name = "tokio-metrics" -version = "0.3.1" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eace09241d62c98b7eeb1107d4c5c64ca3bd7da92e8c218c153ab3a78f9be112" +checksum = "0e0410015c6db7b67b9c9ab2a3af4d74a942d637ff248d0d055073750deac6f9" dependencies = [ "futures-util", "pin-project-lite", @@ -10211,13 +10446,12 @@ dependencies = [ ] [[package]] -name = "tokio-retry" -version = "0.3.0" +name = "tokio-retry2" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" +checksum = "3a0a122635e32bd827df297f311ca5e0292636bbd82f67ff84a4bedeab06dbeb" dependencies = [ "pin-project", - "rand 0.8.5", "tokio", ] @@ -10253,18 +10487,6 @@ dependencies = [ "tokio-util", ] -[[package]] -name = "tokio-tungstenite" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c83b561d025642014097b66e6c1bb422783339e0909e4429cde4749d1990bc38" -dependencies = [ - "futures-util", - "log", - "tokio", - "tungstenite", -] - [[package]] name = "tokio-util" version = "0.7.16" @@ -10275,23 +10497,23 @@ dependencies = [ "futures-core", "futures-io", "futures-sink", - "futures-util", - "hashbrown 0.15.5", "pin-project-lite", - "slab", "tokio", ] [[package]] name = "toml" -version = "0.7.8" +version = "0.9.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" dependencies = [ - "serde", + "indexmap 2.13.0", + "serde_core", "serde_spanned", - "toml_datetime", - "toml_edit 0.19.15", + "toml_datetime 0.7.5+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 0.7.13", ] [[package]] @@ -10299,8 +10521,14 @@ name = "toml_datetime" version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" dependencies = [ - "serde", + "serde_core", ] [[package]] @@ -10309,10 +10537,8 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.11.0", - "serde", - "serde_spanned", - "toml_datetime", + "indexmap 2.13.0", + "toml_datetime 0.6.11", "winnow 0.5.40", ] @@ -10322,97 +10548,79 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.11.0", - "toml_datetime", + "indexmap 2.13.0", + "toml_datetime 0.6.11", "winnow 0.7.13", ] [[package]] -name = "tonic" -version = "0.9.2" +name = "toml_parser" +version = "1.1.0+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" +checksum = "2334f11ee363607eb04df9b8fc8a13ca1715a72ba8662a26ac285c98aabb4011" dependencies = [ - "async-stream", - "async-trait", - "axum 0.6.20", - "base64 0.21.7", - "bytes", - "flate2", - "futures-core", - "futures-util", - "h2 0.3.27", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.32", - "hyper-timeout 0.4.1", - "percent-encoding", - "pin-project", - "prost 0.11.9", - "rustls-pemfile", - "tokio", - "tokio-rustls 0.24.1", - "tokio-stream", - "tower 0.4.13", - "tower-layer", - "tower-service", - "tracing", - "webpki-roots 0.23.1", + "winnow 1.0.1", ] +[[package]] +name = "toml_writer" +version = "1.1.0+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d282ade6016312faf3e41e57ebbba0c073e4056dab1232ab1cb624199648f8ed" + [[package]] name = "tonic" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" dependencies = [ - "async-stream", "async-trait", - "axum 0.7.9", "base64 0.22.1", "bytes", - "h2 0.4.12", - "http 1.3.1", + "flate2", + "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper 1.7.0", - "hyper-timeout 0.5.2", + "hyper 1.8.1", + "hyper-timeout", "hyper-util", "percent-encoding", "pin-project", "prost 0.13.5", - "socket2 0.5.10", + "rustls-pemfile", "tokio", + "tokio-rustls 0.26.2", "tokio-stream", "tower 0.4.13", "tower-layer", "tower-service", "tracing", + "webpki-roots 0.26.11", ] [[package]] name = "tonic" -version = "0.13.1" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" +checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" dependencies = [ "async-trait", - "axum 0.8.4", + "axum", "base64 0.22.1", "bytes", "flate2", "h2 0.4.12", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper 1.7.0", - "hyper-timeout 0.5.2", + "hyper 1.8.1", + "hyper-timeout", "hyper-util", "percent-encoding", "pin-project", - "prost 0.13.5", - "rustls-native-certs 0.8.1", - "socket2 0.5.10", + "rustls-native-certs", + "socket2 0.6.0", + "sync_wrapper", "tokio", "tokio-rustls 0.26.2", "tokio-stream", @@ -10420,46 +10628,73 @@ dependencies = [ "tower-layer", "tower-service", "tracing", - "zstd 0.13.3", + "zstd", ] [[package]] name = "tonic-build" -version = "0.13.1" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac6f67be712d12f0b41328db3137e0d0757645d8904b4cb7d51cd9c2279e847" +checksum = "1882ac3bf5ef12877d7ed57aad87e75154c11931c2ba7e6cde5e22d63522c734" dependencies = [ "prettyplease", "proc-macro2", - "prost-build", - "prost-types 0.13.5", "quote", "syn 2.0.106", ] [[package]] name = "tonic-health" -version = "0.13.1" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb87334d340313fefa513b6e60794d44a86d5f039b523229c99c323e4e19ca4b" +checksum = "f4ff0636fef47afb3ec02818f5bceb4377b8abb9d6a386aeade18bd6212f8eb7" dependencies = [ - "prost 0.13.5", + "prost 0.14.3", "tokio", "tokio-stream", - "tonic 0.13.1", + "tonic 0.14.5", + "tonic-prost", +] + +[[package]] +name = "tonic-prost" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" +dependencies = [ + "bytes", + "prost 0.14.3", + "tonic 0.14.5", +] + +[[package]] +name = "tonic-prost-build" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3144df636917574672e93d0f56d7edec49f90305749c668df5101751bb8f95a" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build 0.14.3", + "prost-types 0.14.3", + "quote", + "syn 2.0.106", + "tempfile", + "tonic-build", ] [[package]] name = "tonic-reflection" -version = "0.13.1" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9687bd5bfeafebdded2356950f278bba8226f0b32109537c4253406e09aafe1" +checksum = "aaf0685a51e6d02b502ba0764002e766b7f3042aed13d9234925b6ffbfa3fca7" dependencies = [ - "prost 0.13.5", - "prost-types 0.13.5", + "prost 0.14.3", + "prost-types 0.14.3", "tokio", "tokio-stream", - "tonic 0.13.1", + "tonic 0.14.5", + "tonic-prost", ] [[package]] @@ -10490,10 +10725,10 @@ checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", - "indexmap 2.11.0", + "indexmap 2.13.0", "pin-project-lite", "slab", - "sync_wrapper 1.0.2", + "sync_wrapper", "tokio", "tokio-util", "tower-layer", @@ -10503,38 +10738,21 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.4.4" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ "async-compression", "bitflags 2.9.4", "bytes", "futures-core", "futures-util", - "http 0.2.12", - "http-body 0.4.6", - "http-range-header", - "pin-project-lite", - "tokio", - "tokio-util", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-http" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" -dependencies = [ - "bitflags 2.9.4", - "bytes", - "futures-util", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "iri-string", "pin-project-lite", + "tokio", + "tokio-util", "tower 0.5.2", "tower-layer", "tower-service", @@ -10554,9 +10772,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.41" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "log", "pin-project-lite", @@ -10566,9 +10784,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", @@ -10577,9 +10795,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.34" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", "valuable", @@ -10598,14 +10816,12 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.28.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97a971f6058498b5c0f1affa23e7ea202057a7301dbff68e968b2d578bcbd053" +checksum = "1ac28f2d093c6c477eaa76b23525478f38de514fa9aeb1285738d4b97a9552fc" dependencies = [ "js-sys", - "once_cell", "opentelemetry", - "opentelemetry_sdk", "smallvec", "tracing", "tracing-core", @@ -10626,9 +10842,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.20" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ "matchers", "nu-ansi-term", @@ -10646,6 +10862,12 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "triomphe" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd69c5aa8f924c7519d6372789a74eac5b94fb0f8fcf0d4a97eb0bfc3e785f39" + [[package]] name = "try-lock" version = "0.2.5" @@ -10662,23 +10884,16 @@ dependencies = [ ] [[package]] -name = "tungstenite" -version = "0.21.0" +name = "twox-hash" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1" -dependencies = [ - "byteorder", - "bytes", - "data-encoding", - "http 1.3.1", - "httparse", - "log", - "rand 0.8.5", - "sha1", - "thiserror 1.0.69", - "url", - "utf-8", -] +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" [[package]] name = "typenum" @@ -10686,6 +10901,30 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" +[[package]] +name = "typetag" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be2212c8a9b9bcfca32024de14998494cf9a5dfa59ea1b829de98bac374b86bf" +dependencies = [ + "erased-serde", + "inventory", + "once_cell", + "serde", + "typetag-impl", +] + +[[package]] +name = "typetag-impl" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27a7a9b72ba121f6f1f6c3632b85604cac41aedb5ddc70accbebb6cac83de846" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + [[package]] name = "tz-rs" version = "0.6.14" @@ -10696,17 +10935,14 @@ dependencies = [ ] [[package]] -name = "uaparser" -version = "0.6.4" +name = "ua-parser" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4c9e1c3f893758f154004195fc2d2c52fbda462df725220ceaef830ac29affa" +checksum = "f01b7ba339f8874b643216c6c957d792047143abe848568131e5d91d2ae2ada1" dependencies = [ - "derive_more", - "lazy_static", "regex", + "regex-filtered", "serde", - "serde_derive", - "serde_yaml", ] [[package]] @@ -10744,6 +10980,12 @@ version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" +[[package]] +name = "unicode-general-category" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f" + [[package]] name = "unicode-ident" version = "1.0.18" @@ -10790,10 +11032,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] -name = "unicode_categories" -version = "0.1.1" +name = "unit-prefix" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" [[package]] name = "universal-hash" @@ -10813,15 +11055,38 @@ checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] name = "untrusted" -version = "0.7.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] -name = "untrusted" -version = "0.9.0" +name = "ureq" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" +dependencies = [ + "base64 0.22.1", + "flate2", + "log", + "percent-encoding", + "rustls 0.23.31", + "rustls-pki-types", + "ureq-proto", + "utf8-zero", + "webpki-roots 1.0.2", +] + +[[package]] +name = "ureq-proto" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" +dependencies = [ + "base64 0.22.1", + "http 1.4.0", + "httparse", + "log", +] [[package]] name = "url" @@ -10851,12 +11116,6 @@ dependencies = [ "winapi 0.2.8", ] -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - [[package]] name = "utf8-ranges" version = "1.0.5" @@ -10869,6 +11128,12 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3" +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -10887,7 +11152,7 @@ version = "4.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5afb1a60e207dca502682537fefcfd9921e71d0b83e9576060f09abc6efab23" dependencies = [ - "indexmap 2.11.0", + "indexmap 2.13.0", "serde", "serde_json", "utoipa-gen", @@ -10908,17 +11173,27 @@ dependencies = [ [[package]] name = "uuid" -version = "1.18.1" +version = "1.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.4.2", "js-sys", - "rand 0.9.2", - "serde", + "rand 0.10.0", + "serde_core", "wasm-bindgen", ] +[[package]] +name = "uuid-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b082222b4f6619906941c17eb2297fff4c2fb96cb60164170522942a200bd8" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "valuable" version = "0.1.1" @@ -10939,15 +11214,15 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "vrl" -version = "0.22.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5c1c142b261b82da01422a842e7b5c49c3907d655423d80529296fb04f2343c" +checksum = "b12fd383709e1187354f818cafa07a28d788eb30351a627891579deedeffce40" dependencies = [ "aes", "aes-siv", "base16", "base62", - "base64 0.22.1", + "base64-simd", "bytes", "cbc", "cfb-mode", @@ -10957,41 +11232,43 @@ dependencies = [ "chrono", "chrono-tz", "ciborium", - "cidr-utils", + "cidr", "codespan-reporting", "community-id", - "convert_case 0.7.1", + "convert_case", "crc", "crypto_secretbox", "csv", "ctr", - "data-encoding", "digest", "dns-lookup", "domain", "dyn-clone", "encoding_rs", - "fancy-regex", + "fancy-regex 0.15.0", "flate2", "grok", "hex", "hmac", - "hostname 0.4.1", + "hostname", "iana-time-zone", "idna", - "indexmap 2.11.0", + "indexmap 2.13.0", "indoc", "influxdb-line-protocol", + "ipcrypt-rs", "itertools 0.14.0", + "jsonschema", "lalrpop", "lalrpop-util", + "lz4_flex 0.11.5", "md-5", - "nom", + "nom 8.0.0", + "nom-language", "ofb", "onig", "ordered-float 4.6.0", "parse-size", - "paste", "peeking_take_while", "percent-encoding", "pest", @@ -11004,11 +11281,15 @@ dependencies = [ "quoted_printable", "rand 0.8.5", "regex", + "reqwest", + "reqwest-middleware", + "reqwest-retry 0.7.0", "roxmltree 0.20.0", "rust_decimal", "seahash", "serde", "serde_json", + "serde_yaml", "sha-1", "sha2", "sha3", @@ -11021,13 +11302,14 @@ dependencies = [ "thiserror 2.0.16", "tokio", "tracing", - "uaparser", + "ua-parser", "unicode-segmentation", "url", "utf8-width", "uuid", "woothee", - "zstd 0.13.3", + "xxhash-rust", + "zstd", ] [[package]] @@ -11036,36 +11318,16 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" -[[package]] -name = "vte" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cbce692ab4ca2f1f3047fcf732430249c0e971bfdd2b234cf2c47ad93af5983" -dependencies = [ - "arrayvec 0.5.2", - "utf8parse", - "vte_generate_state_changes", -] - [[package]] name = "vte" version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" dependencies = [ + "arrayvec", "memchr", ] -[[package]] -name = "vte_generate_state_changes" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e369bee1b05d510a7b4ed645f5faa90619e05437111783ea5848f28d97d3c2e" -dependencies = [ - "proc-macro2", - "quote", -] - [[package]] name = "wait-timeout" version = "0.2.1" @@ -11102,21 +11364,22 @@ dependencies = [ [[package]] name = "warp" -version = "0.3.7" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4378d202ff965b011c64817db11d5829506d3404edeadb61f190d111da3f231c" +checksum = "51d06d9202adc1f15d709c4f4a2069be5428aa912cc025d6f268ac441ab066b0" dependencies = [ - "async-compression", "bytes", "futures-channel", "futures-util", "headers", - "http 0.2.12", - "hyper 0.14.32", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.8.1", + "hyper-util", "log", "mime", "mime_guess", - "multer", "percent-encoding", "pin-project", "scoped-tls", @@ -11124,7 +11387,6 @@ dependencies = [ "serde_json", "serde_urlencoded", "tokio", - "tokio-tungstenite", "tokio-util", "tower-service", "tracing", @@ -11143,10 +11405,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] -name = "wasi" -version = "0.14.3+wasi-0.2.4" +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a51ae83037bdd272a9e28ce236db8c07016dd0d50c27038b3f407533c030c95" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" dependencies = [ "wit-bindgen", ] @@ -11238,6 +11509,28 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap 2.13.0", + "wasm-encoder", + "wasmparser", +] + [[package]] name = "wasm-streams" version = "0.4.2" @@ -11266,6 +11559,32 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.9.4", + "hashbrown 0.15.5", + "indexmap 2.13.0", + "semver", +] + +[[package]] +name = "wasmtimer" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c598d6b99ea013e35844697fc4670d08339d5cda15588f193c6beedd12f644b" +dependencies = [ + "futures", + "js-sys", + "parking_lot 0.12.4", + "pin-utils", + "slab", + "wasm-bindgen", +] + [[package]] name = "web-sys" version = "0.3.78" @@ -11288,19 +11607,13 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.23.1" +version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "rustls-webpki 0.100.3", + "webpki-roots 1.0.2", ] -[[package]] -name = "webpki-roots" -version = "0.25.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" - [[package]] name = "webpki-roots" version = "1.0.2" @@ -11322,12 +11635,6 @@ dependencies = [ "rustix 0.38.44", ] -[[package]] -name = "whichlang" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9aa3ad29c3d08283ac6b769e3ec15ad1ddb88af7d2e9bc402c574973b937e7" - [[package]] name = "whoami" version = "1.6.1" @@ -11381,16 +11688,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows" -version = "0.57.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" -dependencies = [ - "windows-core 0.57.0", - "windows-targets 0.52.6", -] - [[package]] name = "windows" version = "0.61.3" @@ -11398,9 +11695,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" dependencies = [ "windows-collections", - "windows-core 0.61.2", + "windows-core", "windows-future", - "windows-link", + "windows-link 0.1.3", "windows-numerics", ] @@ -11410,19 +11707,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" dependencies = [ - "windows-core 0.61.2", -] - -[[package]] -name = "windows-core" -version = "0.57.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" -dependencies = [ - "windows-implement 0.57.0", - "windows-interface 0.57.0", - "windows-result 0.1.2", - "windows-targets 0.52.6", + "windows-core", ] [[package]] @@ -11431,10 +11716,10 @@ version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ - "windows-implement 0.60.0", - "windows-interface 0.59.1", - "windows-link", - "windows-result 0.3.4", + "windows-implement", + "windows-interface", + "windows-link 0.1.3", + "windows-result", "windows-strings", ] @@ -11444,22 +11729,11 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" dependencies = [ - "windows-core 0.61.2", - "windows-link", + "windows-core", + "windows-link 0.1.3", "windows-threading", ] -[[package]] -name = "windows-implement" -version = "0.57.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - [[package]] name = "windows-implement" version = "0.60.0" @@ -11471,17 +11745,6 @@ dependencies = [ "syn 2.0.106", ] -[[package]] -name = "windows-interface" -version = "0.57.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - [[package]] name = "windows-interface" version = "0.59.1" @@ -11500,33 +11763,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" [[package]] -name = "windows-numerics" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" -dependencies = [ - "windows-core 0.61.2", - "windows-link", -] - -[[package]] -name = "windows-registry" -version = "0.5.3" +name = "windows-link" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a9ed28765efc97bbc954883f4e6796c33a06546ebafacbabee9696967499e" -dependencies = [ - "windows-link", - "windows-result 0.3.4", - "windows-strings", -] +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] -name = "windows-result" -version = "0.1.2" +name = "windows-numerics" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" dependencies = [ - "windows-targets 0.52.6", + "windows-core", + "windows-link 0.1.3", ] [[package]] @@ -11535,7 +11784,7 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" dependencies = [ - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -11544,7 +11793,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" dependencies = [ - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -11583,6 +11832,15 @@ dependencies = [ "windows-targets 0.53.3", ] +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -11620,7 +11878,7 @@ version = "0.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" dependencies = [ - "windows-link", + "windows-link 0.1.3", "windows_aarch64_gnullvm 0.53.0", "windows_aarch64_msvc 0.53.0", "windows_i686_gnu 0.53.0", @@ -11637,7 +11895,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" dependencies = [ - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -11789,23 +12047,28 @@ dependencies = [ [[package]] name = "winnow" -version = "0.7.13" +version = "0.6.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" +checksum = "1e90edd2ac1aa278a5c4599b1d89cf03074b610800f866d4026dc199d7929a28" dependencies = [ "memchr", ] [[package]] -name = "winreg" -version = "0.50.0" +name = "winnow" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" dependencies = [ - "cfg-if", - "windows-sys 0.48.0", + "memchr", ] +[[package]] +name = "winnow" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5" + [[package]] name = "wiremock" version = "0.6.5" @@ -11816,9 +12079,9 @@ dependencies = [ "base64 0.22.1", "deadpool", "futures", - "http 1.3.1", + "http 1.4.0", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.1", "hyper-util", "log", "once_cell", @@ -11831,45 +12094,108 @@ dependencies = [ [[package]] name = "wit-bindgen" -version = "0.45.1" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c573471f125075647d03df72e026074b7203790d41351cd6edc96f46bcccd36" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] [[package]] -name = "woothee" -version = "0.13.0" +name = "wit-bindgen-core" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "896174c6a4779d4d7d4523dd27aef7d46609eda2497e370f6c998325c6bf6971" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" dependencies = [ - "lazy_static", - "regex", + "anyhow", + "heck 0.5.0", + "wit-parser", ] [[package]] -name = "writeable" -version = "0.6.1" +name = "wit-bindgen-rust" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck 0.5.0", + "indexmap 2.13.0", + "prettyplease", + "syn 2.0.106", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] [[package]] -name = "wyz" -version = "0.5.1" +name = "wit-bindgen-rust-macro" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" dependencies = [ - "tap", + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.106", + "wit-bindgen-core", + "wit-bindgen-rust", ] [[package]] -name = "xattr" -version = "1.5.1" +name = "wit-component" +version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af3a19837351dc82ba89f8a125e22a3c475f05aba604acc023d62b2739ae2909" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ - "libc", - "rustix 1.0.8", + "anyhow", + "bitflags 2.9.4", + "indexmap 2.13.0", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.13.0", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "woothee" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "896174c6a4779d4d7d4523dd27aef7d46609eda2497e370f6c998325c6bf6971" +dependencies = [ + "lazy_static", + "regex", ] +[[package]] +name = "writeable" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" + [[package]] name = "xmlparser" version = "0.13.6" @@ -11877,10 +12203,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] -name = "yada" -version = "0.5.1" +name = "xxhash-rust" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aed111bd9e48a802518765906cbdadf0b45afb72b9c81ab049a3b86252adffdd" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" [[package]] name = "yansi" @@ -11996,33 +12322,16 @@ dependencies = [ ] [[package]] -name = "zip" -version = "0.6.6" +name = "zlib-rs" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" -dependencies = [ - "aes", - "byteorder", - "bzip2", - "constant_time_eq", - "crc32fast", - "crossbeam-utils", - "flate2", - "hmac", - "pbkdf2 0.11.0", - "sha1", - "time", - "zstd 0.11.2+zstd.1.5.2", -] +checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" [[package]] -name = "zstd" -version = "0.11.2+zstd.1.5.2" +name = "zmij" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" -dependencies = [ - "zstd-safe 5.0.2+zstd.1.5.2", -] +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" [[package]] name = "zstd" @@ -12030,17 +12339,7 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ - "zstd-safe 7.2.4", -] - -[[package]] -name = "zstd-safe" -version = "5.0.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" -dependencies = [ - "libc", - "zstd-sys", + "zstd-safe", ] [[package]] diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml index 86d17907be9..6662193192a 100644 --- a/quickwit/Cargo.toml +++ b/quickwit/Cargo.toml @@ -19,7 +19,8 @@ members = [ "quickwit-integration-tests", "quickwit-jaeger", "quickwit-janitor", - "quickwit-lambda", + "quickwit-lambda-client", + "quickwit-lambda-server", "quickwit-macros", "quickwit-metastore", @@ -27,6 +28,7 @@ members = [ # We can reenable it when we need it. # "quickwit-metastore-utils", "quickwit-opentelemetry", + "quickwit-parquet-engine", "quickwit-proto", "quickwit-query", "quickwit-rest-client", @@ -36,7 +38,7 @@ members = [ "quickwit-telemetry", ] -# The following list excludes `quickwit-metastore-utils` and `quickwit-lambda` +# The following list excludes `quickwit-metastore-utils` # from the default member to ease build/deps. default-members = [ "quickwit-actors", @@ -57,9 +59,12 @@ default-members = [ "quickwit-integration-tests", "quickwit-jaeger", "quickwit-janitor", + "quickwit-lambda-client", + "quickwit-lambda-server", "quickwit-macros", "quickwit-metastore", "quickwit-opentelemetry", + "quickwit-parquet-engine", "quickwit-proto", "quickwit-query", "quickwit-rest-client", @@ -80,114 +85,120 @@ license = "Apache-2.0" [workspace.dependencies] anyhow = "1" -arc-swap = "1.7" +arc-swap = "1.8" +arrow = { version = "57", default-features = false, features = ["ipc"] } assert-json-diff = "2" async-compression = { version = "0.4", features = ["tokio", "gzip"] } async-speed-limit = "0.4" async-trait = "0.1" backtrace = "0.3" base64 = "0.22" -binggan = { version = "0.14" } +binggan = { version = "0.15" } +bitpacking = "0.9.3" bytes = { version = "1", features = ["serde"] } -bytesize = { version = "1.3", features = ["serde"] } -bytestring = "1.4" -chitchat = { git = "https://github.com/quickwit-oss/chitchat.git", rev = "bd54c81" } +bytesize = { version = "2.3.1", features = ["serde"] } +bytestring = "1.5" +chitchat = "0.10.0" chrono = { version = "0.4", default-features = false, features = [ "clock", "std", ] } clap = { version = "4.5", features = ["env", "string"] } coarsetime = "0.1" -colored = "2.2" -console-subscriber = "0.1" -criterion = { version = "0.5", features = ["async_tokio"] } -cron = "0.12" -dialoguer = "0.10" +colored = "3.0" +console-subscriber = "0.5" +criterion = { version = "0.8", features = ["async_tokio"] } +cron = "0.15" +dialoguer = { version = "0.12", default-features = false } dotenvy = "0.15" dyn-clone = "1.0" -enum-iterator = "1.5" -env_logger = "0.10" +enum-iterator = "2.3" +env_logger = { version = "0.11", default-features = false, features = ["auto-color"] } fail = "0.5" flate2 = "1.1" -flume = "0.11" +flume = "0.12" fnv = "1" futures = "0.3" futures-util = { version = "0.3", default-features = false } glob = "0.3" -google-cloud-auth = "0.12" -google-cloud-gax = "0.15" -google-cloud-googleapis = { version = "0.10", features = ["pubsub"] } -google-cloud-pubsub = "0.18" +# We can't directly update google-cloud-auth to 1.3 and google-cloud-gax to 1.4, because the latest version +# of google-cloud-pubsub is "0.30" which explicitly depends on: google-cloud-auth ^0.17 and google-cloud-gax ^0.19. +google-cloud-auth = "0.17.2" +google-cloud-gax = "0.19.2" +google-cloud-googleapis = { version = "0.16", features = ["pubsub"] } +google-cloud-pubsub = "0.30" +governor = "0.10.4" heck = "0.5" hex = "0.4" home = "0.5" -hostname = "0.3" -http = "1.3" -http-body-util = "0.1.3" -http-serde = "1.1" -humantime = "2.2" -hyper = { version = "1.6", features = ["client", "http1", "http2", "server"] } +hostname = "0.4" +http = "1.4" +http-body = "1.0" +http-body-util = "0.1" +http-serde = "2.1" +humantime = "2.3" +hyper = { version = "1.8", features = ["client", "http1", "http2", "server"] } hyper-rustls = "0.27" -hyper-util = { version = "0.1", features = ["full"] } -indexmap = { version = "2.9", features = ["serde"] } -indicatif = "0.17" +hyper-util = { version = "0.1", default-features = false, features = [ + "client-legacy", + "server-auto", + "server-graceful", + "service", + "tokio", +] } +indexmap = { version = "2.12", features = ["serde"] } +indicatif = "0.18" itertools = "0.14" +lambda_runtime = "0.13" json_comments = "0.2" -legacy-http = { version = "0.2", package = "http" } libz-sys = "1.1" -lindera-core = "0.27" -lindera-dictionary = "0.27" -lindera-tokenizer = { version = "0.27", features = [ - "cc-cedict-compress", - "cc-cedict", - "ipadic-compress", - "ipadic", - "ko-dic-compress", - "ko-dic", -] } -lru = "0.13" +lru = "0.16" matches = "0.1" -md5 = "0.7" +md5 = "0.8" mime_guess = "2.0" -mockall = "0.11" +mini-moka = "0.10.3" +mockall = "0.14" mrecordlog = { git = "https://github.com/quickwit-oss/mrecordlog", rev = "306c0a7" } new_string_template = "1.5" -nom = "7.1" -numfmt = "1.1" +nom = "8.0" +numfmt = "1.2" once_cell = "1" oneshot = "0.1" openssl = { version = "0.10", default-features = false } openssl-probe = "0.1" -opentelemetry = "0.27" -opentelemetry_sdk = { version = "0.27", features = ["rt-tokio"] } -opentelemetry-otlp = "0.27" +opentelemetry = "0.31" +opentelemetry-appender-tracing = "0.31" +opentelemetry_sdk = { version = "0.31", features = ["rt-tokio"] } +opentelemetry-otlp = { version = "0.31", features = ["grpc-tonic"] } ouroboros = "0.18" +parquet = { version = "57", default-features = false, features = ["arrow", "zstd", "snap", "variant_experimental"] } percent-encoding = "2.3" pin-project = "1.1" -pnet = { version = "0.33", features = ["std"] } +pnet = { version = "0.35", features = ["std"] } postcard = { version = "1.1", features = [ "use-std", ], default-features = false } -pprof = { version = "0.13", features = ["flamegraph"] } +pprof = { version = "0.15", features = ["flamegraph"] } predicates = "3" prettyplease = "0.2" proc-macro2 = "1.0" -prometheus = { version = "0.13", features = ["process"] } +prometheus = { version = "0.14", default-features = false, features = ["process"] } proptest = "1" -prost = { version = "0.13", default-features = false, features = [ - "prost-derive", +prost = { version = "0.14", default-features = false, features = [ + "derive", ] } -prost-build = "0.13" -prost-types = "0.13" -pulsar = { version = "6.3", default-features = false, features = [ +prost-build = "0.14" +prost-types = "0.14" +pulsar = { version = "6.6", default-features = false, features = [ "auth-oauth2", "compression", "tokio-runtime", ] } +quick_cache = "0.6.18" quote = "1.0" -rand = "0.8" -rand_distr = "0.4" -rayon = "1.10" +rand = "0.9" +rand_distr = "0.5" +rayon = "1.11" rdkafka = { version = "0.38", default-features = false, features = [ "cmake-build", "libz", @@ -195,34 +206,35 @@ rdkafka = { version = "0.38", default-features = false, features = [ "tokio", "zstd", ] } -regex = "1.11" +regex = "1.12" regex-syntax = "0.8" reqwest = { version = "0.12", default-features = false, features = [ "json", "rustls-tls", ] } reqwest-middleware = "0.4" -reqwest-retry = "0.7" -rust-embed = "6.8" +reqwest-retry = "0.8" +rust-embed = "8.9" rustc-hash = "2.1" -rustls = "0.21" -rustls-pemfile = "1.0" -sea-query = { version = "0.30" } -sea-query-binder = { version = "0.5", features = [ +rustls = "0.23" +rustls-pemfile = "2.2" +sea-query = { version = "0.32" } +sea-query-binder = { version = "0.7", features = [ "runtime-tokio-rustls", "sqlx-postgres", ] } # ^1.0.184 due to serde-rs/serde#2538 -serde = { version = "1.0.219", features = ["derive", "rc"] } +serde = { version = "1.0.228", features = ["derive", "rc"] } serde_json = "1.0" -serde_json_borrow = "0.5" -serde_qs = { version = "0.12", features = ["warp"] } -serde_with = "3.12" +serde_json_borrow = "0.9" +serde_qs = { version = "0.15" } +serde_with = "3.16" serde_yaml = "0.9" serial_test = { version = "3.2", features = ["file_locks"] } +sha2 = "0.10" siphasher = "1.0" smallvec = "1" -sqlx = { version = "0.7", features = [ +sqlx = { version = "0.8", features = [ "migrate", "postgres", "runtime-tokio-rustls", @@ -230,29 +242,34 @@ sqlx = { version = "0.7", features = [ ] } syn = { version = "2.0", features = ["extra-traits", "full", "parsing"] } sync_wrapper = "1" -sysinfo = "0.33" -tabled = { version = "0.14", features = ["color"] } +sysinfo = { version = "0.37", default-features = false, features = ["disk"] } +tabled = { version = "0.20", features = ["ansi"] } tempfile = "3" thiserror = "2" thousands = "0.2" -tikv-jemalloc-ctl = "0.5" -tikv-jemallocator = "0.5" +tikv-jemalloc-ctl = { version = "0.6", features = ["stats"] } +tikv-jemallocator = "0.6" time = { version = "0.3", features = ["std", "formatting", "macros"] } -tokio = { version = "1.45", features = ["full"] } -tokio-metrics = { version = "0.3", features = ["rt"] } -tokio-rustls = { version = "0.24", default-features = false } +tokio = { version = "1.48", features = ["full"] } +tokio-metrics = { version = "0.4", features = ["rt"] } +tokio-rustls = { version = "0.26", default-features = false } tokio-stream = { version = "0.1", features = ["sync"] } -tokio-util = { version = "0.7", features = ["full"] } -toml = "0.7" -tonic = { version = "0.13", features = [ +tokio-util = { version = "0.7", default-features = false, features = [ + "compat", + "io-util", +] } +toml = "0.9" +tonic = { version = "0.14", features = [ "_tls-any", "gzip", "tls-native-roots", "zstd", ] } -tonic-build = "0.13" -tonic-health = "0.13" -tonic-reflection = "0.13" +tonic-build = "0.14" +tonic-health = "0.14" +tonic-prost = "0.14" +tonic-prost-build = "0.14" +tonic-reflection = "0.14" tower = { version = "0.5", features = [ "balance", "buffer", @@ -261,13 +278,13 @@ tower = { version = "0.5", features = [ "util", ] } # legacy version because of warp -tower-http = { version = "0.4", features = [ +tower-http = { version = "0.6", features = [ "compression-gzip", "compression-zstd", "cors", ] } tracing = "0.1" -tracing-opentelemetry = "0.28" +tracing-opentelemetry = "0.32" tracing-subscriber = { version = "0.3", features = [ "env-filter", "json", @@ -277,29 +294,34 @@ tracing-subscriber = { version = "0.3", features = [ ttl_cache = "0.5" typetag = "0.2" ulid = "1.2" +ureq = "3" username = "0.2" +# We cannot upgrade to utoipa 5.0+ due to significant breaking changes: +# 1. The `OpenApi` struct structure changed (fields are private), breaking our manual merging logic in openapi.rs +# in `quickwit-serve`. This code is fundamentally incompatible with version 5.x. utoipa = { version = "4.2", features = ["time", "ulid"] } -uuid = { version = "1.17", features = ["v4", "serde"] } -vrl = { version = "0.22", default-features = false, features = [ +uuid = { version = "1.19", features = ["v4", "serde"] } +vrl = { version = "0.29", default-features = false, features = [ "compiler", "diagnostic", "stdlib", "value", ] } -warp = "0.3" -whichlang = "0.1" +warp = { version = "0.4", features = ["server", "test"] } wiremock = "0.6" -zstd = "0.13" +zstd = { version = "0.13", default-features = false } -aws-config = "1.6" +aws-config = "1.8" aws-credential-types = { version = "1.2", features = ["hardcoded-credentials"] } aws-runtime = "1.5" -aws-sdk-kinesis = "1.74" +aws-sdk-kinesis = "1.97" aws-sdk-s3 = "=1.62" -aws-sdk-sqs = "1.70" +aws-sdk-lambda = "1" +aws-sdk-sqs = "1.91" aws-smithy-async = "1.2" -aws-smithy-http-client = { version = "1.0" } -aws-smithy-runtime = "1.8" +aws-smithy-mocks = "0.2" +aws-smithy-http-client = { version = "1.1", features = ["default-client"] } +aws-smithy-runtime = "1.9" aws-smithy-types = { version = "1.3", features = [ "byte-stream-poll-next", "http-body-1-x", @@ -315,8 +337,8 @@ azure_storage_blobs = { version = "0.21", default-features = false, features = [ "enable_reqwest_rustls", ] } -opendal = { version = "0.53", default-features = false } -reqsign = { version = "0.16", default-features = false } +opendal = { version = "0.55", default-features = false } +reqsign = { version = "0.18", default-features = false, features = ["google", "default-context"] } quickwit-actors = { path = "quickwit-actors" } quickwit-aws = { path = "quickwit-aws" } @@ -336,9 +358,12 @@ quickwit-ingest = { path = "quickwit-ingest" } quickwit-integration-tests = { path = "quickwit-integration-tests" } quickwit-jaeger = { path = "quickwit-jaeger" } quickwit-janitor = { path = "quickwit-janitor" } +quickwit-lambda-client = { path = "quickwit-lambda-client" } +quickwit-lambda-server = { path = "quickwit-lambda-server" } quickwit-macros = { path = "quickwit-macros" } quickwit-metastore = { path = "quickwit-metastore" } quickwit-opentelemetry = { path = "quickwit-opentelemetry" } +quickwit-parquet-engine = { path = "quickwit-parquet-engine" } quickwit-proto = { path = "quickwit-proto" } quickwit-query = { path = "quickwit-query" } quickwit-rest-client = { path = "quickwit-rest-client" } @@ -347,11 +372,12 @@ quickwit-serve = { path = "quickwit-serve" } quickwit-storage = { path = "quickwit-storage" } quickwit-telemetry = { path = "quickwit-telemetry" } -tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "80f5f1e", default-features = false, features = [ +tantivy = { git = "https://github.com/indextables/tantivy/", rev = "ffe408f49bb1f605d8eb28de9d18edd29c5245d5", default-features = false, features = [ "lz4-compression", "mmap", "quickwit", "zstd-compression", + "columnar-zstd-compression", ] } tantivy-fst = "0.5" diff --git a/quickwit/Makefile b/quickwit/Makefile index aed3f16a46d..f06d320dbaa 100644 --- a/quickwit/Makefile +++ b/quickwit/Makefile @@ -21,6 +21,12 @@ fix: @echo "Running cargo clippy --fix" @cargo clippy --workspace --all-features --tests --fix --allow-dirty --allow-staged @$(MAKE) fmt + @$(MAKE) unused-deps + +unused-deps: + @echo "Checking for unused dependencies" + @(command -v cargo-machete >/dev/null || cargo --list | grep -q machete || (echo "cargo-machete is not installed. Please install using 'cargo install cargo-machete'." && exit 1)) + @cargo machete # Usage: # `make test-all` starts the Docker services and runs all the tests. @@ -40,14 +46,6 @@ test-all: test-failpoints: cargo nextest run --test failpoints --features fail/failpoints -test-lambda: - AWS_ACCESS_KEY_ID=ignored \ - AWS_SECRET_ACCESS_KEY=ignored \ - AWS_REGION=us-east-1 \ - QW_S3_ENDPOINT=http://localhost:4566 \ - QW_S3_FORCE_PATH_STYLE_ACCESS=1 \ - cargo nextest run --all-features -p quickwit-lambda --retries 1 - # TODO: to be replaced by https://github.com/quickwit-oss/quickwit/issues/237 TARGET ?= x86_64-unknown-linux-gnu .PHONY: build @@ -76,3 +74,8 @@ build-ui: rm-postgres: rm -fr /tmp/quickwit/services/postgres + +update-licenses: + dd-rust-license-tool --config license-tool.toml write + mv LICENSE-3rdparty.csv ../LICENSE-3rdparty.csv + diff --git a/quickwit/license-tool.toml b/quickwit/license-tool.toml index 7cfd0b21fb1..e6beb605438 100644 --- a/quickwit/license-tool.toml +++ b/quickwit/license-tool.toml @@ -5,3 +5,6 @@ # `ring` has a custom license that is mostly "ISC-style" but parts of it also fall under OpenSSL licensing. "ring-0.17.8" = { license = "ISC AND Custom" } + +# openssl-macros doesn't declare a repository in its crate metadata +"openssl-macros" = { origin = "https://github.com/sfackler/rust-openssl" } diff --git a/quickwit/quickwit-actors/examples/ping_actor.rs b/quickwit/quickwit-actors/examples/ping_actor.rs index 88f0ec1c0a6..5e06667a051 100644 --- a/quickwit/quickwit-actors/examples/ping_actor.rs +++ b/quickwit/quickwit-actors/examples/ping_actor.rs @@ -89,7 +89,7 @@ impl Handler for PingSender { type Reply = (); async fn handle(&mut self, _: Loop, ctx: &ActorContext) -> Result<(), ActorExitStatus> { - let random_peer_id_opt = (0..self.peers.len()).choose(&mut rand::thread_rng()); + let random_peer_id_opt = (0..self.peers.len()).choose(&mut rand::rng()); if let Some(random_peer_id) = random_peer_id_opt { match ctx.ask(&self.peers[random_peer_id], Ping).await { Ok(reply_msg) => { diff --git a/quickwit/quickwit-actors/src/actor.rs b/quickwit/quickwit-actors/src/actor.rs index 2fa32d7f2a5..7b879b5de79 100644 --- a/quickwit/quickwit-actors/src/actor.rs +++ b/quickwit/quickwit-actors/src/actor.rs @@ -18,7 +18,6 @@ use std::sync::Arc; use async_trait::async_trait; use thiserror::Error; -use tracing::error; use crate::{ActorContext, QueueCapacity, SendError}; @@ -221,7 +220,8 @@ pub trait Handler: Actor { #[async_trait::async_trait] impl DeferableReplyHandler for H -where H: Handler +where + H: Handler, { type Reply = H::Reply; diff --git a/quickwit/quickwit-actors/src/actor_context.rs b/quickwit/quickwit-actors/src/actor_context.rs index 3186e210647..61d8a3cd209 100644 --- a/quickwit/quickwit-actors/src/actor_context.rs +++ b/quickwit/quickwit-actors/src/actor_context.rs @@ -145,7 +145,9 @@ impl ActorContext { /// Executes a future in a protected zone. pub async fn protect_future(&self, future: Fut) -> T - where Fut: Future { + where + Fut: Future, + { let _guard = self.protect_zone(); future.await } diff --git a/quickwit/quickwit-aws/Cargo.toml b/quickwit/quickwit-aws/Cargo.toml index 72a181605ed..2993b4ad5a2 100644 --- a/quickwit/quickwit-aws/Cargo.toml +++ b/quickwit/quickwit-aws/Cargo.toml @@ -17,14 +17,7 @@ aws-sdk-kinesis = { workspace = true, optional = true } aws-sdk-s3 = { workspace = true } aws-sdk-sqs = { workspace = true, optional = true } aws-smithy-async = { workspace = true } -aws-smithy-http-client = { workspace = true } -aws-smithy-runtime = { workspace = true } -aws-types = { workspace = true } - futures = { workspace = true } -http-body-util = { workspace = true } -hyper = { workspace = true } -hyper-rustls = { workspace = true } tokio = { workspace = true } quickwit-common = { workspace = true } diff --git a/quickwit/quickwit-aws/src/error.rs b/quickwit/quickwit-aws/src/error.rs index 53983e5fcc6..97b8e51910c 100644 --- a/quickwit/quickwit-aws/src/error.rs +++ b/quickwit/quickwit-aws/src/error.rs @@ -29,7 +29,8 @@ use aws_sdk_s3::operation::upload_part::UploadPartError; use crate::retry::AwsRetryable; impl AwsRetryable for SdkError -where E: AwsRetryable +where + E: AwsRetryable, { fn is_retryable(&self) -> bool { match self { diff --git a/quickwit/quickwit-aws/src/lib.rs b/quickwit/quickwit-aws/src/lib.rs index 3fad7aad979..2dec49509bc 100644 --- a/quickwit/quickwit-aws/src/lib.rs +++ b/quickwit/quickwit-aws/src/lib.rs @@ -42,5 +42,5 @@ pub async fn get_aws_config() -> &'static aws_config::SdkConfig { /// Returns the AWS behavior version. pub fn aws_behavior_version() -> BehaviorVersion { - BehaviorVersion::v2025_01_17() + BehaviorVersion::v2026_01_12() } diff --git a/quickwit/quickwit-aws/src/retry.rs b/quickwit/quickwit-aws/src/retry.rs index b52ce35bdc8..020fada2332 100644 --- a/quickwit/quickwit-aws/src/retry.rs +++ b/quickwit/quickwit-aws/src/retry.rs @@ -38,7 +38,8 @@ impl AwsRetryable for Retry { struct AwsRetryableWrapper(E); impl Retryable for AwsRetryableWrapper -where E: AwsRetryable +where + E: AwsRetryable, { fn is_retryable(&self) -> bool { self.0.is_retryable() diff --git a/quickwit/quickwit-cli/Cargo.toml b/quickwit/quickwit-cli/Cargo.toml index 2ddbd21baae..5d9dc955107 100644 --- a/quickwit/quickwit-cli/Cargo.toml +++ b/quickwit/quickwit-cli/Cargo.toml @@ -37,9 +37,11 @@ numfmt = { workspace = true } once_cell = { workspace = true } openssl-probe = { workspace = true, optional = true } opentelemetry = { workspace = true } +opentelemetry-appender-tracing = { workspace = true } opentelemetry_sdk = { workspace = true } opentelemetry-otlp = { workspace = true } reqwest = { workspace = true } +rustls = { workspace = true } serde_json = { workspace = true } tabled = { workspace = true } tempfile = { workspace = true } @@ -58,7 +60,6 @@ quickwit-actors = { workspace = true } quickwit-cluster = { workspace = true } quickwit-common = { workspace = true } quickwit-config = { workspace = true } -quickwit-doc-mapper = { workspace = true } quickwit-index-management = { workspace = true } quickwit-indexing = { workspace = true } quickwit-ingest = { workspace = true } @@ -101,10 +102,10 @@ release-feature-set = [ "quickwit-indexing/pulsar", "quickwit-indexing/sqs", "quickwit-indexing/vrl", + "quickwit-serve/lambda", "quickwit-storage/azure", "quickwit-storage/gcs", "quickwit-metastore/postgres", - "quickwit-doc-mapper/multilang", ] release-feature-vendored-set = [ "jemalloc", @@ -115,10 +116,10 @@ release-feature-vendored-set = [ "quickwit-indexing/sqs", "quickwit-indexing/vrl", "quickwit-indexing/vendored-kafka", + "quickwit-serve/lambda", "quickwit-storage/azure", "quickwit-storage/gcs", "quickwit-metastore/postgres", - "quickwit-doc-mapper/multilang", ] release-macos-feature-vendored-set = [ "jemalloc", @@ -128,16 +129,12 @@ release-macos-feature-vendored-set = [ "quickwit-indexing/sqs", "quickwit-indexing/vrl", "quickwit-indexing/vendored-kafka-macos", + "quickwit-serve/lambda", "quickwit-storage/azure", "quickwit-storage/gcs", "quickwit-metastore/postgres", - "quickwit-doc-mapper/multilang", ] release-jemalloc-profiled = [ "release-feature-set", "jemalloc-profiled", ] - -[package.metadata.cargo-machete] -# used to enable the `multilang` feature -ignored = ["quickwit-doc-mapper"] diff --git a/quickwit/quickwit-cli/src/generate_markdown.rs b/quickwit/quickwit-cli/src/generate_markdown.rs index 98709139553..9e7062f03d5 100644 --- a/quickwit/quickwit-cli/src/generate_markdown.rs +++ b/quickwit/quickwit-cli/src/generate_markdown.rs @@ -81,10 +81,10 @@ fn markdown_for_command_helper( if !about.trim().is_empty() { println!("{about} "); } - } else if let Some(about) = subcommand.get_about() { - if !about.to_string().trim().is_empty() { - println!("{about} "); - } + } else if let Some(about) = subcommand.get_about() + && !about.to_string().trim().is_empty() + { + println!("{about} "); } if let Some(note) = note { @@ -181,10 +181,10 @@ fn generate_markdown_from_clap(command: &Command) { for command in commands { let command_name = command.get_name(); // index, split, source println!("## {command_name}"); - if let Some(about) = command.get_long_about().or_else(|| command.get_about()) { - if !about.to_string().trim().is_empty() { - println!("{about}\n"); - } + if let Some(about) = command.get_long_about().or_else(|| command.get_about()) + && !about.to_string().trim().is_empty() + { + println!("{about}\n"); } if command.get_subcommands().count() == 0 { diff --git a/quickwit/quickwit-cli/src/index.rs b/quickwit/quickwit-cli/src/index.rs index f11d0d16d41..535e217b3d3 100644 --- a/quickwit/quickwit-cli/src/index.rs +++ b/quickwit/quickwit-cli/src/index.rs @@ -39,7 +39,7 @@ use quickwit_serve::{ListSplitsQueryParams, SearchRequestQueryString, SortBy}; use quickwit_storage::{StorageResolver, load_file}; use tabled::settings::object::{FirstRow, Rows, Segment}; use tabled::settings::panel::Footer; -use tabled::settings::{Alignment, Disable, Format, Modify, Panel, Rotate, Style}; +use tabled::settings::{Alignment, Format, Modify, Panel, Remove, Rotate, Style}; use tabled::{Table, Tabled}; use tracing::{Level, debug}; @@ -579,7 +579,9 @@ pub async fn list_index_cli(args: ListIndexesArgs) -> anyhow::Result<()> { } fn make_list_indexes_table(indexes: I) -> Table -where I: IntoIterator { +where + I: IntoIterator, +{ let rows = indexes .into_iter() .map(|index| IndexRow { @@ -799,7 +801,7 @@ impl IndexStats { Table::builder(tables.into_iter().map(|table| table.to_string())) .build() .with(Modify::new(Segment::all()).with(Alignment::center_vertical())) - .with(Disable::row(FirstRow)) + .with(Remove::row(FirstRow)) .with(Style::empty()) .to_string() } @@ -849,7 +851,7 @@ impl DescriptiveStats { table .with(Style::empty()) - .with(Disable::row(FirstRow)) + .with(Remove::row(FirstRow)) // We separate tables with a newline already, this is to separate quantile part of the // table further away from the next table. .with(Footer::new("\n")); @@ -1061,14 +1063,14 @@ pub async fn ingest_docs_cli(args: IngestDocsArgs) -> anyhow::Result<()> { // TODO(#5604) remove unwrap .unwrap_or(response.num_docs_for_processing), ); - if let Some(rejected) = response.num_rejected_docs { - if rejected > 0 { - println!( - "{} Rejected {} document(s).", - "✖".color(RED_COLOR), - rejected - ); - } + if let Some(rejected) = response.num_rejected_docs + && rejected > 0 + { + println!( + "{} Rejected {} document(s).", + "✖".color(RED_COLOR), + rejected + ); } if let Some(parse_failures) = response.parse_failures { if !parse_failures.is_empty() { @@ -1266,7 +1268,7 @@ mod test { let mut split_4 = template_split; split_4.split_metadata = split_metadata_4; - let splits = vec![split_1, split_2, split_3, split_4]; + let splits = [split_1, split_2, split_3, split_4]; let splits_num_docs = splits .iter() diff --git a/quickwit/quickwit-cli/src/lib.rs b/quickwit/quickwit-cli/src/lib.rs index 5a591ac76e3..aaeb4da7e9d 100644 --- a/quickwit/quickwit-cli/src/lib.rs +++ b/quickwit/quickwit-cli/src/lib.rs @@ -16,6 +16,7 @@ use std::collections::HashSet; use std::str::FromStr; +use std::sync::OnceLock; use anyhow::Context; use clap::{Arg, ArgMatches, arg}; @@ -107,6 +108,17 @@ fn client_args() -> Vec { ] } +pub fn install_default_crypto_ring_provider() { + static CALL_ONLY_ONCE: OnceLock> = OnceLock::new(); + CALL_ONLY_ONCE + .get_or_init(|| { + rustls::crypto::ring::default_provider() + .install_default() + .map_err(|_| ()) + }) + .expect("rustls crypto ring default provider installation should not fail"); +} + #[derive(Debug, Eq, PartialEq)] pub struct ClientArgs { pub cluster_endpoint: Url, @@ -315,7 +327,7 @@ pub fn make_table( .with(Modify::new(Rows::new(1..)).with(Alignment::left())) .with(Style::ascii()) .with(Header::new(header)) - .with(Modify::new(Rows::single(0)).with(Alignment::center())); + .with(Modify::new(Rows::new(0..1)).with(Alignment::center())); table } diff --git a/quickwit/quickwit-cli/src/logger.rs b/quickwit/quickwit-cli/src/logger.rs index bf080a8d26f..e1e60a14f93 100644 --- a/quickwit/quickwit-cli/src/logger.rs +++ b/quickwit/quickwit-cli/src/logger.rs @@ -18,8 +18,10 @@ use std::{env, fmt}; use anyhow::Context; use opentelemetry::trace::TracerProvider; use opentelemetry::{KeyValue, global}; +use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge; +use opentelemetry_sdk::logs::SdkLoggerProvider; use opentelemetry_sdk::propagation::TraceContextPropagator; -use opentelemetry_sdk::trace::BatchConfigBuilder; +use opentelemetry_sdk::trace::{BatchConfigBuilder, SdkTracerProvider}; use opentelemetry_sdk::{Resource, trace}; use quickwit_common::{get_bool_from_env, get_from_env_opt}; use quickwit_serve::{BuildInfo, EnvFilterReloadFn}; @@ -56,12 +58,15 @@ pub fn setup_logging_and_tracing( level: Level, ansi_colors: bool, build_info: &BuildInfo, -) -> anyhow::Result { +) -> anyhow::Result<( + EnvFilterReloadFn, + Option<(SdkTracerProvider, SdkLoggerProvider)>, +)> { #[cfg(feature = "tokio-console")] { if get_bool_from_env(QW_ENABLE_TOKIO_CONSOLE_ENV_KEY, false) { console_subscriber::init(); - return Ok(quickwit_serve::do_nothing_env_filter_reload_fn()); + return Ok((quickwit_serve::do_nothing_env_filter_reload_fn(), None)); } } global::set_text_map_propagator(TraceContextPropagator::new()); @@ -92,45 +97,68 @@ pub fn setup_logging_and_tracing( // Note on disabling ANSI characters: setting the ansi boolean on event format is insufficient. // It is thus set on layers, see https://github.com/tokio-rs/tracing/issues/1817 - if get_bool_from_env(QW_ENABLE_OPENTELEMETRY_OTLP_EXPORTER_ENV_KEY, false) { - let otlp_exporter = opentelemetry_otlp::SpanExporter::builder() + let provider_opt = if get_bool_from_env(QW_ENABLE_OPENTELEMETRY_OTLP_EXPORTER_ENV_KEY, false) { + let span_exporter = opentelemetry_otlp::SpanExporter::builder() .with_tonic() .build() .context("failed to initialize OpenTelemetry OTLP exporter")?; - let batch_processor = - trace::BatchSpanProcessor::builder(otlp_exporter, opentelemetry_sdk::runtime::Tokio) - .with_batch_config( - BatchConfigBuilder::default() - // Quickwit can generate a lot of spans, especially in debug mode, and the - // default queue size of 2048 is too small. - .with_max_queue_size(32_768) - .build(), - ) - .build(); - let provider = opentelemetry_sdk::trace::TracerProvider::builder() - .with_span_processor(batch_processor) - .with_resource(Resource::new([ - KeyValue::new("service.name", "quickwit"), - KeyValue::new("service.version", build_info.version.clone()), - ])) + let span_processor = trace::BatchSpanProcessor::builder(span_exporter) + .with_batch_config( + BatchConfigBuilder::default() + // Quickwit can generate a lot of spans, especially in debug mode, and the + // default queue size of 2048 is too small. + .with_max_queue_size(32_768) + .build(), + ) + .build(); + + let resource = Resource::builder() + .with_service_name("quickwit") + .with_attribute(KeyValue::new("service.version", build_info.version.clone())) + .build(); + + let logs_exporter = opentelemetry_otlp::LogExporter::builder() + .with_tonic() + .build() + .context("failed to initialize OpenTelemetry OTLP logs")?; + + let logger_provider = SdkLoggerProvider::builder() + .with_resource(resource.clone()) + .with_batch_exporter(logs_exporter) .build(); - let tracer = provider.tracer("quickwit"); + + let tracing_provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_span_processor(span_processor) + .with_resource(resource) + .build(); + + let tracer = tracing_provider.tracer("quickwit"); let telemetry_layer = tracing_opentelemetry::layer().with_tracer(tracer); + + // Bridge between tracing logs and otel tracing events + let logs_otel_layer = OpenTelemetryTracingBridge::new(&logger_provider); + registry .with(telemetry_layer) + .with(logs_otel_layer) .try_init() .context("failed to register tracing subscriber")?; + Some((tracing_provider, logger_provider)) } else { registry .try_init() .context("failed to register tracing subscriber")?; - } + None + }; - Ok(Arc::new(move |env_filter_def: &str| { - let new_env_filter = EnvFilter::try_new(env_filter_def)?; - reload_handle.reload(new_env_filter)?; - Ok(()) - })) + Ok(( + Arc::new(move |env_filter_def: &str| { + let new_env_filter = EnvFilter::try_new(env_filter_def)?; + reload_handle.reload(new_env_filter)?; + Ok(()) + }), + provider_opt, + )) } /// We do not rely on the RFC3339 implementation, because it has a nanosecond precision. @@ -152,7 +180,7 @@ impl EventFormat<'_> { /// Gets the log format from the environment variable `QW_LOG_FORMAT`. Returns a JSON /// formatter if the variable is set to `json`, otherwise returns a full formatter. fn get_from_env() -> Self { - if get_from_env_opt::("QW_LOG_FORMAT") + if get_from_env_opt::("QW_LOG_FORMAT", false) .map(|log_format| log_format.eq_ignore_ascii_case("json")) .unwrap_or(false) { @@ -266,10 +294,10 @@ pub(super) mod jemalloc_profiled { seen = true; let ext = span.extensions(); - if let Some(fields) = &ext.get::>() { - if !fields.is_empty() { - write!(writer, "{{{fields}}}:")?; - } + if let Some(fields) = &ext.get::>() + && !fields.is_empty() + { + write!(writer, "{{{fields}}}:")?; } } diff --git a/quickwit/quickwit-cli/src/main.rs b/quickwit/quickwit-cli/src/main.rs index 989752dfd05..4a1f9ce036e 100644 --- a/quickwit/quickwit-cli/src/main.rs +++ b/quickwit/quickwit-cli/src/main.rs @@ -18,13 +18,12 @@ use std::collections::BTreeMap; use anyhow::Context; use colored::Colorize; -use opentelemetry::global; -use quickwit_cli::busy_detector; use quickwit_cli::checklist::RED_COLOR; use quickwit_cli::cli::{CliCommand, build_cli}; #[cfg(feature = "jemalloc")] use quickwit_cli::jemalloc::start_jemalloc_metrics_loop; use quickwit_cli::logger::setup_logging_and_tracing; +use quickwit_cli::{busy_detector, install_default_crypto_ring_provider}; use quickwit_common::runtimes::scrape_tokio_runtime_metrics; use quickwit_serve::BuildInfo; use tracing::error; @@ -33,7 +32,11 @@ use tracing::error; /// QW_RUNTIME_NUM_THREADS environment variable. fn get_main_runtime_num_threads() -> usize { let default_num_runtime_threads: usize = quickwit_common::num_cpus().div_ceil(3); - quickwit_common::get_from_env("QW_TOKIO_RUNTIME_NUM_THREADS", default_num_runtime_threads) + quickwit_common::get_from_env( + "QW_TOKIO_RUNTIME_NUM_THREADS", + default_num_runtime_threads, + false, + ) } fn main() -> anyhow::Result<()> { @@ -89,11 +92,13 @@ async fn main_impl() -> anyhow::Result<()> { } }; + install_default_crypto_ring_provider(); + #[cfg(feature = "jemalloc")] start_jemalloc_metrics_loop(); let build_info = BuildInfo::get(); - let env_filter_reload_fn = + let (env_filter_reload_fn, tracer_provider_opt) = setup_logging_and_tracing(command.default_log_level(), ansi_colors, build_info)?; let return_code: i32 = if let Err(command_error) = command.execute(env_filter_reload_fn).await { @@ -108,7 +113,15 @@ async fn main_impl() -> anyhow::Result<()> { 0 }; - global::shutdown_tracer_provider(); + if let Some((trace_provider, logs_provider)) = tracer_provider_opt { + trace_provider + .shutdown() + .context("failed to shutdown OpenTelemetry tracer provider")?; + logs_provider + .shutdown() + .context("failed to shutdown OpenTelemetry logs provider")?; + } + std::process::exit(return_code) } @@ -125,7 +138,6 @@ fn about_text() -> String { #[cfg(test)] mod tests { - use std::path::PathBuf; use std::str::FromStr; use std::time::Duration; @@ -675,7 +687,7 @@ mod tests { split_id, target_dir, .. - })) if &index_id == "wikipedia" && &split_id == "ABC" && target_dir == PathBuf::from("datadir") + })) if &index_id == "wikipedia" && &split_id == "ABC" && target_dir == *"datadir" )); Ok(()) } diff --git a/quickwit/quickwit-cli/src/source.rs b/quickwit/quickwit-cli/src/source.rs index 122e24a6a09..54297db7639 100644 --- a/quickwit/quickwit-cli/src/source.rs +++ b/quickwit/quickwit-cli/src/source.rs @@ -523,7 +523,9 @@ async fn list_sources_cli(args: ListSourcesArgs) -> anyhow::Result<()> { } fn make_list_sources_table(sources: I) -> Table -where I: IntoIterator { +where + I: IntoIterator, +{ let rows = sources .into_iter() .map(|source| SourceRow { diff --git a/quickwit/quickwit-cli/src/tool.rs b/quickwit/quickwit-cli/src/tool.rs index da5521c0e8c..0f34a3017e6 100644 --- a/quickwit/quickwit-cli/src/tool.rs +++ b/quickwit/quickwit-cli/src/tool.rs @@ -45,6 +45,7 @@ use quickwit_indexing::models::{ use quickwit_ingest::IngesterPool; use quickwit_metastore::IndexMetadataResponseExt; use quickwit_proto::indexing::CpuCapacity; +use quickwit_proto::ingest::ingester::IngesterStatus; use quickwit_proto::metastore::{IndexMetadataRequest, MetastoreService, MetastoreServiceClient}; use quickwit_proto::search::{CountHits, SearchResponse}; use quickwit_proto::types::{IndexId, PipelineUid, SourceId, SplitId}; @@ -936,8 +937,10 @@ async fn create_empty_cluster(config: &NodeConfig) -> anyhow::Result { enabled_services: HashSet::new(), gossip_advertise_addr: config.gossip_advertise_addr, grpc_advertise_addr: config.grpc_advertise_addr, - indexing_cpu_capacity: CpuCapacity::zero(), indexing_tasks: Vec::new(), + indexing_cpu_capacity: CpuCapacity::zero(), + ingester_status: IngesterStatus::default(), + availability_zone: None, }; let client_grpc_config = make_client_grpc_config(&config.grpc_config)?; let cluster = Cluster::join( diff --git a/quickwit/quickwit-cli/tests/Pipfile b/quickwit/quickwit-cli/tests/Pipfile new file mode 100644 index 00000000000..f7a61075c69 --- /dev/null +++ b/quickwit/quickwit-cli/tests/Pipfile @@ -0,0 +1,12 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +awscli-local = "*" + +[dev-packages] + +[requires] +python_version = "3.11" diff --git a/quickwit/quickwit-cli/tests/Pipfile.lock b/quickwit/quickwit-cli/tests/Pipfile.lock new file mode 100644 index 00000000000..2287e4f5c82 --- /dev/null +++ b/quickwit/quickwit-cli/tests/Pipfile.lock @@ -0,0 +1,92 @@ +{ + "_meta": { + "hash": { + "sha256": "a267110298185b224bf2b9b6f5c887a813d38c8866cc8396db4ab2fdffb1a32d" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.11" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "awscli-local": { + "hashes": [ + "sha256:07c532c372753bf5f15426451dc91d6eec9de8779748049329a9a882bdac8a0b", + "sha256:1901ebef343ba8cbde06f9f6406b5415ef53e93296718c2c31c9d49b0b09bd9d" + ], + "index": "pypi", + "version": "==0.22.2" + }, + "boto3": { + "hashes": [ + "sha256:75752e7dc445131700a58926a50ca705794232f0f47d0e21edb59fbf1898db95", + "sha256:b1a5a203511e594872b39a129365f02eb5846eea990629e8daf47a3c01e7fd49" + ], + "markers": "python_version >= '3.9'", + "version": "==1.40.59" + }, + "botocore": { + "hashes": [ + "sha256:042dd844ca82155ca1ab9608b9bef36d517515c775d075f57b89257108ae843b", + "sha256:842a466d8735272a30fe5b7f97df559d9e211a18e412f62a17ed249fd62f85fe" + ], + "markers": "python_version >= '3.9'", + "version": "==1.40.59" + }, + "jmespath": { + "hashes": [ + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" + ], + "markers": "python_version >= '3.7'", + "version": "==1.0.1" + }, + "localstack-client": { + "hashes": [ + "sha256:732a07e23fffd6a581af2714bbe006ad6f884ac4f8ac955211a8a63321cdc409" + ], + "version": "==2.10" + }, + "python-dateutil": { + "hashes": [ + "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", + "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==2.9.0.post0" + }, + "s3transfer": { + "hashes": [ + "sha256:ea3b790c7077558ed1f02a3072fb3cb992bbbd253392f4b6e9e8976941c7d456", + "sha256:eff12264e7c8b4985074ccce27a3b38a485bb7f7422cc8046fee9be4983e4125" + ], + "markers": "python_version >= '3.9'", + "version": "==0.14.0" + }, + "six": { + "hashes": [ + "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", + "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==1.17.0" + }, + "urllib3": { + "hashes": [ + "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", + "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4" + ], + "index": "pypi", + "markers": "python_version >= '3.9'", + "version": "==2.6.3" + } + }, + "develop": {} +} diff --git a/quickwit/quickwit-cli/tests/cli.rs b/quickwit/quickwit-cli/tests/cli.rs index 3ecc473a7a8..bbd416cfd7f 100644 --- a/quickwit/quickwit-cli/tests/cli.rs +++ b/quickwit/quickwit-cli/tests/cli.rs @@ -960,18 +960,6 @@ async fn test_all_local_index() { let result: Value = serde_json::from_str(&query_response).unwrap(); assert_eq!(result["num_hits"], Value::Number(Number::from(2i64))); - let search_stream_response = reqwest::get(format!( - "http://127.0.0.1:{}/api/v1/{}/search/stream?query=level:info&output_format=csv&fast_field=ts", - test_env.rest_listen_port, - test_env.index_id - )) - .await - .unwrap() - .text() - .await - .unwrap(); - assert_eq!(search_stream_response, "72057597000000\n72057608000000\n"); - let args = DeleteIndexArgs { client_args: test_env.default_client_args(), index_id, diff --git a/quickwit/quickwit-cluster/src/change.rs b/quickwit/quickwit-cluster/src/change.rs index 1491212281c..77d3165f5f5 100644 --- a/quickwit/quickwit-cluster/src/change.rs +++ b/quickwit/quickwit-cluster/src/change.rs @@ -35,7 +35,10 @@ use crate::member::NodeStateExt; #[derive(Debug, Clone)] pub enum ClusterChange { Add(ClusterNode), - Update(ClusterNode), + Update { + previous: ClusterNode, + updated: ClusterNode, + }, Remove(ClusterNode), } @@ -246,7 +249,10 @@ async fn compute_cluster_change_events_on_updated( ); Some(ClusterChange::Remove(updated_node)) } else if previous_node.is_ready() && updated_node.is_ready() { - Some(ClusterChange::Update(updated_node)) + Some(ClusterChange::Update { + previous: previous_node, + updated: updated_node, + }) } else { None } @@ -681,16 +687,16 @@ pub(crate) mod tests { .await .unwrap(); - let ClusterChange::Update(node) = event else { + let ClusterChange::Update { updated, .. } = event else { panic!("expected `ClusterChange::Remove` event, got `{event:?}`"); }; - assert_eq!(node.chitchat_id(), &updated_chitchat_id); - assert_eq!(node.grpc_advertise_addr(), grpc_advertise_addr); - assert!(!node.is_self_node()); - assert!(node.is_ready()); + assert_eq!(updated.chitchat_id(), &updated_chitchat_id); + assert_eq!(updated.grpc_advertise_addr(), grpc_advertise_addr); + assert!(!updated.is_self_node()); + assert!(updated.is_ready()); assert_eq!( previous_nodes.get(&updated_chitchat_id.node_id).unwrap(), - &node + &updated ); } { @@ -1009,7 +1015,7 @@ pub(crate) mod tests { .await; assert_eq!(events.len(), 1); - let ClusterChange::Update(_node) = events[0].clone() else { + let ClusterChange::Update { .. } = events[0].clone() else { panic!( "Expected `ClusterChange::Update` event, got `{:?}`", events[0] diff --git a/quickwit/quickwit-cluster/src/cluster.rs b/quickwit/quickwit-cluster/src/cluster.rs index 7d461410b79..0ba54f7f71f 100644 --- a/quickwit/quickwit-cluster/src/cluster.rs +++ b/quickwit/quickwit-cluster/src/cluster.rs @@ -39,9 +39,9 @@ use tracing::{info, warn}; use crate::change::{ClusterChange, ClusterChangeStreamFactory, compute_cluster_change_events}; use crate::grpc_gossip::spawn_catchup_callback_task; use crate::member::{ - ClusterMember, ENABLED_SERVICES_KEY, GRPC_ADVERTISE_ADDR_KEY, NodeStateExt, - PIPELINE_METRICS_PREFIX, READINESS_KEY, READINESS_VALUE_NOT_READY, READINESS_VALUE_READY, - build_cluster_member, + AVAILABILITY_ZONE_KEY, ClusterMember, ENABLED_SERVICES_KEY, GRPC_ADVERTISE_ADDR_KEY, + NodeStateExt, PIPELINE_METRICS_PREFIX, READINESS_KEY, READINESS_VALUE_NOT_READY, + READINESS_VALUE_READY, build_cluster_member, }; use crate::metrics::spawn_metrics_task; use crate::{ClusterChangeStream, ClusterNode}; @@ -150,25 +150,26 @@ impl Cluster { catchup_callback: Some(Box::new(catchup_callback)), extra_liveness_predicate: Some(Box::new(extra_liveness_predicate)), }; - let chitchat_handle = spawn_chitchat( - chitchat_config, - vec![ - ( - ENABLED_SERVICES_KEY.to_string(), - self_node.enabled_services.iter().join(","), - ), - ( - GRPC_ADVERTISE_ADDR_KEY.to_string(), - self_node.grpc_advertise_addr.to_string(), - ), - ( - READINESS_KEY.to_string(), - READINESS_VALUE_NOT_READY.to_string(), - ), - ], - transport, - ) - .await?; + let mut initial_key_values = vec![ + ( + ENABLED_SERVICES_KEY.to_string(), + self_node.enabled_services.iter().join(","), + ), + ( + GRPC_ADVERTISE_ADDR_KEY.to_string(), + self_node.grpc_advertise_addr.to_string(), + ), + ( + READINESS_KEY.to_string(), + READINESS_VALUE_NOT_READY.to_string(), + ), + ]; + + if let Some(az) = &self_node.availability_zone { + initial_key_values.push((AVAILABILITY_ZONE_KEY.to_string(), az.clone())); + } + let chitchat_handle = + spawn_chitchat(chitchat_config, initial_key_values, transport).await?; let chitchat = chitchat_handle.chitchat(); let chitchat_guard = chitchat.lock().await; @@ -696,6 +697,7 @@ pub async fn create_cluster_for_test_with_id( self_node_readiness: bool, ) -> anyhow::Result { use quickwit_proto::indexing::PIPELINE_FULL_CAPACITY; + use quickwit_proto::ingest::ingester::IngesterStatus; let gossip_advertise_addr: SocketAddr = ([127, 0, 0, 1], gossip_advertise_port).into(); let self_node = ClusterMember { node_id, @@ -706,6 +708,8 @@ pub async fn create_cluster_for_test_with_id( grpc_advertise_addr: grpc_addr_from_listen_addr_for_test(gossip_advertise_addr), indexing_tasks: Vec::new(), indexing_cpu_capacity: PIPELINE_FULL_CAPACITY, + ingester_status: IngesterStatus::default(), + availability_zone: None, }; let failure_detector_config = create_failure_detector_config_for_test(); let cluster = Cluster::join( @@ -1046,12 +1050,12 @@ mod tests { .await .unwrap(), ); - let mut random_generator = rand::thread_rng(); + let mut random_generator = rand::rng(); // TODO: increase it back to 1000 when https://github.com/quickwit-oss/chitchat/issues/81 is fixed let indexing_tasks = (0..500) .map(|pipeline_id| { - let index_id = random_generator.gen_range(0..=10_000); - let source_id = random_generator.gen_range(0..=100); + let index_id = random_generator.random_range(0..=10_000); + let source_id = random_generator.random_range(0..=100); IndexingTask { pipeline_uid: Some(PipelineUid::for_test(pipeline_id as u128)), index_uid: Some( diff --git a/quickwit/quickwit-cluster/src/grpc_gossip.rs b/quickwit/quickwit-cluster/src/grpc_gossip.rs index e2f0a4d611d..10be33970db 100644 --- a/quickwit/quickwit-cluster/src/grpc_gossip.rs +++ b/quickwit/quickwit-cluster/src/grpc_gossip.rs @@ -147,7 +147,7 @@ async fn perform_grpc_gossip_rounds( }, ) }); - chitchat_guard.reset_node_state( + chitchat_guard.reset_node_state_if_update( &chitchat_id, key_values, proto_node_state.max_version, @@ -193,7 +193,7 @@ fn select_gossip_candidates( find_gossip_candidate_grpc_addr(self_chitchat_id, node_state) .map(|grpc_addr| (&node_state.chitchat_id().node_id, grpc_addr)) }) - .choose_multiple(&mut rand::thread_rng(), MAX_GOSSIP_PEERS) + .choose_multiple(&mut rand::rng(), MAX_GOSSIP_PEERS) .into_iter() .map(|(node_id, grpc_addr)| (node_id.clone(), grpc_addr)) .unzip() diff --git a/quickwit/quickwit-cluster/src/lib.rs b/quickwit/quickwit-cluster/src/lib.rs index 6d5e8bae203..0f2dbebf749 100644 --- a/quickwit/quickwit-cluster/src/lib.rs +++ b/quickwit/quickwit-cluster/src/lib.rs @@ -36,6 +36,7 @@ use quickwit_common::tower::ClientGrpcConfig; use quickwit_config::service::QuickwitService; use quickwit_config::{GrpcConfig, NodeConfig, TlsConfig}; use quickwit_proto::indexing::CpuCapacity; +use quickwit_proto::ingest::ingester::IngesterStatus; use quickwit_proto::tonic::transport::{Certificate, ClientTlsConfig, Identity}; use time::OffsetDateTime; @@ -143,6 +144,8 @@ pub async fn start_cluster_service(node_config: &NodeConfig) -> anyhow::Result bool; fn size_bytes(&self) -> usize; + + fn ingester_status(&self) -> IngesterStatus; + + fn availability_zone(&self) -> Option; } impl NodeStateExt for NodeState { @@ -74,6 +82,16 @@ impl NodeStateExt for NodeState { .map(|(key, value)| key.len() + value.value.len() + SIZE_OF_VERSION + SIZE_OF_TOMBSTONE) .sum() } + + fn ingester_status(&self) -> IngesterStatus { + self.get(INGESTER_STATUS_KEY) + .and_then(IngesterStatus::from_json_str_name) + .unwrap_or_default() + } + + fn availability_zone(&self) -> Option { + self.get(AVAILABILITY_ZONE_KEY).map(|az| az.to_string()) + } } /// Cluster member. @@ -100,7 +118,13 @@ pub struct ClusterMember { pub indexing_tasks: Vec, /// Indexing cpu capacity of the node expressed in milli cpu. pub indexing_cpu_capacity: CpuCapacity, + /// Status of the ingester service running on the node. `IngesterStatus::Unspecified` if the + /// node is not an ingester. + pub ingester_status: IngesterStatus, + /// Whether the node is ready to serve requests. pub is_ready: bool, + /// Availability zone the node is running in, if enabled. + pub availability_zone: Option, } impl ClusterMember { @@ -152,6 +176,9 @@ pub(crate) fn build_cluster_member( let grpc_advertise_addr = node_state.grpc_advertise_addr()?; let indexing_tasks = parse_indexing_tasks(node_state); let indexing_cpu_capacity = parse_indexing_cpu_capacity(node_state); + let ingester_status = node_state.ingester_status(); + let availability_zone = node_state.availability_zone(); + let member = ClusterMember { node_id: chitchat_id.node_id.into(), generation_id: chitchat_id.generation_id.into(), @@ -161,6 +188,8 @@ pub(crate) fn build_cluster_member( grpc_advertise_addr, indexing_tasks, indexing_cpu_capacity, + ingester_status, + availability_zone, }; Ok(member) } diff --git a/quickwit/quickwit-cluster/src/node.rs b/quickwit/quickwit-cluster/src/node.rs index 4a8b11dbafc..fbdedbcc6b4 100644 --- a/quickwit/quickwit-cluster/src/node.rs +++ b/quickwit/quickwit-cluster/src/node.rs @@ -20,6 +20,7 @@ use std::sync::Arc; use chitchat::{ChitchatId, NodeState}; use quickwit_config::service::QuickwitService; use quickwit_proto::indexing::{CpuCapacity, IndexingTask}; +use quickwit_proto::ingest::ingester::IngesterStatus; use quickwit_proto::types::NodeIdRef; use tonic::transport::Channel; @@ -46,8 +47,10 @@ impl ClusterNode { grpc_advertise_addr: member.grpc_advertise_addr, indexing_tasks: member.indexing_tasks, indexing_capacity: member.indexing_cpu_capacity, + ingester_status: member.ingester_status, is_ready: member.is_ready, is_self_node, + availability_zone: member.availability_zone, }; let node = ClusterNode { inner: Arc::new(inner), @@ -62,7 +65,9 @@ impl ClusterNode { is_self_node: bool, enabled_services: &[&str], indexing_tasks: &[IndexingTask], + ingester_status: IngesterStatus, ) -> Self { + use quickwit_common::shared_consts::INGESTER_STATUS_KEY; use quickwit_common::tower::{ClientGrpcConfig, make_channel}; use crate::cluster::set_indexing_tasks_in_node_state; @@ -75,6 +80,7 @@ impl ClusterNode { let mut node_state = NodeState::for_test(); node_state.set(ENABLED_SERVICES_KEY, enabled_services.join(",")); node_state.set(GRPC_ADVERTISE_ADDR_KEY, grpc_advertise_addr.to_string()); + node_state.set(INGESTER_STATUS_KEY, ingester_status.as_json_str_name()); set_indexing_tasks_in_node_state(indexing_tasks, &mut node_state); Self::try_new(chitchat_id, &node_state, channel, is_self_node).unwrap() } @@ -125,6 +131,10 @@ impl ClusterNode { self.inner.indexing_capacity } + pub fn ingester_status(&self) -> IngesterStatus { + self.inner.ingester_status + } + pub fn is_ready(&self) -> bool { self.inner.is_ready } @@ -132,6 +142,10 @@ impl ClusterNode { pub fn is_self_node(&self) -> bool { self.inner.is_self_node } + + pub fn availability_zone(&self) -> Option<&str> { + self.inner.availability_zone.as_deref() + } } impl Debug for ClusterNode { @@ -153,6 +167,7 @@ impl PartialEq for ClusterNode { && self.inner.indexing_tasks == other.inner.indexing_tasks && self.inner.is_ready == other.inner.is_ready && self.inner.is_self_node == other.inner.is_self_node + && self.inner.availability_zone == other.inner.availability_zone } } @@ -163,6 +178,8 @@ struct InnerNode { grpc_advertise_addr: SocketAddr, indexing_tasks: Vec, indexing_capacity: CpuCapacity, + ingester_status: IngesterStatus, is_ready: bool, is_self_node: bool, + availability_zone: Option, } diff --git a/quickwit/quickwit-codegen/Cargo.toml b/quickwit/quickwit-codegen/Cargo.toml index 3063293e0ac..f3124bd2e9e 100644 --- a/quickwit/quickwit-codegen/Cargo.toml +++ b/quickwit/quickwit-codegen/Cargo.toml @@ -18,7 +18,7 @@ proc-macro2 = { workspace = true } prost-build = { workspace = true } quote = { workspace = true } syn = { workspace = true } -tonic-build = { workspace = true } +tonic-prost-build = { workspace = true } [dev-dependencies] futures = { workspace = true } diff --git a/quickwit/quickwit-codegen/example/Cargo.toml b/quickwit/quickwit-codegen/example/Cargo.toml index e6380b1fb20..0d21ff6bf9e 100644 --- a/quickwit/quickwit-codegen/example/Cargo.toml +++ b/quickwit/quickwit-codegen/example/Cargo.toml @@ -23,6 +23,7 @@ thiserror = { workspace = true } tokio = { workspace = true } tokio-stream = { workspace = true } tonic = { workspace = true } +tonic-prost = { workspace = true } tower = { workspace = true } utoipa = { workspace = true } diff --git a/quickwit/quickwit-codegen/example/src/codegen/hello.rs b/quickwit/quickwit-codegen/example/src/codegen/hello.rs index 8f1f59a4a18..d5007fea29b 100644 --- a/quickwit/quickwit-codegen/example/src/codegen/hello.rs +++ b/quickwit/quickwit-codegen/example/src/codegen/hello.rs @@ -1,45 +1,93 @@ // This file is @generated by prost-build. -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive( + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, + Clone, + PartialEq, + Eq, + Hash, + ::prost::Message, +)] pub struct HelloRequest { #[prost(string, tag = "1")] pub name: ::prost::alloc::string::String, } -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive( + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, + Clone, + PartialEq, + Eq, + Hash, + ::prost::Message, +)] pub struct HelloResponse { #[prost(string, tag = "1")] pub message: ::prost::alloc::string::String, } -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive( + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, + Clone, + PartialEq, + Eq, + Hash, + ::prost::Message, +)] pub struct GoodbyeRequest { #[prost(string, tag = "1")] pub name: ::prost::alloc::string::String, } -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive( + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, + Clone, + PartialEq, + Eq, + Hash, + ::prost::Message, +)] pub struct GoodbyeResponse { #[prost(string, tag = "1")] pub message: ::prost::alloc::string::String, } -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive( + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, + Clone, + PartialEq, + Eq, + Hash, + ::prost::Message, +)] pub struct PingRequest { #[prost(string, tag = "1")] pub name: ::prost::alloc::string::String, } -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive( + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, + Clone, + PartialEq, + Eq, + Hash, + ::prost::Message, +)] pub struct PingResponse { #[prost(string, tag = "1")] pub message: ::prost::alloc::string::String, } +use quickwit_common::tower::RpcName; /// BEGIN quickwit-codegen #[allow(unused_imports)] use std::str::FromStr; use tower::{Layer, Service, ServiceExt}; -use quickwit_common::tower::RpcName; impl RpcName for HelloRequest { fn rpc_name() -> &'static str { "hello" @@ -59,14 +107,11 @@ pub type HelloStream = quickwit_common::ServiceStream>; #[cfg_attr(any(test, feature = "testsuite"), mockall::automock)] #[async_trait::async_trait] pub trait Hello: std::fmt::Debug + Send + Sync + 'static { - /// Says hello. + ///Says hello. async fn hello(&self, request: HelloRequest) -> crate::HelloResult; - /// Says goodbye. - async fn goodbye( - &self, - request: GoodbyeRequest, - ) -> crate::HelloResult; - /// Ping pong. + ///Says goodbye. + async fn goodbye(&self, request: GoodbyeRequest) -> crate::HelloResult; + ///Ping pong. async fn ping( &self, request: quickwit_common::ServiceStream, @@ -87,7 +132,7 @@ impl HelloClient { { #[cfg(any(test, feature = "testsuite"))] assert!( - std::any::TypeId::of:: < T > () != std::any::TypeId::of:: < MockHello > (), + std::any::TypeId::of::() != std::any::TypeId::of::(), "`MockHello` must be wrapped in a `MockHelloWrapper`: use `HelloClient::from_mock(mock)` to instantiate the client" ); Self { @@ -113,9 +158,8 @@ impl HelloClient { max_message_size: bytesize::ByteSize, compression_encoding_opt: Option, ) -> Self { - let (_, connection_keys_watcher) = tokio::sync::watch::channel( - std::collections::HashSet::from_iter([addr]), - ); + let (_, connection_keys_watcher) = + tokio::sync::watch::channel(std::collections::HashSet::from_iter([addr])); let mut client = hello_grpc_client::HelloGrpcClient::new(channel) .max_decoding_message_size(max_message_size.0 as usize) .max_encoding_message_size(max_message_size.0 as usize); @@ -171,10 +215,7 @@ impl Hello for HelloClient { async fn hello(&self, request: HelloRequest) -> crate::HelloResult { self.inner.0.hello(request).await } - async fn goodbye( - &self, - request: GoodbyeRequest, - ) -> crate::HelloResult { + async fn goodbye(&self, request: GoodbyeRequest) -> crate::HelloResult { self.inner.0.goodbye(request).await } async fn ping( @@ -225,9 +266,8 @@ pub mod mock_hello { } } } -pub type BoxFuture = std::pin::Pin< - Box> + Send + 'static>, ->; +pub type BoxFuture = + std::pin::Pin> + Send + 'static>>; impl tower::Service for InnerHelloClient { type Response = HelloResponse; type Error = crate::HelloError; @@ -270,10 +310,7 @@ impl tower::Service> for InnerHelloC ) -> std::task::Poll> { std::task::Poll::Ready(Ok(())) } - fn call( - &mut self, - request: quickwit_common::ServiceStream, - ) -> Self::Future { + fn call(&mut self, request: quickwit_common::ServiceStream) -> Self::Future { let svc = self.clone(); let fut = async move { svc.0.ping(request).await }; Box::pin(fut) @@ -284,16 +321,9 @@ impl tower::Service> for InnerHelloC struct HelloTowerServiceStack { #[allow(dead_code)] inner: InnerHelloClient, - hello_svc: quickwit_common::tower::BoxService< - HelloRequest, - HelloResponse, - crate::HelloError, - >, - goodbye_svc: quickwit_common::tower::BoxService< - GoodbyeRequest, - GoodbyeResponse, - crate::HelloError, - >, + hello_svc: quickwit_common::tower::BoxService, + goodbye_svc: + quickwit_common::tower::BoxService, ping_svc: quickwit_common::tower::BoxService< quickwit_common::ServiceStream, HelloStream, @@ -305,10 +335,7 @@ impl Hello for HelloTowerServiceStack { async fn hello(&self, request: HelloRequest) -> crate::HelloResult { self.hello_svc.clone().ready().await?.call(request).await } - async fn goodbye( - &self, - request: GoodbyeRequest, - ) -> crate::HelloResult { + async fn goodbye(&self, request: GoodbyeRequest) -> crate::HelloResult { self.goodbye_svc.clone().ready().await?.call(request).await } async fn ping( @@ -331,11 +358,7 @@ type HelloLayer = quickwit_common::tower::BoxLayer< crate::HelloError, >; type GoodbyeLayer = quickwit_common::tower::BoxLayer< - quickwit_common::tower::BoxService< - GoodbyeRequest, - GoodbyeResponse, - crate::HelloError, - >, + quickwit_common::tower::BoxService, GoodbyeRequest, GoodbyeResponse, crate::HelloError, @@ -360,29 +383,20 @@ impl HelloTowerLayerStack { pub fn stack_layer(mut self, layer: L) -> Self where L: tower::Layer< - quickwit_common::tower::BoxService< - HelloRequest, - HelloResponse, - crate::HelloError, - >, - > + Clone + Send + Sync + 'static, + quickwit_common::tower::BoxService, + > + Clone + + Send + + Sync + + 'static, , - >>::Service: tower::Service< - HelloRequest, - Response = HelloResponse, - Error = crate::HelloError, - > + Clone + Send + Sync + 'static, + quickwit_common::tower::BoxService, + >>::Service: tower::Service + + Clone + + Send + + Sync + + 'static, <, + quickwit_common::tower::BoxService, >>::Service as tower::Service>::Future: Send + 'static, L: tower::Layer< quickwit_common::tower::BoxService< @@ -390,24 +404,19 @@ impl HelloTowerLayerStack { GoodbyeResponse, crate::HelloError, >, - > + Clone + Send + Sync + 'static, + > + Clone + + Send + + Sync + + 'static, , - >>::Service: tower::Service< - GoodbyeRequest, - Response = GoodbyeResponse, - Error = crate::HelloError, - > + Clone + Send + Sync + 'static, + quickwit_common::tower::BoxService, + >>::Service: tower::Service + + Clone + + Send + + Sync + + 'static, <, + quickwit_common::tower::BoxService, >>::Service as tower::Service>::Future: Send + 'static, L: tower::Layer< quickwit_common::tower::BoxService< @@ -415,7 +424,10 @@ impl HelloTowerLayerStack { HelloStream, crate::HelloError, >, - > + Clone + Send + Sync + 'static, + > + Clone + + Send + + Sync + + 'static, , @@ -426,39 +438,43 @@ impl HelloTowerLayerStack { quickwit_common::ServiceStream, Response = HelloStream, Error = crate::HelloError, - > + Clone + Send + Sync + 'static, + > + Clone + + Send + + Sync + + 'static, <, HelloStream, crate::HelloError, >, - >>::Service as tower::Service< - quickwit_common::ServiceStream, - >>::Future: Send + 'static, + >>::Service as tower::Service>>::Future: + Send + 'static, { - self.hello_layers.push(quickwit_common::tower::BoxLayer::new(layer.clone())); - self.goodbye_layers.push(quickwit_common::tower::BoxLayer::new(layer.clone())); - self.ping_layers.push(quickwit_common::tower::BoxLayer::new(layer.clone())); + self.hello_layers + .push(quickwit_common::tower::BoxLayer::new(layer.clone())); + self.goodbye_layers + .push(quickwit_common::tower::BoxLayer::new(layer.clone())); + self.ping_layers + .push(quickwit_common::tower::BoxLayer::new(layer.clone())); self } pub fn stack_hello_layer(mut self, layer: L) -> Self where L: tower::Layer< - quickwit_common::tower::BoxService< - HelloRequest, - HelloResponse, - crate::HelloError, - >, - > + Send + Sync + 'static, - L::Service: tower::Service< - HelloRequest, - Response = HelloResponse, - Error = crate::HelloError, - > + Clone + Send + Sync + 'static, + quickwit_common::tower::BoxService, + > + Send + + Sync + + 'static, + L::Service: tower::Service + + Clone + + Send + + Sync + + 'static, >::Future: Send + 'static, { - self.hello_layers.push(quickwit_common::tower::BoxLayer::new(layer)); + self.hello_layers + .push(quickwit_common::tower::BoxLayer::new(layer)); self } pub fn stack_goodbye_layer(mut self, layer: L) -> Self @@ -469,15 +485,18 @@ impl HelloTowerLayerStack { GoodbyeResponse, crate::HelloError, >, - > + Send + Sync + 'static, - L::Service: tower::Service< - GoodbyeRequest, - Response = GoodbyeResponse, - Error = crate::HelloError, - > + Clone + Send + Sync + 'static, + > + Send + + Sync + + 'static, + L::Service: tower::Service + + Clone + + Send + + Sync + + 'static, >::Future: Send + 'static, { - self.goodbye_layers.push(quickwit_common::tower::BoxLayer::new(layer)); + self.goodbye_layers + .push(quickwit_common::tower::BoxLayer::new(layer)); self } pub fn stack_ping_layer(mut self, layer: L) -> Self @@ -488,17 +507,22 @@ impl HelloTowerLayerStack { HelloStream, crate::HelloError, >, - > + Send + Sync + 'static, + > + Send + + Sync + + 'static, L::Service: tower::Service< quickwit_common::ServiceStream, Response = HelloStream, Error = crate::HelloError, - > + Clone + Send + Sync + 'static, - , - >>::Future: Send + 'static, + > + Clone + + Send + + Sync + + 'static, + >>::Future: + Send + 'static, { - self.ping_layers.push(quickwit_common::tower::BoxLayer::new(layer)); + self.ping_layers + .push(quickwit_common::tower::BoxLayer::new(layer)); self } pub fn build(self, instance: T) -> HelloClient @@ -515,12 +539,8 @@ impl HelloTowerLayerStack { max_message_size: bytesize::ByteSize, compression_encoding_opt: Option, ) -> HelloClient { - let client = HelloClient::from_channel( - addr, - channel, - max_message_size, - compression_encoding_opt, - ); + let client = + HelloClient::from_channel(addr, channel, max_message_size, compression_encoding_opt); let inner_client = client.inner; self.build_from_inner_client(inner_client) } @@ -538,17 +558,12 @@ impl HelloTowerLayerStack { let inner_client = client.inner; self.build_from_inner_client(inner_client) } - pub fn build_from_mailbox( - self, - mailbox: quickwit_actors::Mailbox, - ) -> HelloClient + pub fn build_from_mailbox(self, mailbox: quickwit_actors::Mailbox) -> HelloClient where A: quickwit_actors::Actor + std::fmt::Debug + Send + 'static, HelloMailbox: Hello, { - let inner_client = InnerHelloClient( - std::sync::Arc::new(HelloMailbox::new(mailbox)), - ); + let inner_client = InnerHelloClient(std::sync::Arc::new(HelloMailbox::new(mailbox))); self.build_from_inner_client(inner_client) } #[cfg(any(test, feature = "testsuite"))] @@ -558,30 +573,18 @@ impl HelloTowerLayerStack { self.build_from_inner_client(inner_client) } fn build_from_inner_client(self, inner_client: InnerHelloClient) -> HelloClient { - let hello_svc = self - .hello_layers - .into_iter() - .rev() - .fold( - quickwit_common::tower::BoxService::new(inner_client.clone()), - |svc, layer| layer.layer(svc), - ); - let goodbye_svc = self - .goodbye_layers - .into_iter() - .rev() - .fold( - quickwit_common::tower::BoxService::new(inner_client.clone()), - |svc, layer| layer.layer(svc), - ); - let ping_svc = self - .ping_layers - .into_iter() - .rev() - .fold( - quickwit_common::tower::BoxService::new(inner_client.clone()), - |svc, layer| layer.layer(svc), - ); + let hello_svc = self.hello_layers.into_iter().rev().fold( + quickwit_common::tower::BoxService::new(inner_client.clone()), + |svc, layer| layer.layer(svc), + ); + let goodbye_svc = self.goodbye_layers.into_iter().rev().fold( + quickwit_common::tower::BoxService::new(inner_client.clone()), + |svc, layer| layer.layer(svc), + ); + let ping_svc = self.ping_layers.into_iter().rev().fold( + quickwit_common::tower::BoxService::new(inner_client.clone()), + |svc, layer| layer.layer(svc), + ); let tower_svc_stack = HelloTowerServiceStack { inner: inner_client, hello_svc, @@ -630,7 +633,8 @@ impl Clone for HelloMailbox { impl tower::Service for HelloMailbox where A: quickwit_actors::Actor - + quickwit_actors::DeferableReplyHandler> + Send + + quickwit_actors::DeferableReplyHandler> + + Send + 'static, M: std::fmt::Debug + Send + 'static, T: Send + 'static, @@ -652,7 +656,10 @@ where fn call(&mut self, message: M) -> Self::Future { let mailbox = self.inner.clone(); let fut = async move { - mailbox.ask_for_res(message).await.map_err(|error| error.into()) + mailbox + .ask_for_res(message) + .await + .map_err(|error| error.into()) }; Box::pin(fut) } @@ -661,21 +668,17 @@ where impl Hello for HelloMailbox where A: quickwit_actors::Actor + std::fmt::Debug, - HelloMailbox< - A, - >: tower::Service< + HelloMailbox: tower::Service< HelloRequest, Response = HelloResponse, Error = crate::HelloError, Future = BoxFuture, - > - + tower::Service< + > + tower::Service< GoodbyeRequest, Response = GoodbyeResponse, Error = crate::HelloError, Future = BoxFuture, - > - + tower::Service< + > + tower::Service< quickwit_common::ServiceStream, Response = HelloStream, Error = crate::HelloError, @@ -685,10 +688,7 @@ where async fn hello(&self, request: HelloRequest) -> crate::HelloResult { self.clone().call(request).await } - async fn goodbye( - &self, - request: GoodbyeRequest, - ) -> crate::HelloResult { + async fn goodbye(&self, request: GoodbyeRequest) -> crate::HelloResult { self.clone().call(request).await } async fn ping( @@ -705,8 +705,11 @@ where } fn endpoints(&self) -> Vec { vec![ - quickwit_common::uri::Uri::from_str(& format!("actor://localhost/{}", self - .inner.actor_instance_id())).expect("URI should be valid") + quickwit_common::uri::Uri::from_str(&format!( + "actor://localhost/{}", + self.inner.actor_instance_id() + )) + .expect("URI should be valid"), ] } } @@ -714,9 +717,8 @@ where pub struct HelloGrpcClientAdapter { inner: T, #[allow(dead_code)] - connection_addrs_rx: tokio::sync::watch::Receiver< - std::collections::HashSet, - >, + connection_addrs_rx: + tokio::sync::watch::Receiver>, } impl HelloGrpcClientAdapter { pub fn new( @@ -734,11 +736,14 @@ impl HelloGrpcClientAdapter { #[async_trait::async_trait] impl Hello for HelloGrpcClientAdapter> where - T: tonic::client::GrpcService + std::fmt::Debug + Clone + Send - + Sync + 'static, + T: tonic::client::GrpcService + + std::fmt::Debug + + Clone + + Send + + Sync + + 'static, T::ResponseBody: tonic::codegen::Body + Send + 'static, - ::Error: Into - + Send, + ::Error: Into + Send, T::Future: Send, { async fn hello(&self, request: HelloRequest) -> crate::HelloResult { @@ -747,24 +752,19 @@ where .hello(request) .await .map(|response| response.into_inner()) - .map_err(|status| crate::error::grpc_status_to_service_error( - status, - HelloRequest::rpc_name(), - )) + .map_err(|status| { + crate::error::grpc_status_to_service_error(status, HelloRequest::rpc_name()) + }) } - async fn goodbye( - &self, - request: GoodbyeRequest, - ) -> crate::HelloResult { + async fn goodbye(&self, request: GoodbyeRequest) -> crate::HelloResult { self.inner .clone() .goodbye(request) .await .map(|response| response.into_inner()) - .map_err(|status| crate::error::grpc_status_to_service_error( - status, - GoodbyeRequest::rpc_name(), - )) + .map_err(|status| { + crate::error::grpc_status_to_service_error(status, GoodbyeRequest::rpc_name()) + }) } async fn ping( &self, @@ -777,16 +777,13 @@ where .map(|response| { let streaming: tonic::Streaming<_> = response.into_inner(); let stream = quickwit_common::ServiceStream::from(streaming); - stream - .map_err(|status| crate::error::grpc_status_to_service_error( - status, - PingRequest::rpc_name(), - )) + stream.map_err(|status| { + crate::error::grpc_status_to_service_error(status, PingRequest::rpc_name()) + }) + }) + .map_err(|status| { + crate::error::grpc_status_to_service_error(status, PingRequest::rpc_name()) }) - .map_err(|status| crate::error::grpc_status_to_service_error( - status, - PingRequest::rpc_name(), - )) } async fn check_connectivity(&self) -> anyhow::Result<()> { if self.connection_addrs_rx.borrow().is_empty() { @@ -798,9 +795,12 @@ where self.connection_addrs_rx .borrow() .iter() - .flat_map(|addr| quickwit_common::uri::Uri::from_str( - &format!("grpc://{addr}/{}.{}", "hello", "Hello"), - )) + .flat_map(|addr| { + quickwit_common::uri::Uri::from_str(&format!( + "grpc://{addr}/{}.{}", + "hello", "Hello" + )) + }) .collect() } } @@ -854,9 +854,9 @@ impl hello_grpc_server::HelloGrpc for HelloGrpcServerAdapter { quickwit_common::ServiceStream::from(streaming) }) .await - .map(|stream| tonic::Response::new( - stream.map_err(crate::error::grpc_error_to_grpc_status), - )) + .map(|stream| { + tonic::Response::new(stream.map_err(crate::error::grpc_error_to_grpc_status)) + }) .map_err(crate::error::grpc_error_to_grpc_status) } } @@ -867,10 +867,10 @@ pub mod hello_grpc_client { dead_code, missing_docs, clippy::wildcard_imports, - clippy::let_unit_value, + clippy::let_unit_value )] - use tonic::codegen::*; use tonic::codegen::http::Uri; + use tonic::codegen::*; #[derive(Debug, Clone)] pub struct HelloGrpcClient { inner: tonic::client::Grpc, @@ -909,14 +909,13 @@ pub mod hello_grpc_client { F: tonic::service::Interceptor, T::ResponseBody: Default, T: tonic::codegen::Service< - http::Request, - Response = http::Response< - >::ResponseBody, + http::Request, + Response = http::Response< + >::ResponseBody, + >, >, - >, - , - >>::Error: Into + std::marker::Send + std::marker::Sync, + >>::Error: + Into + std::marker::Send + std::marker::Sync, { HelloGrpcClient::new(InterceptedService::new(inner, interceptor)) } @@ -956,40 +955,29 @@ pub mod hello_grpc_client { &mut self, request: impl tonic::IntoRequest, ) -> std::result::Result, tonic::Status> { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static("/hello.Hello/Hello"); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new("hello.Hello", "Hello")); + req.extensions_mut() + .insert(GrpcMethod::new("hello.Hello", "Hello")); self.inner.unary(req, path, codec).await } /// Says goodbye. pub async fn goodbye( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static("/hello.Hello/Goodbye"); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new("hello.Hello", "Goodbye")); + req.extensions_mut() + .insert(GrpcMethod::new("hello.Hello", "Goodbye")); self.inner.unary(req, path, codec).await } /// Ping pong. @@ -1000,18 +988,14 @@ pub mod hello_grpc_client { tonic::Response>, tonic::Status, > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static("/hello.Hello/Ping"); let mut req = request.into_streaming_request(); - req.extensions_mut().insert(GrpcMethod::new("hello.Hello", "Ping")); + req.extensions_mut() + .insert(GrpcMethod::new("hello.Hello", "Ping")); self.inner.streaming(req, path, codec).await } } @@ -1023,7 +1007,7 @@ pub mod hello_grpc_server { dead_code, missing_docs, clippy::wildcard_imports, - clippy::let_unit_value, + clippy::let_unit_value )] use tonic::codegen::*; /// Generated trait containing gRPC methods that should be implemented for use with HelloGrpcServer. @@ -1042,8 +1026,7 @@ pub mod hello_grpc_server { /// Server streaming response type for the Ping method. type PingStream: tonic::codegen::tokio_stream::Stream< Item = std::result::Result, - > - + std::marker::Send + > + std::marker::Send + 'static; /// Ping pong. async fn ping( @@ -1072,10 +1055,7 @@ pub mod hello_grpc_server { max_encoding_message_size: None, } } - pub fn with_interceptor( - inner: T, - interceptor: F, - ) -> InterceptedService + pub fn with_interceptor(inner: T, interceptor: F) -> InterceptedService where F: tonic::service::Interceptor, { @@ -1130,21 +1110,15 @@ pub mod hello_grpc_server { "/hello.Hello/Hello" => { #[allow(non_camel_case_types)] struct HelloSvc(pub Arc); - impl tonic::server::UnaryService - for HelloSvc { + impl tonic::server::UnaryService for HelloSvc { type Response = super::HelloResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; + type Future = BoxFuture, tonic::Status>; fn call( &mut self, request: tonic::Request, ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::hello(&inner, request).await - }; + let fut = async move { ::hello(&inner, request).await }; Box::pin(fut) } } @@ -1155,7 +1129,7 @@ pub mod hello_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = HelloSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -1173,21 +1147,16 @@ pub mod hello_grpc_server { "/hello.Hello/Goodbye" => { #[allow(non_camel_case_types)] struct GoodbyeSvc(pub Arc); - impl tonic::server::UnaryService - for GoodbyeSvc { + impl tonic::server::UnaryService for GoodbyeSvc { type Response = super::GoodbyeResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; + type Future = BoxFuture, tonic::Status>; fn call( &mut self, request: tonic::Request, ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::goodbye(&inner, request).await - }; + let fut = + async move { ::goodbye(&inner, request).await }; Box::pin(fut) } } @@ -1198,7 +1167,7 @@ pub mod hello_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = GoodbyeSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -1216,24 +1185,17 @@ pub mod hello_grpc_server { "/hello.Hello/Ping" => { #[allow(non_camel_case_types)] struct PingSvc(pub Arc); - impl< - T: HelloGrpc, - > tonic::server::StreamingService - for PingSvc { + impl tonic::server::StreamingService for PingSvc { type Response = super::PingResponse; type ResponseStream = T::PingStream; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; + type Future = + BoxFuture, tonic::Status>; fn call( &mut self, request: tonic::Request>, ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::ping(&inner, request).await - }; + let fut = async move { ::ping(&inner, request).await }; Box::pin(fut) } } @@ -1244,7 +1206,7 @@ pub mod hello_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = PingSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -1259,25 +1221,19 @@ pub mod hello_grpc_server { }; Box::pin(fut) } - _ => { - Box::pin(async move { - let mut response = http::Response::new( - tonic::body::Body::default(), - ); - let headers = response.headers_mut(); - headers - .insert( - tonic::Status::GRPC_STATUS, - (tonic::Code::Unimplemented as i32).into(), - ); - headers - .insert( - http::header::CONTENT_TYPE, - tonic::metadata::GRPC_CONTENT_TYPE, - ); - Ok(response) - }) - } + _ => Box::pin(async move { + let mut response = http::Response::new(tonic::body::Body::default()); + let headers = response.headers_mut(); + headers.insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers.insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) + }), } } } diff --git a/quickwit/quickwit-codegen/example/src/error.rs b/quickwit/quickwit-codegen/example/src/error.rs index 096a9ba42b3..4056f7639dc 100644 --- a/quickwit/quickwit-codegen/example/src/error.rs +++ b/quickwit/quickwit-codegen/example/src/error.rs @@ -67,7 +67,8 @@ impl GrpcServiceError for HelloError { } impl From> for HelloError -where E: fmt::Debug +where + E: fmt::Debug, { fn from(error: AskError) -> Self { HelloError::Internal(format!("{error:?}")) diff --git a/quickwit/quickwit-codegen/example/src/lib.rs b/quickwit/quickwit-codegen/example/src/lib.rs index 75a1ada2f0d..f4782889a9e 100644 --- a/quickwit/quickwit-codegen/example/src/lib.rs +++ b/quickwit/quickwit-codegen/example/src/lib.rs @@ -40,7 +40,8 @@ struct Counter { } impl Service for Counter -where S: Service +where + S: Service, { type Response = S::Response; type Error = S::Error; diff --git a/quickwit/quickwit-codegen/src/codegen.rs b/quickwit/quickwit-codegen/src/codegen.rs index 8c6249f6190..a7d6d311be0 100644 --- a/quickwit/quickwit-codegen/src/codegen.rs +++ b/quickwit/quickwit-codegen/src/codegen.rs @@ -138,7 +138,7 @@ impl QuickwitServiceGenerator { ) -> Self { let inner = Box::new(WithSuffixServiceGenerator::new( "Grpc", - tonic_build::configure().service_generator(), + tonic_prost_build::configure().service_generator(), )); Self { result_type_path, diff --git a/quickwit/quickwit-common/Cargo.toml b/quickwit/quickwit-common/Cargo.toml index d281cd6eaeb..72b8a9ed8f8 100644 --- a/quickwit/quickwit-common/Cargo.toml +++ b/quickwit/quickwit-common/Cargo.toml @@ -22,6 +22,7 @@ dyn-clone = { workspace = true } env_logger = { workspace = true } fnv = { workspace = true } futures = { workspace = true } +governor = { workspace = true } home = { workspace = true } hostname = { workspace = true } http = { workspace = true } @@ -40,11 +41,13 @@ siphasher = { workspace = true } sysinfo = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } -tikv-jemalloc-ctl = { workspace = true, optional = true } tikv-jemallocator = { workspace = true, optional = true } tokio = { workspace = true } tokio-metrics = { workspace = true } tokio-stream = { workspace = true } + +[package.metadata.cargo-machete] +ignored = ["tokio-metrics"] tonic = { workspace = true, features = [ "tls-native-roots", "server", @@ -60,7 +63,6 @@ jemalloc-profiled = [ "named_tasks", "dep:backtrace", "dep:tikv-jemallocator", - "dep:tikv-jemalloc-ctl", ] [dev-dependencies] diff --git a/quickwit/quickwit-common/src/binary_heap.rs b/quickwit/quickwit-common/src/binary_heap.rs index 1910a14c673..eb2473fea8b 100644 --- a/quickwit/quickwit-common/src/binary_heap.rs +++ b/quickwit/quickwit-common/src/binary_heap.rs @@ -103,7 +103,7 @@ impl Ord for OrderItemPair { impl PartialOrd for OrderItemPair { fn partial_cmp(&self, other: &Self) -> Option { - Some(self.order.cmp(&other.order)) + Some(self.cmp(other)) } } diff --git a/quickwit/quickwit-common/src/coolid.rs b/quickwit/quickwit-common/src/coolid.rs index c38b318ee11..1141ed3d989 100644 --- a/quickwit/quickwit-common/src/coolid.rs +++ b/quickwit/quickwit-common/src/coolid.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use rand::distributions::Alphanumeric; +use rand::distr::Alphanumeric; use rand::prelude::*; const ADJECTIVES: &[&str] = &[ @@ -84,8 +84,8 @@ const ADJECTIVES: &[&str] = &[ /// Returns a randomly generated id pub fn new_coolid(name: &str) -> String { - let mut rng = rand::thread_rng(); - let adjective = ADJECTIVES[rng.gen_range(0..ADJECTIVES.len())]; + let mut rng = rand::rng(); + let adjective = ADJECTIVES[rng.random_range(0..ADJECTIVES.len())]; let slug: String = rng .sample_iter(&Alphanumeric) .take(4) diff --git a/quickwit/quickwit-common/src/cpus.rs b/quickwit/quickwit-common/src/cpus.rs new file mode 100644 index 00000000000..dba84f326d2 --- /dev/null +++ b/quickwit/quickwit-common/src/cpus.rs @@ -0,0 +1,143 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::num::NonZero; +use std::sync::OnceLock; + +use tracing::{error, info, warn}; + +const QW_NUM_CPUS_ENV_KEY: &str = "QW_NUM_CPUS"; +const KUBERNETES_LIMITS_CPU: &str = "KUBERNETES_LIMITS_CPU"; + +/// Return the number of vCPU/hyperthreads available. +/// The following methods are used in order: +/// - from the `QW_NUM_CPUS` environment variable +/// - from the `KUBERNETES_LIMITS_CPU` environment variable +/// - from the operating system +/// - default to 2. +pub fn num_cpus() -> usize { + static NUM_CPUS: OnceLock = OnceLock::new(); + *NUM_CPUS.get_or_init(num_cpus_aux) +} + +fn num_cpus_aux() -> usize { + let num_cpus_from_os_opt = std::thread::available_parallelism() + .map(NonZero::get) + .inspect_err(|err| { + error!(error=?err, "failed to detect the number of threads available: arbitrarily returning 2"); + }) + .ok(); + let num_cpus_from_env_opt = get_num_cpus_from_env(QW_NUM_CPUS_ENV_KEY); + let num_cpus_from_k8s_limit = get_num_cpus_from_env(KUBERNETES_LIMITS_CPU); + + if let Some(num_cpus) = num_cpus_from_env_opt { + return num_cpus; + } + + if let Some(num_cpus_from_k8s_limit) = num_cpus_from_k8s_limit { + info!( + "num cpus from k8s limit: {}, possibly overriding os value {:?}", + num_cpus_from_k8s_limit, num_cpus_from_env_opt + ); + return num_cpus_from_k8s_limit; + } + + if let Some(num_cpus_from_os_opt) = num_cpus_from_os_opt { + info!("num cpus from os: {}", num_cpus_from_os_opt); + return num_cpus_from_os_opt; + } + + warn!("failed to detect number of cpus. defaulting to 2"); + 2 +} + +fn parse_cpu_to_mcpu(cpu_string: &str) -> Result { + let trimmed_str = cpu_string.trim(); + + if trimmed_str.is_empty() { + return Err("input cpu_string cannot be empty"); + } + + if let Some(val_str) = trimmed_str.strip_suffix('m') { + // The value is already in millicores. + val_str + .parse::() + .map_err(|_| "invalid millicore value") + } else { + // The value is in CPU cores. + let value = trimmed_str + .parse::() + .map_err(|_| "invalid float value")?; + Ok((value * 1000.0f64) as usize) + } +} + +// Get the number of CPUs from an environment variable. +// The value is expected to be in k8s format (200m means 200 millicores, 2 means 2 cores) +// +// We then get the number of vCPUs by ceiling any non integer value. +fn get_num_cpus_from_env(env_key: &str) -> Option { + let k8s_cpu_limit_str: String = crate::get_from_env_opt(env_key, false)?; + let mcpus = parse_cpu_to_mcpu(&k8s_cpu_limit_str) + .inspect_err(|err_msg| { + warn!( + "failed to parse k8s cpu limit (`{}`): {}", + k8s_cpu_limit_str, err_msg + ); + }) + .ok()?; + let num_vcpus = mcpus.div_ceil(1000); + Some(num_vcpus) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_millicores() { + assert_eq!(parse_cpu_to_mcpu("500m").unwrap(), 500); + assert_eq!(parse_cpu_to_mcpu("100m").unwrap(), 100); + assert_eq!(parse_cpu_to_mcpu("2500m").unwrap(), 2500); + } + + #[test] + fn test_cores() { + assert_eq!(parse_cpu_to_mcpu("1").unwrap(), 1000); + assert_eq!(parse_cpu_to_mcpu("2").unwrap(), 2000); + } + + #[test] + fn test_fractional_cores() { + assert_eq!(parse_cpu_to_mcpu("0.5").unwrap(), 500); + assert_eq!(parse_cpu_to_mcpu("1.5").unwrap(), 1500); + assert_eq!(parse_cpu_to_mcpu("0.25").unwrap(), 250); + } + + #[test] + fn test_with_whitespace() { + assert_eq!(parse_cpu_to_mcpu(" 750m ").unwrap(), 750); + assert_eq!(parse_cpu_to_mcpu(" 0.75 ").unwrap(), 750); + } + + #[test] + fn test_invalid_input() { + assert!(parse_cpu_to_mcpu("").is_err()); + assert!(parse_cpu_to_mcpu(" ").is_err()); + assert!(parse_cpu_to_mcpu("abc").is_err()); + assert!(parse_cpu_to_mcpu("1a").is_err()); + assert!(parse_cpu_to_mcpu("m500").is_err()); + assert!(parse_cpu_to_mcpu("500m1").is_err()); + } +} diff --git a/quickwit/quickwit-common/src/debug.rs b/quickwit/quickwit-common/src/debug.rs index 5621253b387..52b2f8d3a01 100644 --- a/quickwit/quickwit-common/src/debug.rs +++ b/quickwit/quickwit-common/src/debug.rs @@ -1,8 +1,23 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Debug utilities for conditional logging based on TANTIVY4JAVA_DEBUG environment variable -use once_cell::sync::Lazy; use std::env; + use chrono; +use once_cell::sync::Lazy; /// Global debug flag for tantivy4java, evaluated once at startup pub static TANTIVY4JAVA_DEBUG_ENABLED: Lazy = Lazy::new(|| { @@ -24,4 +39,4 @@ macro_rules! tantivy4java_debug { eprintln!("[{}] {}", $crate::debug::format_timestamp(), format!($($arg)*)); } }; -} \ No newline at end of file +} diff --git a/quickwit/quickwit-common/src/io.rs b/quickwit/quickwit-common/src/io.rs index 102393f33ce..d560ca86dff 100644 --- a/quickwit/quickwit-common/src/io.rs +++ b/quickwit/quickwit-common/src/io.rs @@ -294,18 +294,22 @@ pub trait IoControlsAccess: Sized { } fn apply(&self, f: F) -> R - where F: Fn(&IoControls) -> R; + where + F: Fn(&IoControls) -> R; } impl IoControlsAccess for IoControls { fn apply(&self, f: F) -> R - where F: Fn(&IoControls) -> R { + where + F: Fn(&IoControls) -> R, + { f(self) } } impl ControlledWrite -where A: IoControlsAccess +where + A: IoControlsAccess, { pub fn underlying_wrt(&mut self) -> &mut W { &mut self.underlying_wrt @@ -318,7 +322,8 @@ where A: IoControlsAccess } impl io::Write for ControlledWrite -where A: IoControlsAccess +where + A: IoControlsAccess, { fn write(&mut self, buf: &[u8]) -> io::Result { let buf = truncate_bytes(buf); @@ -377,7 +382,7 @@ mod tests { } controlled_write.flush().await.unwrap(); let elapsed = start.elapsed(); - assert!(elapsed <= Duration::from_millis(5)); + assert!(elapsed <= Duration::from_millis(10)); assert_eq!(io_controls.num_bytes(), 2_000_000u64); } diff --git a/quickwit/quickwit-common/src/lib.rs b/quickwit/quickwit-common/src/lib.rs index 17e8f615f1b..9fa6881c7e2 100644 --- a/quickwit/quickwit-common/src/lib.rs +++ b/quickwit/quickwit-common/src/lib.rs @@ -20,6 +20,7 @@ pub mod debug; #[cfg(feature = "jemalloc-profiled")] pub(crate) mod alloc_tracker; pub mod binary_heap; +mod cpus; pub mod fs; pub mod io; #[cfg(feature = "jemalloc-profiled")] @@ -36,6 +37,7 @@ pub mod rate_limited_tracing; pub mod rate_limiter; pub mod rendezvous_hasher; pub mod retry; +pub mod ring_buffer; pub mod runtimes; pub mod shared_consts; pub mod sorted_iter; @@ -57,6 +59,7 @@ use std::ops::{Range, RangeInclusive}; use std::str::FromStr; pub use coolid::new_coolid; +pub use cpus::num_cpus; pub use kill_switch::KillSwitch; pub use path_hasher::PathHasher; pub use progress::{Progress, ProtectedZoneGuard}; @@ -64,6 +67,18 @@ pub use socket_addr_legacy_hash::SocketAddrLegacyHash; pub use stream_utils::{BoxStream, ServiceStream}; use tracing::{error, info}; +/// Returns true at compile time. This function is mostly used with serde to initialize boolean +/// fields to true. +pub const fn true_fn() -> bool { + true +} + +/// Returns whether the given boolean value is true. This function is mostly used with serde to skip +/// serializing boolean fields with `skip_serializing_if = "is_true"` when the value is true. +pub fn is_true(value: &bool) -> bool { + *value +} + pub fn chunk_range(range: Range, chunk_size: usize) -> impl Iterator> { range.clone().step_by(chunk_size).map(move |block_start| { let block_end = (block_start + chunk_size).min(range.end); @@ -83,43 +98,47 @@ pub fn split_file(split_id: impl Display) -> String { format!("{split_id}.split") } -pub fn get_from_env(key: &str, default_value: T) -> T { - if let Ok(value_str) = std::env::var(key) { - if let Ok(value) = T::from_str(&value_str) { - info!(value=?value, "using environment variable `{key}` value"); - return value; - } else { - error!(value=%value_str, "failed to parse environment variable `{key}` value"); - } +fn get_from_env_opt_aux( + key: &str, + parse_fn: impl FnOnce(&str) -> Option, + sensitive: bool, +) -> Option { + let value_str = std::env::var(key).ok()?; + let Some(value) = parse_fn(&value_str) else { + error!(value=%value_str, "failed to parse environment variable `{key}` value"); + return None; + }; + if sensitive { + info!("using environment variable `{key}` value"); + } else { + info!(value=?value, "using environment variable `{key}` value"); } - info!(value=?default_value, "using environment variable `{key}` default value"); - default_value + Some(value) } -pub fn get_bool_from_env(key: &str, default_value: bool) -> bool { - if let Ok(value_str) = std::env::var(key) { - if let Some(value) = parse_bool_lenient(&value_str) { - info!(value=%value, "using environment variable `{key}` value"); - return value; - } else { - error!(value=%value_str, "failed to parse environment variable `{key}` value"); - } +pub fn get_from_env(key: &str, default_value: T, sensitive: bool) -> T { + if let Some(value) = get_from_env_opt(key, sensitive) { + value + } else { + info!(default_value=?default_value, "using environment variable `{key}` default value"); + default_value } - info!(value=?default_value, "using environment variable `{key}` default value"); - default_value } -pub fn get_from_env_opt(key: &str) -> Option { - let Some(value_str) = std::env::var(key).ok() else { - info!("environment variable `{key}` is not set"); - return None; - }; - if let Ok(value) = T::from_str(&value_str) { - info!(value=?value, "using environment variable `{key}` value"); - Some(value) +pub fn get_from_env_opt(key: &str, sensitive: bool) -> Option { + get_from_env_opt_aux(key, |val_str| val_str.parse().ok(), sensitive) +} + +pub fn get_bool_from_env_opt(key: &str) -> Option { + get_from_env_opt_aux(key, parse_bool_lenient, false) +} + +pub fn get_bool_from_env(key: &str, default_value: bool) -> bool { + if let Some(flag_value) = get_bool_from_env_opt(key) { + flag_value } else { - error!(value=%value_str, "failed to parse environment variable `{key}` value"); - None + info!(default_value=%default_value, "using environment variable `{key}` default value"); + default_value } } @@ -171,6 +190,40 @@ pub fn no_color() -> bool { matches!(env::var("NO_COLOR"), Ok(value) if !value.is_empty()) } +#[macro_export] +macro_rules! assert_eventually { + ($cond:expr, $timeout:expr, $interval:expr) => { + let start = std::time::Instant::now(); + loop { + if $cond { + break; + } + if start.elapsed() > $timeout { + panic!( + "assertion failed: condition `{}` never became true within {} ms", + stringify!($cond), + $timeout.as_millis() + ); + } + tokio::time::sleep($interval).await; + } + }; + ($cond:expr, $timeout:expr) => { + assert_eventually!($cond, $timeout, std::time::Duration::from_millis(50)); + }; + ($cond:expr) => { + assert_eventually!($cond, std::time::Duration::from_secs(1)); + }; +} + +/// Returns whether the given index ID corresponds to a metrics index. +/// +/// Metrics indexes use the Parquet/DataFusion pipeline instead of the Tantivy pipeline. +/// An index is considered a metrics index if it starts with "otel-metrics" or "metrics-". +pub fn is_metrics_index(index_id: &str) -> bool { + index_id.starts_with("otel-metrics") || index_id.starts_with("metrics-") +} + #[macro_export] macro_rules! ignore_error_kind { ($kind:path, $expr:expr) => { @@ -200,18 +253,6 @@ pub const fn div_ceil(lhs: i64, rhs: i64) -> i64 { } } -/// Return the number of vCPU/hyperthreads available. -/// This number is usually not equal to the number of cpu cores -pub fn num_cpus() -> usize { - match std::thread::available_parallelism() { - Ok(num_cpus) => num_cpus.get(), - Err(io_error) => { - error!(error=?io_error, "failed to detect the number of threads available: arbitrarily returning 2"); - 2 - } - } -} - // The following are helpers to build named tasks. // // Named tasks require the tokio feature `tracing` to be enabled. If the @@ -317,11 +358,11 @@ mod tests { // way, we are keeping it that way const TEST_KEY: &str = "TEST_KEY"; - assert_eq!(super::get_from_env(TEST_KEY, 10), 10); + assert_eq!(super::get_from_env(TEST_KEY, 10, false), 10); unsafe { std::env::set_var(TEST_KEY, "15") }; - assert_eq!(super::get_from_env(TEST_KEY, 10), 15); + assert_eq!(super::get_from_env(TEST_KEY, 10, false), 15); unsafe { std::env::set_var(TEST_KEY, "1invalidnumber") }; - assert_eq!(super::get_from_env(TEST_KEY, 10), 10); + assert_eq!(super::get_from_env(TEST_KEY, 10, false), 10); } #[test] @@ -383,6 +424,27 @@ mod tests { assert_eq!(div_ceil_u32(0, 3), 0); } + #[test] + fn test_is_metrics_index() { + // OpenTelemetry metrics indexes + assert!(is_metrics_index("otel-metrics-v0_7")); + assert!(is_metrics_index("otel-metrics")); + assert!(is_metrics_index("otel-metrics-custom")); + + // Generic metrics indexes + assert!(is_metrics_index("metrics-default")); + assert!(is_metrics_index("metrics-")); + assert!(is_metrics_index("metrics-my-app")); + + // Non-metrics indexes + assert!(!is_metrics_index("otel-logs-v0_7")); + assert!(!is_metrics_index("otel-traces-v0_7")); + assert!(!is_metrics_index("my-index")); + assert!(!is_metrics_index("logs-default")); + assert!(!is_metrics_index("metrics")); // No hyphen after "metrics" + assert!(!is_metrics_index("my-metrics-index")); // Not prefixed + } + #[test] fn test_parse_bool_lenient() { assert_eq!(parse_bool_lenient("true"), Some(true)); diff --git a/quickwit/quickwit-common/src/metrics.rs b/quickwit/quickwit-common/src/metrics.rs index c59bf953937..193def5e01a 100644 --- a/quickwit/quickwit-common/src/metrics.rs +++ b/quickwit/quickwit-common/src/metrics.rs @@ -13,9 +13,8 @@ // limitations under the License. use std::collections::{BTreeMap, HashMap}; -use std::sync::OnceLock; +use std::sync::{LazyLock, OnceLock}; -use once_cell::sync::Lazy; use prometheus::{Gauge, HistogramOpts, Opts, TextEncoder}; pub use prometheus::{ Histogram, HistogramTimer, HistogramVec as PrometheusHistogramVec, IntCounter, @@ -40,6 +39,26 @@ pub struct IntCounterVec { } impl IntCounterVec { + pub fn new( + name: &str, + help: &str, + subsystem: &str, + const_labels: &[(&str, &str)], + label_names: [&str; N], + ) -> IntCounterVec { + let owned_const_labels: HashMap = const_labels + .iter() + .map(|(label_name, label_value)| (label_name.to_string(), label_value.to_string())) + .collect(); + let counter_opts = Opts::new(name, help) + .namespace("quickwit") + .subsystem(subsystem) + .const_labels(owned_const_labels); + let underlying = PrometheusIntCounterVec::new(counter_opts, &label_names) + .expect("failed to create counter vec"); + IntCounterVec { underlying } + } + pub fn with_label_values(&self, label_values: [&str; N]) -> IntCounter { self.underlying.with_label_values(&label_values) } @@ -92,21 +111,10 @@ pub fn new_counter_vec( const_labels: &[(&str, &str)], label_names: [&str; N], ) -> IntCounterVec { - let owned_const_labels: HashMap = const_labels - .iter() - .map(|(label_name, label_value)| (label_name.to_string(), label_value.to_string())) - .collect(); - let counter_opts = Opts::new(name, help) - .namespace("quickwit") - .subsystem(subsystem) - .const_labels(owned_const_labels); - let underlying = PrometheusIntCounterVec::new(counter_opts, &label_names) - .expect("failed to create counter vec"); - - let collector = Box::new(underlying.clone()); + let int_counter_vec = IntCounterVec::new(name, help, subsystem, const_labels, label_names); + let collector = Box::new(int_counter_vec.underlying.clone()); prometheus::register(collector).expect("failed to register counter vec"); - - IntCounterVec { underlying } + int_counter_vec } pub fn new_float_gauge( @@ -429,16 +437,17 @@ impl InFlightDataGauges { } } -/// This function returns `index_name` or projects it to `` if per-index metrics are disabled. -pub fn index_label(index_name: &str) -> &str { - static PER_INDEX_METRICS_ENABLED: OnceLock = OnceLock::new(); - let per_index_metrics_enabled: bool = *PER_INDEX_METRICS_ENABLED - .get_or_init(|| !crate::get_bool_from_env("QW_DISABLE_PER_INDEX_METRICS", false)); - if per_index_metrics_enabled { - index_name +/// This function returns `index_id` as is if per-index metrics are enabled, or projects it to +/// `"__any__"` otherwise. +pub fn index_label(index_id: &str) -> &str { + static PER_INDEX_METRICS_ENABLED: LazyLock = + LazyLock::new(|| !crate::get_bool_from_env("QW_DISABLE_PER_INDEX_METRICS", false)); + + if *PER_INDEX_METRICS_ENABLED { + index_id } else { "__any__" } } -pub static MEMORY_METRICS: Lazy = Lazy::new(MemoryMetrics::default); +pub static MEMORY_METRICS: LazyLock = LazyLock::new(MemoryMetrics::default); diff --git a/quickwit/quickwit-common/src/net.rs b/quickwit/quickwit-common/src/net.rs index a06ee9af2e1..28e162baeb0 100644 --- a/quickwit/quickwit-common/src/net.rs +++ b/quickwit/quickwit-common/src/net.rs @@ -78,7 +78,9 @@ impl Display for Host { impl Serialize for Host { fn serialize(&self, serializer: S) -> Result - where S: Serializer { + where + S: Serializer, + { match self { Host::Hostname(hostname) => hostname.serialize(serializer), Host::IpAddr(ip_addr) => ip_addr.serialize(serializer), @@ -88,7 +90,9 @@ impl Serialize for Host { impl<'de> Deserialize<'de> for Host { fn deserialize(deserializer: D) -> Result - where D: serde::Deserializer<'de> { + where + D: serde::Deserializer<'de>, + { let host_str: String = Deserialize::deserialize(deserializer)?; host_str.parse().map_err(serde::de::Error::custom) } diff --git a/quickwit/quickwit-common/src/pretty.rs b/quickwit/quickwit-common/src/pretty.rs index 77f1b9c9c5f..ae2ce56113a 100644 --- a/quickwit/quickwit-common/src/pretty.rs +++ b/quickwit/quickwit-common/src/pretty.rs @@ -23,14 +23,15 @@ impl PrettySample { } } -impl fmt::Debug for PrettySample +impl fmt::Debug for PrettySample where - I: IntoIterator + Clone, - T: fmt::Debug, + I: IntoIterator + Clone, + I::Item: fmt::Debug, { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { write!(formatter, "[")?; - // in general we will get passed a reference (&[...], &HashMap...) or a Map<_> of them. + + // In general, we will receive a reference (&[...], &HashMap...) or a Map<_> of them. // So we either perform a Copy, or a cheap Clone of a simple struct let mut iter = self.0.clone().into_iter().enumerate(); for (i, item) in &mut iter { @@ -55,9 +56,9 @@ pub trait PrettyDisplay { fn pretty_display(&self) -> impl fmt::Display; } -struct PrettyDurationDisplay<'a>(&'a Duration); +struct DurationPrettyDisplay<'a>(&'a Duration); -impl fmt::Display for PrettyDurationDisplay<'_> { +impl fmt::Display for DurationPrettyDisplay<'_> { fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { // This is enough for my current use cases. To be extended as you see fit. let duration_millis = self.0.as_millis(); @@ -76,7 +77,37 @@ impl fmt::Display for PrettyDurationDisplay<'_> { impl PrettyDisplay for Duration { fn pretty_display(&self) -> impl fmt::Display { - PrettyDurationDisplay(self) + DurationPrettyDisplay(self) + } +} + +struct SequencePrettyDisplay(I); + +impl fmt::Display for SequencePrettyDisplay +where + I: IntoIterator + Clone, + I::Item: fmt::Display, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "[")?; + + // In general, we will receive a reference (&[...], &HashMap...) or a Map<_> of them. + // So we either perform a Copy, or a cheap Clone of a simple struct + let mut iter = self.0.clone().into_iter().peekable(); + + while let Some(item) = iter.next() { + write!(f, "{item}")?; + if iter.peek().is_some() { + write!(f, ", ")?; + } + } + write!(f, "]") + } +} + +impl PrettyDisplay for &[T] { + fn pretty_display(&self) -> impl fmt::Display { + SequencePrettyDisplay(*self) } } @@ -103,17 +134,29 @@ mod tests { } #[test] - fn test_pretty_duration() { - let pretty_duration = Duration::from_millis(0); - assert_eq!(format!("{}", pretty_duration.pretty_display()), "0ms"); + fn test_duration_pretty_display() { + let duration = Duration::from_millis(0); + assert_eq!(format!("{}", duration.pretty_display()), "0ms"); - let pretty_duration = Duration::from_millis(125); - assert_eq!(format!("{}", pretty_duration.pretty_display()), "125ms"); + let duration = Duration::from_millis(125); + assert_eq!(format!("{}", duration.pretty_display()), "125ms"); + + let duration = Duration::from_millis(1_000); + assert_eq!(format!("{}", duration.pretty_display()), "1.0s"); + + let duration = Duration::from_millis(1_125); + assert_eq!(format!("{}", duration.pretty_display()), "1.12s"); + } + + #[test] + fn test_sequence_pretty_display() { + let empty_slice: &[i32] = &[]; + assert_eq!(format!("{}", empty_slice.pretty_display()), "[]"); - let pretty_duration = Duration::from_millis(1_000); - assert_eq!(format!("{}", pretty_duration.pretty_display()), "1.0s"); + let slice_one: &[i32] = &[1]; + assert_eq!(format!("{}", slice_one.pretty_display()), "[1]"); - let pretty_duration = Duration::from_millis(1_125); - assert_eq!(format!("{}", pretty_duration.pretty_display()), "1.12s"); + let slice_two: &[i32] = &[1, 2]; + assert_eq!(format!("{}", slice_two.pretty_display()), "[1, 2]"); } } diff --git a/quickwit/quickwit-common/src/progress.rs b/quickwit/quickwit-common/src/progress.rs index 4fb3bb19238..57fd9e7bda2 100644 --- a/quickwit/quickwit-common/src/progress.rs +++ b/quickwit/quickwit-common/src/progress.rs @@ -82,7 +82,9 @@ impl Progress { /// Executes a future in a protected zone. pub async fn protect_future(&self, future: Fut) -> T - where Fut: Future { + where + Fut: Future, + { let _guard = self.protect_zone(); future.await } diff --git a/quickwit/quickwit-common/src/pubsub.rs b/quickwit/quickwit-common/src/pubsub.rs index ccfab8d438e..25cbaf3e71e 100644 --- a/quickwit/quickwit-common/src/pubsub.rs +++ b/quickwit/quickwit-common/src/pubsub.rs @@ -125,7 +125,9 @@ impl EventBroker { /// the callback future will be aborted. #[must_use] pub fn subscribe(&self, subscriber: impl EventSubscriber) -> EventSubscriptionHandle - where E: Event { + where + E: Event, + { self.subscribe_aux(subscriber, true) } @@ -145,7 +147,9 @@ impl EventBroker { /// Publishes an event. pub fn publish(&self, event: E) - where E: Event { + where + E: Event, + { let subscriptions = self .inner .subscriptions diff --git a/quickwit/quickwit-common/src/rand.rs b/quickwit/quickwit-common/src/rand.rs index a8bfde5a96e..ce8478d4228 100644 --- a/quickwit/quickwit-common/src/rand.rs +++ b/quickwit/quickwit-common/src/rand.rs @@ -13,11 +13,11 @@ // limitations under the License. use rand::Rng; -use rand::distributions::Alphanumeric; +use rand::distr::Alphanumeric; /// Appends a random suffix composed of a hyphen and five random alphanumeric characters. pub fn append_random_suffix(string: &str) -> String { - let rng = rand::thread_rng(); + let rng = rand::rng(); let mut randomized_string = String::with_capacity(string.len() + 6); randomized_string.push_str(string); randomized_string.push('-'); diff --git a/quickwit/quickwit-common/src/rate_limited_tracing.rs b/quickwit/quickwit-common/src/rate_limited_tracing.rs index c9a323f9ec2..198c2bf8bdd 100644 --- a/quickwit/quickwit-common/src/rate_limited_tracing.rs +++ b/quickwit/quickwit-common/src/rate_limited_tracing.rs @@ -179,12 +179,13 @@ fn _check_macro_works() { #[doc(hidden)] pub use coarsetime::Instant as CoarsetimeInstant; +pub use rate_limited_debug; +pub use rate_limited_error; +pub use rate_limited_info; +pub use rate_limited_trace; #[doc(hidden)] pub use rate_limited_tracing; -pub use { - rate_limited_debug, rate_limited_error, rate_limited_info, rate_limited_trace, - rate_limited_warn, -}; +pub use rate_limited_warn; #[cfg(test)] mod tests { diff --git a/quickwit/quickwit-common/src/rate_limiter.rs b/quickwit/quickwit-common/src/rate_limiter.rs index b4b690f83ca..d288a58604e 100644 --- a/quickwit/quickwit-common/src/rate_limiter.rs +++ b/quickwit/quickwit-common/src/rate_limiter.rs @@ -12,9 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::time::{Duration, Instant}; +use std::ops::Add; +use std::time::Duration; use bytesize::ByteSize; +use governor::clock::{Clock, DefaultClock, Reference}; +use governor::nanos::Nanos; use crate::tower::{ConstantRate, Rate}; @@ -53,65 +56,61 @@ impl Default for RateLimiterSettings { /// A bursty token-based rate limiter. #[derive(Debug, Clone)] -pub struct RateLimiter { +pub struct RateLimiter { // Maximum number of permits that can be accumulated. max_capacity: u64, // Number of permits available. available_permits: u64, refill_amount: u64, refill_period: Duration, - refill_period_micros: u64, - refill_at: Instant, + refill_period_nanos: u64, + refill_at: C::Instant, + clock: C, } -impl RateLimiter { - /// Creates a new rate limiter from the given settings. +#[cfg(any(test, feature = "testsuite"))] +impl Default for RateLimiter { + fn default() -> Self { + Self::from_settings(RateLimiterSettings::default()) + } +} + +impl RateLimiter { + /// Creates a new rate limiter from the given settings using the default clock. pub fn from_settings(settings: RateLimiterSettings) -> Self { + Self::from_settings_with_clock(settings, DefaultClock::default()) + } +} + +impl RateLimiter { + /// Creates a new rate limiter from the given settings with a custom clock. + pub fn from_settings_with_clock(settings: RateLimiterSettings, clock: C) -> Self { let max_capacity = settings.burst_limit; let refill_period = settings.refill_period; let rate_limit = settings.rate_limit.rescale(refill_period); - let now = Instant::now(); + let now = clock.now(); Self { max_capacity, available_permits: max_capacity, refill_amount: rate_limit.work(), refill_period, - refill_period_micros: refill_period.as_micros() as u64, - refill_at: now + refill_period, + refill_period_nanos: refill_period.as_nanos() as u64, + refill_at: now.add(Nanos::from(refill_period)), + clock, } } /// Returns the number of permits available. - pub fn available_permits(&self) -> u64 { + pub fn available_permits(&mut self) -> u64 { + self.refill(self.clock.now()); self.available_permits } /// Acquires some permits from the rate limiter. Returns whether the permits were acquired. pub fn acquire(&mut self, num_permits: u64) -> bool { - if self.acquire_inner(num_permits) { - true - } else { - self.refill(Instant::now()); - self.acquire_inner(num_permits) - } - } - - /// Acquires some permits from the rate limiter. - /// If the permits are not available, returns the duration to wait before trying again. - /// - /// This method is currently only used in simian. - pub fn acquire_with_duration(&mut self, num_permits: u64) -> Result<(), Duration> { - if self.acquire_inner(num_permits) { - return Ok(()); - } - self.refill(Instant::now()); - if self.acquire_inner(num_permits) { - return Ok(()); - } - let missing = num_permits - self.available_permits; - let wait = Duration::from_micros(missing * self.refill_period_micros / self.refill_amount); - Err(wait) + self.refill(self.clock.now()); + self.acquire_inner(num_permits) } /// Acquires some permits expressed in bytes from the rate limiter. Returns whether the permits @@ -124,7 +123,7 @@ impl RateLimiter { /// guarded by the rate limiter for one refill period. pub fn drain(&mut self) { self.available_permits = 0; - self.refill_at = Instant::now() + self.refill_period; + self.refill_at = self.clock.now().add(Nanos::from(self.refill_period)); } /// Gives back some unused permits to the rate limiter. @@ -141,20 +140,23 @@ impl RateLimiter { } } - fn refill(&mut self, now: Instant) { - if now < self.refill_at { + fn refill(&mut self, now: C::Instant) { + if now.lt(&self.refill_at) { return; } - let elapsed = (now - self.refill_at).as_micros() as u64; + let elapsed_nanos = now.duration_since(self.refill_at).as_u64(); // More than one refill period may have elapsed so we need to take that into account. - let refill = self.refill_amount + self.refill_amount * elapsed / self.refill_period_micros; + let refill = + self.refill_amount + self.refill_amount * elapsed_nanos / self.refill_period_nanos; self.available_permits = self.max_capacity.min(self.available_permits + refill); - self.refill_at = now + self.refill_period; + self.refill_at = now.add(Nanos::from(self.refill_period)); } } #[cfg(test)] mod tests { + use governor::clock::FakeRelativeClock; + use super::*; #[test] @@ -164,7 +166,8 @@ mod tests { rate_limit: ConstantRate::bytes_per_sec(ByteSize::mb(1)), refill_period: Duration::from_millis(100), }; - let mut rate_limiter = RateLimiter::from_settings(settings); + let clock = FakeRelativeClock::default(); + let mut rate_limiter = RateLimiter::from_settings_with_clock(settings, clock.clone()); assert_eq!(rate_limiter.max_capacity, ByteSize::mb(2).as_u64()); assert_eq!(rate_limiter.available_permits, ByteSize::mb(2).as_u64()); assert_eq!(rate_limiter.refill_amount, ByteSize::kb(100).as_u64()); @@ -174,12 +177,12 @@ mod tests { assert!(rate_limiter.acquire_bytes(ByteSize::mb(1))); assert!(!rate_limiter.acquire_bytes(ByteSize::kb(1))); - std::thread::sleep(Duration::from_millis(100)); + clock.advance(Duration::from_millis(100)); assert!(rate_limiter.acquire_bytes(ByteSize::kb(100))); assert!(!rate_limiter.acquire_bytes(ByteSize::kb(20))); - std::thread::sleep(Duration::from_millis(250)); + clock.advance(Duration::from_millis(250)); assert!(rate_limiter.acquire_bytes(ByteSize::kb(125))); assert!(rate_limiter.acquire_bytes(ByteSize::kb(125))); @@ -193,14 +196,17 @@ mod tests { rate_limit: ConstantRate::bytes_per_sec(ByteSize::mb(1)), refill_period: Duration::from_millis(100), }; - let mut rate_limiter = RateLimiter::from_settings(settings); + let clock = FakeRelativeClock::default(); + let mut rate_limiter = RateLimiter::from_settings_with_clock(settings, clock.clone()); rate_limiter.drain(); assert_eq!(rate_limiter.available_permits, 0); - rate_limiter.refill(Instant::now() + Duration::from_millis(50)); + clock.advance(Duration::from_millis(50)); + rate_limiter.refill(clock.now()); assert_eq!(rate_limiter.available_permits, 0); - rate_limiter.refill(Instant::now() + Duration::from_millis(100)); + clock.advance(Duration::from_millis(50)); + rate_limiter.refill(clock.now()); assert!(rate_limiter.available_permits >= ByteSize::kb(100).as_u64()); } @@ -229,26 +235,40 @@ mod tests { rate_limit: ConstantRate::bytes_per_sec(ByteSize::mb(1)), refill_period: Duration::from_millis(100), }; - let mut rate_limiter = RateLimiter::from_settings(settings); + let clock = FakeRelativeClock::default(); + let mut rate_limiter = RateLimiter::from_settings_with_clock(settings, clock.clone()); rate_limiter.available_permits = 0; - let now = Instant::now(); - rate_limiter.refill(now); assert_eq!(rate_limiter.available_permits, 0); rate_limiter.available_permits = 0; - let now = now + Duration::from_millis(100); - rate_limiter.refill(now); + clock.advance(Duration::from_millis(100)); + rate_limiter.refill(clock.now()); assert_eq!(rate_limiter.available_permits, ByteSize::kb(100).as_u64()); rate_limiter.available_permits = 0; - let now = now + Duration::from_millis(110); - rate_limiter.refill(now); + clock.advance(Duration::from_millis(110)); + rate_limiter.refill(clock.now()); assert_eq!(rate_limiter.available_permits, ByteSize::kb(110).as_u64()); rate_limiter.available_permits = 0; - let now = now + Duration::from_millis(210); - rate_limiter.refill(now); + clock.advance(Duration::from_millis(210)); + rate_limiter.refill(clock.now()); assert_eq!(rate_limiter.available_permits, ByteSize::kb(210).as_u64()); } + + #[test] + fn test_rate_limiter_available_permits() { + let settings = RateLimiterSettings { + burst_limit: ByteSize::mb(2).as_u64(), + rate_limit: ConstantRate::bytes_per_sec(ByteSize::mb(1)), + refill_period: Duration::from_millis(100), + }; + let clock = FakeRelativeClock::default(); + let mut rate_limiter = RateLimiter::from_settings_with_clock(settings, clock.clone()); + + rate_limiter.available_permits = 0; + clock.advance(Duration::from_millis(100)); + assert_eq!(rate_limiter.available_permits(), ByteSize::kb(100).as_u64()); + } } diff --git a/quickwit/quickwit-common/src/retry.rs b/quickwit/quickwit-common/src/retry.rs index 77e3f4d8d0e..93e1c3d099b 100644 --- a/quickwit/quickwit-common/src/retry.rs +++ b/quickwit/quickwit-common/src/retry.rs @@ -106,7 +106,7 @@ impl RetryParams { let capped_delay_ms = delay_ms.min(self.max_delay.as_millis() as u64); let half_delay_ms = capped_delay_ms.div_ceil(2); let jitter_range = half_delay_ms..capped_delay_ms + 1; - let jittered_delay_ms = rand::thread_rng().gen_range(jitter_range); + let jittered_delay_ms = rand::rng().random_range(jitter_range); Duration::from_millis(jittered_delay_ms) } diff --git a/quickwit/quickwit-common/src/ring_buffer.rs b/quickwit/quickwit-common/src/ring_buffer.rs new file mode 100644 index 00000000000..5d884d8188f --- /dev/null +++ b/quickwit/quickwit-common/src/ring_buffer.rs @@ -0,0 +1,170 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fmt::{Debug, Formatter}; + +/// Fixed-size buffer that keeps the last N elements pushed into it. +/// +/// `head` is the write cursor. It advances by one on each push and wraps +/// back to 0 when it reaches N, overwriting the oldest element. +/// +/// ```text +/// RingBuffer after pushing 1, 2, 3, 4, 5, 6: +/// +/// buffer = [5, 6, 3, 4] head = 2 len = 4 +/// ^ +/// next write goes here +/// +/// logical view (oldest → newest): [3, 4, 5, 6] +/// ``` +pub struct RingBuffer { + buffer: [T; N], + head: usize, + len: usize, +} + +impl Default for RingBuffer { + fn default() -> Self { + Self { + buffer: [T::default(); N], + head: 0, + len: 0, + } + } +} + +impl Debug for RingBuffer { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_list().entries(self.iter()).finish() + } +} + +impl RingBuffer { + pub fn push_back(&mut self, value: T) { + self.buffer[self.head] = value; + self.head = (self.head + 1) % N; + if self.len < N { + self.len += 1; + } + } + + pub fn last(&self) -> Option { + if self.len == 0 { + return None; + } + Some(self.buffer[(self.head + N - 1) % N]) + } + + pub fn front(&self) -> Option { + if self.len == 0 { + return None; + } + Some(self.buffer[(self.head + N - self.len) % N]) + } + + pub fn len(&self) -> usize { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + pub fn iter(&self) -> impl Iterator + '_ { + let start = (self.head + N - self.len) % N; + (0..self.len).map(move |i| &self.buffer[(start + i) % N]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_empty() { + let rb = RingBuffer::::default(); + assert!(rb.is_empty()); + assert_eq!(rb.len(), 0); + assert_eq!(rb.last(), None); + assert_eq!(rb.front(), None); + assert_eq!(rb.iter().count(), 0); + } + + #[test] + fn test_single_push() { + let mut rb = RingBuffer::::default(); + rb.push_back(10); + assert_eq!(rb.len(), 1); + assert!(!rb.is_empty()); + assert_eq!(rb.last(), Some(10)); + assert_eq!(rb.front(), Some(10)); + assert_eq!(rb.iter().copied().collect::>(), vec![10]); + } + + #[test] + fn test_partial_fill() { + let mut rb = RingBuffer::::default(); + rb.push_back(1); + rb.push_back(2); + rb.push_back(3); + assert_eq!(rb.len(), 3); + assert_eq!(rb.last(), Some(3)); + assert_eq!(rb.front(), Some(1)); + assert_eq!(rb.iter().copied().collect::>(), vec![1, 2, 3]); + } + + #[test] + fn test_exactly_full() { + let mut rb = RingBuffer::::default(); + for i in 1..=4 { + rb.push_back(i); + } + assert_eq!(rb.len(), 4); + assert_eq!(rb.last(), Some(4)); + assert_eq!(rb.front(), Some(1)); + assert_eq!(rb.iter().copied().collect::>(), vec![1, 2, 3, 4]); + } + + #[test] + fn test_wrap_around() { + let mut rb = RingBuffer::::default(); + for i in 1..=6 { + rb.push_back(i); + } + assert_eq!(rb.len(), 4); + assert_eq!(rb.last(), Some(6)); + assert_eq!(rb.front(), Some(3)); + assert_eq!(rb.iter().copied().collect::>(), vec![3, 4, 5, 6]); + } + + #[test] + fn test_many_wraps() { + let mut rb = RingBuffer::::default(); + for i in 1..=100 { + rb.push_back(i); + } + assert_eq!(rb.len(), 3); + assert_eq!(rb.last(), Some(100)); + assert_eq!(rb.front(), Some(98)); + assert_eq!(rb.iter().copied().collect::>(), vec![98, 99, 100]); + } + + #[test] + fn test_debug() { + let mut rb = RingBuffer::::default(); + rb.push_back(1); + rb.push_back(2); + assert_eq!(format!("{:?}", rb), "[1, 2]"); + } +} diff --git a/quickwit/quickwit-common/src/runtimes.rs b/quickwit/quickwit-common/src/runtimes.rs index 813b159b461..e5d68117e00 100644 --- a/quickwit/quickwit-common/src/runtimes.rs +++ b/quickwit/quickwit-common/src/runtimes.rs @@ -62,14 +62,26 @@ impl RuntimesConfig { } pub fn with_num_cpus(num_cpus: usize) -> Self { - // Non blocking task are supposed to be io intensive, and not require many threads... - let num_threads_non_blocking = if num_cpus > 6 { 2 } else { 1 }; + // Non blocking task are supposed to be io intensive, and not require many threads. // On the other hand the blocking actors are cpu intensive. We allocate // almost all of the threads to them. - let num_threads_blocking = (num_cpus - num_threads_non_blocking).max(1); - RuntimesConfig { - num_threads_non_blocking, - num_threads_blocking, + match num_cpus { + 0..=3 => { + // We do not have enough vCPUs to allocate a full thread to + // non-blocking. + RuntimesConfig { + num_threads_non_blocking: 1, + num_threads_blocking: num_cpus, + } + } + 4..=6 => RuntimesConfig { + num_threads_non_blocking: 1, + num_threads_blocking: num_cpus - 1, + }, + 7.. => RuntimesConfig { + num_threads_non_blocking: 2, + num_threads_blocking: num_cpus - 2, + }, } } } @@ -84,7 +96,7 @@ impl Default for RuntimesConfig { fn start_runtimes(config: RuntimesConfig) -> HashMap { let mut runtimes = HashMap::with_capacity(2); - //let disable_lifo_slot = crate::get_bool_from_env("QW_DISABLE_TOKIO_LIFO_SLOT", false); + let _disable_lifo_slot = crate::get_bool_from_env("QW_DISABLE_TOKIO_LIFO_SLOT", true); let mut blocking_runtime_builder = tokio::runtime::Builder::new_multi_thread(); //if disable_lifo_slot { @@ -160,26 +172,27 @@ pub fn scrape_tokio_runtime_metrics(handle: &tokio::runtime::Handle, label: &'st loop { interval.tick().await; let metrics = handle_clone.metrics(); - + // Calculate cumulative busy duration across all workers let mut total_busy_duration = Duration::ZERO; let mut total_local_queue_depth = 0; - + for worker_id in 0..metrics.num_workers() { total_busy_duration += metrics.worker_total_busy_duration(worker_id); total_local_queue_depth += metrics.worker_local_queue_depth(worker_id); } - + // Calculate busy ratio as the delta since last measurement - let busy_duration_delta = total_busy_duration.saturating_sub(last_total_busy_duration); + let busy_duration_delta = + total_busy_duration.saturating_sub(last_total_busy_duration); let busy_ratio = busy_duration_delta.as_secs_f64() / 1.0; // 1 second interval last_total_busy_duration = total_busy_duration; - + prometheus_runtime_metrics.update_with_values( total_local_queue_depth, busy_duration_delta, busy_ratio, - metrics.num_workers() + metrics.num_workers(), ); } }); @@ -191,7 +204,7 @@ pub fn scrape_tokio_runtime_metrics(handle: &tokio::runtime::Handle, label: &'st handle.spawn(async move { let _interval = tokio::time::interval(Duration::from_secs(1)); let _prometheus_runtime_metrics = PrometheusRuntimeMetrics::new(label); - + // Without tokio_unstable, we can't access detailed runtime metrics // This is a no-op implementation to maintain compatibility tracing::warn!("Runtime metrics collection requires tokio_unstable feature"); @@ -237,19 +250,18 @@ impl PrometheusRuntimeMetrics { } } - pub fn update_with_values(&mut self, + pub fn update_with_values( + &mut self, total_local_queue_depth: usize, busy_duration_delta: Duration, busy_ratio: f64, - num_workers: usize + num_workers: usize, ) { - self.scheduled_tasks - .set(total_local_queue_depth as i64); + self.scheduled_tasks.set(total_local_queue_depth as i64); self.worker_busy_duration_milliseconds_total .inc_by(busy_duration_delta.as_millis() as u64); self.worker_busy_ratio.set(busy_ratio.min(1.0)); - self.worker_threads - .set(num_workers as i64); + self.worker_threads.set(num_workers as i64); } } @@ -274,7 +286,7 @@ mod tests { #[test] fn test_runtimes_with_given_num_cpus_3() { let runtime = RuntimesConfig::with_num_cpus(3); - assert_eq!(runtime.num_threads_blocking, 2); + assert_eq!(runtime.num_threads_blocking, 3); assert_eq!(runtime.num_threads_non_blocking, 1); } } diff --git a/quickwit/quickwit-common/src/shared_consts.rs b/quickwit/quickwit-common/src/shared_consts.rs index 531121f9696..611636a5aa9 100644 --- a/quickwit/quickwit-common/src/shared_consts.rs +++ b/quickwit/quickwit-common/src/shared_consts.rs @@ -40,6 +40,7 @@ pub fn split_deletion_grace_period() -> Duration { let deletion_grace_period_secs: u64 = crate::get_from_env( "QW_SPLIT_DELETION_GRACE_PERIOD_SECS", DEFAULT_DELETION_GRACE_PERIOD.as_secs(), + false, ); let deletion_grace_period_secs_clamped: u64 = deletion_grace_period_secs.clamp( MINIMUM_DELETION_GRACE_PERIOD.as_secs(), @@ -60,9 +61,15 @@ pub fn split_deletion_grace_period() -> Duration { /// being requested. pub const SCROLL_BATCH_LEN: usize = 1_000; -/// Prefix used in chitchat to broadcast the list of primary shards hosted by a leader. +/// Key prefix used in chitchat to broadcast the list of primary shards hosted by a leader. pub const INGESTER_PRIMARY_SHARDS_PREFIX: &str = "ingester.primary_shards:"; +/// Key used in chitchat to broadcast the status of an ingester. +pub const INGESTER_STATUS_KEY: &str = "ingester.status"; + +/// Prefix used in chitchat to broadcast per-source ingester capacity scores and open shard counts. +pub const INGESTER_CAPACITY_SCORE_PREFIX: &str = "ingester.capacity_score:"; + /// File name for the encoded list of fields in the split pub const SPLIT_FIELDS_FILE_NAME: &str = "split_fields"; diff --git a/quickwit/quickwit-common/src/sorted_iter.rs b/quickwit/quickwit-common/src/sorted_iter.rs index 26fee863b55..823829792c7 100644 --- a/quickwit/quickwit-common/src/sorted_iter.rs +++ b/quickwit/quickwit-common/src/sorted_iter.rs @@ -24,7 +24,9 @@ trait Sorted {} pub trait SortedIterator: Iterator + Sized { /// Compares two sorted iterators and returns the diff. fn diff(self, other: U) -> DiffIterator - where U: SortedIterator { + where + U: SortedIterator, + { DiffIterator { left: self.peekable(), right: other.peekable(), @@ -110,7 +112,9 @@ impl Sorted for btree_set::Iter<'_, K> {} pub trait SortedByKeyIterator: Iterator + Sized { /// Compares the keys of two sorted key-value iterators and returns the diff. fn diff_by_key(self, other: U) -> DiffByKeyIterator - where U: SortedByKeyIterator { + where + U: SortedByKeyIterator, + { DiffByKeyIterator { left: self.peekable(), right: other.peekable(), diff --git a/quickwit/quickwit-common/src/stream_utils.rs b/quickwit/quickwit-common/src/stream_utils.rs index e0fc126b465..5c88cc5cc8b 100644 --- a/quickwit/quickwit-common/src/stream_utils.rs +++ b/quickwit/quickwit-common/src/stream_utils.rs @@ -34,7 +34,8 @@ pub struct ServiceStream { } impl ServiceStream -where T: Send + 'static +where + T: Send + 'static, { pub fn new(inner: BoxStream) -> Self { Self { inner } @@ -58,7 +59,8 @@ where T: Send + 'static } impl fmt::Debug for ServiceStream -where T: 'static +where + T: 'static, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "ServiceStream<{:?}>", TypeId::of::()) @@ -68,7 +70,8 @@ where T: 'static impl Unpin for ServiceStream {} impl ServiceStream -where T: Send + 'static +where + T: Send + 'static, { pub fn new_bounded(capacity: usize) -> (mpsc::Sender, Self) { let (sender, receiver) = mpsc::channel(capacity); @@ -107,7 +110,8 @@ where T: Send + 'static } impl ServiceStream -where T: Clone + Send + Sync + 'static +where + T: Clone + Send + Sync + 'static, { pub fn new_watch(init: T) -> (watch::Sender, Self) { let (sender, receiver) = watch::channel(init); @@ -143,7 +147,8 @@ impl Stream for ServiceStream { } impl From> for ServiceStream -where T: Send + 'static +where + T: Send + 'static, { fn from(receiver: mpsc::Receiver) -> Self { Self { @@ -153,7 +158,8 @@ where T: Send + 'static } impl From> for ServiceStream -where T: Send + 'static +where + T: Send + 'static, { fn from(receiver: mpsc::UnboundedReceiver) -> Self { Self { @@ -163,7 +169,8 @@ where T: Send + 'static } impl From> for ServiceStream -where T: Clone + Send + Sync + 'static +where + T: Clone + Send + Sync + 'static, { fn from(receiver: watch::Receiver) -> Self { Self { @@ -176,7 +183,8 @@ where T: Clone + Send + Sync + 'static /// an error is encountered, the stream will be closed and subsequent calls to `poll_next` will /// return `None`. impl From> for ServiceStream> -where T: Send + 'static +where + T: Send + 'static, { fn from(streaming: tonic::Streaming) -> Self { Self { @@ -188,7 +196,8 @@ where T: Send + 'static /// Adapts a client-side tonic::Streaming into a ServiceStream of `T`. Once an error is encountered, /// the stream will be closed and subsequent calls to `poll_next` will return `None`. impl From> for ServiceStream -where T: Send + 'static +where + T: Send + 'static, { fn from(streaming: tonic::Streaming) -> Self { let message_stream = stream::unfold(streaming, |mut streaming| { @@ -211,7 +220,8 @@ where T: Send + 'static #[cfg(any(test, feature = "testsuite"))] impl From> for ServiceStream -where T: Send + 'static +where + T: Send + 'static, { fn from(values: Vec) -> Self { Self { @@ -221,7 +231,8 @@ where T: Send + 'static } impl RpcName for ServiceStream -where T: RpcName +where + T: RpcName, { fn rpc_name() -> &'static str { T::rpc_name() @@ -231,7 +242,8 @@ where T: RpcName pub struct InFlightValue(T, #[allow(dead_code)] GaugeGuard<'static>); impl fmt::Debug for InFlightValue -where T: fmt::Debug +where + T: fmt::Debug, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self.0) diff --git a/quickwit/quickwit-common/src/temp_dir.rs b/quickwit/quickwit-common/src/temp_dir.rs index 2502441d23b..daf70f852d5 100644 --- a/quickwit/quickwit-common/src/temp_dir.rs +++ b/quickwit/quickwit-common/src/temp_dir.rs @@ -335,16 +335,16 @@ mod tests { #[test] fn test_prefix_random() { - let mut rng = rand::thread_rng(); + let mut rng = rand::rng(); let template = "A".repeat(100); for _ in 0..10000 { - let rand_bytes = rng.r#gen::() % 4; - let parts_num = rng.r#gen::() % 10; + let rand_bytes = rng.random_range(0..4); + let parts_num = rng.random_range(0..10); let mut builder = Builder::default(); builder.rand_bytes(rand_bytes); let mut max_size = 0; for _ in 0..parts_num { - let size = 1 + rand::random::() % 10; + let size = 1 + rng.random_range(0..10); builder.join(&template[0..size]); max_size += size + 1; } @@ -360,14 +360,14 @@ mod tests { } }; let limit_threshold = parts_num + separator_count + rand_bytes; - if parts_num > 0 && rng.r#gen::() { - builder.max_length(rand::random::() % limit_threshold); + if parts_num > 0 && rng.random() { + builder.max_length(rng.random_range(0..limit_threshold)); assert_eq!( "the filename limit is too small", builder.prefix().unwrap_err().to_string() ); } else { - let len = limit_threshold + rand::random::() % 100; + let len = limit_threshold + rng.random_range(0..100); builder.max_length(len); let builder_debug = format!("{builder:?}, len {len}"); let builder_prefix = builder.prefix().unwrap(); diff --git a/quickwit/quickwit-common/src/tower/box_service.rs b/quickwit/quickwit-common/src/tower/box_service.rs index cc5a545319a..d90202160eb 100644 --- a/quickwit/quickwit-common/src/tower/box_service.rs +++ b/quickwit/quickwit-common/src/tower/box_service.rs @@ -30,7 +30,8 @@ trait CloneService: dyn_clone::clone_trait_object!( CloneService); -impl CloneService for S where S: Service> +impl CloneService for S where + S: Service> + Clone + Send + Sync diff --git a/quickwit/quickwit-common/src/tower/buffer.rs b/quickwit/quickwit-common/src/tower/buffer.rs index 0a1ffe4595a..1c38e3c5fa6 100644 --- a/quickwit/quickwit-common/src/tower/buffer.rs +++ b/quickwit/quickwit-common/src/tower/buffer.rs @@ -34,7 +34,8 @@ pub enum BufferError { /// A wrapper around [`tower::buffer::Buffer`] service that preserves the original error type. pub struct Buffer -where S: Service +where + S: Service, { bound: usize, inner: TowerBuffer>::Future>, @@ -83,7 +84,9 @@ where /// Downcasts an error boxed as [`tower::BoxError`] by the buffer service back into the original /// error `E`. fn downcast_error(error: BoxError) -> E -where E: error::Error + From + Clone + 'static { +where + E: error::Error + From + Clone + 'static, +{ if let Some(error) = error.downcast_ref::() { return error.clone(); } @@ -92,19 +95,19 @@ where E: error::Error + From + Clone + 'static { return BufferError::Closed.into(); } // This happens when the inner service returns an error on `poll_ready`. - if let Some(service_error) = error.downcast_ref::() { - if let Some(source) = service_error.source() { - if let Some(inner) = source.downcast_ref::() { - return inner.clone(); - } - } + if let Some(service_error) = error.downcast_ref::() + && let Some(source) = service_error.source() + && let Some(inner) = source.downcast_ref::() + { + return inner.clone(); } // This will happen only if the buffer service implementation adds a new error type. BufferError::Unknown.into() } impl fmt::Debug for Buffer -where S: Service +where + S: Service, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Buffer") diff --git a/quickwit/quickwit-common/src/tower/delay.rs b/quickwit/quickwit-common/src/tower/delay.rs index cabb9e66e80..765e1235d87 100644 --- a/quickwit/quickwit-common/src/tower/delay.rs +++ b/quickwit/quickwit-common/src/tower/delay.rs @@ -29,7 +29,8 @@ pub struct Delay { } impl Service for Delay -where S: Service +where + S: Service, { type Response = S::Response; type Error = S::Error; @@ -59,7 +60,8 @@ pub struct DelayFuture { } impl Future for DelayFuture -where F: Future> +where + F: Future>, { type Output = Result; diff --git a/quickwit/quickwit-common/src/tower/estimate_rate.rs b/quickwit/quickwit-common/src/tower/estimate_rate.rs index e5ce1be45dc..f5ff40dd156 100644 --- a/quickwit/quickwit-common/src/tower/estimate_rate.rs +++ b/quickwit/quickwit-common/src/tower/estimate_rate.rs @@ -29,7 +29,8 @@ pub struct Handle { } impl Drop for Handle -where T: RateEstimator +where + T: RateEstimator, { fn drop(&mut self) { let ended_at = Instant::now(); @@ -48,7 +49,8 @@ pub struct EstimateRate { } impl EstimateRate -where T: RateEstimator +where + T: RateEstimator, { /// Creates a new rate estimator. pub fn new(service: S, estimator: T) -> Self { diff --git a/quickwit/quickwit-common/src/tower/load_shed.rs b/quickwit/quickwit-common/src/tower/load_shed.rs index d6f2e6eb4ea..f22353f6b92 100644 --- a/quickwit/quickwit-common/src/tower/load_shed.rs +++ b/quickwit/quickwit-common/src/tower/load_shed.rs @@ -31,7 +31,8 @@ pub struct LoadShed { } impl Clone for LoadShed -where S: Clone +where + S: Clone, { fn clone(&self) -> Self { Self { @@ -88,7 +89,8 @@ pub struct LoadShedFuture { } impl Future for LoadShedFuture -where F: Future> +where + F: Future>, { type Output = Result; diff --git a/quickwit/quickwit-common/src/tower/metrics.rs b/quickwit/quickwit-common/src/tower/metrics.rs index b2d093adbe3..87c29a2b46c 100644 --- a/quickwit/quickwit-common/src/tower/metrics.rs +++ b/quickwit/quickwit-common/src/tower/metrics.rs @@ -148,7 +148,8 @@ impl PinnedDrop for ResponseFuture { } impl Future for ResponseFuture -where F: Future> +where + F: Future>, { type Output = Result; diff --git a/quickwit/quickwit-common/src/tower/one_task_per_call_layer.rs b/quickwit/quickwit-common/src/tower/one_task_per_call_layer.rs index 0a9a6225c08..6e71fba7760 100644 --- a/quickwit/quickwit-common/src/tower/one_task_per_call_layer.rs +++ b/quickwit/quickwit-common/src/tower/one_task_per_call_layer.rs @@ -88,7 +88,8 @@ pub struct UnwrapOrElseFuture { } impl Future for UnwrapOrElseFuture -where E: From +where + E: From, { type Output = Result; diff --git a/quickwit/quickwit-common/src/tower/pool.rs b/quickwit/quickwit-common/src/tower/pool.rs index 87a6f15e879..32d34a83015 100644 --- a/quickwit/quickwit-common/src/tower/pool.rs +++ b/quickwit/quickwit-common/src/tower/pool.rs @@ -49,7 +49,8 @@ impl Clone for Pool { } impl Default for Pool -where K: Eq + PartialEq + Hash +where + K: Eq + PartialEq + Hash, { fn default() -> Self { Self { @@ -109,6 +110,16 @@ where .collect() } + /// Returns all the key-value pairs in the pool. + pub fn keys_values(&self) -> Vec<(K, V)> { + self.pool + .read() + .expect("lock should not be poisoned") + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect() + } + /// Returns all the values in the pool. pub fn values(&self) -> Vec { self.pool @@ -182,10 +193,13 @@ where } impl FromIterator<(K, V)> for Pool -where K: Eq + PartialEq + Hash +where + K: Eq + PartialEq + Hash, { fn from_iter(iter: I) -> Self - where I: IntoIterator { + where + I: IntoIterator, + { Self { pool: Arc::new(RwLock::new(HashMap::from_iter(iter))), } diff --git a/quickwit/quickwit-common/src/tower/rate_limit.rs b/quickwit/quickwit-common/src/tower/rate_limit.rs index 855c89c73ee..0c87920d177 100644 --- a/quickwit/quickwit-common/src/tower/rate_limit.rs +++ b/quickwit/quickwit-common/src/tower/rate_limit.rs @@ -43,7 +43,8 @@ enum State { } impl RateLimit -where T: Rate +where + T: Rate, { /// Creates a new rate limiter. pub fn new(inner: S, rate: T) -> Self { @@ -171,7 +172,8 @@ impl RateLimitLayer { } impl Layer for RateLimitLayer -where T: Rate +where + T: Rate, { type Service = RateLimit; diff --git a/quickwit/quickwit-common/src/tower/retry.rs b/quickwit/quickwit-common/src/tower/retry.rs index 4594aaa1cda..7d0fa8a2aec 100644 --- a/quickwit/quickwit-common/src/tower/retry.rs +++ b/quickwit/quickwit-common/src/tower/retry.rs @@ -25,7 +25,8 @@ use crate::retry::{RetryParams, Retryable}; /// Retry layer copy/pasted from `tower::retry::RetryLayer` /// but which implements `Clone`. impl Layer for RetryLayer

-where P: Clone +where + P: Clone, { type Service = Retry; diff --git a/quickwit/quickwit-common/src/tower/transport.rs b/quickwit/quickwit-common/src/tower/transport.rs index 79bd8707c61..6201fafe0fc 100644 --- a/quickwit/quickwit-common/src/tower/transport.rs +++ b/quickwit/quickwit-common/src/tower/transport.rs @@ -46,7 +46,8 @@ struct ChangeStreamAdapter { // A blanket `Discover` implementation exists for any `Stream, E>>` impl Stream for ChangeStreamAdapter -where K: Hash + Eq + Clone +where + K: Hash + Eq + Clone, { type Item = Result, Infallible>; @@ -91,7 +92,8 @@ pub struct BalanceChannel { } impl BalanceChannel -where K: Hash + Eq + Send + Sync + Clone + 'static +where + K: Hash + Eq + Send + Sync + Clone + 'static, { pub fn new() -> (Self, mpsc::UnboundedSender>) { let (change_tx, change_rx) = mpsc::unbounded_channel(); @@ -105,7 +107,9 @@ where K: Hash + Eq + Send + Sync + Clone + 'static } pub fn from_stream(changes: S) -> Self - where S: Stream> + Send + Unpin + 'static { + where + S: Stream> + Send + Unpin + 'static, + { let (connection_keys_tx, connection_keys_rx) = watch::channel(HashSet::new()); let change_stream = unlazy_stream(ChangeStreamAdapter:: { changes: Box::pin(changes), @@ -168,7 +172,8 @@ where } impl Service for BalanceChannel -where K: Hash + Eq + Clone + Send +where + K: Hash + Eq + Clone + Send, { type Response = HttpResponse; type Error = BoxError; @@ -184,7 +189,8 @@ where K: Hash + Eq + Clone + Send } impl fmt::Debug for BalanceChannel -where K: Hash + Eq + Clone + Send + Sync + 'static +where + K: Hash + Eq + Clone + Send + Sync + 'static, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("BalanceChannel") diff --git a/quickwit/quickwit-common/src/uri.rs b/quickwit/quickwit-common/src/uri.rs index 92a702267bf..c3b6bc17b40 100644 --- a/quickwit/quickwit-common/src/uri.rs +++ b/quickwit/quickwit-common/src/uri.rs @@ -139,7 +139,7 @@ impl Uri { } /// Strips sensitive information such as credentials from URI. - fn as_redacted_str(&self) -> Cow { + fn as_redacted_str(&self) -> Cow<'_, str> { if self.protocol().is_database() { static DATABASE_URI_PATTERN: OnceCell = OnceCell::new(); DATABASE_URI_PATTERN @@ -344,7 +344,9 @@ impl PartialEq for Uri { impl<'de> Deserialize<'de> for Uri { fn deserialize(deserializer: D) -> Result - where D: serde::Deserializer<'de> { + where + D: serde::Deserializer<'de>, + { let uri_str: Cow<'de, str> = Deserialize::deserialize(deserializer)?; let uri = Uri::from_str(&uri_str).map_err(D::Error::custom)?; Ok(uri) @@ -353,7 +355,9 @@ impl<'de> Deserialize<'de> for Uri { impl Serialize for Uri { fn serialize(&self, serializer: S) -> Result - where S: Serializer { + where + S: Serializer, + { serializer.serialize_str(&self.uri) } } diff --git a/quickwit/quickwit-config/Cargo.toml b/quickwit/quickwit-config/Cargo.toml index df52a40ced7..a1877661490 100644 --- a/quickwit/quickwit-config/Cargo.toml +++ b/quickwit/quickwit-config/Cargo.toml @@ -17,11 +17,11 @@ bytesize = { workspace = true } chrono = { workspace = true } cron = { workspace = true } enum-iterator = { workspace = true } +http = { workspace = true } http-serde = { workspace = true } humantime = { workspace = true } itertools = { workspace = true } json_comments = { workspace = true } -legacy-http = { workspace = true } new_string_template = { workspace = true } once_cell = { workspace = true } regex = { workspace = true } @@ -43,6 +43,7 @@ quickwit-proto = { workspace = true } tokio = { workspace = true } quickwit-proto = { workspace = true, features = ["testsuite"] } +quickwit-common = { workspace = true, features = ["testsuite"] } [features] testsuite = [] diff --git a/quickwit/quickwit-config/resources/tests/node_config/quickwit.json b/quickwit/quickwit-config/resources/tests/node_config/quickwit.json index 0b881f85a13..7269b37ae22 100644 --- a/quickwit/quickwit-config/resources/tests/node_config/quickwit.json +++ b/quickwit/quickwit-config/resources/tests/node_config/quickwit.json @@ -3,6 +3,7 @@ "version": "0.7", "cluster_id": "quickwit-cluster", "node_id": "my-unique-node-id", + "availability_zone": "az-1", "enabled_services": [ "janitor", "metastore" @@ -69,6 +70,16 @@ "min_throughtput_bytes_per_secs": 100000, "timeout_millis": 2000, "max_num_retries": 2 + }, + "lambda": { + "function_name": "quickwit-lambda-leaf-search", + "max_splits_per_invocation": 10, + "offload_threshold": 30, + "auto_deploy": { + "execution_role_arn": "arn:aws:iam::123456789012:role/quickwit-lambda-role", + "memory_size": "5 GiB", + "invocation_timeout_secs": 15 + } } }, "jaeger": { diff --git a/quickwit/quickwit-config/resources/tests/node_config/quickwit.toml b/quickwit/quickwit-config/resources/tests/node_config/quickwit.toml index 97b44376a75..ea715dcffe0 100644 --- a/quickwit/quickwit-config/resources/tests/node_config/quickwit.toml +++ b/quickwit/quickwit-config/resources/tests/node_config/quickwit.toml @@ -2,6 +2,7 @@ version = "0.7" cluster_id = "quickwit-cluster" node_id = "my-unique-node-id" +availability_zone = "az-1" enabled_services = [ "janitor", "metastore" ] listen_address = "0.0.0.0" advertise_address = "172.0.0.12" @@ -61,6 +62,16 @@ min_throughtput_bytes_per_secs = 100000 timeout_millis = 2000 max_num_retries = 2 +[searcher.lambda] +function_name = "quickwit-lambda-leaf-search" +max_splits_per_invocation = 10 +offload_threshold = 30 + +[searcher.lambda.auto_deploy] +execution_role_arn = "arn:aws:iam::123456789012:role/quickwit-lambda-role" +memory_size = "5 GiB" +invocation_timeout_secs = 15 + [jaeger] enable_endpoint = true lookback_period_hours = 24 diff --git a/quickwit/quickwit-config/resources/tests/node_config/quickwit.yaml b/quickwit/quickwit-config/resources/tests/node_config/quickwit.yaml index c23ec0c0e3f..face0852972 100644 --- a/quickwit/quickwit-config/resources/tests/node_config/quickwit.yaml +++ b/quickwit/quickwit-config/resources/tests/node_config/quickwit.yaml @@ -2,6 +2,7 @@ version: 0.8 cluster_id: quickwit-cluster node_id: my-unique-node-id +availability_zone: az-1 enabled_services: - janitor - metastore @@ -63,6 +64,14 @@ searcher: min_throughtput_bytes_per_secs: 100000 timeout_millis: 2000 max_num_retries: 2 + lambda: + function_name: quickwit-lambda-leaf-search + max_splits_per_invocation: 10 + offload_threshold: 30 + auto_deploy: + execution_role_arn: arn:aws:iam::123456789012:role/quickwit-lambda-role + memory_size: 5 GiB + invocation_timeout_secs: 15 jaeger: enable_endpoint: true diff --git a/quickwit/quickwit-config/src/config_value.rs b/quickwit/quickwit-config/src/config_value.rs index d7183e72737..243fab346ae 100644 --- a/quickwit/quickwit-config/src/config_value.rs +++ b/quickwit/quickwit-config/src/config_value.rs @@ -18,6 +18,7 @@ use std::{any, fmt}; use anyhow::{self, Context}; use serde::{Deserialize, Deserializer}; +use tracing::warn; use crate::qw_env_vars::{QW_ENV_VARS, QW_NONE}; @@ -41,18 +42,17 @@ where } } - #[cfg(test)] - pub(crate) fn for_test(value: T) -> Self { + pub(crate) fn none() -> Self { Self { - provided: Some(value), + provided: None, default: None, } } #[cfg(test)] - pub(crate) fn none() -> Self { + pub(crate) fn for_test(value: T) -> Self { Self { - provided: None, + provided: Some(value), default: None, } } @@ -67,19 +67,28 @@ where env_vars: &HashMap, ) -> anyhow::Result> { // QW env vars take precedence over the config file values. - if E > QW_NONE { - if let Some(env_var_key) = QW_ENV_VARS.get(&E) { - if let Some(env_var_value) = env_vars.get(*env_var_key) { - let value = env_var_value.parse::().map_err(|error| { - anyhow::anyhow!( - "failed to convert value `{env_var_value}` read from environment \ - variable `{env_var_key}` to type `{}`: {error:?}", - any::type_name::(), - ) - })?; - return Ok(Some(value)); + if E > QW_NONE + && let Some(env_var_key) = QW_ENV_VARS.get(&E) + && let Some(env_var_value) = env_vars.get(*env_var_key).filter(|val| { + if val.is_empty() { + warn!( + "environment variable `{}` is set but value is empty", + *env_var_key + ); + false + } else { + true } - } + }) + { + let value = env_var_value.parse::().map_err(|error| { + anyhow::anyhow!( + "failed to convert value `{env_var_value}` read from environment variable \ + `{env_var_key}` to type `{}`: {error:?}", + any::type_name::(), + ) + })?; + return Ok(Some(value)); } Ok(self.provided.or(self.default)) } @@ -93,7 +102,8 @@ where } impl Default for ConfigValue -where T: Default +where + T: Default, { fn default() -> Self { Self { @@ -104,10 +114,13 @@ where T: Default } impl<'de, T, const E: usize> Deserialize<'de> for ConfigValue -where T: Deserialize<'de> +where + T: Deserialize<'de>, { fn deserialize(deserializer: D) -> Result, D::Error> - where D: Deserializer<'de> { + where + D: Deserializer<'de>, + { let value: Option = Deserialize::deserialize(deserializer)?; Ok(ConfigValue { provided: value, @@ -120,7 +133,7 @@ where T: Deserialize<'de> mod tests { use super::*; use crate::qw_env_vars::{ - QW_CLUSTER_ID, QW_GOSSIP_LISTEN_PORT, QW_NODE_ID, QW_REST_LISTEN_PORT, + QW_AVAILABILITY_ZONE, QW_CLUSTER_ID, QW_GOSSIP_LISTEN_PORT, QW_NODE_ID, QW_REST_LISTEN_PORT, }; #[test] @@ -193,6 +206,14 @@ mod tests { rest_listen_port.resolve(&env_vars).unwrap_err(); } + #[test] + fn test_config_value_resolve_optional_empty_string() { + let mut env_vars = HashMap::new(); + env_vars.insert("QW_AVAILABILITY_ZONE".to_string(), "".to_string()); + let az = ConfigValue::::none(); + assert!(az.resolve_optional(&env_vars).unwrap().is_none()); + } + #[test] fn test_config_value_deserialize() { fn default_cluster_id() -> ConfigValue { diff --git a/quickwit/quickwit-config/src/index_config/mod.rs b/quickwit/quickwit-config/src/index_config/mod.rs index e62e556bce3..286f2d695be 100644 --- a/quickwit/quickwit-config/src/index_config/mod.rs +++ b/quickwit/quickwit-config/src/index_config/mod.rs @@ -14,6 +14,7 @@ pub(crate) mod serialize; +use std::collections::HashSet; use std::hash::{Hash, Hasher}; use std::num::NonZeroUsize; use std::str::FromStr; @@ -26,6 +27,7 @@ use chrono::Utc; use cron::Schedule; use humantime::parse_duration; use quickwit_common::uri::Uri; +use quickwit_common::{is_true, true_fn}; use quickwit_doc_mapper::{DocMapper, DocMapperBuilder, DocMapping}; use quickwit_proto::types::IndexId; use serde::{Deserialize, Serialize}; @@ -41,6 +43,7 @@ use crate::merge_policy_config::MergePolicyConfig; pub struct IndexingResources { #[schema(value_type = String, default = "2 GB")] #[serde(default = "IndexingResources::default_heap_size")] + #[serde(with = "crate::serde_utils::bytesize_serde")] pub heap_size: ByteSize, // DEPRECATED: See #4439 #[schema(value_type = String)] @@ -170,6 +173,16 @@ pub struct IngestSettings { #[schema(default = 1, value_type = usize)] #[serde(default = "IngestSettings::default_min_shards")] pub min_shards: NonZeroUsize, + /// Whether to validate documents against the current doc mapping during ingestion. + /// Defaults to true. When false, documents will be written directly to the WAL without + /// validation, but might still be rejected during indexing when applying the doc mapping + /// in the doc processor, in that case the documents are dropped and a warning is logged. + /// + /// Note that when a source has a VRL transform configured, documents are not validated against + /// the doc mapping during ingestion either. + #[schema(default = true, value_type = bool)] + #[serde(default = "true_fn", skip_serializing_if = "is_true")] + pub validate_docs: bool, } impl IngestSettings { @@ -182,6 +195,7 @@ impl Default for IngestSettings { fn default() -> Self { Self { min_shards: Self::default_min_shards(), + validate_docs: true, } } } @@ -481,6 +495,7 @@ impl crate::TestableForRegression for IndexConfig { }; let ingest_settings = IngestSettings { min_shards: NonZeroUsize::new(12).unwrap(), + validate_docs: true, }; let search_settings = SearchSettings { default_search_fields: vec!["message".to_string()], @@ -552,11 +567,67 @@ pub(super) fn validate_index_config( Ok(()) } +/// Returns the updated doc mapping and a boolean indicating whether a mutation occurred. +/// +/// The logic goes as follows: +/// 1. If the new doc mapping is the same as the current doc mapping, ignoring their UIDs, returns +/// the current doc mapping and `false`, indicating that no mutation occurred. +/// 2. If the new doc mapping is different from the current doc mapping, verifies the following +/// constraints before returning the new doc mapping and `true`, indicating that a mutation +/// occurred: +/// - The doc mapping UID should differ from the current one +/// - The timestamp field should remain the same +/// - The tokenizers should be a superset of the current tokenizers +/// - A doc mapper can be built from the new doc mapping +pub fn prepare_doc_mapping_update( + mut new_doc_mapping: DocMapping, + current_doc_mapping: &DocMapping, + search_settings: &SearchSettings, +) -> anyhow::Result<(DocMapping, bool)> { + // Save the new doc mapping UID in a temporary variable and override it with the current doc + // mapping UID to compare the two doc mappings, ignoring their UIDs. + let new_doc_mapping_uid = new_doc_mapping.doc_mapping_uid; + new_doc_mapping.doc_mapping_uid = current_doc_mapping.doc_mapping_uid; + + if new_doc_mapping == *current_doc_mapping { + return Ok((new_doc_mapping, false)); + } + // Restore the new doc mapping UID. + new_doc_mapping.doc_mapping_uid = new_doc_mapping_uid; + + ensure!( + new_doc_mapping.doc_mapping_uid != current_doc_mapping.doc_mapping_uid, + "new doc mapping UID should differ from the current one, current UID `{}`, new UID `{}`", + current_doc_mapping.doc_mapping_uid, + new_doc_mapping.doc_mapping_uid, + ); + let new_timestamp_field = new_doc_mapping.timestamp_field.as_deref(); + let current_timestamp_field = current_doc_mapping.timestamp_field.as_deref(); + ensure!( + new_timestamp_field == current_timestamp_field, + "updating timestamp field is not allowed, current timestamp field `{}`, new timestamp \ + field `{}`", + current_timestamp_field.unwrap_or("none"), + new_timestamp_field.unwrap_or("none"), + ); + // TODO: Unsure this constraint is required, should we relax it? + let new_tokenizers: HashSet<_> = new_doc_mapping.tokenizers.iter().collect(); + let current_tokenizers: HashSet<_> = current_doc_mapping.tokenizers.iter().collect(); + ensure!( + new_tokenizers.is_superset(¤t_tokenizers), + "updating tokenizers is allowed only if adding new tokenizers, current tokenizers \ + `{current_tokenizers:?}`, new tokenizers `{new_tokenizers:?}`", + ); + build_doc_mapper(&new_doc_mapping, search_settings).context("invalid doc mapping")?; + Ok((new_doc_mapping, true)) +} + #[cfg(test)] mod tests { use cron::TimeUnitSpec; - use quickwit_doc_mapper::ModeType; + use quickwit_doc_mapper::{Mode, ModeType, TokenizerEntry}; + use quickwit_proto::types::DocMappingUid; use super::*; use crate::ConfigFormat; @@ -942,18 +1013,122 @@ mod tests { #[test] fn test_ingest_settings_serde() { - let ingest_settings = IngestSettings { + let settings = IngestSettings { + min_shards: NonZeroUsize::MIN, + validate_docs: false, + }; + let settings_yaml = serde_yaml::to_string(&settings).unwrap(); + assert!(settings_yaml.contains("validate_docs")); + + let expected_settings: IngestSettings = serde_yaml::from_str(&settings_yaml).unwrap(); + assert_eq!(settings, expected_settings); + + let settings = IngestSettings { min_shards: NonZeroUsize::MIN, + validate_docs: true, }; - let ingest_settings_yaml = serde_yaml::to_string(&ingest_settings).unwrap(); - let ingest_settings_roundtrip: IngestSettings = - serde_yaml::from_str(&ingest_settings_yaml).unwrap(); - assert_eq!(ingest_settings, ingest_settings_roundtrip); + let settings_yaml = serde_yaml::to_string(&settings).unwrap(); + assert!(!settings_yaml.contains("validate_docs")); - let ingest_settings_yaml = r#" + let expected_settings: IngestSettings = serde_yaml::from_str(&settings_yaml).unwrap(); + assert_eq!(settings, expected_settings); + + let settings_yaml = r#" min_shards: 0 "#; - let error = serde_yaml::from_str::(ingest_settings_yaml).unwrap_err(); + let error = serde_yaml::from_str::(settings_yaml).unwrap_err(); assert!(error.to_string().contains("expected a nonzero")); } + + #[test] + fn test_prepare_doc_mapping_update() { + let current_index_config = IndexConfig::for_test("test-index", "s3://test-index"); + let mut current_doc_mapping = current_index_config.doc_mapping; + let search_settings = current_index_config.search_settings; + + let tokenizer_json = r#" + { + "name": "breton-tokenizer", + "type": "regex", + "pattern": "crêpes*" + } + "#; + let tokenizer: TokenizerEntry = serde_json::from_str(tokenizer_json).unwrap(); + + current_doc_mapping.tokenizers.push(tokenizer.clone()); + + // The new doc mapping should have a different doc mapping UID. + let mut new_doc_mapping = current_doc_mapping.clone(); + new_doc_mapping.store_source = false; // This is set to `true` for the current doc mapping. + let error = + prepare_doc_mapping_update(new_doc_mapping, ¤t_doc_mapping, &search_settings) + .unwrap_err() + .to_string(); + assert!(error.contains("doc mapping UID should differ")); + + // The new doc mapping should not change the timestamp field. + let mut new_doc_mapping = current_doc_mapping.clone(); + new_doc_mapping.doc_mapping_uid = DocMappingUid::random(); + new_doc_mapping.timestamp_field = Some("ts".to_string()); // This is set to `timestamp` for the current doc mapping. + let error = + prepare_doc_mapping_update(new_doc_mapping, ¤t_doc_mapping, &search_settings) + .unwrap_err() + .to_string(); + assert!(error.contains("timestamp field")); + + // The new doc mapping should not remove the timestamp field. + let mut new_doc_mapping = current_doc_mapping.clone(); + new_doc_mapping.doc_mapping_uid = DocMappingUid::random(); + new_doc_mapping.timestamp_field = None; + let error = + prepare_doc_mapping_update(new_doc_mapping, ¤t_doc_mapping, &search_settings) + .unwrap_err() + .to_string(); + assert!(error.contains("timestamp field")); + + // The new doc mapping should not remove tokenizers. + let mut new_doc_mapping = current_doc_mapping.clone(); + new_doc_mapping.doc_mapping_uid = DocMappingUid::random(); + new_doc_mapping.tokenizers.clear(); + let error = + prepare_doc_mapping_update(new_doc_mapping, ¤t_doc_mapping, &search_settings) + .unwrap_err() + .to_string(); + assert!(error.contains("tokenizers")); + + // The new doc mapping should be "buildable" into a doc mapper. + let mut new_doc_mapping = current_doc_mapping.clone(); + new_doc_mapping.doc_mapping_uid = DocMappingUid::random(); + new_doc_mapping.tokenizers.push(tokenizer); + let error = + prepare_doc_mapping_update(new_doc_mapping, ¤t_doc_mapping, &search_settings) + .unwrap_err() + .source() + .unwrap() + .to_string(); + assert!(error.contains("duplicated custom tokenizer")); + + let mut new_doc_mapping = current_doc_mapping.clone(); + new_doc_mapping.doc_mapping_uid = DocMappingUid::random(); + let (updated_doc_mapping, mutation_occurred) = + prepare_doc_mapping_update(new_doc_mapping, ¤t_doc_mapping, &search_settings) + .unwrap(); + assert!(!mutation_occurred); + assert_eq!( + updated_doc_mapping.doc_mapping_uid, + current_doc_mapping.doc_mapping_uid + ); + assert_eq!(updated_doc_mapping, current_doc_mapping); + + let mut new_doc_mapping = current_doc_mapping.clone(); + let new_doc_mapping_uid = DocMappingUid::random(); + new_doc_mapping.doc_mapping_uid = new_doc_mapping_uid; + new_doc_mapping.mode = Mode::Strict; + let (updated_doc_mapping, mutation_occurred) = + prepare_doc_mapping_update(new_doc_mapping, ¤t_doc_mapping, &search_settings) + .unwrap(); + assert!(mutation_occurred); + assert_eq!(updated_doc_mapping.doc_mapping_uid, new_doc_mapping_uid); + assert_eq!(updated_doc_mapping.mode, Mode::Strict); + } } diff --git a/quickwit/quickwit-config/src/index_config/serialize.rs b/quickwit/quickwit-config/src/index_config/serialize.rs index 5e04e8dbf3c..24fcc6d1ac2 100644 --- a/quickwit/quickwit-config/src/index_config/serialize.rs +++ b/quickwit/quickwit-config/src/index_config/serialize.rs @@ -12,11 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashSet; - use anyhow::{Context, ensure}; use quickwit_common::uri::Uri; -use quickwit_doc_mapper::DocMapperBuilder; use quickwit_proto::types::{DocMappingUid, IndexId}; use serde::{Deserialize, Serialize}; use tracing::info; @@ -24,7 +21,7 @@ use tracing::info; use super::{IngestSettings, validate_index_config}; use crate::{ ConfigFormat, DocMapping, IndexConfig, IndexingSettings, RetentionPolicy, SearchSettings, - validate_identifier, + prepare_doc_mapping_update, validate_identifier, }; /// Alias for the latest serialization format. @@ -92,63 +89,12 @@ pub fn load_index_config_update( current_index_config.index_uri, new_index_config.index_uri ); - - // verify the new mapping is coherent - let doc_mapper_builder = DocMapperBuilder { - doc_mapping: new_index_config.doc_mapping.clone(), - default_search_fields: new_index_config - .search_settings - .default_search_fields - .clone(), - legacy_type_tag: None, - }; - doc_mapper_builder - .try_build() - .context("invalid mapping update")?; - - { - let new_mapping_uid = new_index_config.doc_mapping.doc_mapping_uid; - // we verify whether they are equal ignoring the mapping uid as it is generated at random: - // we don't want to record a mapping change when nothing really happened. - new_index_config.doc_mapping.doc_mapping_uid = - current_index_config.doc_mapping.doc_mapping_uid; - if new_index_config.doc_mapping != current_index_config.doc_mapping { - new_index_config.doc_mapping.doc_mapping_uid = new_mapping_uid; - ensure!( - current_index_config.doc_mapping.doc_mapping_uid - != new_index_config.doc_mapping.doc_mapping_uid, - "`doc_mapping_doc_mapping_uid` must change when the doc mapping is updated", - ); - ensure!( - current_index_config.doc_mapping.timestamp_field - == new_index_config.doc_mapping.timestamp_field, - "`doc_mapping.timestamp_field` cannot be updated, current value {}, new expected \ - value {}", - current_index_config - .doc_mapping - .timestamp_field - .as_deref() - .unwrap_or(""), - new_index_config - .doc_mapping - .timestamp_field - .as_deref() - .unwrap_or(""), - ); - // TODO: i'm not sure this is necessary, we can relax this requirement once we know - // for sure - let current_tokenizers: HashSet<_> = - current_index_config.doc_mapping.tokenizers.iter().collect(); - let new_tokenizers: HashSet<_> = - new_index_config.doc_mapping.tokenizers.iter().collect(); - ensure!( - new_tokenizers.is_superset(¤t_tokenizers), - "`.doc_mapping.tokenizers` must be a superset of previously available tokenizers" - ); - } else { - // the docmapping is unchanged, keep the old uid - } - } + let (updated_doc_mapping, _mutation_occurred) = prepare_doc_mapping_update( + new_index_config.doc_mapping, + ¤t_index_config.doc_mapping, + &new_index_config.search_settings, + )?; + new_index_config.doc_mapping = updated_doc_mapping; Ok(new_index_config) } diff --git a/quickwit/quickwit-config/src/lib.rs b/quickwit/quickwit-config/src/lib.rs index f7589995f5d..5fccd9ab7b4 100644 --- a/quickwit/quickwit-config/src/lib.rs +++ b/quickwit/quickwit-config/src/lib.rs @@ -47,6 +47,7 @@ use index_config::serialize::{IndexConfigV0_8, VersionedIndexConfig}; pub use index_config::{ IndexConfig, IndexingResources, IndexingSettings, IngestSettings, RetentionPolicy, SearchSettings, build_doc_mapper, load_index_config_from_user_config, load_index_config_update, + prepare_doc_mapping_update, }; pub use quickwit_doc_mapper::DocMapping; use serde::Serialize; @@ -72,9 +73,9 @@ pub use crate::metastore_config::{ MetastoreBackend, MetastoreConfig, MetastoreConfigs, PostgresMetastoreConfig, }; pub use crate::node_config::{ - DEFAULT_QW_CONFIG_PATH, GrpcConfig, IndexerConfig, IngestApiConfig, JaegerConfig, - KeepAliveConfig, NodeConfig, RestConfig, SearcherConfig, SplitCacheLimits, - StorageTimeoutPolicy, TlsConfig, + CacheConfig, CachePolicy, DEFAULT_QW_CONFIG_PATH, GrpcConfig, IndexerConfig, IngestApiConfig, + JaegerConfig, KeepAliveConfig, LambdaConfig, LambdaDeployConfig, NodeConfig, RestConfig, + SearcherConfig, SplitCacheLimits, StorageTimeoutPolicy, TlsConfig, }; use crate::source_config::serialize::{SourceConfigV0_7, SourceConfigV0_8, VersionedSourceConfig}; pub use crate::storage_config::{ @@ -226,7 +227,9 @@ impl ConfigFormat { } pub fn parse(&self, payload: &[u8]) -> anyhow::Result - where T: DeserializeOwned { + where + T: DeserializeOwned, + { match self { ConfigFormat::Json => { let mut json_value: JsonValue = diff --git a/quickwit/quickwit-config/src/merge_policy_config.rs b/quickwit/quickwit-config/src/merge_policy_config.rs index 3e4e5dad0ce..75f749dbf05 100644 --- a/quickwit/quickwit-config/src/merge_policy_config.rs +++ b/quickwit/quickwit-config/src/merge_policy_config.rs @@ -120,7 +120,9 @@ impl Default for StableLogMergePolicyConfig { } fn parse_human_duration<'de, D>(deserializer: D) -> Result -where D: Deserializer<'de> { +where + D: Deserializer<'de>, +{ let value: String = Deserialize::deserialize(deserializer)?; let duration = humantime::parse_duration(&value).map_err(|error| { de::Error::custom(format!( @@ -131,7 +133,9 @@ where D: Deserializer<'de> { } fn serialize_duration(value: &Duration, s: S) -> Result -where S: Serializer { +where + S: Serializer, +{ let value_str = humantime::format_duration(*value).to_string(); s.serialize_str(&value_str) } diff --git a/quickwit/quickwit-config/src/node_config/mod.rs b/quickwit/quickwit-config/src/node_config/mod.rs index d22631074a9..54dfb16db65 100644 --- a/quickwit/quickwit-config/src/node_config/mod.rs +++ b/quickwit/quickwit-config/src/node_config/mod.rs @@ -23,7 +23,7 @@ use std::time::Duration; use anyhow::{bail, ensure}; use bytesize::ByteSize; -use legacy_http::HeaderMap; +use http::HeaderMap; use quickwit_common::net::HostAddr; use quickwit_common::shared_consts::{ DEFAULT_SHARD_BURST_LIMIT, DEFAULT_SHARD_SCALE_UP_FACTOR, DEFAULT_SHARD_THROUGHPUT_LIMIT, @@ -32,7 +32,7 @@ use quickwit_common::uri::Uri; use quickwit_proto::indexing::CpuCapacity; use quickwit_proto::tonic::codec::CompressionEncoding; use quickwit_proto::types::NodeId; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use tracing::{info, warn}; use crate::node_config::serialize::load_node_config_with_env; @@ -264,11 +264,30 @@ impl SplitCacheLimits { pub struct SearcherConfig { pub aggregation_memory_limit: ByteSize, pub aggregation_bucket_limit: u32, - pub fast_field_cache_capacity: ByteSize, - pub split_footer_cache_capacity: ByteSize, - pub partial_request_cache_capacity: ByteSize, + + #[serde(alias = "fast_field_cache_capacity")] + #[serde( + deserialize_with = "CacheConfig::deserialize_with_default::<_, {ByteSize::gb(1).as_u64()}>" + )] + pub fast_field_cache: CacheConfig, + #[serde(alias = "split_footer_cache_capacity")] + #[serde(deserialize_with = "CacheConfig::deserialize_with_default::<_, \ + {ByteSize::mb(500).as_u64()}>")] + pub split_footer_cache: CacheConfig, + #[serde(alias = "partial_request_cache_capacity")] + #[serde(deserialize_with = "CacheConfig::deserialize_with_default::<_, \ + {ByteSize::mb(64).as_u64()}>")] + pub partial_request_cache: CacheConfig, + #[serde(alias = "predicate_cache_capacity")] + #[serde(deserialize_with = "CacheConfig::deserialize_with_default::<_, \ + {ByteSize::mb(256).as_u64()}>")] + pub predicate_cache: CacheConfig, + pub max_num_concurrent_split_searches: usize, - pub max_num_concurrent_split_streams: usize, + pub max_splits_per_search: Option, + // Deprecated: stream search requests are no longer supported. + #[serde(alias = "max_num_concurrent_split_streams", default, skip_serializing)] + pub _max_num_concurrent_split_streams: Option, // Strangely, if None, this will also have the effect of not forwarding // to searcher. // TODO document and fix if necessary. @@ -281,6 +300,178 @@ pub struct SearcherConfig { pub storage_timeout_policy: Option, pub warmup_memory_budget: ByteSize, pub warmup_single_split_initial_allocation: ByteSize, + /// Lambda configuration for serverless leaf search execution. + /// If set, enables Lambda execution for leaf search. + /// + /// If set, and Quickwit cannot access the Lambda (after a deploy attempt if + /// auto deploy is set up), Quickwit will log an error and + /// fail on startup. + #[serde(default)] + pub lambda: Option, +} + +/// Configuration for AWS Lambda leaf search execution. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LambdaConfig { + /// AWS Lambda function name. + #[serde(default = "LambdaConfig::default_function_name")] + pub function_name: String, + /// Maximum number of splits per Lambda invocation. + #[serde(default = "LambdaConfig::default_max_splits_per_invocation")] + pub max_splits_per_invocation: NonZeroUsize, + /// Maximum number of splits to process locally before offloading to Lambda. + /// When the number of pending split searches exceeds this threshold, + /// new splits are offloaded to Lambda instead of being queued locally. + /// A value of 0 offloads everything to Lambda. + #[serde(default = "LambdaConfig::default_offload_threshold")] + pub offload_threshold: usize, + /// Auto-deploy configuration. If set, Quickwit will automatically deploy + /// the Lambda function at startup. + /// If deploying a lambda fails, Quickwit will log an error and fail. + #[serde(default)] + pub auto_deploy: Option, +} + +impl LambdaConfig { + #[cfg(feature = "testsuite")] + pub fn for_test() -> Self { + Self { + function_name: Self::default_function_name(), + max_splits_per_invocation: Self::default_max_splits_per_invocation(), + offload_threshold: Self::default_offload_threshold(), + auto_deploy: None, + } + } + + fn default_function_name() -> String { + "quickwit-lambda-search".to_string() + } + fn default_max_splits_per_invocation() -> NonZeroUsize { + NonZeroUsize::new(10).unwrap() + } + fn default_offload_threshold() -> usize { + 100 + } +} + +/// Configuration for automatic Lambda function deployment. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LambdaDeployConfig { + /// IAM execution role ARN for the Lambda function. + /// The role only requires GetObject permission to the targeted S3 bucket. + pub execution_role_arn: String, + /// Memory size for the Lambda function. + /// It will be rounded up to the nearest multiple of 1MiB. + #[serde(default = "LambdaDeployConfig::default_memory_size")] + pub memory_size: ByteSize, + /// Timeout for Lambda invocations in seconds. + #[serde(default = "LambdaDeployConfig::default_invocation_timeout_secs")] + pub invocation_timeout_secs: u64, +} + +impl LambdaDeployConfig { + fn default_memory_size() -> ByteSize { + // Empirically this implies between 4 and 6 vCPUs. + ByteSize::gib(5) + } + fn default_invocation_timeout_secs() -> u64 { + 15 + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CacheConfig { + #[serde(default)] + capacity: Option, + #[serde(default)] + policy: Option, + + // Cache configs inside the virtual cache aren't allowed to contain virtual cache + #[serde(default)] + pub virtual_caches: Vec, +} + +impl CacheConfig { + pub fn no_cache() -> Self { + CacheConfig { + capacity: None, + policy: None, + virtual_caches: Vec::new(), + } + } + + pub fn default_with_capacity(capacity: ByteSize) -> Self { + CacheConfig { + capacity: Some(capacity), + policy: None, + virtual_caches: Vec::new(), + } + } + + pub fn capacity(&self) -> ByteSize { + // this should always be there + self.capacity.unwrap_or_default() + } + + pub fn capacity_for_virtual_cache(&mut self, real_capacity: ByteSize) -> ByteSize { + let capacity = self.capacity.unwrap_or(real_capacity); + self.capacity = Some(capacity); + capacity + } + + pub fn policy(&self) -> CachePolicy { + self.policy.unwrap_or_default() + } + + pub fn policy_for_virtual_cache(&mut self, real_policy: CachePolicy) -> CachePolicy { + let policy = self.policy.unwrap_or(real_policy); + self.policy = Some(policy); + policy + } + + fn deserialize_with_default<'de, D, const DEFAULT_CAPACITY: u64>( + deserializer: D, + ) -> Result + where + D: Deserializer<'de>, + { + use serde_with::{DeserializeAs, FromInto, PickFirst, Same}; + + let mut cache_config: CacheConfig = + PickFirst::<(Same, FromInto)>::deserialize_as(deserializer)?; + if cache_config.capacity.is_none() { + cache_config.capacity = Some(ByteSize::b(DEFAULT_CAPACITY)); + } + Ok(cache_config) + } +} + +impl From for CacheConfig { + fn from(capacity: ByteSize) -> Self { + CacheConfig::default_with_capacity(capacity) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "kebab-case")] +pub enum CachePolicy { + #[default] + Lru, + S3Fifo, + TinyLfu, +} + +impl std::fmt::Display for CachePolicy { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + CachePolicy::Lru => f.write_str("lru"), + CachePolicy::S3Fifo => f.write_str("s3-fifo"), + CachePolicy::TinyLfu => f.write_str("tiny-lfu"), + } + } } /// Configuration controlling how fast a searcher should timeout a `get_slice` @@ -319,18 +510,21 @@ impl StorageTimeoutPolicy { impl Default for SearcherConfig { fn default() -> Self { SearcherConfig { - fast_field_cache_capacity: ByteSize::gb(1), - split_footer_cache_capacity: ByteSize::mb(500), - partial_request_cache_capacity: ByteSize::mb(64), - max_num_concurrent_split_streams: 100, + fast_field_cache: CacheConfig::default_with_capacity(ByteSize::gb(1)), + split_footer_cache: CacheConfig::default_with_capacity(ByteSize::mb(500)), + partial_request_cache: CacheConfig::default_with_capacity(ByteSize::mb(64)), + predicate_cache: CacheConfig::default_with_capacity(ByteSize::mb(256)), max_num_concurrent_split_searches: 100, + max_splits_per_search: None, + _max_num_concurrent_split_streams: None, aggregation_memory_limit: ByteSize::mb(500), aggregation_bucket_limit: 65000, split_cache: None, request_timeout_secs: Self::default_request_timeout_secs(), storage_timeout_policy: None, warmup_memory_budget: ByteSize::gb(100), - warmup_single_split_initial_allocation: ByteSize::gb(1), + warmup_single_split_initial_allocation: ByteSize::mb(300), + lambda: None, } } } @@ -355,16 +549,6 @@ impl SearcherConfig { split_cache_limits.max_file_descriptors ); } - if self.max_num_concurrent_split_streams - > split_cache_limits.max_file_descriptors.get() as usize - { - anyhow::bail!( - "max_num_concurrent_split_streams ({}) must be lower or equal to \ - split_cache.max_file_descriptors ({})", - self.max_num_concurrent_split_streams, - split_cache_limits.max_file_descriptors - ); - } if self.warmup_single_split_initial_allocation > self.warmup_memory_budget { anyhow::bail!( "warmup_single_split_initial_allocation ({}) must be lower or equal to \ @@ -461,23 +645,23 @@ impl IngestApiConfig { ensure!( self.max_queue_disk_usage > ByteSize::mib(256), "max_queue_disk_usage must be at least 256 MiB, got `{}`", - self.max_queue_disk_usage + self.max_queue_disk_usage.display().si() ); ensure!( self.max_queue_disk_usage >= self.max_queue_memory_usage, "max_queue_disk_usage ({}) must be at least max_queue_memory_usage ({})", - self.max_queue_disk_usage, - self.max_queue_memory_usage + self.max_queue_disk_usage.display().si(), + self.max_queue_memory_usage.display().si() ); info!( "ingestion shard throughput limit: {}", - self.shard_throughput_limit + self.shard_throughput_limit.display().si() ); ensure!( self.shard_throughput_limit >= ByteSize::mib(1) && self.shard_throughput_limit <= ByteSize::mib(20), "shard_throughput_limit ({}) must be within 1mb and 20mb", - self.shard_throughput_limit + self.shard_throughput_limit.display().si() ); // The newline delimited format is persisted as something a bit larger // (lines prefixed with their length) @@ -570,6 +754,7 @@ impl Default for JaegerConfig { pub struct NodeConfig { pub cluster_id: String, pub node_id: NodeId, + pub availability_zone: Option, pub enabled_services: HashSet, pub gossip_listen_addr: SocketAddr, pub grpc_listen_addr: SocketAddr, @@ -618,15 +803,17 @@ impl NodeConfig { // validation purposes. Additionally, we need to append a default port if necessary and // finally return the addresses as strings, which is tricky for IPv6. We let the logic baked // in `HostAddr` handle this complexity. + let mut found_something = false; for peer_seed in &self.peer_seeds { let peer_seed_addr = HostAddr::parse_with_default_port(peer_seed, default_gossip_port)?; if let Err(error) = peer_seed_addr.resolve().await { warn!(peer_seed = %peer_seed_addr, error = ?error, "failed to resolve peer seed address"); - continue; + } else { + found_something = true; } peer_seed_addrs.push(peer_seed_addr.to_string()) } - if !self.peer_seeds.is_empty() && peer_seed_addrs.is_empty() { + if !self.peer_seeds.is_empty() && !found_something { warn!("failed to resolve all the peer seed addresses") } Ok(peer_seed_addrs) diff --git a/quickwit/quickwit-config/src/node_config/serialize.rs b/quickwit/quickwit-config/src/node_config/serialize.rs index 9ab90e53ee3..faf15c76c6d 100644 --- a/quickwit/quickwit-config/src/node_config/serialize.rs +++ b/quickwit/quickwit-config/src/node_config/serialize.rs @@ -19,7 +19,7 @@ use std::time::Duration; use anyhow::{Context, bail}; use bytesize::ByteSize; -use legacy_http::HeaderMap; +use http::HeaderMap; use quickwit_common::fs::get_disk_size; use quickwit_common::net::{Host, find_private_ip, get_short_hostname}; use quickwit_common::new_coolid; @@ -66,6 +66,10 @@ fn default_node_id() -> ConfigValue { ConfigValue::with_default(node_id) } +fn default_availability_zone() -> ConfigValue { + ConfigValue::none() +} + #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] struct List(Vec); @@ -169,6 +173,8 @@ struct NodeConfigBuilder { cluster_id: ConfigValue, #[serde(default = "default_node_id")] node_id: ConfigValue, + #[serde(default = "default_availability_zone")] + availability_zone: ConfigValue, #[serde(default = "default_enabled_services")] enabled_services: ConfigValue, #[serde(default = "default_listen_address")] @@ -218,6 +224,7 @@ impl NodeConfigBuilder { env_vars: &HashMap, ) -> anyhow::Result { let node_id = self.node_id.resolve(env_vars).map(NodeId::new)?; + let availability_zone = self.availability_zone.resolve_optional(env_vars)?; let enabled_services = self .enabled_services @@ -305,6 +312,7 @@ impl NodeConfigBuilder { let node_config = NodeConfig { cluster_id: self.cluster_id.resolve(env_vars)?, node_id, + availability_zone, enabled_services, gossip_listen_addr, grpc_listen_addr, @@ -401,6 +409,7 @@ impl Default for NodeConfigBuilder { Self { cluster_id: default_cluster_id(), node_id: default_node_id(), + availability_zone: ConfigValue::none(), enabled_services: default_enabled_services(), listen_address: default_listen_address(), rest_listen_port: None, @@ -469,6 +478,7 @@ pub fn node_config_for_tests_from_ports( ) -> NodeConfig { let node_id = NodeId::new(default_node_id().unwrap()); let enabled_services = QuickwitService::supported_services(); + let availability_zone = Some(String::from("az-1")); let listen_address = Host::default(); let rest_listen_addr = listen_address .with_port(rest_listen_port) @@ -499,6 +509,7 @@ pub fn node_config_for_tests_from_ports( NodeConfig { cluster_id: default_cluster_id().unwrap(), node_id, + availability_zone, enabled_services, gossip_advertise_addr: gossip_listen_addr, grpc_advertise_addr: grpc_listen_addr, @@ -532,6 +543,7 @@ mod tests { use super::*; use crate::storage_config::StorageBackendFlavor; + use crate::{CacheConfig, LambdaConfig, LambdaDeployConfig}; fn get_config_filepath(config_filename: &str) -> String { format!( @@ -553,6 +565,7 @@ mod tests { assert!(config.is_service_enabled(QuickwitService::Janitor)); assert!(config.is_service_enabled(QuickwitService::Metastore)); + assert_eq!(config.availability_zone.unwrap(), "az-1"); assert_eq!( config.rest_config.listen_addr, SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 1111) @@ -658,11 +671,13 @@ mod tests { SearcherConfig { aggregation_memory_limit: ByteSize::gb(1), aggregation_bucket_limit: 500_000, - fast_field_cache_capacity: ByteSize::gb(10), - split_footer_cache_capacity: ByteSize::gb(1), - partial_request_cache_capacity: ByteSize::mb(64), + fast_field_cache: CacheConfig::default_with_capacity(ByteSize::gb(10)), + split_footer_cache: CacheConfig::default_with_capacity(ByteSize::gb(1)), + partial_request_cache: CacheConfig::default_with_capacity(ByteSize::mb(64)), + predicate_cache: CacheConfig::default_with_capacity(ByteSize::mb(256)), max_num_concurrent_split_searches: 150, - max_num_concurrent_split_streams: 120, + max_splits_per_search: None, + _max_num_concurrent_split_streams: Some(serde::de::IgnoredAny), split_cache: None, request_timeout_secs: NonZeroU64::new(30).unwrap(), storage_timeout_policy: Some(crate::StorageTimeoutPolicy { @@ -671,7 +686,18 @@ mod tests { max_num_retries: 2 }), warmup_memory_budget: ByteSize::gb(100), - warmup_single_split_initial_allocation: ByteSize::gb(1), + warmup_single_split_initial_allocation: ByteSize::mb(300), + lambda: Some(LambdaConfig { + function_name: "quickwit-lambda-leaf-search".to_string(), + max_splits_per_invocation: NonZeroUsize::new(10).unwrap(), + offload_threshold: 30, + auto_deploy: Some(LambdaDeployConfig { + execution_role_arn: "arn:aws:iam::123456789012:role/quickwit-lambda-role" + .to_string(), + memory_size: ByteSize::gib(5), + invocation_timeout_secs: 15, + }), + }), } ); assert_eq!( @@ -736,6 +762,7 @@ mod tests { .unwrap(); assert_eq!(config.cluster_id, DEFAULT_CLUSTER_ID); assert_eq!(config.node_id, get_short_hostname().unwrap()); + assert_eq!(config.availability_zone, None); assert_eq!( config.enabled_services, QuickwitService::supported_services() @@ -948,6 +975,7 @@ mod tests { assert_eq!( node_config.peer_seed_addrs().await.unwrap(), vec![ + "unresolvable.example.com:1789".to_string(), "localhost:1789".to_string(), "localhost:1337".to_string(), "127.0.0.1:1789".to_string(), diff --git a/quickwit/quickwit-config/src/qw_env_vars.rs b/quickwit/quickwit-config/src/qw_env_vars.rs index a217e7a6353..7f36fdc2911 100644 --- a/quickwit/quickwit-config/src/qw_env_vars.rs +++ b/quickwit/quickwit-config/src/qw_env_vars.rs @@ -45,6 +45,7 @@ qw_env_vars!( QW_NONE, QW_CLUSTER_ID, QW_NODE_ID, + QW_AVAILABILITY_ZONE, QW_ENABLED_SERVICES, QW_LISTEN_ADDRESS, QW_ADVERTISE_ADDRESS, diff --git a/quickwit/quickwit-config/src/serde_utils.rs b/quickwit/quickwit-config/src/serde_utils.rs index 34aefebbf22..1a4d89c799e 100644 --- a/quickwit/quickwit-config/src/serde_utils.rs +++ b/quickwit/quickwit-config/src/serde_utils.rs @@ -18,6 +18,29 @@ use std::time::Duration; use serde::{Deserialize, Serialize}; +/// Custom serde module for ByteSize that serializes as raw byte count (u64). +/// This ensures perfect roundtrip consistency regardless of display format changes +/// in the bytesize crate. Deserialization still accepts human-readable strings +/// like "2 GB" via bytesize's default deserializer. +pub mod bytesize_serde { + use bytesize::ByteSize; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(byte_size: &ByteSize, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_u64(byte_size.as_u64()) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + ByteSize::deserialize(deserializer) + } +} + #[derive(Serialize, Deserialize, Clone)] #[serde(try_from = "String", into = "String")] pub struct DurationAsStr { diff --git a/quickwit/quickwit-config/src/source_config/mod.rs b/quickwit/quickwit-config/src/source_config/mod.rs index cfebf6efdda..11981876882 100644 --- a/quickwit/quickwit-config/src/source_config/mod.rs +++ b/quickwit/quickwit-config/src/source_config/mod.rs @@ -386,12 +386,12 @@ impl TryFrom for FileSourceParams { value.notifications.remove(0), )) } else if value.notifications.len() > 1 { - return Err("Only one notification can be specified for now".into()); + Err("Only one notification can be specified for now".into()) } else { - return Err( + Err( "Either `notifications` or `filepath` must be specified as file source parameters" .into(), - ); + ) } } } @@ -602,7 +602,9 @@ pub enum PulsarSourceAuth { // Deserializing a string into an pulsar uri. fn pulsar_uri<'de, D>(deserializer: D) -> Result -where D: Deserializer<'de> { +where + D: Deserializer<'de>, +{ let uri: String = Deserialize::deserialize(deserializer)?; let re: Regex = Regex::new(r"pulsar(\+ssl)?://.*").expect("regular expression should compile"); diff --git a/quickwit/quickwit-config/src/source_config/serialize.rs b/quickwit/quickwit-config/src/source_config/serialize.rs index 224689019d2..3f580ee8fa3 100644 --- a/quickwit/quickwit-config/src/source_config/serialize.rs +++ b/quickwit/quickwit-config/src/source_config/serialize.rs @@ -129,7 +129,9 @@ impl SourceConfigForSerialization { | SourceParams::File(FileSourceParams::Notifications(_)) => {} _ => { if self.num_pipelines > 1 { - bail!("Quickwit currently supports multiple pipelines only for GCP PubSub or Kafka sources. open an issue https://github.com/quickwit-oss/quickwit/issues if you need the feature for other source types"); + bail!( + "Quickwit currently supports multiple pipelines only for GCP PubSub or Kafka sources. open an issue https://github.com/quickwit-oss/quickwit/issues if you need the feature for other source types" + ); } } } diff --git a/quickwit/quickwit-control-plane/Cargo.toml b/quickwit/quickwit-control-plane/Cargo.toml index a9b2da54615..abaf672e908 100644 --- a/quickwit/quickwit-control-plane/Cargo.toml +++ b/quickwit/quickwit-control-plane/Cargo.toml @@ -24,9 +24,7 @@ rand = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } smallvec = { workspace = true } -time = { workspace = true } tokio = { workspace = true } -tower = { workspace = true } tracing = { workspace = true } ulid = { workspace = true } @@ -49,6 +47,7 @@ quickwit-cluster = { workspace = true, features = ["testsuite"] } quickwit-common = { workspace = true, features = ["testsuite"] } quickwit-config = { workspace = true, features = ["testsuite"] } quickwit-indexing = { workspace = true } +quickwit-ingest = { workspace = true, features = ["testsuite"] } quickwit-metastore = { workspace = true, features = ["testsuite"] } quickwit-proto = { workspace = true, features = ["testsuite"] } diff --git a/quickwit/quickwit-control-plane/src/control_plane.rs b/quickwit/quickwit-control-plane/src/control_plane.rs index 9ee2f101bc5..1262e4b1ebb 100644 --- a/quickwit/quickwit-control-plane/src/control_plane.rs +++ b/quickwit/quickwit-control-plane/src/control_plane.rs @@ -12,7 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::{BTreeSet, HashMap}; +use std::collections::btree_map::Entry; +use std::collections::{BTreeMap, BTreeSet}; use std::fmt; use std::fmt::Formatter; use std::num::NonZeroUsize; @@ -22,17 +23,16 @@ use anyhow::Context; use async_trait::async_trait; use futures::stream::FuturesUnordered; use futures::{Future, StreamExt}; +use itertools::Itertools; use quickwit_actors::{ Actor, ActorContext, ActorExitStatus, ActorHandle, DeferableReplyHandler, Handler, Mailbox, Supervisor, Universe, WeakMailbox, }; -use quickwit_cluster::{ - ClusterChange, ClusterChangeStream, ClusterChangeStreamFactory, ClusterNode, -}; +use quickwit_cluster::{ClusterChange, ClusterChangeStream, ClusterChangeStreamFactory}; +use quickwit_common::pretty::PrettyDisplay; use quickwit_common::pubsub::EventSubscriber; use quickwit_common::uri::Uri; use quickwit_common::{Progress, shared_consts}; -use quickwit_config::service::QuickwitService; use quickwit_config::{ClusterConfig, IndexConfig, IndexTemplate, SourceConfig}; use quickwit_ingest::{IngesterPool, LocalShardsUpdate}; use quickwit_metastore::{CreateIndexRequestExt, CreateIndexResponseExt, IndexMetadataResponseExt}; @@ -41,6 +41,7 @@ use quickwit_proto::control_plane::{ GetOrCreateOpenShardsRequest, GetOrCreateOpenShardsResponse, GetOrCreateOpenShardsSubrequest, }; use quickwit_proto::indexing::ShardPositionsUpdate; +use quickwit_proto::ingest::ingester::IngesterStatus; use quickwit_proto::metastore::{ AddSourceRequest, CreateIndexRequest, CreateIndexResponse, DeleteIndexRequest, DeleteShardsRequest, DeleteSourceRequest, EmptyResponse, FindIndexTemplateMatchesRequest, @@ -52,7 +53,7 @@ use quickwit_proto::types::{IndexId, IndexUid, NodeId, ShardId, SourceId, Source use serde::Serialize; use serde_json::{Value as JsonValue, json}; use tokio::sync::watch; -use tracing::{debug, error, info}; +use tracing::{Level, debug, enabled, error, info}; use crate::IndexerPool; use crate::cooldown_map::{CooldownMap, CooldownStatus}; @@ -219,6 +220,7 @@ impl Actor for ControlPlane { async fn initialize(&mut self, ctx: &ActorContext) -> Result<(), ActorExitStatus> { crate::metrics::CONTROL_PLANE_METRICS.restart_total.inc(); + self.model .load_from_metastore(&mut self.metastore, ctx.progress()) .await @@ -312,7 +314,12 @@ impl ControlPlane { shard_ids: &[ShardId], progress: &Progress, ) -> anyhow::Result<()> { - info!(shard_ids=?shard_ids, source_uid=?source_uid, "deleting shards"); + debug!( + index_uid=%source_uid.index_uid, + source_id=%source_uid.source_id, + shard_ids=%shard_ids.pretty_display(), + "deleting shards" + ); let delete_shards_request = DeleteShardsRequest { index_uid: Some(source_uid.index_uid.clone()), source_id: source_uid.source_id.clone(), @@ -329,11 +336,41 @@ impl ControlPlane { .protect_future(self.metastore.delete_shards(delete_shards_request)) .await .context("failed to delete shards from metastore")?; + self.model.delete_shards(source_uid, shard_ids); + + info!( + index_uid=%source_uid.index_uid, + source_id=%source_uid.source_id, + shard_ids=%shard_ids.pretty_display(), + "deleted shards" + ); Ok(()) } fn debug_info(&self) -> JsonValue { + // Build the union of ingesters tracked by ingester pool and the model. + let mut ingesters: BTreeMap = BTreeMap::new(); + + for (ingester_id, ingester) in self.ingest_controller.ingester_pool.keys_values() { + let ingester_json = json!({ + "available": true, + "status": ingester.status.as_json_str_name(), + }); + ingesters.insert(ingester_id.clone(), ingester_json); + } + for shard in self.model.all_shards() { + let ingester_id = NodeId::from(shard.leader_id.clone()); + + if let Entry::Vacant(entry) = ingesters.entry(ingester_id.clone()) { + let ingester_json = json!({ + "available": false, + "status": IngesterStatus::default(), + }); + entry.insert(ingester_json); + } + } + let physical_indexing_plan: Vec = self .indexing_scheduler .observable_state() @@ -351,30 +388,34 @@ impl ControlPlane { }) .unwrap_or_default(); - let mut per_index_shards_json: HashMap> = HashMap::new(); + let mut per_index_and_leader_shards_json: BTreeMap< + IndexUid, + BTreeMap>, + > = BTreeMap::new(); for (source_uid, shard_entries) in self.model.all_shards_with_source() { - let index_uid = source_uid.index_uid.clone(); - let source_id = source_uid.source_id.clone(); - let shards_json = shard_entries.map(|shard_entry| { - json!({ - "index_uid": index_uid, - "source_id": source_id, - "shard_id": shard_entry.shard_id.clone(), + for shard_entry in shard_entries { + let shard_json = json!({ + "index_uid": source_uid.index_uid, + "source_id": source_uid.source_id, + "shard_id": shard_entry.shard_id, "shard_state": shard_entry.shard_state().as_json_str_name(), - "leader_id": shard_entry.leader_id.clone(), - "follower_id": shard_entry.follower_id.clone(), + "leader_id": shard_entry.leader_id, + "follower_id": shard_entry.follower_id, "publish_position_inclusive": shard_entry.publish_position_inclusive(), - }) - }); - per_index_shards_json - .entry(index_uid.clone()) - .or_default() - .extend(shards_json); + }); + per_index_and_leader_shards_json + .entry(source_uid.index_uid.clone()) + .or_default() + .entry(shard_entry.leader_id.clone()) + .or_default() + .push(shard_json); + } } json!({ + "ingesters": ingesters, "physical_indexing_plan": physical_indexing_plan, - "shard_table": per_index_shards_json, + "shard_table": per_index_and_leader_shards_json, }) } @@ -422,7 +463,21 @@ impl Handler for ControlPlane { shard_positions_update: ShardPositionsUpdate, ctx: &ActorContext, ) -> Result<(), ActorExitStatus> { - debug!(shard_positions_update=?shard_positions_update, "shard positions update"); + if enabled!(Level::DEBUG) { + let pretty_positions: Vec = shard_positions_update + .updated_shard_positions + .iter() + .map(|(shard_id, position)| format!("{shard_id}:{}", position.pretty_display())) + .sorted() + .collect(); + + debug!( + index_uid=%shard_positions_update.source_uid.index_uid, + source_id=%shard_positions_update.source_uid.source_id, + positions=%pretty_positions.as_slice().pretty_display(), + "received shard positions update" + ); + } let Some(shard_entries) = self .model .get_shards_for_source_mut(&shard_positions_update.source_uid) @@ -438,7 +493,13 @@ impl Handler for ControlPlane { Some(shard.publish_position_inclusive().max(position.clone())); if position.is_eof() { // identify shards that have reached EOF but have not yet been removed. - info!(shard_id=%shard_id, position=?position, "received eof shard via gossip"); + info!( + index_uid=%shard_positions_update.source_uid.index_uid, + source_id=%shard_positions_update.source_uid.source_id, + %shard_id, + position=%position.pretty_display(), + "received shard eof via gossip" + ); shard_ids_to_close.push(shard_id); } } @@ -534,6 +595,8 @@ impl DeferableReplyHandler for ControlPlane { reply: impl FnOnce(Self::Reply) + Send + Sync + 'static, ctx: &ActorContext, ) -> Result<(), ActorExitStatus> { + debug!("creating index"); + let response = match ctx .protect_future(self.metastore.create_index(request)) .await @@ -551,6 +614,8 @@ impl DeferableReplyHandler for ControlPlane { return Err(ActorExitStatus::from(anyhow::anyhow!(serde_error))); } }; + let index_uid = index_metadata.index_uid.clone(); + // Now, create index can also add sources to support creating indexes automatically from // index and source config templates. let should_rebuild_plan = !index_metadata.sources.is_empty(); @@ -565,6 +630,7 @@ impl DeferableReplyHandler for ControlPlane { } else { reply(Ok(response)); } + info!(%index_uid, "created index"); Ok(()) } } @@ -630,8 +696,6 @@ impl Handler for ControlPlane { { return convert_metastore_error(metastore_error); }; - info!(%index_uid, "deleted index"); - let ingester_needing_resync: BTreeSet = self .model .list_shards_for_index(&index_uid) @@ -648,6 +712,7 @@ impl Handler for ControlPlane { // the metastore. We should update the state of the control plane. let _rebuild_plan_waiter = self.rebuild_plan_debounced(ctx); + info!(%index_uid, "deleted index"); let response = EmptyResponse {}; Ok(Ok(response)) } @@ -669,6 +734,7 @@ impl Handler for ControlPlane { match serde_utils::from_json_str(&request.source_config_json) { Ok(source_config) => source_config, Err(error) => { + error!(%error, "failed to deserialize source config"); return Ok(Err(ControlPlaneError::from(error))); } }; @@ -707,6 +773,7 @@ impl Handler for ControlPlane { match serde_utils::from_json_str(&request.source_config_json) { Ok(source_config) => source_config, Err(error) => { + error!(%error, "failed to deserialize source config"); return Ok(Err(ControlPlaneError::from(error))); } }; @@ -723,12 +790,11 @@ impl Handler for ControlPlane { .update_source(&index_uid, source_config) .context("failed to add source")?; - info!(%index_uid, source_id, "updated source"); - // TODO: Refine the event. Notify index will have the effect to reload the entire state from // the metastore. We should update the state of the control plane. let _rebuild_plan_waiter = self.rebuild_plan_debounced(ctx); + info!(%index_uid, source_id, "updated source"); let response = EmptyResponse {}; Ok(Ok(response)) } @@ -756,7 +822,6 @@ impl Handler for ControlPlane { { return Ok(Err(ControlPlaneError::from(error))); }; - info!(%index_uid, source_id, enabled=enable, "toggled source"); let mutation_occurred = self .model @@ -766,7 +831,9 @@ impl Handler for ControlPlane { if mutation_occurred { let _rebuild_plan_waiter = self.rebuild_plan_debounced(ctx); } - Ok(Ok(EmptyResponse {})) + info!(%index_uid, source_id, enabled=enable, "toggled source"); + let response = EmptyResponse {}; + Ok(Ok(response)) } } @@ -783,6 +850,7 @@ impl Handler for ControlPlane { ) -> Result, ActorExitStatus> { let index_uid: IndexUid = request.index_uid().clone(); let source_id = request.source_id.clone(); + debug!(%index_uid, source_id, "deleting source"); let source_uid = SourceUid { index_uid: index_uid.clone(), @@ -815,10 +883,14 @@ impl Handler for ControlPlane { .sync_with_ingesters(&ingesters_needing_resync, &self.model); self.model.delete_source(&source_uid); - let _rebuild_plan_waiter = self.rebuild_plan_debounced(ctx); - let response = EmptyResponse {}; + info!( + index_uid=%source_uid.index_uid, + source_id=%source_uid.source_id, + "deleted source" + ); + let response = EmptyResponse {}; Ok(Ok(response)) } } @@ -845,13 +917,14 @@ impl Handler for ControlPlane { ), interval, ); - if let CooldownStatus::Ready = status { - if let Err(metastore_error) = self.metastore.prune_shards(request).await { - return convert_metastore_error(metastore_error); - }; - } + if let CooldownStatus::Ready = status + && let Err(metastore_error) = self.metastore.prune_shards(request).await + { + return convert_metastore_error(metastore_error); + }; // Return ok regardless of whether the call was successful or debounced - Ok(Ok(EmptyResponse {})) + let response = EmptyResponse {}; + Ok(Ok(response)) } } @@ -951,13 +1024,12 @@ impl ControlPlaneEventSubscriber { #[async_trait] impl EventSubscriber for ControlPlaneEventSubscriber { async fn handle_event(&mut self, local_shards_update: LocalShardsUpdate) { - if let Some(control_plane_mailbox) = self.0.upgrade() { - if let Err(error) = control_plane_mailbox + if let Some(control_plane_mailbox) = self.0.upgrade() + && let Err(error) = control_plane_mailbox .send_message(local_shards_update) .await - { - error!(error=%error, "failed to forward local shards update to control plane"); - } + { + error!(%error, "failed to forward local shards update to control plane"); } } } @@ -965,13 +1037,12 @@ impl EventSubscriber for ControlPlaneEventSubscriber { #[async_trait] impl EventSubscriber for ControlPlaneEventSubscriber { async fn handle_event(&mut self, shard_positions_update: ShardPositionsUpdate) { - if let Some(control_plane_mailbox) = self.0.upgrade() { - if let Err(error) = control_plane_mailbox + if let Some(control_plane_mailbox) = self.0.upgrade() + && let Err(error) = control_plane_mailbox .send_message(shard_positions_update) .await - { - error!(error=%error, "failed to forward shard positions update to control plane"); - } + { + error!(%error, "failed to forward shard positions update to control plane"); } } } @@ -991,61 +1062,25 @@ fn apply_index_template_match( Ok(index_config) } -/// The indexer joined the cluster. #[derive(Debug)] -struct IndexerJoined(ClusterNode); +struct RebalanceShards; #[async_trait] -impl Handler for ControlPlane { +impl Handler for ControlPlane { type Reply = (); async fn handle( &mut self, - message: IndexerJoined, + _message: RebalanceShards, ctx: &ActorContext, ) -> Result { - info!( - "indexer `{}` joined the cluster: rebalancing shards and rebuilding indexing plan", - message.0.node_id() - ); - // TODO: Update shard table. - if let Err(metastore_error) = self + if let Err(error) = self .ingest_controller .rebalance_shards(&mut self.model, ctx.mailbox(), ctx.progress()) .await { - return convert_metastore_error::<()>(metastore_error).map(|_| ()); - } - self.indexing_scheduler.rebuild_plan(&self.model); - Ok(()) - } -} - -/// The indexer left the cluster. -#[derive(Debug)] -struct IndexerLeft(ClusterNode); - -#[async_trait] -impl Handler for ControlPlane { - type Reply = (); - - async fn handle( - &mut self, - message: IndexerLeft, - ctx: &ActorContext, - ) -> Result { - info!( - "indexer `{}` left the cluster: rebalancing shards and rebuilding indexing plan", - message.0.node_id() - ); - // TODO: Update shard table. - if let Err(metastore_error) = self - .ingest_controller - .rebalance_shards(&mut self.model, ctx.mailbox(), ctx.progress()) - .await - { - return convert_metastore_error::<()>(metastore_error).map(|_| ()); - } + return convert_metastore_error::<()>(error).map(|_| ()); + }; self.indexing_scheduler.rebuild_plan(&self.model); Ok(()) } @@ -1060,10 +1095,9 @@ impl Handler for ControlPlane { message: RebalanceShardsCallback, _ctx: &ActorContext, ) -> Result { - info!( - "closing {} shards after rebalance", - message.closed_shards.len() - ); + let num_closed_shards = message.closed_shards.len(); + debug!("closing {num_closed_shards} shards after rebalance"); + for closed_shard in message.closed_shards { let shard_id = closed_shard.shard_id().clone(); let source_uid = SourceUid { @@ -1072,7 +1106,7 @@ impl Handler for ControlPlane { }; self.model.close_shards(&source_uid, &[shard_id]); } - // We drop the rebalance guard explicitly here to put some emphasis on where a the rebalance + // We drop the rebalance guard explicitly here to put some emphasis on where the rebalance // lock is released. drop(message.rebalance_guard); Ok(()) @@ -1094,24 +1128,50 @@ async fn watcher_indexers( let Some(mailbox) = weak_mailbox.upgrade() else { return; }; + + // Ingesters have two readiness levels: + // 1. Cluster connectivity: node is up and can reach the metastore (similar to other nodes) + // 2. Shard readiness: IngesterStatus::Ready indicates the ingester can accept new shards + // We rebalance shards when either readiness level changes. + let mut trigger_rebalance = false; match cluster_change { - ClusterChange::Add(node) => { - if node.enabled_services().contains(&QuickwitService::Indexer) { - if let Err(error) = mailbox.send_message(IndexerJoined(node)).await { - error!(error=%error, "failed to forward `IndexerJoined` event to control plane"); - } + ClusterChange::Add(node) if node.is_indexer() => { + if node.ingester_status().is_ready() { + info!( + "indexer `{}` with status `{}` joined the cluster: rebalancing shards and \ + rebuilding indexing plan", + node.node_id(), + node.ingester_status().as_json_str_name() + ); + trigger_rebalance = true; } } - ClusterChange::Remove(node) => { - if node.enabled_services().contains(&QuickwitService::Indexer) { - if let Err(error) = mailbox.send_message(IndexerLeft(node)).await { - error!(error=%error, "failed to forward `IndexerLeft` event to control plane"); - } - } + ClusterChange::Remove(node) if node.is_indexer() => { + info!( + "indexer `{}` left the cluster: rebalancing shards and rebuilding indexing \ + plan", + node.node_id() + ); + trigger_rebalance = true } - ClusterChange::Update(_) => { - // We are not interested in updates (yet). + ClusterChange::Update { previous, updated } if updated.is_indexer() => { + let was_ready = previous.ingester_status().is_ready(); + let is_ready = updated.ingester_status().is_ready(); + + if was_ready ^ is_ready { + info!( + "indexer `{}` status changed to `{}`: rebalancing shards and rebuilding \ + indexing plan", + updated.node_id(), + updated.ingester_status().as_json_str_name() + ); + trigger_rebalance = true; + } } + _ => {} + } + if trigger_rebalance && mailbox.send_message(RebalanceShards).await.is_err() { + return; } } } @@ -1123,11 +1183,12 @@ mod tests { use mockall::Sequence; use quickwit_actors::{AskError, Observe, SupervisorMetrics}; - use quickwit_cluster::ClusterChangeStreamFactoryForTest; + use quickwit_cluster::{ClusterChangeStreamFactoryForTest, ClusterNode}; use quickwit_config::{ CLI_SOURCE_ID, INGEST_V2_SOURCE_ID, IndexConfig, KafkaSourceParams, SourceParams, }; use quickwit_indexing::IndexingService; + use quickwit_ingest::IngesterPoolEntry; use quickwit_metastore::{ CreateIndexRequestExt, IndexMetadata, ListIndexesMetadataResponseExt, }; @@ -1139,8 +1200,8 @@ mod tests { MockIndexingService, }; use quickwit_proto::ingest::ingester::{ - IngesterServiceClient, InitShardSuccess, InitShardsResponse, MockIngesterService, - RetainShardsResponse, + IngesterServiceClient, IngesterStatus, InitShardSuccess, InitShardsResponse, + MockIngesterService, RetainShardsResponse, }; use quickwit_proto::ingest::{Shard, ShardPKey, ShardState}; use quickwit_proto::metastore::{ @@ -1757,14 +1818,13 @@ mod tests { #[tokio::test] async fn test_delete_shard_on_eof() { - quickwit_common::setup_logging_for_tests(); let universe = Universe::with_accelerated_time(); - let node_id = NodeId::new("control-plane-node".to_string()); + let node_id = NodeId::new("test-control-plane".to_string()); let indexer_pool = IndexerPool::default(); let (client_mailbox, client_inbox) = universe.create_test_mailbox(); let client = IndexingServiceClient::from_mailbox::(client_mailbox); let indexer_node_info = IndexerNodeInfo { - node_id: NodeId::new("indexer-node-1".to_string()), + node_id: NodeId::new("test-indexer".to_string()), generation_id: 0, client, indexing_tasks: Vec::new(), @@ -1810,7 +1870,7 @@ mod tests { index_uid: Some(index_0.index_uid.clone()), source_id: INGEST_V2_SOURCE_ID.to_string(), shard_id: Some(ShardId::from(17)), - leader_id: "test_node".to_string(), + leader_id: "test-ingester".to_string(), publish_position_inclusive: Some(Position::Beginning), ..Default::default() }; @@ -1863,14 +1923,14 @@ mod tests { .unwrap(); let indexing_tasks = last_applied_physical_plan .indexing_tasks_per_indexer() - .get("indexer-node-1") + .get("test-indexer") .unwrap(); assert_eq!(indexing_tasks.len(), 1); assert_eq!(indexing_tasks[0].shard_ids, [ShardId::from(17)]); let control_plane_debug_info = control_plane_mailbox.ask(GetDebugInfo).await.unwrap(); - let shard = - &control_plane_debug_info["shard_table"]["test-index-0:00000000000000000000000000"][0]; + let shard = &control_plane_debug_info["shard_table"]["test-index-0:00000000000000000000000000"] + ["test-ingester"][0]; assert_eq!(shard["shard_id"], "00000000000000000017"); assert_eq!(shard["publish_position_inclusive"], "00000000000000001000"); @@ -1894,16 +1954,13 @@ mod tests { .unwrap(); let indexing_tasks = last_applied_physical_plan .indexing_tasks_per_indexer() - .get("indexer-node-1") + .get("test-indexer") .unwrap(); assert!(indexing_tasks.is_empty()); let apply_plan_requests = client_inbox.drain_for_test_typed::(); - assert!(!apply_plan_requests.is_empty()); - - for apply_plan_request in &apply_plan_requests { - assert!(apply_plan_request.indexing_tasks.is_empty()); - } + let last_apply_plan_request = apply_plan_requests.last().unwrap(); + assert!(last_apply_plan_request.indexing_tasks.is_empty()); universe.assert_quit().await; } @@ -1911,12 +1968,12 @@ mod tests { #[tokio::test] async fn test_fill_shard_table_position_from_metastore_on_startup() { let universe = Universe::with_accelerated_time(); - let node_id = NodeId::new("control-plane-node".to_string()); + let node_id = NodeId::new("test-control-plane".to_string()); let indexer_pool = IndexerPool::default(); let (client_mailbox, _client_inbox) = universe.create_test_mailbox(); let client = IndexingServiceClient::from_mailbox::(client_mailbox); let indexer_node_info = IndexerNodeInfo { - node_id: NodeId::new("indexer-node-1".to_string()), + node_id: NodeId::new("test-indexer".to_string()), generation_id: 0, client, indexing_tasks: Vec::new(), @@ -1945,7 +2002,7 @@ mod tests { index_uid: Some(index_metadata.index_uid.clone()), source_id: INGEST_V2_SOURCE_ID.to_string(), shard_id: Some(ShardId::from(17)), - leader_id: "test_node".to_string(), + leader_id: "test-ingester".to_string(), publish_position_inclusive: Some(Position::Offset(1234u64.into())), ..Default::default() }; @@ -1977,8 +2034,8 @@ mod tests { MetastoreServiceClient::from_mock(mock_metastore), ); let control_plane_debug_info = control_plane_mailbox.ask(GetDebugInfo).await.unwrap(); - let shard = - &control_plane_debug_info["shard_table"]["test-index:00000000000000000000000000"][0]; + let shard = &control_plane_debug_info["shard_table"]["test-index:00000000000000000000000000"] + ["test-ingester"][0]; assert_eq!(shard["shard_id"], "00000000000000000017"); assert_eq!(shard["publish_position_inclusive"], "00000000000000001234"); @@ -1989,12 +2046,12 @@ mod tests { async fn test_delete_non_existing_shard() { quickwit_common::setup_logging_for_tests(); let universe = Universe::default(); - let node_id = NodeId::new("control-plane-node".to_string()); + let node_id = NodeId::new("test-control-plane".to_string()); let indexer_pool = IndexerPool::default(); let (client_mailbox, _client_inbox) = universe.create_test_mailbox(); let client = IndexingServiceClient::from_mailbox::(client_mailbox); let indexer_node_info = IndexerNodeInfo { - node_id: NodeId::new("indexer-node-1".to_string()), + node_id: NodeId::new("test-indexer".to_string()), generation_id: 0, client, indexing_tasks: Vec::new(), @@ -2082,7 +2139,7 @@ mod tests { async fn test_delete_index() { quickwit_common::setup_logging_for_tests(); let universe = Universe::default(); - let node_id = NodeId::new("control-plane-node".to_string()); + let node_id = NodeId::new("test-control-plane".to_string()); let indexer_pool = IndexerPool::default(); let ingester_pool = IngesterPool::default(); @@ -2166,7 +2223,8 @@ mod tests { assert!(&retain_shards_for_source.shard_ids.is_empty()); Ok(RetainShardsResponse {}) }); - let ingester = IngesterServiceClient::from_mock(mock_ingester); + let ingester = + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester)); ingester_pool.insert("node1".into(), ingester); let cluster_config = ClusterConfig::for_test(); @@ -2196,7 +2254,7 @@ mod tests { async fn test_delete_source() { quickwit_common::setup_logging_for_tests(); let universe = Universe::default(); - let node_id = NodeId::new("control-plane-node".to_string()); + let node_id = NodeId::new("test-control-plane".to_string()); let indexer_pool = IndexerPool::default(); let ingester_pool = IngesterPool::default(); @@ -2212,7 +2270,8 @@ mod tests { ); Ok(RetainShardsResponse {}) }); - let ingester = IngesterServiceClient::from_mock(mock_ingester); + let ingester = + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester)); ingester_pool.insert("node1".into(), ingester); let mut index_0 = IndexMetadata::for_test("test-index-0", "ram:///test-index-0"); @@ -2399,24 +2458,96 @@ mod tests { let cluster_change_stream_tx = cluster_change_stream_factory.change_stream_tx(); - let metastore_node = - ClusterNode::for_test("test-metastore", 1337, false, &["metastore"], &[]).await; + // a non-indexer node status change doesn't trigger a shard rebalancing. + let metastore_node = ClusterNode::for_test( + "test-metastore", + 1515, + false, + &["metastore"], + &[], + IngesterStatus::Unspecified, + ) + .await; let cluster_change = ClusterChange::Add(metastore_node); cluster_change_stream_tx.send(cluster_change).unwrap(); - let indexer_node = - ClusterNode::for_test("test-indexer", 1515, false, &["indexer"], &[]).await; + tokio::time::sleep(Duration::from_millis(1)).await; + assert!( + control_plane_inbox + .drain_for_test_typed::() + .is_empty() + ); + + // an indexer initializing doesn't trigger a shard rebalancing. + let indexer_node_initializing: ClusterNode = ClusterNode::for_test( + "test-indexer", + 1515, + false, + &["indexer"], + &[], + IngesterStatus::Initializing, + ) + .await; + let cluster_change = ClusterChange::Add(indexer_node_initializing); + cluster_change_stream_tx.send(cluster_change).unwrap(); + + tokio::time::sleep(Duration::from_millis(1)).await; + assert!( + control_plane_inbox + .drain_for_test_typed::() + .is_empty() + ); + + // an indexer ready triggers a shard rebalancing. + let indexer_node: ClusterNode = ClusterNode::for_test( + "test-indexer", + 1515, + false, + &["indexer"], + &[], + IngesterStatus::Ready, + ) + .await; let cluster_change = ClusterChange::Add(indexer_node.clone()); cluster_change_stream_tx.send(cluster_change).unwrap(); + tokio::time::sleep(Duration::from_millis(1)).await; + let RebalanceShards = control_plane_inbox.recv_typed_message().await.unwrap(); + + // removing an indexer node triggers a shard rebalancing. let cluster_change = ClusterChange::Remove(indexer_node.clone()); cluster_change_stream_tx.send(cluster_change).unwrap(); - let IndexerJoined(joined) = control_plane_inbox.recv_typed_message().await.unwrap(); - assert_eq!(joined.grpc_advertise_addr().port(), 1516); + tokio::time::sleep(Duration::from_millis(1)).await; + let RebalanceShards = control_plane_inbox.recv_typed_message().await.unwrap(); + + // a change in IngesterStatus readiness triggers a shard rebalancing. + let node_ready = ClusterNode::for_test( + "test-indexer", + 1515, + false, + &["indexer"], + &[], + IngesterStatus::Ready, + ) + .await; + let node_retiring = ClusterNode::for_test( + "test-indexer", + 1515, + false, + &["indexer"], + &[], + IngesterStatus::Retiring, + ) + .await; + let cluster_change = ClusterChange::Update { + previous: node_ready, + updated: node_retiring, + }; + cluster_change_stream_tx.send(cluster_change).unwrap(); - let IndexerLeft(left) = control_plane_inbox.recv_typed_message().await.unwrap(); - assert_eq!(left.grpc_advertise_addr().port(), 1516); + tokio::time::sleep(Duration::from_millis(1)).await; + let RebalanceShards = control_plane_inbox.recv_typed_message().await.unwrap(); universe.assert_quit().await; } @@ -2449,8 +2580,15 @@ mod tests { disable_control_loop, ); let cluster_change_stream_tx = cluster_change_stream_factory.change_stream_tx(); - let indexer_node = - ClusterNode::for_test("test-indexer", 1515, false, &["indexer"], &[]).await; + let indexer_node: ClusterNode = ClusterNode::for_test( + "test-indexer", + 1515, + false, + &["indexer"], + &[], + IngesterStatus::Ready, + ) + .await; let cluster_change = ClusterChange::Add(indexer_node.clone()); cluster_change_stream_tx.send(cluster_change).unwrap(); @@ -2508,7 +2646,8 @@ mod tests { }; Ok(response) }); - let ingester = IngesterServiceClient::from_mock(mock_ingester); + let ingester = + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester)); ingester_pool.insert(ingester_id, ingester); let mut mock_metastore = MockMetastoreService::new(); @@ -2612,8 +2751,8 @@ mod tests { control_plane_mailbox.ask(callback).await.unwrap(); let control_plane_debug_info = control_plane_mailbox.ask(GetDebugInfo).await.unwrap(); - let shard = - &control_plane_debug_info["shard_table"]["test-index:00000000000000000000000000"][0]; + let shard = &control_plane_debug_info["shard_table"]["test-index:00000000000000000000000000"] + ["test-ingester"][0]; assert_eq!(shard["shard_id"], "00000000000000000000"); assert_eq!(shard["shard_state"], "closed"); @@ -2662,7 +2801,8 @@ mod tests { }; Ok(response) }); - let ingester = IngesterServiceClient::from_mock(mock_ingester); + let ingester = + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester)); ingester_pool.insert(ingester_id, ingester); let mut mock_metastore = MockMetastoreService::new(); @@ -2751,8 +2891,8 @@ mod tests { control_plane_debug_info["physical_indexing_plan"][0]["node_id"], "test-ingester" ); - let shard = - &control_plane_debug_info["shard_table"]["test-index:00000000000000000000000000"][0]; + let shard = &control_plane_debug_info["shard_table"]["test-index:00000000000000000000000000"] + ["test-ingester"][0]; assert_eq!(shard["index_uid"], "test-index:00000000000000000000000000"); assert_eq!(shard["source_id"], INGEST_V2_SOURCE_ID); assert_eq!(shard["shard_id"], "00000000000000000000"); @@ -2761,7 +2901,7 @@ mod tests { assert_eq!(shard["follower_id"], JsonValue::Null); assert_eq!( shard["publish_position_inclusive"], - Position::Beginning.to_string() + json!(Position::Beginning) ); universe.assert_quit().await; diff --git a/quickwit/quickwit-control-plane/src/cooldown_map.rs b/quickwit/quickwit-control-plane/src/cooldown_map.rs index 1d498cc80c5..0a9197516b8 100644 --- a/quickwit/quickwit-control-plane/src/cooldown_map.rs +++ b/quickwit/quickwit-control-plane/src/cooldown_map.rs @@ -53,13 +53,12 @@ impl CooldownMap { } } else { let capacity: usize = self.0.cap().into(); - if self.0.len() == capacity { - if let Some((_, deadline)) = self.0.peek_lru() { - if *deadline > now { - // the oldest entry is not outdated, grow the LRU - self.0.resize(NonZeroUsize::new(capacity * 2).unwrap()); - } - } + if self.0.len() == capacity + && let Some((_, deadline)) = self.0.peek_lru() + && *deadline > now + { + // the oldest entry is not outdated, grow the LRU + self.0.resize(NonZeroUsize::new(capacity * 2).unwrap()); } self.0.push(key, now + cooldown_interval); CooldownStatus::Ready diff --git a/quickwit/quickwit-control-plane/src/debouncer.rs b/quickwit/quickwit-control-plane/src/debouncer.rs index 42f68ce4f16..8baa2c8661d 100644 --- a/quickwit/quickwit-control-plane/src/debouncer.rs +++ b/quickwit/quickwit-control-plane/src/debouncer.rs @@ -112,7 +112,9 @@ impl Debouncer { } pub fn self_send_with_cooldown(&self, ctx: &ActorContext>) - where M: Default + std::fmt::Debug + Send + Sync + 'static { + where + M: Default + std::fmt::Debug + Send + Sync + 'static, + { let cooldown_state = self.accept_transition(Transition::Emit); match cooldown_state { DebouncerState::NoCooldown => { diff --git a/quickwit/quickwit-control-plane/src/indexing_scheduler/mod.rs b/quickwit/quickwit-control-plane/src/indexing_scheduler/mod.rs index 5d51dcac13f..9f79032d7c5 100644 --- a/quickwit/quickwit-control-plane/src/indexing_scheduler/mod.rs +++ b/quickwit/quickwit-control-plane/src/indexing_scheduler/mod.rs @@ -18,12 +18,13 @@ mod scheduling; use std::cmp::Ordering; use std::fmt; use std::num::NonZeroU32; -use std::sync::Arc; +use std::sync::{Arc, OnceLock}; use std::time::{Duration, Instant}; use fnv::{FnvHashMap, FnvHashSet}; use itertools::Itertools; use once_cell::sync::OnceCell; +use quickwit_common::is_metrics_index; use quickwit_common::pretty::PrettySample; use quickwit_config::{FileSourceParams, SourceParams, indexing_pipeline_params_fingerprint}; use quickwit_proto::indexing::{ @@ -42,6 +43,8 @@ use crate::metrics::ShardLocalityMetrics; use crate::model::{ControlPlaneModel, ShardEntry, ShardLocations}; use crate::{IndexerNodeInfo, IndexerPool}; +const DEFAULT_ENABLE_VARIABLE_SHARD_LOAD: bool = false; + pub(crate) const MIN_DURATION_BETWEEN_SCHEDULING: Duration = if cfg!(any(test, feature = "testsuite")) { Duration::from_millis(50) @@ -121,7 +124,30 @@ impl fmt::Debug for IndexingScheduler { fn enable_variable_shard_load() -> bool { static IS_SHARD_LOAD_CP_ENABLED: OnceCell = OnceCell::new(); *IS_SHARD_LOAD_CP_ENABLED.get_or_init(|| { - !quickwit_common::get_bool_from_env("QW_DISABLE_VARIABLE_SHARD_LOAD", false) + if let Some(enable_flag) = + quickwit_common::get_bool_from_env_opt("QW_ENABLE_VARIABLE_SHARD_LOAD") + { + return enable_flag; + } + // For backward compatibility, if QW_DISABLE_VARIABLE_SHARD_LOAD is set, we accept this + // value too. + if let Some(disable_flag) = + quickwit_common::get_bool_from_env_opt("QW_DISABLE_VARIABLE_SHARD_LOAD") + { + warn!( + disable = disable_flag, + "QW_DISABLE_VARIABLE_SHARD_LOAD is deprecated. Please use \ + QW_ENABLE_VARIABLE_SHARD_LOAD instead. We will use your setting in this version, \ + but will likely ignore it in future versions." + ); + return !disable_flag; + } + // Defaulting to false + info!( + "QW_ENABLE_VARIABLE_SHARD_LOAD not set, defaulting to {}", + DEFAULT_ENABLE_VARIABLE_SHARD_LOAD + ); + DEFAULT_ENABLE_VARIABLE_SHARD_LOAD }) } @@ -152,10 +178,22 @@ fn compute_load_per_shard(shard_entries: &[&ShardEntry]) -> NonZeroU32 { const MIN_CPU_LOAD_PER_SHARD: u32 = 50u32; NonZeroU32::new((num_cpu_millis as u32).max(MIN_CPU_LOAD_PER_SHARD)).unwrap() } else { - NonZeroU32::new(PIPELINE_FULL_CAPACITY.cpu_millis() / 4).unwrap() + get_default_load_per_shard() } } +fn get_default_load_per_shard() -> NonZeroU32 { + static DEFAULT_LOAD_PER_SHARD: OnceLock = OnceLock::new(); + *DEFAULT_LOAD_PER_SHARD.get_or_init(|| { + let default_load_per_shard = quickwit_common::get_from_env( + "QW_DEFAULT_LOAD_PER_SHARD", + PIPELINE_FULL_CAPACITY.cpu_millis() / 4, + false, + ); + NonZeroU32::new(default_load_per_shard).unwrap() + }) +} + fn get_sources_to_schedule(model: &ControlPlaneModel) -> Vec { let mut sources = Vec::new(); @@ -178,6 +216,11 @@ fn get_sources_to_schedule(model: &ControlPlaneModel) -> Vec { } SourceParams::IngestApi => { + // Metrics indexes should use IngestV2 only, not IngestV1. + // The ParquetSourceLoader doesn't support IngestV1. + if is_metrics_index(&source_uid.index_uid.index_id) { + continue; + } // TODO ingest v1 is scheduled differently sources.push(SourceToSchedule { source_uid, @@ -318,12 +361,11 @@ impl IndexingScheduler { self.rebuild_plan(model); return; }; - if let Some(last_applied_plan_timestamp) = self.state.last_applied_plan_timestamp { - if Instant::now().duration_since(last_applied_plan_timestamp) + if let Some(last_applied_plan_timestamp) = self.state.last_applied_plan_timestamp + && Instant::now().duration_since(last_applied_plan_timestamp) < MIN_DURATION_BETWEEN_SCHEDULING - { - return; - } + { + return; } let indexers: Vec = self.get_indexers_from_indexer_pool(); let running_indexing_tasks_by_node_id: FnvHashMap> = indexers @@ -356,7 +398,7 @@ impl IndexingScheduler { notify_on_drop: Option>, ) { debug!(new_physical_plan=?new_physical_plan, "apply physical indexing plan"); - crate::metrics::CONTROL_PLANE_METRICS.apply_total.inc(); + crate::metrics::CONTROL_PLANE_METRICS.apply_plan_total.inc(); for (node_id, indexing_tasks) in new_physical_plan.indexing_tasks_per_indexer() { // We don't want to block on a slow indexer so we apply this change asynchronously // TODO not blocking is cool, but we need to make sure there is not accumulation @@ -377,7 +419,7 @@ impl IndexingScheduler { .await { warn!( - error=%error, + %error, node_id=%indexer.node_id, generation_id=indexer.generation_id, "failed to apply indexing plan to indexer" @@ -945,7 +987,7 @@ mod tests { index_uid: IndexUid::for_test("index-2", 0), source_id: "source2".to_string(), }; - let sources = vec![ + let sources = [ SourceToSchedule { source_uid: source_1.clone(), source_type: SourceToScheduleType::NonSharded { diff --git a/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/mod.rs b/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/mod.rs index ca5ac4d1fc7..8a6b0f025c7 100644 --- a/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/mod.rs +++ b/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/mod.rs @@ -750,7 +750,7 @@ mod tests { use itertools::Itertools; use quickwit_proto::indexing::{CpuCapacity, IndexingTask, mcpu}; use quickwit_proto::types::{IndexUid, NodeId, PipelineUid, ShardId, SourceUid}; - use rand::seq::SliceRandom; + use rand::prelude::IndexedRandom; use super::{ SourceToSchedule, SourceToScheduleType, build_physical_indexing_plan, @@ -875,7 +875,7 @@ mod tests { for indexer in &indexers { indexer_id_to_cpu_capacities.insert(indexer.as_str().to_string(), mcpu(16_000)); } - let mut rng = rand::thread_rng(); + let mut rng = rand::rng(); let mut shard_locations = ShardLocations::default(); for shard_id in &shard_ids { diff --git a/quickwit/quickwit-control-plane/src/ingest/ingest_controller.rs b/quickwit/quickwit-control-plane/src/ingest/ingest_controller.rs index bc6242cf5d1..53d3ee4034f 100644 --- a/quickwit/quickwit-control-plane/src/ingest/ingest_controller.rs +++ b/quickwit/quickwit-control-plane/src/ingest/ingest_controller.rs @@ -13,7 +13,7 @@ // limitations under the License. use std::collections::btree_map::Entry; -use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::fmt; use std::future::Future; use std::num::NonZeroUsize; @@ -23,6 +23,7 @@ use std::time::Duration; use fnv::FnvHashSet; use futures::StreamExt; use futures::stream::FuturesUnordered; +use itertools::{Itertools as _, MinMaxResult}; use quickwit_actors::Mailbox; use quickwit_common::Progress; use quickwit_common::pretty::PrettySample; @@ -33,7 +34,7 @@ use quickwit_proto::control_plane::{ GetOrCreateOpenShardsSuccess, }; use quickwit_proto::ingest::ingester::{ - CloseShardsRequest, CloseShardsResponse, IngesterService, InitShardFailure, + CloseShardsRequest, CloseShardsResponse, IngesterService, IngesterStatus, InitShardFailure, InitShardSubrequest, InitShardsRequest, InitShardsResponse, RetainShardsForSource, RetainShardsRequest, }; @@ -45,13 +46,13 @@ use quickwit_proto::metastore::{ OpenShardsRequest, OpenShardsResponse, serde_utils, }; use quickwit_proto::types::{IndexUid, NodeId, NodeIdRef, Position, ShardId, SourceUid}; +use rand::prelude::IndexedRandom; use rand::rngs::ThreadRng; use rand::seq::SliceRandom; -use rand::{Rng, RngCore, thread_rng}; +use rand::{Rng, RngCore, rng}; use serde::{Deserialize, Serialize}; use tokio::sync::{Mutex, OwnedMutexGuard}; -use tokio::task::JoinHandle; -use tracing::{Level, debug, enabled, error, info, warn}; +use tracing::{Level, debug, enabled, error, info, instrument, warn}; use ulid::Ulid; use super::scaling_arbiter::ScalingArbiter; @@ -81,11 +82,11 @@ const FIRE_AND_FORGET_TIMEOUT: Duration = Duration::from_secs(3); /// All errors are ignored, and not even logged. fn fire_and_forget( fut: impl Future + Send + 'static, - operation: impl std::fmt::Display + Send + Sync + 'static, + operation: impl std::fmt::Display + Send + 'static, ) { tokio::spawn(async move { if let Err(_timeout_elapsed) = tokio::time::timeout(FIRE_AND_FORGET_TIMEOUT, fut).await { - error!(operation=%operation, "timeout elapsed"); + error!(%operation, "timeout elapsed"); } }); } @@ -100,14 +101,14 @@ fn pick_position( let except_pos_opt = except_el_opt.and_then(|except_el| els.iter().position(|el| *el == except_el)); if let Some(except_pos) = except_pos_opt { - let pos = rng.gen_range(0..els.len() - 1); + let pos = rng.random_range(0..els.len() - 1); if pos >= except_pos { Some(pos + 1) } else { Some(pos) } } else { - Some(rng.gen_range(0..els.len())) + Some(rng.random_range(0..els.len())) } } @@ -179,7 +180,7 @@ fn allocate_shards( .or_default() .push(node_id.as_ref()); } - let mut rng = thread_rng(); + let mut rng = rng(); let mut shard_allocations: Vec<(&NodeIdRef, Option<&NodeIdRef>)> = Vec::with_capacity(num_shards); for _ in 0..num_shards { @@ -200,12 +201,12 @@ pub struct IngestControllerStats { } pub struct IngestController { - ingester_pool: IngesterPool, + pub(crate) ingester_pool: IngesterPool, + pub(crate) stats: IngestControllerStats, metastore: MetastoreServiceClient, replication_factor: usize, // This lock ensures that only one rebalance operation is performed at a time. rebalance_lock: Arc>, - pub stats: IngestControllerStats, scaling_arbiter: ScalingArbiter, } @@ -324,17 +325,17 @@ impl IngestController { /// Syncs the ingester in a fire and forget manner. /// /// The returned oneshot is just here for unit test to wait for the operation to terminate. - fn sync_with_ingester(&self, ingester: &NodeId, model: &ControlPlaneModel) -> WaitHandle { - info!(ingester = %ingester, "sync_with_ingester"); + fn sync_with_ingester(&self, ingester_id: &NodeId, model: &ControlPlaneModel) -> WaitHandle { + info!(ingester = %ingester_id, "sync_with_ingester"); let (wait_drop_guard, wait_handle) = WaitHandle::new(); - let Some(ingester_client) = self.ingester_pool.get(ingester) else { + let Some(ingester) = self.ingester_pool.get(ingester_id) else { // TODO: (Maybe) We should mark the ingester as unavailable, and stop advertise its // shard to routers. - warn!("failed to sync with ingester `{ingester}`: not available"); + warn!("failed to sync with ingester `{ingester_id}`: not available"); return wait_handle; }; let mut retain_shards_req = RetainShardsRequest::default(); - for (source_uid, shard_ids) in &*model.list_shards_for_node(ingester) { + for (source_uid, shard_ids) in &*model.list_shards_for_node(ingester_id) { let shards_for_source = RetainShardsForSource { index_uid: Some(source_uid.index_uid.clone()), source_id: source_uid.source_id.clone(), @@ -344,12 +345,12 @@ impl IngestController { .retain_shards_for_sources .push(shards_for_source); } - info!(ingester = %ingester, "retain shards ingester"); - let operation: String = format!("retain shards `{ingester}`"); + info!(%ingester_id, "retain shards ingester"); + let operation: String = format!("retain shards `{ingester_id}`"); fire_and_forget( async move { if let Err(retain_shards_err) = - ingester_client.retain_shards(retain_shards_req).await + ingester.client.retain_shards(retain_shards_req).await { error!(%retain_shards_err, "retain shards error"); } @@ -546,10 +547,12 @@ impl IngestController { // shards). let mut per_node_num_open_shards: HashMap = self .ingester_pool - .keys() + .keys_values() .into_iter() - .filter(|ingester| !unavailable_leaders.contains(ingester)) - .map(|ingester| (ingester, 0)) + .filter(|(ingester_id, ingester)| { + ingester.status.is_ready() && !unavailable_leaders.contains(ingester_id) + }) + .map(|(ingester_id, _)| (ingester_id, 0)) .collect(); let num_ingesters = per_node_num_open_shards.len(); @@ -646,7 +649,7 @@ impl IngestController { let init_shards_future = async move { let init_shards_result = tokio::time::timeout( INIT_SHARDS_REQUEST_TIMEOUT, - leader.init_shards(init_shards_request), + leader.client.init_shards(init_shards_request), ) .await; (leader_id.clone(), init_shards_result, init_shard_failures) @@ -789,11 +792,13 @@ impl IngestController { let index_metadata = model .index_metadata(&source_uid.index_uid) .expect("index should exist"); - let validate_docs = model + let has_transform = model .source_metadata(source_uid) .expect("source should exist") .transform_config - .is_none(); + .is_some(); + let validate_docs = + index_metadata.index_config.ingest_settings.validate_docs && !has_transform; let doc_mapping = &index_metadata.index_config.doc_mapping; let doc_mapping_uid = doc_mapping.doc_mapping_uid; let doc_mapping_json = serde_utils::to_json_str(doc_mapping)?; @@ -906,7 +911,7 @@ impl IngestController { let close_shards_request = CloseShardsRequest { shard_pkeys }; if let Err(error) = progress - .protect_future(ingester.close_shards(close_shards_request)) + .protect_future(ingester.client.close_shards(close_shards_request)) .await { warn!("failed to scale down number of shards: {error}"); @@ -922,7 +927,10 @@ impl IngestController { request: AdviseResetShardsRequest, model: &ControlPlaneModel, ) -> AdviseResetShardsResponse { - info!("advise reset shards"); + info!( + "received advise reset shards request from `{}`", + request.ingester_id + ); debug!(shard_ids=?summarize_shard_ids(&request.shard_ids), "advise reset shards"); let mut shards_to_delete: Vec = Vec::new(); @@ -995,130 +1003,86 @@ impl IngestController { } } - /// This method just "computes"" the number of shards to move for rebalance. - /// It does not run any side effect except logging. - /// - /// TODO we consider the number of alive ingesters for this computation, - /// but deal with entire number of shards here. - /// This could cause problems when dealing with a lot of unavailable ingesters. - /// - /// On the other hand it biases thing the "right way": - /// If we are missing some ingesters, their shards should still be in the model, but they should - /// be missing from the ingester pool. - /// - /// As a result `num_open_shards_per_leader_threshold` should be inflated. - /// - /// TODO this implementation does not consider replica. - fn rebalance_compute_shards_to_move(&self, model: &ControlPlaneModel) -> Vec { - let num_ingesters = self.ingester_pool.len(); - let mut num_open_shards: usize = 0; - - if num_ingesters == 0 { - debug!("no ingester available"); - return Vec::new(); - } - - let mut per_leader_open_shards: HashMap<&str, Vec<&ShardEntry>> = HashMap::new(); - - for shard in model.all_shards() { - if shard.is_open() { - num_open_shards += 1; - per_leader_open_shards - .entry(&shard.leader_id) - .or_default() - .push(shard); - } - } - - // We tolerate an ingester with 10% more shards than the average. - // Let's first identify the list of shards we want to "move". - let num_open_shards_per_leader_threshold = - (num_open_shards * 11).div_ceil(10 * num_ingesters); - let mut shards_to_move: Vec = Vec::new(); - - let mut rng = thread_rng(); - for open_shards in per_leader_open_shards.values() { - if let Some(num_shards_to_move) = open_shards - .len() - .checked_sub(num_open_shards_per_leader_threshold) - { - shards_to_move.extend( - open_shards[..] - .choose_multiple(&mut rng, num_shards_to_move) - .map(|shard_entry| shard_entry.shard.clone()), - ); - } - } - - shards_to_move - } - - /// Moves shards from ingesters with too many shards to ingesters with too few shards. Moving a - /// shard consists of closing the shard on the source ingester and opening a new one on the - /// target ingester. + /// Rebalances shards from ingesters with too many shards to ingesters with too few shards. + /// Moving a shard consists of closing the shard on the source ingester and opening a new + /// one on the target ingester. /// /// This method is guarded by a lock to ensure that only one rebalance operation is performed at /// a time. + #[instrument(skip_all)] pub(crate) async fn rebalance_shards( &mut self, model: &mut ControlPlaneModel, mailbox: &Mailbox, progress: &Progress, - ) -> MetastoreResult>> { + ) -> MetastoreResult { let Ok(rebalance_guard) = self.rebalance_lock.clone().try_lock_owned() else { - return Ok(None); + debug!("skipping rebalance: another rebalance is already in progress"); + return Ok(0); }; self.stats.num_rebalance_shards_ops += 1; - let shards_to_move: Vec = self.rebalance_compute_shards_to_move(model); - if shards_to_move.is_empty() { - return Ok(None); - } + let shards_to_rebalance: Vec = self.compute_shards_to_rebalance(model); - let num_shards_to_move = shards_to_move.len(); - info!("rebalancing {} shards", num_shards_to_move); + crate::metrics::CONTROL_PLANE_METRICS + .rebalance_shards + .set(shards_to_rebalance.len() as i64); + + if shards_to_rebalance.is_empty() { + debug!("skipping rebalance: no shards to rebalance"); + return Ok(0); + } + let mut per_source_num_shards_to_open: HashMap = HashMap::new(); - let mut new_shards_source_uids: HashMap = HashMap::new(); - for shard in &shards_to_move { - *new_shards_source_uids + for shard in &shards_to_rebalance { + *per_source_num_shards_to_open .entry(shard.source_uid()) .or_default() += 1; } + let mut per_source_num_opened_shards: HashMap = self + .try_open_shards( + per_source_num_shards_to_open, + model, + &Default::default(), + progress, + ) + .await + .inspect_err(|error| { + error!(%error, "failed to open shards during rebalance"); + crate::metrics::CONTROL_PLANE_METRICS + .rebalance_shards + .set(0); + })?; - let mut successfully_source_uids: HashMap = self - .try_open_shards(new_shards_source_uids, model, &Default::default(), progress) - .await?; + let num_opened_shards: usize = per_source_num_opened_shards.values().sum(); + + crate::metrics::CONTROL_PLANE_METRICS + .rebalance_shards + .set(num_opened_shards as i64); - for source_uid in successfully_source_uids.keys() { + for source_uid in per_source_num_opened_shards.keys() { // We temporarily disable the ability the scale down the number of shards for // the source to avoid closing the shards we just opened. model.drain_scaling_permits(source_uid, ScalingMode::Down); } - // Let's close one of the shard to move for every successfully newly opened shards. - let mut shards_to_close = Vec::new(); - for shard in shards_to_move { + // Close as many shards as we opened. Because `try_open_shards` might fail partially, we + // must only close the shards that we successfully opened. + let mut shards_to_close = Vec::with_capacity(shards_to_rebalance.len()); + + for shard in shards_to_rebalance { let source_uid = shard.source_uid(); - let Some(num_open_shards) = successfully_source_uids.get_mut(&source_uid) else { + let Some(num_open_shards) = per_source_num_opened_shards.get_mut(&source_uid) else { continue; }; if *num_open_shards == 0 { continue; }; *num_open_shards -= 1; - - let leader_id = NodeId::from(shard.leader_id.clone()); - let shard_pkey = ShardPKey { - index_uid: shard.index_uid.clone(), - source_id: shard.source_id.clone(), - shard_id: shard.shard_id.clone(), - }; - shards_to_close.push((leader_id, shard_pkey)); + shards_to_close.push(shard); } - - let mailbox_clone = mailbox.clone(); - let close_shards_fut = self.close_shards(shards_to_close); + let mailbox_clone = mailbox.clone(); let close_shards_and_send_callback_fut = async move { // We wait for a few seconds before closing the shards to give the ingesters some time @@ -1136,17 +1100,122 @@ impl IngestController { }; let _ = mailbox_clone.send_message(callback).await; }; + tokio::spawn(close_shards_and_send_callback_fut); - Ok(Some(tokio::spawn(close_shards_and_send_callback_fut))) + if num_opened_shards > 0 { + info!("rebalance opened {num_opened_shards} new shards"); + } + Ok(num_opened_shards) } + /// Computes shards that need to be rebalanced. + /// + /// This function identifies which shards should be moved to achieve a balance across available + /// ingesters. + /// It does not mutate any state. It just identifies the list of shards + /// that need to be rebalanced. + /// + /// Unfortunately, we cannot move shards that are on unavailable ingesters. + /// The closing operation can only be done by the leader of that shard. + /// For these reason, we exclude these shards from the rebalance process. + fn compute_shards_to_rebalance(&self, model: &ControlPlaneModel) -> Vec { + let mut per_ready_ingester_shards: HashMap> = HashMap::new(); + let mut retiring_ingesters: HashSet = HashSet::new(); + + for (ingester_id, ingester) in self.ingester_pool.keys_values() { + if ingester.status.is_ready() { + per_ready_ingester_shards.insert(ingester_id, Vec::new()); + } else if ingester.status == IngesterStatus::Retiring { + retiring_ingesters.insert(ingester_id); + } + } + + let mut shards_to_rebalance: Vec = Vec::new(); + let mut num_ready_shards: usize = 0; + + for shard in model.all_shards() { + if !shard.is_open() { + continue; + } + let leader_id_ref = NodeIdRef::from_str(&shard.leader_id); + + if let Some(shards) = per_ready_ingester_shards.get_mut(leader_id_ref) { + // Shards on ready ingesters participate in the balancing logic. + num_ready_shards += 1; + shards.push(&shard.shard); + } else if retiring_ingesters.contains(leader_id_ref) { + // All open shards on retiring ingesters must be rebalanced. + shards_to_rebalance.push(shard.shard.clone()); + } + } + + let num_retiring_shards = shards_to_rebalance.len(); + let num_ready_ingesters = per_ready_ingester_shards.len(); + + let mut rng = rng(); + let mut per_leader_open_shards_shuffled: Vec> = per_ready_ingester_shards + .into_values() + .map(|mut shards| { + shards.shuffle(&mut rng); + shards + }) + .collect(); + + // This is more of a loop-loop, but since we know it should exit before + // `num_ready_shards`, we defensively use a for-loop. + for _ in 0..num_ready_shards { + let MinMaxResult::MinMax(min_shards, max_shards) = per_leader_open_shards_shuffled + .iter_mut() + .minmax_by_key(|shards| shards.len()) + else { + // There are less than 2 ingesters. + // Nothing to do here. + break; + }; + + // We leave a tolerance of 1/10 between the min and max number of shards per leader + const TOLERANCE_INV_RATIO: usize = 10; + if max_shards.len() + < min_shards.len() + min_shards.len().div_ceil(TOLERANCE_INV_RATIO).max(2) + { + break; + } + + let shard = max_shards.pop().expect("shards should not be empty"); + shards_to_rebalance.push(shard.clone()); + min_shards.push(shard); + } + + if shards_to_rebalance.is_empty() { + debug!("no shards to rebalance"); + } else { + info!( + num_ready_shards, + num_ready_ingesters, + num_retiring_shards, + num_shards_to_rebalance = shards_to_rebalance.len(), + "rebalancing shards" + ); + } + shards_to_rebalance + } + + /// Attempts to close the list of shards passed as argument. + /// + /// If ingesters are not available, the shards are not closed. fn close_shards( &self, - shards_to_close: Vec<(LeaderId, ShardPKey)>, + shards_to_close: Vec, ) -> impl Future> + Send + 'static { let mut per_leader_shards_to_close: HashMap> = HashMap::new(); - for (leader_id, shard_pkey) in shards_to_close { + for shard in shards_to_close { + let shard_pkey = ShardPKey { + index_uid: shard.index_uid, + source_id: shard.source_id, + shard_id: shard.shard_id, + }; + let leader_id = NodeId::from(shard.leader_id); per_leader_shards_to_close .entry(leader_id) .or_default() @@ -1163,7 +1232,7 @@ impl IngestController { let close_shards_future = async move { tokio::time::timeout( CLOSE_SHARDS_REQUEST_TIMEOUT, - ingester.close_shards(shards_to_close_request), + ingester.client.close_shards(shards_to_close_request), ) .await }; @@ -1221,7 +1290,7 @@ fn find_scale_down_candidate( model: &ControlPlaneModel, ) -> Option<(NodeId, ShardId)> { let mut per_leader_shard_entries: HashMap<&String, Vec<&ShardEntry>> = HashMap::new(); - let mut rng = thread_rng(); + let mut rng = rng(); for shard in model.get_shards_for_source(source_uid)?.values() { if shard.is_open() { @@ -1246,7 +1315,6 @@ fn find_scale_down_candidate( #[cfg(test)] mod tests { - use std::collections::BTreeSet; use std::str::FromStr; use std::sync::Arc; @@ -1258,12 +1326,12 @@ mod tests { use quickwit_common::shared_consts::DEFAULT_SHARD_THROUGHPUT_LIMIT; use quickwit_common::tower::DelayLayer; use quickwit_config::{DocMapping, INGEST_V2_SOURCE_ID, SourceConfig}; - use quickwit_ingest::{RateMibPerSec, ShardInfo}; + use quickwit_ingest::{IngesterPoolEntry, RateMibPerSec, ShardInfo}; use quickwit_metastore::IndexMetadata; use quickwit_proto::control_plane::GetOrCreateOpenShardsSubrequest; use quickwit_proto::ingest::ingester::{ - CloseShardsResponse, IngesterServiceClient, InitShardSuccess, InitShardsResponse, - MockIngesterService, RetainShardsResponse, + CloseShardsResponse, IngesterServiceClient, IngesterStatus, InitShardSuccess, + InitShardsResponse, MockIngesterService, RetainShardsResponse, }; use quickwit_proto::ingest::{IngestV2Error, Shard, ShardState}; use quickwit_proto::metastore::{ @@ -1327,10 +1395,13 @@ mod tests { let metastore = MetastoreServiceClient::from_mock(mock_metastore); let mock_ingester = MockIngesterService::new(); - let ingester = IngesterServiceClient::from_mock(mock_ingester); + let client = IngesterServiceClient::from_mock(mock_ingester); let ingester_pool = IngesterPool::default(); - ingester_pool.insert(NodeId::from("test-ingester-1"), ingester.clone()); + ingester_pool.insert( + NodeId::from("test-ingester-1"), + IngesterPoolEntry::ready_with_client(client.clone()), + ); let mut mock_ingester = MockIngesterService::new(); let index_uid_1_clone = index_uid_1.clone(); @@ -1358,7 +1429,10 @@ mod tests { Ok(response) }); let ingester = IngesterServiceClient::from_mock(mock_ingester); - ingester_pool.insert(NodeId::from("test-ingester-2"), ingester.clone()); + ingester_pool.insert( + NodeId::from("test-ingester-2"), + IngesterPoolEntry::ready_with_client(ingester.clone()), + ); let replication_factor = 2; let mut controller = IngestController::new( @@ -1541,10 +1615,13 @@ mod tests { }; Ok(response) }); - let ingester = IngesterServiceClient::from_mock(mock_ingester); + let client = IngesterServiceClient::from_mock(mock_ingester); let ingester_pool = IngesterPool::default(); - ingester_pool.insert(NodeId::from("test-ingester-1"), ingester.clone()); + ingester_pool.insert( + NodeId::from("test-ingester-1"), + IngesterPoolEntry::ready_with_client(client.clone()), + ); let replication_factor = 1; let mut controller = IngestController::new( @@ -1657,7 +1734,7 @@ mod tests { ingester_pool.insert( NodeId::from("test-ingester-1"), - IngesterServiceClient::mocked(), + IngesterPoolEntry::ready_with_client(IngesterServiceClient::mocked()), ); let leader_follower_pairs_opt = @@ -1667,7 +1744,10 @@ mod tests { // find any solution. assert!(leader_follower_pairs_opt.is_none()); - ingester_pool.insert("test-ingester-2".into(), IngesterServiceClient::mocked()); + ingester_pool.insert( + "test-ingester-2".into(), + IngesterPoolEntry::ready_with_client(IngesterServiceClient::mocked()), + ); let leader_follower_pairs = controller .allocate_shards(0, &FnvHashSet::default(), &model) @@ -1786,7 +1866,10 @@ mod tests { Some(NodeId::from("test-ingester-1")) ); - ingester_pool.insert("test-ingester-3".into(), IngesterServiceClient::mocked()); + ingester_pool.insert( + "test-ingester-3".into(), + IngesterPoolEntry::ready_with_client(IngesterServiceClient::mocked()), + ); let unavailable_leaders = FnvHashSet::from_iter([NodeId::from("test-ingester-2")]); let leader_follower_pairs = controller .allocate_shards(4, &unavailable_leaders, &model) @@ -1879,7 +1962,10 @@ mod tests { Ok(response) }); let ingester_0 = IngesterServiceClient::from_mock(mock_ingester_0); - ingester_pool.insert(ingester_id_0, ingester_0); + ingester_pool.insert( + ingester_id_0, + IngesterPoolEntry::ready_with_client(ingester_0), + ); let ingester_id_1 = NodeId::from("test-ingester-1"); let mut mock_ingester_1 = MockIngesterService::new(); @@ -1900,17 +1986,21 @@ mod tests { Err(IngestV2Error::Internal("internal error".to_string())) }); - let ingester_1 = IngesterServiceClient::from_mock(mock_ingester_1); + let ingester_1 = + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester_1)); ingester_pool.insert(ingester_id_1, ingester_1); let ingester_id_2 = NodeId::from("test-ingester-2"); let mut mock_ingester_2 = MockIngesterService::new(); mock_ingester_2.expect_init_shards().never(); - let ingester_2 = IngesterServiceClient::tower() + let client_2 = IngesterServiceClient::tower() .stack_init_shards_layer(DelayLayer::new(INIT_SHARDS_REQUEST_TIMEOUT * 2)) .build_from_mock(mock_ingester_2); - ingester_pool.insert(ingester_id_2, ingester_2); + ingester_pool.insert( + ingester_id_2, + IngesterPoolEntry::ready_with_client(client_2), + ); let init_shards_response = controller .init_shards(Vec::new(), &Progress::default()) @@ -2118,7 +2208,7 @@ mod tests { ingester_pool.insert( NodeId::from("test-ingester-1"), - IngesterServiceClient::from_mock(mock_ingester), + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester)), ); let source_uids: HashMap = HashMap::from_iter([(source_uid.clone(), 1)]); let unavailable_leaders = FnvHashSet::default(); @@ -2288,7 +2378,8 @@ mod tests { "failed to close shards".to_string(), )) }); - let ingester = IngesterServiceClient::from_mock(mock_ingester); + let ingester = + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester)); ingester_pool.insert("test-ingester".into(), ingester); let shard_infos = BTreeSet::from_iter([ @@ -2441,7 +2532,8 @@ mod tests { }, ); - let ingester = IngesterServiceClient::from_mock(mock_ingester); + let ingester = + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester)); ingester_pool.insert("test-ingester".into(), ingester); let shard_infos = BTreeSet::from_iter([ShardInfo { @@ -2587,7 +2679,8 @@ mod tests { }; Ok(response) }); - let ingester = IngesterServiceClient::from_mock(mock_ingester); + let ingester = + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester)); ingester_pool.insert("test-ingester".into(), ingester); // Test failed to open shards. @@ -2709,7 +2802,8 @@ mod tests { }; Ok(response) }); - let ingester = IngesterServiceClient::from_mock(mock_ingester); + let ingester = + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester)); ingester_pool.insert("test-ingester".into(), ingester); // Test failed to close shard. @@ -2941,15 +3035,15 @@ mod tests { }); ingester_pool.insert( "node-1".into(), - IngesterServiceClient::from_mock(mock_ingester_1), + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester_1)), ); ingester_pool.insert( "node-2".into(), - IngesterServiceClient::from_mock(mock_ingester_2), + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester_2)), ); ingester_pool.insert( "node-3".into(), - IngesterServiceClient::from_mock(mock_ingester_3), + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester_3)), ); let node_id = "node-1".into(); let wait_handle = controller.sync_with_ingester(&node_id, &model); @@ -2988,6 +3082,7 @@ mod tests { model.insert_shards(&index_uid, &source_id_00, shards); let advise_reset_shards_request = AdviseResetShardsRequest { + ingester_id: "test-ingester".to_string(), shard_ids: vec![ ShardIds { index_uid: Some(index_uid.clone()), @@ -3078,7 +3173,10 @@ mod tests { Ok(response) }); let ingester_0 = IngesterServiceClient::from_mock(mock_ingester_0); - ingester_pool.insert(ingester_id_0.clone(), ingester_0); + ingester_pool.insert( + ingester_id_0.clone(), + IngesterPoolEntry::ready_with_client(ingester_0), + ); let ingester_id_1 = NodeId::from("test-ingester-1"); let mut mock_ingester_1 = MockIngesterService::new(); @@ -3096,16 +3194,22 @@ mod tests { Err(IngestV2Error::Internal("internal error".to_string())) }); let ingester_1 = IngesterServiceClient::from_mock(mock_ingester_1); - ingester_pool.insert(ingester_id_1.clone(), ingester_1); + ingester_pool.insert( + ingester_id_1.clone(), + IngesterPoolEntry::ready_with_client(ingester_1), + ); let ingester_id_2 = NodeId::from("test-ingester-2"); let mut mock_ingester_2 = MockIngesterService::new(); mock_ingester_2.expect_close_shards().never(); - let ingester_2 = IngesterServiceClient::tower() + let client_2 = IngesterServiceClient::tower() .stack_close_shards_layer(DelayLayer::new(CLOSE_SHARDS_REQUEST_TIMEOUT * 2)) .build_from_mock(mock_ingester_2); - ingester_pool.insert(ingester_id_2.clone(), ingester_2); + ingester_pool.insert( + ingester_id_2.clone(), + IngesterPoolEntry::ready_with_client(client_2), + ); // In this test: // - ingester 0 will close shard 0 successfully and fail to close shard 1; @@ -3114,46 +3218,41 @@ mod tests { // - ingester 3 will be unavailable. let shards_to_close = vec![ - ( - ingester_id_0.clone(), - ShardPKey { - index_uid: Some(IndexUid::for_test("test-index", 0)), - source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(0)), - }, - ), - ( - ingester_id_0, - ShardPKey { - index_uid: Some(IndexUid::for_test("test-index", 0)), - source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), - }, - ), - ( - ingester_id_1, - ShardPKey { - index_uid: Some(IndexUid::for_test("test-index", 0)), - source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(2)), - }, - ), - ( - ingester_id_2, - ShardPKey { - index_uid: Some(IndexUid::for_test("test-index", 0)), - source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(3)), - }, - ), - ( - NodeId::from("test-ingester-3"), - ShardPKey { - index_uid: Some(IndexUid::for_test("test-index", 0)), - source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(4)), - }, - ), + Shard { + index_uid: Some(IndexUid::for_test("test-index", 0)), + source_id: "test-source".to_string(), + shard_id: Some(ShardId::from(0)), + leader_id: ingester_id_0.to_string(), + ..Default::default() + }, + Shard { + index_uid: Some(IndexUid::for_test("test-index", 0)), + source_id: "test-source".to_string(), + shard_id: Some(ShardId::from(1)), + leader_id: ingester_id_0.to_string(), + ..Default::default() + }, + Shard { + index_uid: Some(IndexUid::for_test("test-index", 0)), + source_id: "test-source".to_string(), + shard_id: Some(ShardId::from(2)), + leader_id: ingester_id_1.to_string(), + ..Default::default() + }, + Shard { + index_uid: Some(IndexUid::for_test("test-index", 0)), + source_id: "test-source".to_string(), + shard_id: Some(ShardId::from(3)), + leader_id: ingester_id_2.to_string(), + ..Default::default() + }, + Shard { + index_uid: Some(IndexUid::for_test("test-index", 0)), + source_id: "test-source".to_string(), + shard_id: Some(ShardId::from(4)), + leader_id: "test-ingester-3".to_string(), + ..Default::default() + }, ]; let closed_shards = controller.close_shards(shards_to_close).await; assert_eq!(closed_shards.len(), 1); @@ -3210,11 +3309,11 @@ mod tests { let (control_plane_mailbox, control_plane_inbox) = universe.create_test_mailbox(); let progress = Progress::default(); - let close_shards_task_opt = controller + let num_opened_shards = controller .rebalance_shards(&mut model, &control_plane_mailbox, &progress) .await .unwrap(); - assert!(close_shards_task_opt.is_none()); + assert_eq!(num_opened_shards, 0); let index_metadata = IndexMetadata::for_test("test-index", "ram://indexes/test-index"); let index_uid = index_metadata.index_uid.clone(); @@ -3290,7 +3389,10 @@ mod tests { Ok(response) }); let ingester_0 = IngesterServiceClient::from_mock(mock_ingester_0); - ingester_pool.insert(ingester_id_0.clone(), ingester_0); + ingester_pool.insert( + ingester_id_0.clone(), + IngesterPoolEntry::ready_with_client(ingester_0), + ); let ingester_id_1 = NodeId::from("test-ingester-1"); let mut mock_ingester_1 = MockIngesterService::new(); @@ -3331,24 +3433,23 @@ mod tests { }; Ok(response) }); - let ingester_1 = IngesterServiceClient::from_mock(mock_ingester_1); + let ingester_1 = + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester_1)); ingester_pool.insert(ingester_id_1.clone(), ingester_1); - let close_shards_task = controller + let num_opened_shards = controller .rebalance_shards(&mut model, &control_plane_mailbox, &progress) .await - .unwrap() - .unwrap(); - - tokio::time::timeout(CLOSE_SHARDS_REQUEST_TIMEOUT * 2, close_shards_task) - .await - .unwrap() .unwrap(); + assert_eq!(num_opened_shards, 1); - let callbacks: Vec = control_plane_inbox.drain_for_test_typed(); - assert_eq!(callbacks.len(), 1); - - let callback = &callbacks[0]; + let callback: RebalanceShardsCallback = tokio::time::timeout( + CLOSE_SHARDS_REQUEST_TIMEOUT * 2, + control_plane_inbox.recv_typed_message(), + ) + .await + .unwrap() + .unwrap(); assert_eq!(callback.closed_shards.len(), 1); } @@ -3460,7 +3561,7 @@ mod tests { 1, vec![NodeIdRef::from_str("node1"), NodeIdRef::from_str("node2")], ); - let mut rng = rand::thread_rng(); + let mut rng = rand::rng(); let node = pick_one( &mut shard_counts, Some(NodeIdRef::from_str("node2")), @@ -3485,4 +3586,233 @@ mod tests { &[NodeIdRef::from_str("node1"), NodeIdRef::from_str("node2")] ); } + + /// Test helper for compute_shards_to_rebalance. + /// The reason for testing both available and unavailable ingesters with open shards is to + /// ensure the algorithm holds up when there are open shards + /// + /// - `available_ingester_shards`: open shards per available ingester + /// - `unavailable_ingester_shards`: open shards on unavailable ingesters + fn test_compute_shards_to_rebalance_aux( + ready_ingester_shards: &[usize], + unavailable_ingester_shards: &[usize], + retiring_ingester_shards: &[usize], + ) { + let index_id = "test-index"; + let index_metadata = IndexMetadata::for_test(index_id, "ram://indexes/test-index"); + let index_uid = index_metadata.index_uid.clone(); + let source_id: SourceId = "test-source".to_string(); + + let mut model = ControlPlaneModel::default(); + model.add_index(index_metadata.clone()); + + let mut source_config = SourceConfig::ingest_v2(); + source_config.source_id = source_id.to_string(); + model.add_source(&index_uid, source_config).unwrap(); + + let ingester_pool = IngesterPool::default(); + let mock_ingester = MockIngesterService::new(); + let ingester_client = IngesterServiceClient::from_mock(mock_ingester); + + let ready_ids: Vec = (0..ready_ingester_shards.len()) + .map(|i| format!("ready-ingester-{}", i)) + .collect(); + + for ingester_id in &ready_ids { + let ingester = IngesterPoolEntry { + client: ingester_client.clone(), + status: IngesterStatus::Ready, + availability_zone: None, + }; + ingester_pool.insert(NodeId::from(ingester_id.clone()), ingester); + } + + let unavailable_ids: Vec = (0..unavailable_ingester_shards.len()) + .map(|i| format!("unavailable-ingester-{}", i)) + .collect(); + + let retiring_ids: Vec = (0..retiring_ingester_shards.len()) + .map(|i| format!("retiring-ingester-{}", i)) + .collect(); + + for ingester_id in &retiring_ids { + let ingester = IngesterPoolEntry { + client: ingester_client.clone(), + status: IngesterStatus::Retiring, + availability_zone: None, + }; + ingester_pool.insert(NodeId::from(ingester_id.clone()), ingester); + } + + let mut shards: Vec = Vec::new(); + let mut shard_id: u64 = 0; + + for (idx, &num_shards) in ready_ingester_shards.iter().enumerate() { + for _ in 0..num_shards { + shards.push(Shard { + index_uid: Some(index_uid.clone()), + source_id: source_id.clone(), + shard_id: Some(ShardId::from(shard_id)), + leader_id: ready_ids[idx].clone(), + shard_state: ShardState::Open as i32, + ..Default::default() + }); + shard_id += 1; + } + } + + // Shards on unavailable ingesters - these shouldn't affect rebalancing calculations + for (idx, &num_shards) in unavailable_ingester_shards.iter().enumerate() { + for _ in 0..num_shards { + shards.push(Shard { + index_uid: Some(index_uid.clone()), + source_id: source_id.clone(), + shard_id: Some(ShardId::from(shard_id)), + leader_id: unavailable_ids[idx].clone(), + shard_state: ShardState::Open as i32, + ..Default::default() + }); + shard_id += 1; + } + } + + let num_retiring_shards: usize = retiring_ingester_shards.iter().sum(); + + // Shards on retiring ingesters - all of these should be rebalanced + for (idx, &num_shards) in retiring_ingester_shards.iter().enumerate() { + for _ in 0..num_shards { + shards.push(Shard { + index_uid: Some(index_uid.clone()), + source_id: source_id.clone(), + shard_id: Some(ShardId::from(shard_id)), + leader_id: retiring_ids[idx].clone(), + shard_state: ShardState::Open as i32, + ..Default::default() + }); + shard_id += 1; + } + } + + model.insert_shards(&index_uid, &source_id, shards.clone()); + + let controller = IngestController::new( + MetastoreServiceClient::mocked(), + ingester_pool.clone(), + 2, // replication_factor + TEST_SHARD_THROUGHPUT_LIMIT_MIB, + 1.001, + ); + let shards_to_rebalance = controller.compute_shards_to_rebalance(&model); + + // All shards on retiring ingesters must be rebalanced. + let num_retiring_shards_to_rebalance = shards_to_rebalance + .iter() + .filter(|shard| shard.leader_id.starts_with("retiring-")) + .count(); + assert_eq!(num_retiring_shards_to_rebalance, num_retiring_shards); + + let source_uid = SourceUid { + index_uid: index_uid.clone(), + source_id: source_id.clone(), + }; + let shard_ids_to_rebalance: Vec = shards_to_rebalance + .iter() + .flat_map(|shard| shard.shard_id.clone()) + .collect(); + + let closed_shard_ids = model.close_shards(&source_uid, &shard_ids_to_rebalance); + assert_eq!(closed_shard_ids.len(), shards_to_rebalance.len()); + + let mut per_ready_ingester_num_shards: HashMap<&str, usize> = ready_ids + .iter() + .map(|ready_id| (ready_id.as_str(), 0)) + .collect(); + + for shard in model.all_shards() { + if !shard.is_open() { + continue; + } + if let Some(count_shard) = + per_ready_ingester_num_shards.get_mut(shard.leader_id.as_str()) + { + *count_shard += 1; + } + } + + // Now we move the different shards to ready ingesters (not retiring ones). + // We can only simulate this if there are ready ingesters to receive shards. + if !ready_ids.is_empty() { + let mut per_ingester_num_shards_sorted: BTreeSet<(usize, &str)> = + per_ready_ingester_num_shards + .into_iter() + .map(|(ingester_id, num_shards)| (num_shards, ingester_id)) + .collect(); + let mut opened_shards: Vec = Vec::new(); + for _ in 0..shards_to_rebalance.len() { + let (num_shards, ingester_id) = per_ingester_num_shards_sorted.pop_first().unwrap(); + let opened_shard = Shard { + index_uid: Some(index_uid.clone()), + source_id: source_id.to_string(), + shard_id: Some(ShardId::from(shard_id)), + leader_id: ingester_id.to_string(), + shard_state: ShardState::Open as i32, + ..Default::default() + }; + per_ingester_num_shards_sorted.insert((num_shards + 1, ingester_id)); + opened_shards.push(opened_shard); + shard_id += 1; + } + + if let Some((min_shards, max_shards)) = per_ingester_num_shards_sorted + .iter() + .map(|(num_shards, _)| num_shards) + .copied() + .minmax() + .into_option() + { + assert!(min_shards + min_shards.div_ceil(10).max(2) >= max_shards); + } + + // Test stability of the algorithm: mark the retiring ingesters as + // decommissioned, insert the new shards, and verify no further rebalance is + // needed among the ready ingesters. + for ingester_id in &retiring_ids { + let ingester = IngesterPoolEntry { + client: ingester_client.clone(), + status: IngesterStatus::Decommissioned, + availability_zone: None, + }; + ingester_pool.insert(NodeId::from(ingester_id.clone()), ingester); + } + model.insert_shards(&index_uid, &source_id, opened_shards); + + let shards_to_rebalance = controller.compute_shards_to_rebalance(&model); + assert!(shards_to_rebalance.is_empty()); + } + } + + proptest! { + #[test] + fn test_compute_shards_to_rebalance_proptest( + ready_shards in proptest::collection::vec(0..13usize, 0..13usize), + unavailable_shards in proptest::collection::vec(0..13usize, 0..5usize), + retiring_shards in proptest::collection::vec(0..5usize, 0..5usize), + ) { + test_compute_shards_to_rebalance_aux(&ready_shards, &unavailable_shards, &retiring_shards); + } + } + + #[test] + fn test_compute_shards_to_rebalance() { + test_compute_shards_to_rebalance_aux(&[], &[], &[]); + test_compute_shards_to_rebalance_aux(&[0], &[], &[]); + test_compute_shards_to_rebalance_aux(&[1], &[], &[]); + test_compute_shards_to_rebalance_aux(&[0, 1], &[], &[]); + test_compute_shards_to_rebalance_aux(&[0, 1], &[1], &[]); + test_compute_shards_to_rebalance_aux(&[0, 1, 2], &[3, 4], &[]); + // Retiring ingesters: all their shards must be rebalanced + test_compute_shards_to_rebalance_aux(&[1, 1], &[], &[3]); + test_compute_shards_to_rebalance_aux(&[0, 0, 0], &[], &[5]); + test_compute_shards_to_rebalance_aux(&[2], &[], &[1, 2]); + } } diff --git a/quickwit/quickwit-control-plane/src/ingest/scaling_arbiter.rs b/quickwit/quickwit-control-plane/src/ingest/scaling_arbiter.rs index 3cd6911fbaa..7294b9f9176 100644 --- a/quickwit/quickwit-control-plane/src/ingest/scaling_arbiter.rs +++ b/quickwit/quickwit-control-plane/src/ingest/scaling_arbiter.rs @@ -129,6 +129,7 @@ mod tests { scaling_arbiter.should_scale( ShardStats { num_open_shards: 0, + num_closed_shards: 0, avg_short_term_ingestion_rate: 0.0, avg_long_term_ingestion_rate: 0.0, }, @@ -140,6 +141,7 @@ mod tests { scaling_arbiter.should_scale( ShardStats { num_open_shards: 1, + num_closed_shards: 0, avg_short_term_ingestion_rate: 5.0, avg_long_term_ingestion_rate: 6.0, }, @@ -151,6 +153,7 @@ mod tests { scaling_arbiter.should_scale( ShardStats { num_open_shards: 1, + num_closed_shards: 0, avg_short_term_ingestion_rate: 8.1, avg_long_term_ingestion_rate: 8.1, }, @@ -162,6 +165,7 @@ mod tests { scaling_arbiter.should_scale( ShardStats { num_open_shards: 2, + num_closed_shards: 0, avg_short_term_ingestion_rate: 8.1, avg_long_term_ingestion_rate: 8.1, }, @@ -173,6 +177,7 @@ mod tests { scaling_arbiter.should_scale( ShardStats { num_open_shards: 2, + num_closed_shards: 0, avg_short_term_ingestion_rate: 3.0, avg_long_term_ingestion_rate: 1.5, }, @@ -184,6 +189,7 @@ mod tests { scaling_arbiter.should_scale( ShardStats { num_open_shards: 1, + num_closed_shards: 0, avg_short_term_ingestion_rate: 3.0, avg_long_term_ingestion_rate: 1.5, }, @@ -195,6 +201,7 @@ mod tests { scaling_arbiter.should_scale( ShardStats { num_open_shards: 1, + num_closed_shards: 0, avg_short_term_ingestion_rate: 8.0, avg_long_term_ingestion_rate: 3.0, }, @@ -213,6 +220,7 @@ mod tests { scaling_arbiter.should_scale( ShardStats { num_open_shards: 0, + num_closed_shards: 0, avg_short_term_ingestion_rate: 0.0, avg_long_term_ingestion_rate: 0.0, }, @@ -224,6 +232,7 @@ mod tests { scaling_arbiter.should_scale( ShardStats { num_open_shards: 2, + num_closed_shards: 0, avg_short_term_ingestion_rate: 5.0, avg_long_term_ingestion_rate: 6.0, }, @@ -235,6 +244,7 @@ mod tests { scaling_arbiter.should_scale( ShardStats { num_open_shards: 1, + num_closed_shards: 0, avg_short_term_ingestion_rate: 8.1, avg_long_term_ingestion_rate: 8.1, }, @@ -246,6 +256,7 @@ mod tests { scaling_arbiter.should_scale( ShardStats { num_open_shards: 2, + num_closed_shards: 0, avg_short_term_ingestion_rate: 8.1, avg_long_term_ingestion_rate: 8.1, }, @@ -257,6 +268,7 @@ mod tests { scaling_arbiter.should_scale( ShardStats { num_open_shards: 2, + num_closed_shards: 0, avg_short_term_ingestion_rate: 3.0, avg_long_term_ingestion_rate: 1.5, }, @@ -268,6 +280,7 @@ mod tests { scaling_arbiter.should_scale( ShardStats { num_open_shards: 1, + num_closed_shards: 0, avg_short_term_ingestion_rate: 3.0, avg_long_term_ingestion_rate: 1.5, }, @@ -279,6 +292,7 @@ mod tests { scaling_arbiter.should_scale( ShardStats { num_open_shards: 1, + num_closed_shards: 0, avg_short_term_ingestion_rate: 8.0, avg_long_term_ingestion_rate: 3.1, }, @@ -291,6 +305,7 @@ mod tests { scaling_arbiter.should_scale( ShardStats { num_open_shards: 2, + num_closed_shards: 0, avg_short_term_ingestion_rate: 8.1, avg_long_term_ingestion_rate: 5., }, @@ -308,6 +323,7 @@ mod tests { let shard_stats = ShardStats { num_open_shards: 0, + num_closed_shards: 0, avg_short_term_ingestion_rate: 0., avg_long_term_ingestion_rate: 0., }; @@ -322,6 +338,7 @@ mod tests { let shard_stats = ShardStats { num_open_shards: 1, + num_closed_shards: 0, avg_short_term_ingestion_rate: 5.0, avg_long_term_ingestion_rate: 6.1, }; @@ -336,6 +353,7 @@ mod tests { let shard_stats = ShardStats { num_open_shards: 2, + num_closed_shards: 0, avg_short_term_ingestion_rate: 5.0, avg_long_term_ingestion_rate: 1.1, }; @@ -350,6 +368,7 @@ mod tests { let shard_stats = ShardStats { num_open_shards: 2, + num_closed_shards: 0, avg_short_term_ingestion_rate: 5.0, avg_long_term_ingestion_rate: 6.1, }; @@ -364,6 +383,7 @@ mod tests { let shard_stats = ShardStats { num_open_shards: 5, + num_closed_shards: 0, avg_short_term_ingestion_rate: 5.0, avg_long_term_ingestion_rate: 1.1, }; @@ -384,6 +404,7 @@ mod tests { let shard_stats = ShardStats { num_open_shards: 0, + num_closed_shards: 0, avg_short_term_ingestion_rate: 0.0, avg_long_term_ingestion_rate: 0.0, }; @@ -393,6 +414,7 @@ mod tests { let shard_stats = ShardStats { num_open_shards: 1, + num_closed_shards: 0, avg_short_term_ingestion_rate: 0.0, avg_long_term_ingestion_rate: 0.0, }; @@ -408,6 +430,7 @@ mod tests { let shard_stats = ShardStats { num_open_shards: 1, + num_closed_shards: 0, avg_short_term_ingestion_rate: 5.0, avg_long_term_ingestion_rate: 1.0, }; diff --git a/quickwit/quickwit-control-plane/src/metrics.rs b/quickwit/quickwit-control-plane/src/metrics.rs index 119ea4af0b6..5e534c4f176 100644 --- a/quickwit/quickwit-control-plane/src/metrics.rs +++ b/quickwit/quickwit-control-plane/src/metrics.rs @@ -24,13 +24,22 @@ pub struct ShardLocalityMetrics { } pub struct ControlPlaneMetrics { + // Indexes and shards tracked by the control plane. pub indexes_total: IntGauge, + pub open_shards: IntGaugeVec<1>, + pub closed_shards: IntGaugeVec<1>, + + // Operations performed by the control plane. + pub apply_plan_total: IntCounter, + pub rebalance_shards: IntGauge, pub restart_total: IntCounter, pub schedule_total: IntCounter, - pub apply_total: IntCounter, + + // Metastore errors. pub metastore_error_aborted: IntCounter, pub metastore_error_maybe_executed: IntCounter, - pub open_shards_total: IntGaugeVec<1>, + + // Indexing plan metrics. pub local_shards: IntGauge, pub remote_shards: IntGauge, } @@ -46,20 +55,54 @@ impl ControlPlaneMetrics { impl Default for ControlPlaneMetrics { fn default() -> Self { - let shards = new_gauge_vec( + let open_shards = new_gauge_vec( "shards", + "Number of open and closed shards tracked by the ingest controller", + "control_plane", + &[("state", "open")], + ["index_id"], + ); + let closed_shards = new_gauge_vec( + "shards", + "Number of open and closed shards tracked by the ingest controller", + "control_plane", + &[("state", "closed")], + ["index_id"], + ); + let indexed_shards = new_gauge_vec( + "indexed_shards", "Number of (remote/local) shards in the indexing plan", "control_plane", &[], ["locality"], ); - let local_shards = shards.with_label_values(["local"]); - let remote_shards = shards.with_label_values(["remote"]); + let local_shards = indexed_shards.with_label_values(["local"]); + let remote_shards = indexed_shards.with_label_values(["remote"]); + ControlPlaneMetrics { - indexes_total: new_gauge("indexes_total", "Number of indexes.", "control_plane", &[]), + indexes_total: new_gauge( + "indexes_total", + "Number of indexes tracked by the control plane.", + "control_plane", + &[], + ), + open_shards, + closed_shards, + apply_plan_total: new_counter( + "apply_plan_total", + "Number of control plane `apply plan` operations.", + "control_plane", + &[], + ), + rebalance_shards: new_gauge( + "rebalance_shards", + "Number of shards rebalanced by the control plane.", + "control_plane", + &[], + ), restart_total: new_counter( "restart_total", - "Number of control plane restart.", + "Number of control plane restarts.", "control_plane", &[], ), @@ -69,12 +112,6 @@ impl Default for ControlPlaneMetrics { "control_plane", &[], ), - apply_total: new_counter( - "apply_total", - "Number of control plane `apply plan` operations.", - "control_plane", - &[], - ), metastore_error_aborted: new_counter( "metastore_error_aborted", "Number of aborted metastore transaction (= do not trigger a control plane \ @@ -89,13 +126,6 @@ impl Default for ControlPlaneMetrics { "control_plane", &[], ), - open_shards_total: new_gauge_vec( - "open_shards_total", - "Number of open shards per source.", - "control_plane", - &[], - ["index_id"], - ), local_shards, remote_shards, } diff --git a/quickwit/quickwit-control-plane/src/model/mod.rs b/quickwit/quickwit-control-plane/src/model/mod.rs index bc1e545c4a3..0d0431a67ce 100644 --- a/quickwit/quickwit-control-plane/src/model/mod.rs +++ b/quickwit/quickwit-control-plane/src/model/mod.rs @@ -69,7 +69,7 @@ impl ControlPlaneModel { self.shard_table.num_sources() } - pub fn shard_locations(&self) -> ShardLocations { + pub fn shard_locations(&self) -> ShardLocations<'_> { self.shard_table.shard_locations() } @@ -160,7 +160,7 @@ impl ControlPlaneModel { self.index_table.get(index_uid) } - pub fn source_metadata(&self, source_uid: &SourceUid) -> Option<&SourceConfig> { + pub(crate) fn source_metadata(&self, source_uid: &SourceUid) -> Option<&SourceConfig> { self.index_metadata(&source_uid.index_uid)? .sources .get(&source_uid.source_id) @@ -216,7 +216,6 @@ impl ControlPlaneModel { }; let fp_changed = !index_model.index_config.equals_fingerprint(&index_config); index_model.index_config = index_config; - self.update_metrics(); Ok(fp_changed) } @@ -377,18 +376,34 @@ impl ControlPlaneModel { source_uid: &SourceUid, shard_infos: &ShardInfos, ) -> ShardStats { + debug!( + index_uid=%source_uid.index_uid, + source_id=%source_uid.source_id, + "updating shards" + ); self.shard_table.update_shards(source_uid, shard_infos) } /// Sets the state of the shards identified by their index UID, source ID, and shard IDs to /// `Closed`. pub fn close_shards(&mut self, source_uid: &SourceUid, shard_ids: &[ShardId]) -> Vec { + debug!( + index_uid=%source_uid.index_uid, + source_id=%source_uid.source_id, + shard_ids=%shard_ids.pretty_display(), + "closing shards" + ); self.shard_table.close_shards(source_uid, shard_ids) } /// Removes the shards identified by their index UID, source ID, and shard IDs. pub fn delete_shards(&mut self, source_uid: &SourceUid, shard_ids: &[ShardId]) { - info!(source_uid=%source_uid, shard_ids=?shard_ids, "removing shards from model"); + debug!( + index_uid=%source_uid.index_uid, + source_id=%source_uid.source_id, + shard_ids=%shard_ids.pretty_display(), + "deleting shards" + ); self.shard_table.delete_shards(source_uid, shard_ids); } diff --git a/quickwit/quickwit-control-plane/src/model/shard_table.rs b/quickwit/quickwit-control-plane/src/model/shard_table.rs index 304b5f4d2ac..623ae3e6224 100644 --- a/quickwit/quickwit-control-plane/src/model/shard_table.rs +++ b/quickwit/quickwit-control-plane/src/model/shard_table.rs @@ -18,6 +18,7 @@ use std::ops::{Deref, DerefMut}; use std::time::Duration; use fnv::{FnvHashMap, FnvHashSet}; +use quickwit_common::metrics::index_label; use quickwit_common::rate_limiter::{RateLimiter, RateLimiterSettings}; use quickwit_common::tower::ConstantRate; use quickwit_ingest::{RateMibPerSec, ShardInfo, ShardInfos}; @@ -102,11 +103,37 @@ impl ShardTableEntry { self.shard_entries.is_empty() } - fn num_open_shards(&self) -> usize { - self.shard_entries - .values() - .filter(|shard_entry| shard_entry.is_open()) - .count() + fn shards_stats(&self) -> ShardStats { + let mut num_open_shards = 0; + let mut num_closed_shards = 0; + let mut short_term_ingestion_rate_sum = 0; + let mut long_term_ingestion_rate_sum = 0; + + for shard_entry in self.shard_entries.values() { + if shard_entry.is_open() { + num_open_shards += 1; + short_term_ingestion_rate_sum += shard_entry.short_term_ingestion_rate.0 as usize; + long_term_ingestion_rate_sum += shard_entry.long_term_ingestion_rate.0 as usize; + } else if shard_entry.is_closed() { + num_closed_shards += 1; + } + } + let avg_short_term_ingestion_rate = if num_open_shards > 0 { + short_term_ingestion_rate_sum as f32 / num_open_shards as f32 + } else { + 0.0 + }; + let avg_long_term_ingestion_rate = if num_open_shards > 0 { + long_term_ingestion_rate_sum as f32 / num_open_shards as f32 + } else { + 0.0 + }; + ShardStats { + num_open_shards, + num_closed_shards, + avg_short_term_ingestion_rate, + avg_long_term_ingestion_rate, + } } } @@ -138,7 +165,7 @@ impl<'a> ShardLocations<'a> { // A table that keeps track of the existing shards for each index and source, // and for each ingester, the list of shards it is supposed to host. // -// (All mutable methods must maintain the two consistent) +// (All mutable methods must maintain these two invariants.) #[derive(Debug, Default)] pub(crate) struct ShardTable { table_entries: FnvHashMap, @@ -172,7 +199,7 @@ fn remove_shard_from_ingesters_internal( impl ShardTable { /// Returns a ShardLocations object that maps each shard to the list of ingesters hosting it. /// All shards are considered regardless of their state (including unavailable). - pub fn shard_locations(&self) -> ShardLocations { + pub fn shard_locations(&self) -> ShardLocations<'_> { let mut shard_locations = ShardLocations::default(); for (ingester_id, source_shards) in &self.ingester_shards { for shard_ids in source_shards.values() { @@ -275,13 +302,13 @@ impl ShardTable { }; let table_entry = ShardTableEntry::default(); let previous_table_entry_opt = self.table_entries.insert(source_uid, table_entry); - if let Some(previous_table_entry) = previous_table_entry_opt { - if !previous_table_entry.is_empty() { - error!( - "shard table entry for index `{}` and source `{}` already exists", - index_uid.index_id, source_id - ); - } + if let Some(previous_table_entry) = previous_table_entry_opt + && !previous_table_entry.is_empty() + { + error!( + "shard table entry for index `{}` and source `{}` already exists", + index_uid.index_id, source_id + ); } self.check_invariant(); } @@ -424,18 +451,45 @@ impl ShardTable { } pub fn update_shard_metrics_for_source_uid(&self, source_uid: &SourceUid) { - let num_open_shards: usize = - if let Some(shard_table_entry) = self.table_entries.get(source_uid) { - shard_table_entry.num_open_shards() - } else { - 0 - }; - let index_label = - quickwit_common::metrics::index_label(source_uid.index_uid.index_id.as_str()); + let Some(table_entry) = self.table_entries.get(source_uid) else { + return; + }; + let index_id = source_uid.index_uid.index_id.as_str(); + let index_label = index_label(index_id); + + // If `index_label(index_id)` returns `index_id`, then per-index metrics are enabled and we + // can update the metrics for this specific index. + if index_label == index_id { + let shard_stats = table_entry.shards_stats(); + crate::metrics::CONTROL_PLANE_METRICS + .open_shards + .with_label_values([index_label]) + .set(shard_stats.num_open_shards as i64); + crate::metrics::CONTROL_PLANE_METRICS + .closed_shards + .with_label_values([index_label]) + .set(shard_stats.num_closed_shards as i64); + return; + } + // Per-index metrics are disabled, so we update the metrics for all sources. + let mut num_open_shards = 0; + let mut num_closed_shards = 0; + + for shard_entry in self.all_shards() { + if shard_entry.is_open() { + num_open_shards += 1; + } else if shard_entry.is_closed() { + num_closed_shards += 1; + } + } crate::metrics::CONTROL_PLANE_METRICS - .open_shards_total + .open_shards .with_label_values([index_label]) .set(num_open_shards as i64); + crate::metrics::CONTROL_PLANE_METRICS + .closed_shards + .with_label_values([index_label]) + .set(num_closed_shards as i64); } pub fn update_shards( @@ -443,71 +497,51 @@ impl ShardTable { source_uid: &SourceUid, shard_infos: &ShardInfos, ) -> ShardStats { - let mut num_open_shards = 0; - let mut short_term_ingestion_rate_sum = RateMibPerSec::default(); - let mut long_term_ingestion_rate_sum = RateMibPerSec::default(); - - if let Some(table_entry) = self.table_entries.get_mut(source_uid) { - for shard_info in shard_infos { - let ShardInfo { - shard_id, - shard_state, - short_term_ingestion_rate, - long_term_ingestion_rate, - } = shard_info; - - if let Some(shard_entry) = table_entry.shard_entries.get_mut(shard_id) { - shard_entry.short_term_ingestion_rate = *short_term_ingestion_rate; - shard_entry.long_term_ingestion_rate = *long_term_ingestion_rate; - // `ShardInfos` are broadcasted via Chitchat and eventually consistent. As a - // result, we can only trust the `Closed` state, which is final. - if shard_state.is_closed() { - shard_entry.set_shard_state(ShardState::Closed); - } - } - } - for shard_entry in table_entry.shard_entries.values() { - if shard_entry.is_open() { - num_open_shards += 1; - short_term_ingestion_rate_sum += shard_entry.short_term_ingestion_rate; - long_term_ingestion_rate_sum += shard_entry.long_term_ingestion_rate; + let Some(table_entry) = self.table_entries.get_mut(source_uid) else { + return ShardStats::default(); + }; + for shard_info in shard_infos { + let ShardInfo { + shard_id, + shard_state, + short_term_ingestion_rate, + long_term_ingestion_rate, + } = shard_info; + + if let Some(shard_entry) = table_entry.shard_entries.get_mut(shard_id) { + shard_entry.short_term_ingestion_rate = *short_term_ingestion_rate; + shard_entry.long_term_ingestion_rate = *long_term_ingestion_rate; + // `ShardInfos` are broadcasted via Chitchat and eventually consistent. As a + // result, we can only trust the `Closed` state, which is final. + if shard_state.is_closed() { + shard_entry.set_shard_state(ShardState::Closed); } } } - let avg_short_term_ingestion_rate = if num_open_shards > 0 { - short_term_ingestion_rate_sum.0 as f32 / num_open_shards as f32 - } else { - 0.0 - }; - - let avg_long_term_ingestion_rate = if num_open_shards > 0 { - long_term_ingestion_rate_sum.0 as f32 / num_open_shards as f32 - } else { - 0.0 - }; - - ShardStats { - num_open_shards, - avg_short_term_ingestion_rate, - avg_long_term_ingestion_rate, - } + table_entry.shards_stats() } /// Sets the state of the shards identified by their index UID, source ID, and shard IDs to /// `Closed`. pub fn close_shards(&mut self, source_uid: &SourceUid, shard_ids: &[ShardId]) -> Vec { + let Some(table_entry) = self.table_entries.get_mut(source_uid) else { + return Vec::new(); + }; let mut closed_shard_ids = Vec::new(); - if let Some(table_entry) = self.table_entries.get_mut(source_uid) { - for shard_id in shard_ids { - if let Some(shard_entry) = table_entry.shard_entries.get_mut(shard_id) { - if !shard_entry.is_closed() { - shard_entry.set_shard_state(ShardState::Closed); - closed_shard_ids.push(shard_id.clone()); - } - } else { - info!(shard=%shard_id, "ignoring attempt to close shard: it is unknown (probably because it has been deleted)"); + for shard_id in shard_ids { + if let Some(shard_entry) = table_entry.shard_entries.get_mut(shard_id) { + if !shard_entry.is_closed() { + shard_entry.set_shard_state(ShardState::Closed); + closed_shard_ids.push(shard_id.clone()); } + } else { + info!( + index_id=%source_uid.index_uid.index_id, + source_id=%source_uid.source_id, + %shard_id, + "ignoring attempt to close shard: it is unknown (probably because it has been deleted)" + ); } } self.update_shard_metrics_for_source_uid(source_uid); @@ -516,14 +550,15 @@ impl ShardTable { /// Removes the shards identified by their index UID, source ID, and shard IDs. pub fn delete_shards(&mut self, source_uid: &SourceUid, shard_ids: &[ShardId]) { + let Some(table_entry) = self.table_entries.get_mut(source_uid) else { + return; + }; let mut shard_entries_to_remove: Vec = Vec::new(); - if let Some(table_entry) = self.table_entries.get_mut(source_uid) { - for shard_id in shard_ids { - if let Some(shard_entry) = table_entry.shard_entries.remove(shard_id) { - shard_entries_to_remove.push(shard_entry); - } else { - warn!(shard=%shard_id, "deleting a non-existing shard"); - } + for shard_id in shard_ids { + if let Some(shard_entry) = table_entry.shard_entries.remove(shard_id) { + shard_entries_to_remove.push(shard_entry); + } else { + warn!(shard=%shard_id, "deleting a non-existing shard"); } } for shard_entry in shard_entries_to_remove { @@ -533,8 +568,8 @@ impl ShardTable { &mut self.ingester_shards, ); } - self.update_shard_metrics_for_source_uid(source_uid); self.check_invariant(); + self.update_shard_metrics_for_source_uid(source_uid); } pub fn acquire_scaling_permits( @@ -574,9 +609,10 @@ impl ShardTable { #[derive(Clone, Copy, Default)] pub(crate) struct ShardStats { pub num_open_shards: usize, - /// Average short-term ingestion rate (MiB/s) per open shard + pub num_closed_shards: usize, + /// Average short-term ingestion rate (MiB/s) over all open shards. pub avg_short_term_ingestion_rate: f32, - /// Average long-term ingestion rate (MiB/s) per open shard + /// Average long-term ingestion rate (MiB/s) over all open shards. pub avg_long_term_ingestion_rate: f32, } @@ -1060,7 +1096,7 @@ mod tests { let previous_available_permits = shard_table .table_entries - .get(&source_uid) + .get_mut(&source_uid) .unwrap() .scaling_up_rate_limiter .available_permits(); @@ -1073,7 +1109,7 @@ mod tests { let new_available_permits = shard_table .table_entries - .get(&source_uid) + .get_mut(&source_uid) .unwrap() .scaling_up_rate_limiter .available_permits(); @@ -1102,7 +1138,7 @@ mod tests { let previous_available_permits = shard_table .table_entries - .get(&source_uid) + .get_mut(&source_uid) .unwrap() .scaling_down_rate_limiter .available_permits(); @@ -1115,7 +1151,7 @@ mod tests { let new_available_permits = shard_table .table_entries - .get(&source_uid) + .get_mut(&source_uid) .unwrap() .scaling_down_rate_limiter .available_permits(); @@ -1138,7 +1174,7 @@ mod tests { }; let previous_available_permits = shard_table .table_entries - .get(&source_uid) + .get_mut(&source_uid) .unwrap() .scaling_up_rate_limiter .available_permits(); @@ -1153,7 +1189,7 @@ mod tests { let new_available_permits = shard_table .table_entries - .get(&source_uid) + .get_mut(&source_uid) .unwrap() .scaling_up_rate_limiter .available_permits(); @@ -1176,7 +1212,7 @@ mod tests { }; let previous_available_permits = shard_table .table_entries - .get(&source_uid) + .get_mut(&source_uid) .unwrap() .scaling_up_rate_limiter .available_permits(); @@ -1191,7 +1227,7 @@ mod tests { let new_available_permits = shard_table .table_entries - .get(&source_uid) + .get_mut(&source_uid) .unwrap() .scaling_up_rate_limiter .available_permits(); diff --git a/quickwit/quickwit-datetime/Cargo.toml b/quickwit/quickwit-datetime/Cargo.toml index 004e959a348..57c9901f262 100644 --- a/quickwit/quickwit-datetime/Cargo.toml +++ b/quickwit/quickwit-datetime/Cargo.toml @@ -11,7 +11,6 @@ authors.workspace = true license.workspace = true [dependencies] -anyhow = { workspace = true } itertools = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/quickwit/quickwit-datetime/src/date_time_format.rs b/quickwit/quickwit-datetime/src/date_time_format.rs index 6892dbd78fc..ecb1c624aa2 100644 --- a/quickwit/quickwit-datetime/src/date_time_format.rs +++ b/quickwit/quickwit-datetime/src/date_time_format.rs @@ -78,14 +78,18 @@ impl FromStr for DateTimeInputFormat { impl Serialize for DateTimeInputFormat { fn serialize(&self, serializer: S) -> Result - where S: serde::Serializer { + where + S: serde::Serializer, + { serializer.serialize_str(self.as_str()) } } impl<'de> Deserialize<'de> for DateTimeInputFormat { fn deserialize(deserializer: D) -> Result - where D: Deserializer<'de> { + where + D: Deserializer<'de>, + { let date_time_format_str: String = Deserialize::deserialize(deserializer)?; let date_time_format = date_time_format_str.parse().map_err(D::Error::custom)?; Ok(date_time_format) @@ -180,14 +184,18 @@ impl FromStr for DateTimeOutputFormat { impl Serialize for DateTimeOutputFormat { fn serialize(&self, serializer: S) -> Result - where S: serde::Serializer { + where + S: serde::Serializer, + { serializer.serialize_str(self.as_str()) } } impl<'de> Deserialize<'de> for DateTimeOutputFormat { fn deserialize(deserializer: D) -> Result - where D: Deserializer<'de> { + where + D: Deserializer<'de>, + { let date_time_format_str: String = Deserialize::deserialize(deserializer)?; let date_time_format = date_time_format_str.parse().map_err(D::Error::custom)?; Ok(date_time_format) diff --git a/quickwit/quickwit-datetime/src/date_time_parsing.rs b/quickwit/quickwit-datetime/src/date_time_parsing.rs index 064d31fd514..2e56b6b3d3f 100644 --- a/quickwit/quickwit-datetime/src/date_time_parsing.rs +++ b/quickwit/quickwit-datetime/src/date_time_parsing.rs @@ -106,17 +106,17 @@ pub fn parse_timestamp_str(timestamp_str: &str) -> Option { if subsecond_digits_str.is_empty() { return parse_timestamp_str(timestamp_secs_str); } - if let Ok(timestamp_secs) = timestamp_secs_str.parse::() { - if (MIN_TIMESTAMP_SECONDS..=MAX_TIMESTAMP_SECONDS).contains(×tamp_secs) { - let num_subsecond_digits = subsecond_digits_str.len().min(9); - - if let Ok(subsecond_digits) = - subsecond_digits_str[..num_subsecond_digits].parse::() - { - let nanos = subsecond_digits * 10i64.pow(9 - num_subsecond_digits as u32); - let timestamp_nanos = timestamp_secs * 1_000_000_000 + nanos; - return Some(TantivyDateTime::from_timestamp_nanos(timestamp_nanos)); - } + if let Ok(timestamp_secs @ MIN_TIMESTAMP_SECONDS..=MAX_TIMESTAMP_SECONDS) = + timestamp_secs_str.parse::() + { + let num_subsecond_digits = subsecond_digits_str.len().min(9); + + if let Ok(subsecond_digits) = + subsecond_digits_str[..num_subsecond_digits].parse::() + { + let nanos = subsecond_digits * 10i64.pow(9 - num_subsecond_digits as u32); + let timestamp_nanos = timestamp_secs * 1_000_000_000 + nanos; + return Some(TantivyDateTime::from_timestamp_nanos(timestamp_nanos)); } } } diff --git a/quickwit/quickwit-directories/src/bundle_directory.rs b/quickwit/quickwit-directories/src/bundle_directory.rs index b9413dda9fb..105e3065099 100644 --- a/quickwit/quickwit-directories/src/bundle_directory.rs +++ b/quickwit/quickwit-directories/src/bundle_directory.rs @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +use quickwit_common::tantivy4java_debug; use std::convert::TryInto; use std::fmt::Debug; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::{fmt, io}; -use quickwit_common::tantivy4java_debug; use quickwit_storage::{BundleStorageFileOffsets, OwnedBytes, Storage, StorageResult}; use tantivy::directory::error::OpenReadError; @@ -76,21 +76,42 @@ pub async fn read_split_footer( /// Return two slices for given split: `[body and bundle meta data] [hotcache]` pub fn split_footer(file_slice: FileSlice) -> io::Result<(FileSlice, FileSlice)> { let thread_id = std::thread::current().id(); - tantivy4java_debug!("QUICKWIT DEBUG: [Thread {:?}] split_footer called with file_slice size: {}", thread_id, file_slice.len()); - tantivy4java_debug!("QUICKWIT DEBUG: [Thread {:?}] split_footer file_slice len: {}", thread_id, file_slice.len()); - + tantivy4java_debug!( + "QUICKWIT DEBUG: [Thread {:?}] split_footer called with file_slice size: {}", + thread_id, + file_slice.len() + ); + tantivy4java_debug!( + "QUICKWIT DEBUG: [Thread {:?}] split_footer file_slice len: {}", + thread_id, + file_slice.len() + ); + let (body_and_footer_slice, footer_len_slice) = file_slice.split_from_end(4); - tantivy4java_debug!("QUICKWIT DEBUG: [Thread {:?}] split_footer after split_from_end(4): body_and_footer_slice size: {}, footer_len_slice size: {}", - thread_id, body_and_footer_slice.len(), footer_len_slice.len()); - + tantivy4java_debug!( + "QUICKWIT DEBUG: [Thread {:?}] split_footer after split_from_end(4): body_and_footer_slice size: {}, footer_len_slice size: {}", + thread_id, + body_and_footer_slice.len(), + footer_len_slice.len() + ); + let footer_len_bytes = footer_len_slice.read_bytes()?; let footer_len = u32::from_le_bytes(footer_len_bytes.as_slice().try_into().unwrap()); - tantivy4java_debug!("QUICKWIT DEBUG: [Thread {:?}] split_footer footer_len: {}", thread_id, footer_len); - - let (body_and_bundle_metadata, hotcache) = body_and_footer_slice.split_from_end(footer_len as usize); - tantivy4java_debug!("QUICKWIT DEBUG: [Thread {:?}] split_footer final result: body_and_bundle_metadata size: {}, hotcache size: {}", - thread_id, body_and_bundle_metadata.len(), hotcache.len()); - + tantivy4java_debug!( + "QUICKWIT DEBUG: [Thread {:?}] split_footer footer_len: {}", + thread_id, + footer_len + ); + + let (body_and_bundle_metadata, hotcache) = + body_and_footer_slice.split_from_end(footer_len as usize); + tantivy4java_debug!( + "QUICKWIT DEBUG: [Thread {:?}] split_footer final result: body_and_bundle_metadata size: {}, hotcache size: {}", + thread_id, + body_and_bundle_metadata.len(), + hotcache.len() + ); + Ok((body_and_bundle_metadata, hotcache)) } @@ -125,36 +146,63 @@ impl BundleDirectory { /// Opens a split file. pub fn open_split(split_file: FileSlice) -> io::Result { - tantivy4java_debug!("QUICKWIT DEBUG: BundleDirectory::open_split - starting with file size: {}", split_file.len()); - + tantivy4java_debug!( + "QUICKWIT DEBUG: BundleDirectory::open_split - starting with file size: {}", + split_file.len() + ); + // First we remove the hotcache from our file slice. tantivy4java_debug!("QUICKWIT DEBUG: Calling split_footer to extract body and hotcache..."); let (body_and_bundle_metadata, _hot_cache) = split_footer(split_file)?; - tantivy4java_debug!("QUICKWIT DEBUG: split_footer succeeded, body size: {}", body_and_bundle_metadata.len()); - + tantivy4java_debug!( + "QUICKWIT DEBUG: split_footer succeeded, body size: {}", + body_and_bundle_metadata.len() + ); + tantivy4java_debug!("QUICKWIT DEBUG: Calling BundleDirectory::open_bundle..."); - let result = BundleDirectory::open_bundle(body_and_bundle_metadata).map_err(io::Error::other); - + let result = + BundleDirectory::open_bundle(body_and_bundle_metadata).map_err(io::Error::other); + match &result { Ok(_) => tantivy4java_debug!("QUICKWIT DEBUG: BundleDirectory::open_bundle succeeded"), - Err(e) => tantivy4java_debug!("QUICKWIT DEBUG: BundleDirectory::open_bundle failed: {}", e), + Err(e) => { + tantivy4java_debug!("QUICKWIT DEBUG: BundleDirectory::open_bundle failed: {}", e) + } } - + result } /// Opens a BundleDirectory, given a file containing the bundle data. pub fn open_bundle(file: FileSlice) -> anyhow::Result { let thread_id = std::thread::current().id(); - tantivy4java_debug!("QUICKWIT DEBUG: [Thread {:?}] BundleDirectory::open_bundle - opening file offsets with file size: {}", thread_id, file.len()); - tantivy4java_debug!("QUICKWIT DEBUG: [Thread {:?}] BundleDirectory::open_bundle - file slice len: {}", thread_id, file.len()); - tantivy4java_debug!("QUICKWIT DEBUG: [Thread {:?}] BundleDirectory::open_bundle - about to call BundleStorageFileOffsets::open with cloned file", thread_id); - + tantivy4java_debug!( + "QUICKWIT DEBUG: [Thread {:?}] BundleDirectory::open_bundle - opening file offsets with file size: {}", + thread_id, + file.len() + ); + tantivy4java_debug!( + "QUICKWIT DEBUG: [Thread {:?}] BundleDirectory::open_bundle - file slice len: {}", + thread_id, + file.len() + ); + tantivy4java_debug!( + "QUICKWIT DEBUG: [Thread {:?}] BundleDirectory::open_bundle - about to call BundleStorageFileOffsets::open with cloned file", + thread_id + ); + let cloned_file = file.clone(); - tantivy4java_debug!("QUICKWIT DEBUG: [Thread {:?}] BundleDirectory::open_bundle - cloned file size: {}", thread_id, cloned_file.len()); - + tantivy4java_debug!( + "QUICKWIT DEBUG: [Thread {:?}] BundleDirectory::open_bundle - cloned file size: {}", + thread_id, + cloned_file.len() + ); + let file_offsets = BundleStorageFileOffsets::open(cloned_file)?; - tantivy4java_debug!("QUICKWIT DEBUG: [Thread {:?}] BundleStorageFileOffsets::open succeeded", thread_id); + tantivy4java_debug!( + "QUICKWIT DEBUG: [Thread {:?}] BundleStorageFileOffsets::open succeeded", + thread_id + ); Ok(BundleDirectory { file, file_offsets }) } } diff --git a/quickwit/quickwit-directories/src/lib.rs b/quickwit/quickwit-directories/src/lib.rs index a371a46e270..d7ec338ffd2 100644 --- a/quickwit/quickwit-directories/src/lib.rs +++ b/quickwit/quickwit-directories/src/lib.rs @@ -31,10 +31,15 @@ mod hot_directory; mod storage_directory; mod union_directory; -pub use self::bundle_directory::{BundleDirectory, get_hotcache_from_split, read_split_footer, split_footer}; +pub use self::bundle_directory::{ + BundleDirectory, get_hotcache_from_split, read_split_footer, split_footer, +}; pub use self::caching_directory::CachingDirectory; pub use self::debug_proxy_directory::{DebugProxyDirectory, ReadOperation}; -pub use self::hot_directory::{HotDirectory, write_hotcache, StaticDirectoryCacheBuilder, StaticSliceCacheBuilder, list_index_files}; +pub use self::hot_directory::{ + HotDirectory, StaticDirectoryCacheBuilder, StaticSliceCacheBuilder, list_index_files, + write_hotcache, +}; pub use self::storage_directory::StorageDirectory; pub use self::union_directory::UnionDirectory; diff --git a/quickwit/quickwit-doc-mapper/Cargo.toml b/quickwit/quickwit-doc-mapper/Cargo.toml index ae0239e53c5..92c977fe4da 100644 --- a/quickwit/quickwit-doc-mapper/Cargo.toml +++ b/quickwit/quickwit-doc-mapper/Cargo.toml @@ -42,10 +42,9 @@ serde_yaml = { workspace = true } time = { workspace = true } quickwit-common = { workspace = true, features = ["testsuite"] } -quickwit-query = { workspace = true, features = ["multilang"] } +quickwit-query = { workspace = true } [features] -multilang = ["quickwit-query/multilang"] testsuite = [] [[bench]] diff --git a/quickwit/quickwit-doc-mapper/src/doc_mapper/date_time_type.rs b/quickwit/quickwit-doc-mapper/src/doc_mapper/date_time_type.rs index cf10e06a735..9f18f5769b3 100644 --- a/quickwit/quickwit-doc-mapper/src/doc_mapper/date_time_type.rs +++ b/quickwit/quickwit-doc-mapper/src/doc_mapper/date_time_type.rs @@ -13,13 +13,12 @@ // limitations under the License. use indexmap::IndexSet; +use quickwit_common::true_fn; use quickwit_datetime::{DateTimeInputFormat, DateTimeOutputFormat, TantivyDateTime}; use serde::{Deserialize, Deserializer, Serialize}; use serde_json::Value as JsonValue; use tantivy::schema::{DateTimePrecision, OwnedValue as TantivyValue}; -use super::default_as_true; - /// A struct holding DateTime field options. #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -41,10 +40,10 @@ pub struct QuickwitDateTimeOptions { #[serde(alias = "precision")] pub fast_precision: DateTimePrecision, - #[serde(default = "default_as_true")] + #[serde(default = "true_fn")] pub indexed: bool, - #[serde(default = "default_as_true")] + #[serde(default = "true_fn")] pub stored: bool, #[serde(default)] @@ -164,7 +163,9 @@ impl Default for InputFormats { impl<'de> Deserialize<'de> for InputFormats { fn deserialize(deserializer: D) -> Result - where D: Deserializer<'de> { + where + D: Deserializer<'de>, + { let date_time_formats = IndexSet::::deserialize(deserializer)?; if date_time_formats.is_empty() { diff --git a/quickwit/quickwit-doc-mapper/src/doc_mapper/doc_mapper_impl.rs b/quickwit/quickwit-doc-mapper/src/doc_mapper/doc_mapper_impl.rs index e0a24594d0a..087fe2839ab 100644 --- a/quickwit/quickwit-doc-mapper/src/doc_mapper/doc_mapper_impl.rs +++ b/quickwit/quickwit-doc-mapper/src/doc_mapper/doc_mapper_impl.rs @@ -14,12 +14,13 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::num::NonZeroU32; +use std::sync::Arc; use anyhow::{Context, bail}; use fnv::FnvHashSet; use quickwit_proto::types::DocMappingUid; use quickwit_query::create_default_quickwit_tokenizer_manager; -use quickwit_query::query_ast::QueryAst; +use quickwit_query::query_ast::{BuildTantivyAstContext, QueryAst}; use quickwit_query::tokenizers::TokenizerManager; use serde::{Deserialize, Serialize}; use serde_json::{self, Value as JsonValue}; @@ -34,7 +35,7 @@ use super::field_presence::populate_field_presence; use super::tantivy_val_to_json::tantivy_value_to_json; use crate::doc_mapper::mapping_tree::{ JsonValueIterator, MappingNode, MappingNodeRoot, build_field_path_from_str, build_mapping_tree, - map_primitive_json_to_tantivy, + map_primitive_json_to_concatenate_value, }; use crate::doc_mapper::{FieldMappingType, JsonObject, Partition}; use crate::query_builder::build_query; @@ -367,14 +368,14 @@ fn validate_fields_tokenizers( .map(|text_options: &tantivy::schema::TextFieldIndexing| text_options.tokenizer()), _ => None, }; - if let Some(tokenizer_name) = tokenizer_name_opt { - if tokenizer_manager.get_tokenizer(tokenizer_name).is_none() { - bail!( - "unknown tokenizer `{}` for field `{}`", - tokenizer_name, - field_entry.name() - ); - } + if let Some(tokenizer_name) = tokenizer_name_opt + && tokenizer_manager.get_tokenizer(tokenizer_name).is_none() + { + bail!( + "unknown tokenizer `{}` for field `{}`", + tokenizer_name, + field_entry.name() + ); } } Ok(()) @@ -523,27 +524,27 @@ impl DocMapper { &mut dynamic_json_obj, )?; - if let Some(dynamic_field) = self.dynamic_field { - if !dynamic_json_obj.is_empty() { - if !self.concatenate_dynamic_fields.is_empty() { - let json_obj_values = - JsonValueIterator::new(serde_json::Value::Object(dynamic_json_obj.clone())) - .flat_map(map_primitive_json_to_tantivy); - - for value in json_obj_values { - for concatenate_dynamic_field in self.concatenate_dynamic_fields.iter() { - document.add_field_value(*concatenate_dynamic_field, &value); - } + if let Some(dynamic_field) = self.dynamic_field + && !dynamic_json_obj.is_empty() + { + if !self.concatenate_dynamic_fields.is_empty() { + let json_obj_values = + JsonValueIterator::new(serde_json::Value::Object(dynamic_json_obj.clone())) + .flat_map(map_primitive_json_to_concatenate_value); + + for value in json_obj_values { + for concatenate_dynamic_field in self.concatenate_dynamic_fields.iter() { + document.add_field_value(*concatenate_dynamic_field, &value); } } - document.add_object( - dynamic_field, - dynamic_json_obj - .into_iter() - .map(|(key, val)| (key, TantivyValue::from(val))) - .collect(), - ); } + document.add_object( + dynamic_field, + dynamic_json_obj + .into_iter() + .map(|(key, val)| (key, TantivyValue::from(val))) + .collect(), + ); } if let Some(document_size_field) = self.document_size_field { @@ -636,15 +637,19 @@ impl DocMapper { pub fn query( &self, split_schema: Schema, - query_ast: &QueryAst, + query_ast: QueryAst, with_validation: bool, + cache_context: Option<(Arc, String)>, ) -> Result<(Box, WarmupInfo), QueryParserError> { build_query( query_ast, - split_schema, - self.tokenizer_manager(), - &self.default_search_field_names[..], - with_validation, + &BuildTantivyAstContext { + schema: &split_schema, + tokenizer_manager: self.tokenizer_manager(), + search_fields: &self.default_search_field_names[..], + with_validation, + }, + cache_context, ) } @@ -1849,7 +1854,8 @@ mod tests { }"#, "concat", r#"{"some_int": 25}"#, - vec![25_u64.into()], + // i64 comes before u64 + vec![25_i64.into()], ); } @@ -2068,7 +2074,7 @@ mod tests { .parse_user_query(doc_mapper.default_search_fields()) .map_err(|err| err.to_string())?; let (query, _) = doc_mapper - .query(doc_mapper.schema(), &query_ast, true) + .query(doc_mapper.schema(), query_ast, true, None) .map_err(|err| err.to_string())?; Ok(format!("{query:?}")) } diff --git a/quickwit/quickwit-doc-mapper/src/doc_mapper/field_mapping_entry.rs b/quickwit/quickwit-doc-mapper/src/doc_mapper/field_mapping_entry.rs index 45f7916db5d..e69d337a616 100644 --- a/quickwit/quickwit-doc-mapper/src/doc_mapper/field_mapping_entry.rs +++ b/quickwit/quickwit-doc-mapper/src/doc_mapper/field_mapping_entry.rs @@ -18,6 +18,7 @@ use std::convert::TryFrom; use anyhow::bail; use base64::prelude::{BASE64_STANDARD, Engine}; use once_cell::sync::Lazy; +use quickwit_common::true_fn; use regex::Regex; use serde::{Deserialize, Serialize}; use serde_json::Value as JsonValue; @@ -26,8 +27,8 @@ use tantivy::schema::{ TextOptions, Type, }; +use super::FieldMappingType; use super::date_time_type::QuickwitDateTimeOptions; -use super::{FieldMappingType, default_as_true}; use crate::doc_mapper::field_mapping_type::QuickwitFieldType; use crate::{Cardinality, QW_RESERVED_FIELD_NAMES}; @@ -85,13 +86,13 @@ pub struct QuickwitNumericOptions { #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub description: Option, - #[serde(default = "default_as_true")] + #[serde(default = "true_fn")] pub stored: bool, - #[serde(default = "default_as_true")] + #[serde(default = "true_fn")] pub indexed: bool, #[serde(default)] pub fast: bool, - #[serde(default = "default_as_true")] + #[serde(default = "true_fn")] pub coerce: bool, #[serde(default)] pub output_format: NumericOutputFormat, @@ -116,9 +117,9 @@ pub struct QuickwitBoolOptions { #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub description: Option, - #[serde(default = "default_as_true")] + #[serde(default = "true_fn")] pub stored: bool, - #[serde(default = "default_as_true")] + #[serde(default = "true_fn")] pub indexed: bool, #[serde(default)] pub fast: bool, @@ -144,10 +145,10 @@ pub struct QuickwitBytesOptions { #[serde(skip_serializing_if = "Option::is_none")] pub description: Option, /// If true, the field will be stored in the doc store. - #[serde(default = "default_as_true")] + #[serde(default = "true_fn")] pub stored: bool, /// If true, the field will be indexed. - #[serde(default = "default_as_true")] + #[serde(default = "true_fn")] pub indexed: bool, /// If true, the field will be stored in columnar format. #[serde(default)] @@ -245,9 +246,9 @@ pub struct QuickwitIpAddrOptions { #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub description: Option, - #[serde(default = "default_as_true")] + #[serde(default = "true_fn")] pub stored: bool, - #[serde(default = "default_as_true")] + #[serde(default = "true_fn")] pub indexed: bool, #[serde(default)] pub fast: bool, @@ -433,7 +434,7 @@ pub struct QuickwitTextOptions { deserializer = TextIndexingOptions::from_parts_text, serializer = TextIndexingOptions::to_parts_text, fields = ( - #[serde(default = "default_as_true")] + #[serde(default = "true_fn")] pub indexed: bool, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] @@ -447,7 +448,7 @@ pub struct QuickwitTextOptions { ), )] pub indexing_options: Option, - #[serde(default = "default_as_true")] + #[serde(default = "true_fn")] pub stored: bool, #[serde(default)] pub fast: FastFieldOptions, @@ -577,7 +578,7 @@ pub struct QuickwitJsonOptions { serializer = TextIndexingOptions::to_parts_json, fields = ( /// If true, all of the element in the json object will be indexed. - #[serde(default = "default_as_true")] + #[serde(default = "true_fn")] pub indexed: bool, /// Sets the tokenize that should be used with the text fields in the /// json object. @@ -597,10 +598,10 @@ pub struct QuickwitJsonOptions { /// Options for indexing text in a Json field. pub indexing_options: Option, /// If true, the field will be stored in the doc store. - #[serde(default = "default_as_true")] + #[serde(default = "true_fn")] pub stored: bool, /// If true, the '.' in json keys will be expanded. - #[serde(default = "default_as_true")] + #[serde(default = "true_fn")] pub expand_dots: bool, /// If true, the json object will be stored in columnar format. #[serde(default)] @@ -1151,7 +1152,7 @@ mod tests { "type": "text", "stored": true, "record": "basic", - "tokenizer": "en_stem" + "tokenizer": "lowercase" } "#, )?; @@ -1160,7 +1161,7 @@ mod tests { FieldMappingType::Text(options, _) => { assert_eq!(options.stored, true); let indexing_options = options.indexing_options.unwrap(); - assert_eq!(indexing_options.tokenizer.name(), "en_stem"); + assert_eq!(indexing_options.tokenizer.name(), "lowercase"); assert_eq!(indexing_options.record, IndexRecordOption::Basic); } _ => panic!("wrong property type"), diff --git a/quickwit/quickwit-doc-mapper/src/doc_mapper/mapping_tree.rs b/quickwit/quickwit-doc-mapper/src/doc_mapper/mapping_tree.rs index f9d427ed0d5..30b16fea4c8 100644 --- a/quickwit/quickwit-doc-mapper/src/doc_mapper/mapping_tree.rs +++ b/quickwit/quickwit-doc-mapper/src/doc_mapper/mapping_tree.rs @@ -139,16 +139,22 @@ impl> Iterator for OneOrIter { } } -pub(crate) fn map_primitive_json_to_tantivy(value: JsonValue) -> Option { +/// Similar to the native `From for TantivyValue` implementation, with a +/// subtle difference: no automatic parsing to DateTime is performed when the string +/// is a valid RFC3339 date. This enables some level of range querying through prefix +/// queries despite concatenate fields not supporting fast fields. +pub(crate) fn map_primitive_json_to_concatenate_value(value: JsonValue) -> Option { match value { JsonValue::Array(_) | JsonValue::Object(_) | JsonValue::Null => None, JsonValue::String(text) => Some(TantivyValue::Str(text)), JsonValue::Bool(val) => Some((val).into()), JsonValue::Number(number) => { - if let Some(val) = u64::from_json_number(&number) { + if let Some(val) = i64::from_json_number(&number) { + Some((val).into()) + } else if let Some(val) = u64::from_json_number(&number) { Some((val).into()) } else { - i64::from_json_number(&number).map(|val| (val).into()) + f64::from_json_number(&number).map(|val| (val).into()) } } } @@ -258,7 +264,7 @@ impl LeafType { } } - fn tantivy_value_from_json( + fn concatenate_values_from_json( &self, json_val: JsonValue, ) -> Result, String> { @@ -300,7 +306,7 @@ impl LeafType { json_obj .into_iter() .flat_map(|(_key, val)| JsonValueIterator::new(val)) - .flat_map(map_primitive_json_to_tantivy), + .flat_map(map_primitive_json_to_concatenate_value), )) } else { Err(format!("expected object, got `{json_val}`")) @@ -386,7 +392,7 @@ impl MappingLeaf { if !self.concatenate.is_empty() { let concat_values = self .typ - .tantivy_value_from_json(el_json_val.clone()) + .concatenate_values_from_json(el_json_val.clone()) .map_err(|err_msg| DocParsingError::ValueError(path.join("."), err_msg))?; for concat_value in concat_values { for field in &self.concatenate { @@ -406,7 +412,7 @@ impl MappingLeaf { if !self.concatenate.is_empty() { let concat_values = self .typ - .tantivy_value_from_json(json_val.clone()) + .concatenate_values_from_json(json_val.clone()) .map_err(|err_msg| DocParsingError::ValueError(path.join("."), err_msg))?; for concat_value in concat_values { for field in &self.concatenate { diff --git a/quickwit/quickwit-doc-mapper/src/doc_mapper/mod.rs b/quickwit/quickwit-doc-mapper/src/doc_mapper/mod.rs index de79188f3a8..749dde228a7 100644 --- a/quickwit/quickwit-doc-mapper/src/doc_mapper/mod.rs +++ b/quickwit/quickwit-doc-mapper/src/doc_mapper/mod.rs @@ -28,8 +28,6 @@ use std::ops::Bound; pub use doc_mapper_builder::DocMapperBuilder; pub use doc_mapper_impl::DocMapper; -#[cfg(all(test, feature = "multilang"))] -pub(crate) use field_mapping_entry::TextIndexingOptions; pub use field_mapping_entry::{ BinaryFormat, FastFieldOptions, FieldMappingEntry, QuickwitBytesOptions, QuickwitJsonOptions, QuickwitTextNormalizer, @@ -48,11 +46,6 @@ pub(crate) use tokenizer_entry::{ }; pub use tokenizer_entry::{TokenizerConfig, TokenizerEntry, analyze_text}; -/// Function used with serde to initialize boolean value at true if there is no value in json. -fn default_as_true() -> bool { - true -} - pub type Partition = u64; /// An alias for serde_json's object type. @@ -295,7 +288,7 @@ mod tests { } .parse_user_query(&[]) .unwrap(); - let (query, _) = doc_mapper.query(schema, &query_ast, true).unwrap(); + let (query, _) = doc_mapper.query(schema, query_ast, true, None).unwrap(); assert_eq!( format!("{query:?}"), r#"TermQuery(Term(field=2, type=Json, path=toto.titi, type=Str, "hello"))"# @@ -309,7 +302,7 @@ mod tests { let query_ast = query_ast_from_user_text("toto.titi:hello", None) .parse_user_query(doc_mapper.default_search_fields()) .unwrap(); - let (query, _) = doc_mapper.query(schema, &query_ast, true).unwrap(); + let (query, _) = doc_mapper.query(schema, query_ast, true, None).unwrap(); assert_eq!( format!("{query:?}"), r#"TermQuery(Term(field=1, type=Json, path=toto.titi, type=Str, "hello"))"# @@ -323,7 +316,7 @@ mod tests { let query_ast = query_ast_from_user_text("toto:5", None) .parse_user_query(&[]) .unwrap(); - let (query, _) = doc_mapper.query(schema, &query_ast, true).unwrap(); + let (query, _) = doc_mapper.query(schema, query_ast, true, None).unwrap(); assert_eq!( format!("{query:?}"), r#"BooleanQuery { subqueries: [(Should, TermQuery(Term(field=1, type=Json, path=toto, type=I64, 5))), (Should, TermQuery(Term(field=1, type=Json, path=toto, type=Str, "5")))], minimum_number_should_match: 1 }"# @@ -817,55 +810,4 @@ mod tests { warmup_info.simplify(); assert_eq!(warmup_info, expected); } - - #[test] - #[cfg(feature = "multilang")] - fn test_doc_mapper_query_with_multilang_field() { - use quickwit_query::query_ast::TermQuery; - use tantivy::schema::IndexRecordOption; - - use crate::doc_mapper::{ - QuickwitTextOptions, QuickwitTextTokenizer, TextIndexingOptions, TokenizerType, - }; - use crate::{TokenizerConfig, TokenizerEntry}; - let mut doc_mapper_builder = DocMapperBuilder::default(); - doc_mapper_builder - .doc_mapping - .field_mappings - .push(FieldMappingEntry { - name: "multilang".to_string(), - mapping_type: FieldMappingType::Text( - QuickwitTextOptions { - indexing_options: Some(TextIndexingOptions { - tokenizer: QuickwitTextTokenizer::from_static("multilang"), - record: IndexRecordOption::Basic, - fieldnorms: false, - }), - ..Default::default() - }, - Cardinality::SingleValued, - ), - }); - doc_mapper_builder - .doc_mapping - .tokenizers - .push(TokenizerEntry { - name: "multilang".to_string(), - config: TokenizerConfig { - tokenizer_type: TokenizerType::Multilang, - filters: Vec::new(), - }, - }); - let doc_mapper = doc_mapper_builder.try_build().unwrap(); - let schema = doc_mapper.schema(); - let query_ast = quickwit_query::query_ast::QueryAst::Term(TermQuery { - field: "multilang".to_string(), - value: "JPN:す".to_string(), - }); - let (query, _) = doc_mapper.query(schema, &query_ast, false).unwrap(); - assert_eq!( - format!("{query:?}"), - r#"TermQuery(Term(field=2, type=Str, "JPN:す"))"# - ); - } } diff --git a/quickwit/quickwit-doc-mapper/src/doc_mapper/tokenizer_entry.rs b/quickwit/quickwit-doc-mapper/src/doc_mapper/tokenizer_entry.rs index b9793dc9548..0488d118c9f 100644 --- a/quickwit/quickwit-doc-mapper/src/doc_mapper/tokenizer_entry.rs +++ b/quickwit/quickwit-doc-mapper/src/doc_mapper/tokenizer_entry.rs @@ -44,10 +44,6 @@ impl TokenizerConfig { pub fn text_analyzer(&self) -> anyhow::Result { let mut text_analyzer_builder = match &self.tokenizer_type { TokenizerType::Simple => TextAnalyzer::builder(SimpleTokenizer::default()).dynamic(), - #[cfg(any(test, feature = "multilang"))] - TokenizerType::Multilang => { - TextAnalyzer::builder(quickwit_query::MultiLangTokenizer::default()).dynamic() - } TokenizerType::SourceCode => TextAnalyzer::builder(CodeTokenizer::default()).dynamic(), TokenizerType::Ngram(options) => { let tokenizer = @@ -120,8 +116,6 @@ impl TokenFilterType { #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, utoipa::ToSchema)] #[serde(tag = "type", rename_all = "snake_case")] pub enum TokenizerType { - #[cfg(any(test, feature = "multilang"))] - Multilang, Ngram(NgramTokenizerOption), Regex(RegexTokenizerOption), Simple, diff --git a/quickwit/quickwit-doc-mapper/src/doc_mapping.rs b/quickwit/quickwit-doc-mapper/src/doc_mapping.rs index 984932c9a5b..2fae5d45452 100644 --- a/quickwit/quickwit-doc-mapper/src/doc_mapping.rs +++ b/quickwit/quickwit-doc-mapper/src/doc_mapping.rs @@ -166,20 +166,6 @@ impl DocMapping { pub fn default_max_num_partitions() -> NonZeroU32 { NonZeroU32::new(200).unwrap() } - - /// Returns whether the `other` doc mapping is equal to `self` leaving their respective doc - /// mapping UIDs out of the comparison. - pub fn eq_ignore_doc_mapping_uid(&self, other: &Self) -> bool { - let doc_mapping_uid = DocMappingUid::default(); - - let mut left = self.clone(); - left.doc_mapping_uid = doc_mapping_uid; - - let mut right = other.clone(); - right.doc_mapping_uid = doc_mapping_uid; - - left == right - } } #[cfg(test)] diff --git a/quickwit/quickwit-doc-mapper/src/query_builder.rs b/quickwit/quickwit-doc-mapper/src/query_builder.rs index e78f9419b49..5900b577795 100644 --- a/quickwit/quickwit-doc-mapper/src/query_builder.rs +++ b/quickwit/quickwit-doc-mapper/src/query_builder.rs @@ -15,10 +15,12 @@ use std::collections::{HashMap, HashSet}; use std::convert::Infallible; use std::ops::Bound; +use std::sync::Arc; +use quickwit_proto::types::SplitId; use quickwit_query::query_ast::{ - FieldPresenceQuery, FullTextQuery, PhrasePrefixQuery, QueryAst, QueryAstVisitor, RangeQuery, - RegexQuery, TermSetQuery, WildcardQuery, + BuildTantivyAstContext, FieldPresenceQuery, FullTextQuery, PhrasePrefixQuery, QueryAst, + QueryAstTransformer, QueryAstVisitor, RangeQuery, RegexQuery, TermSetQuery, WildcardQuery, }; use quickwit_query::tokenizers::TokenizerManager; use quickwit_query::{InvalidQuery, find_field_or_hit_dynamic}; @@ -45,12 +47,84 @@ impl<'a> QueryAstVisitor<'a> for RangeQueryFields { } } -struct ExistsQueryFastFields { - fields: HashSet, +/// Term Queries on fields which are fast but not indexed. +struct TermSearchOnColumnar<'f> { + fields: &'f mut HashSet, schema: Schema, } +impl<'a, 'f> QueryAstVisitor<'a> for TermSearchOnColumnar<'f> { + type Err = Infallible; + + fn visit_term_set(&mut self, term_set_query: &'a TermSetQuery) -> Result<(), Infallible> { + for field in term_set_query.terms_per_field.keys() { + if let Some((_field, field_entry, path)) = + find_field_or_hit_dynamic(field, &self.schema) + && field_entry.is_fast() + && !field_entry.is_indexed() + { + self.fields.insert(FastFieldWarmupInfo { + name: if path.is_empty() { + field_entry.name().to_string() + } else { + format!("{}.{}", field_entry.name(), path) + }, + with_subfields: false, + }); + } + } + Ok(()) + } + + fn visit_term( + &mut self, + term_query: &'a quickwit_query::query_ast::TermQuery, + ) -> Result<(), Infallible> { + if let Some((_field, field_entry, path)) = + find_field_or_hit_dynamic(&term_query.field, &self.schema) + && field_entry.is_fast() + && !field_entry.is_indexed() + { + self.fields.insert(FastFieldWarmupInfo { + name: if path.is_empty() { + field_entry.name().to_string() + } else { + format!("{}.{}", field_entry.name(), path) + }, + with_subfields: false, + }); + } + Ok(()) + } + /// We also need to visit full text queries because they can be converted to term queries + /// on fast fields. We only care about the field being fast and not indexed AND the tokenizer + /// being `raw` or None. + fn visit_full_text(&mut self, full_text_query: &'a FullTextQuery) -> Result<(), Infallible> { + if let Some((_field, field_entry, path)) = + find_field_or_hit_dynamic(&full_text_query.field, &self.schema) + && field_entry.is_fast() + && !field_entry.is_indexed() + && (full_text_query.params.tokenizer.is_none() + || full_text_query.params.tokenizer.as_deref() == Some("raw")) + { + self.fields.insert(FastFieldWarmupInfo { + name: if path.is_empty() { + field_entry.name().to_string() + } else { + format!("{}.{}", field_entry.name(), path) + }, + with_subfields: false, + }); + } + Ok(()) + } +} -impl<'a> QueryAstVisitor<'a> for ExistsQueryFastFields { +struct ExistsQueryFastFields<'f> { + fields: &'f mut HashSet, + schema: Schema, +} + +impl<'a, 'f> QueryAstVisitor<'a> for ExistsQueryFastFields<'f> { type Err = Infallible; fn visit_exists(&mut self, exists_query: &'a FieldPresenceQuery) -> Result<(), Infallible> { @@ -82,24 +156,24 @@ impl<'a> QueryAstVisitor<'a> for ExistsQueryFastFields { /// Build a `Query` with field resolution & forbidding range clauses. pub(crate) fn build_query( - query_ast: &QueryAst, - schema: Schema, - tokenizer_manager: &TokenizerManager, - search_fields: &[String], - with_validation: bool, + query_ast: QueryAst, + context: &BuildTantivyAstContext, + cache_context: Option<(Arc, SplitId)>, ) -> Result<(Box, WarmupInfo), QueryParserError> { - let mut range_query_fields = RangeQueryFields::default(); - // This cannot fail. The error type is Infallible. - let _: Result<(), Infallible> = range_query_fields.visit(query_ast); - - let mut exists_query_fields = ExistsQueryFastFields { - fields: HashSet::new(), - schema: schema.clone(), + let mut fast_fields: HashSet = HashSet::new(); + + let query_ast = if let Some((cache, split_id)) = cache_context { + let Ok(query_ast) = quickwit_query::query_ast::PredicateCacheInjector { cache, split_id } + .transform(query_ast); + // this transformer isn't supposed to ever remove a node + query_ast.unwrap_or(QueryAst::MatchAll) + } else { + query_ast }; - // This cannot fail. The error type is Infallible. - let _: Result<(), Infallible> = exists_query_fields.visit(query_ast); - let mut fast_fields = HashSet::new(); + let mut range_query_fields = RangeQueryFields::default(); + // This cannot fail. The error type is Infallible. + let Ok(_) = range_query_fields.visit(&query_ast); let range_query_fast_fields = range_query_fields .range_query_field_names @@ -109,22 +183,35 @@ pub(crate) fn build_query( with_subfields: false, }); fast_fields.extend(range_query_fast_fields); - fast_fields.extend(exists_query_fields.fields); - let query = query_ast.build_tantivy_query( - &schema, - tokenizer_manager, - search_fields, - with_validation, - )?; + let Ok(_) = TermSearchOnColumnar { + fields: &mut fast_fields, + schema: context.schema.clone(), + } + .visit(&query_ast); - let term_set_query_fields = extract_term_set_query_fields(query_ast, &schema)?; + let Ok(_) = ExistsQueryFastFields { + fields: &mut fast_fields, + schema: context.schema.clone(), + } + .visit(&query_ast); + + let query = query_ast.build_tantivy_query(context)?; + + let term_set_query_fields = extract_term_set_query_fields(&query_ast, context.schema)?; let (term_ranges_grouped_by_field, automatons_grouped_by_field) = - extract_prefix_term_ranges_and_automaton(query_ast, &schema, tokenizer_manager)?; + extract_prefix_term_ranges_and_automaton( + &query_ast, + context.schema, + context.tokenizer_manager, + )?; let mut terms_grouped_by_field: HashMap> = Default::default(); query.query_terms(&mut |term, need_position| { let field = term.field(); + if !context.schema.get_field_entry(field).is_indexed() { + return; + } *terms_grouped_by_field .entry(field) .or_default() @@ -184,20 +271,32 @@ fn extract_term_set_query_fields( Ok(visitor.term_dict_fields_to_warm_up) } +/// Converts a `prefix` term into the equivalent term range. +/// +/// The resulting range is `[prefix, next_prefix)`, that is: +/// - start bound: `Included(prefix)` +/// - end bound: `Excluded(next lexicographic term after the prefix)` +/// +/// "abc" -> start: "abc", end: "abd" (excluded) +/// "ab\xFF" -> start: "ab\xFF", end: "ac" (excluded) +/// "\xFF\xFF" -> start: "\xFF\xFF", end: Unbounded fn prefix_term_to_range(prefix: Term) -> (Bound, Bound) { - let mut end_bound = prefix.serialized_term().to_vec(); - while !end_bound.is_empty() { - let last_byte = end_bound.last_mut().unwrap(); + // Start from the given prefix and try to find the successor + let mut end_bound = prefix.clone(); + let mut end_bound_value_bytes = prefix.serialized_value_bytes().to_vec(); + while !end_bound_value_bytes.is_empty() { + let last_byte = end_bound_value_bytes.last_mut().unwrap(); if *last_byte != u8::MAX { *last_byte += 1; - return ( - Bound::Included(prefix), - Bound::Excluded(Term::wrap(end_bound)), - ); + // The last non-`u8::MAX` byte incremented + // gives us the exclusive upper bound. + end_bound.set_bytes(&end_bound_value_bytes); + return (Bound::Included(prefix), Bound::Excluded(end_bound)); } - end_bound.pop(); + // pop u8::MAX byte and try next + end_bound_value_bytes.pop(); } - // prefix is something like [255, 255, ..] + // All bytes were `u8::MAX`: there is no successor, so the upper bound is unbounded. (Bound::Included(prefix), Bound::Unbounded) } @@ -328,8 +427,8 @@ mod test { use quickwit_common::shared_consts::FIELD_PRESENCE_FIELD_NAME; use quickwit_query::query_ast::{ - FullTextMode, FullTextParams, PhrasePrefixQuery, QueryAstVisitor, UserInputQuery, - query_ast_from_user_text, + BuildTantivyAstContext, FullTextMode, FullTextParams, PhrasePrefixQuery, QueryAstVisitor, + UserInputQuery, query_ast_from_user_text, }; use quickwit_query::{ BooleanOperand, MatchAllOrNone, create_default_quickwit_tokenizer_manager, @@ -415,13 +514,7 @@ mod test { .parse_user_query(&[]) .map_err(|err| err.to_string())?; let schema = make_schema(dynamic_mode); - let query_result = build_query( - &query_ast, - schema, - &create_default_quickwit_tokenizer_manager(), - &[], - true, - ); + let query_result = build_query(query_ast, &BuildTantivyAstContext::for_test(&schema), None); query_result .map(|query| format!("{query:?}")) .map_err(|err| err.to_string()) @@ -795,14 +888,10 @@ mod test { .parse_user_query(&[]) .unwrap(); - let (_, warmup_info) = build_query( - &query_with_set, - make_schema(true), - &create_default_quickwit_tokenizer_manager(), - &[], - true, - ) - .unwrap(); + let schema = make_schema(true); + let context = BuildTantivyAstContext::for_test(&schema); + + let (_, warmup_info) = build_query(query_with_set, &context, None).unwrap(); assert_eq!(warmup_info.term_dict_fields.len(), 1); assert!( warmup_info @@ -810,14 +899,7 @@ mod test { .contains(&tantivy::schema::Field::from_field_id(2)) ); - let (_, warmup_info) = build_query( - &query_without_set, - make_schema(true), - &create_default_quickwit_tokenizer_manager(), - &[], - true, - ) - .unwrap(); + let (_, warmup_info) = build_query(query_without_set, &context, None).unwrap(); assert!(warmup_info.term_dict_fields.is_empty()); } diff --git a/quickwit/quickwit-doc-mapper/src/routing_expression/mod.rs b/quickwit/quickwit-doc-mapper/src/routing_expression/mod.rs index 28c3a6442fd..e35b2095a7c 100644 --- a/quickwit/quickwit-doc-mapper/src/routing_expression/mod.rs +++ b/quickwit/quickwit-doc-mapper/src/routing_expression/mod.rs @@ -355,8 +355,8 @@ mod expression_dsl { use nom::combinator::{eof, map, opt}; use nom::error::ErrorKind; use nom::multi::separated_list0; - use nom::sequence::{delimited, tuple}; - use nom::{AsChar, Finish, IResult, InputTakeAtPosition}; + use nom::sequence::delimited; + use nom::{AsChar, Finish, IResult, Input, Parser}; // this is a RoutingSubExpr in our DSL. #[derive(Debug, PartialEq, Eq, Clone)] @@ -383,7 +383,7 @@ mod expression_dsl { // tag, but ignore leading and trailing whitespaces pub fn wtag<'a, Error: nom::error::ParseError<&'a str>>( t: &'a str, - ) -> impl FnMut(&'a str) -> IResult<&'a str, &'a str, Error> { + ) -> impl Parser<&'a str, Output = &'a str, Error = Error> { delimited(multispace0, tag(t), multispace0) } @@ -401,13 +401,13 @@ mod expression_dsl { /// An entire routing expression, containing comma separated routing sub-expressions fn routing_expr(input: &str) -> IResult<&str, Vec> { - separated_list0(wtag(","), routing_sub_expr)(input) + separated_list0(wtag(","), routing_sub_expr).parse(input) } /// A sub-part of a routing expression fn routing_sub_expr(input: &str) -> IResult<&str, ExpressionAst> { let (input, identifier) = identifier(input)?; - let (input, args) = opt(tuple((wtag("("), arguments, wtag(")"))))(input)?; + let (input, args) = opt((wtag("("), arguments, wtag(")"))).parse(input)?; let res = if let Some((_, args, _)) = args { ExpressionAst::Function { name: identifier.to_owned(), @@ -430,15 +430,14 @@ mod expression_dsl { /// Arguments for a function fn arguments(input: &str) -> IResult<&str, Vec> { - separated_list0(wtag(","), argument)(input) + separated_list0(wtag(","), argument).parse(input) } /// A single argument for a function fn argument(input: &str) -> IResult<&str, Argument> { if let Ok((input, number)) = number(input) { Ok((input, Argument::Number(number))) - } else if let Ok((input, (_, arg, _))) = tuple((wtag("("), routing_expr, wtag(")")))(input) - { + } else if let Ok((input, (_, arg, _))) = (wtag("("), routing_expr, wtag(")")).parse(input) { Ok((input, Argument::Expression(arg))) } else { routing_sub_expr(input).map(|(input, arg)| (input, Argument::Expression(vec![arg]))) @@ -462,19 +461,21 @@ mod expression_dsl { } /// Parse a single path component, separated by dots. De-escape any escaped dot it may contain. - fn escaped_key(input: &str) -> IResult<&str, Cow> { + fn escaped_key(input: &str) -> IResult<&str, Cow<'_, str>> { map(escaped(key_identifier, '\\', tag(".")), |s: &str| { if s.contains("\\.") { Cow::Owned(s.replace("\\.", ".")) } else { Cow::Borrowed(s) } - })(input) + }) + .parse(input) } /// Parse a field name into a path, de-escaping where appropriate. - pub(crate) fn parse_field_name(input: &str) -> anyhow::Result>> { - let (i, res) = separated_list0(tag("."), escaped_key)(input) + pub(crate) fn parse_field_name(input: &str) -> anyhow::Result>> { + let (i, res) = separated_list0(tag("."), escaped_key) + .parse(input) .finish() .map_err(|e| anyhow::anyhow!("error parsing key expression: {e}"))?; eof::<_, ()>(i)?; diff --git a/quickwit/quickwit-doc-mapper/src/tag_pruning.rs b/quickwit/quickwit-doc-mapper/src/tag_pruning.rs index e75e5e2e2dc..ad6ded9444c 100644 --- a/quickwit/quickwit-doc-mapper/src/tag_pruning.rs +++ b/quickwit/quickwit-doc-mapper/src/tag_pruning.rs @@ -114,6 +114,7 @@ fn extract_unsimplified_tags_filter_ast(query_ast: QueryAst) -> UnsimplifiedTagF } QueryAst::FieldPresence(_) => UnsimplifiedTagFilterAst::Uninformative, QueryAst::Regex(_) => UnsimplifiedTagFilterAst::Uninformative, + QueryAst::Cache(cache_node) => extract_unsimplified_tags_filter_ast(*cache_node.inner), } } diff --git a/quickwit/quickwit-index-management/src/garbage_collection.rs b/quickwit/quickwit-index-management/src/garbage_collection.rs index 51569fcd7fc..1883c26e074 100644 --- a/quickwit/quickwit-index-management/src/garbage_collection.rs +++ b/quickwit/quickwit-index-management/src/garbage_collection.rs @@ -73,7 +73,9 @@ pub struct DeleteSplitsError { } async fn protect_future(progress: Option<&Progress>, future: Fut) -> T -where Fut: Future { +where + Fut: Future, +{ match progress { None => future.await, Some(progress) => { @@ -292,14 +294,15 @@ async fn list_splits_metadata( fn get_maximum_split_deletion_rate_per_sec() -> Option { static MAX_SPLIT_DELETION_RATE_PER_SEC: OnceLock> = OnceLock::new(); *MAX_SPLIT_DELETION_RATE_PER_SEC.get_or_init(|| { - quickwit_common::get_from_env_opt::("QW_MAX_SPLIT_DELETION_RATE_PER_SEC") + quickwit_common::get_from_env_opt::("QW_MAX_SPLIT_DELETION_RATE_PER_SEC", false) }) } fn get_index_gc_concurrency() -> Option { static INDEX_GC_CONCURRENCY: OnceLock> = OnceLock::new(); - *INDEX_GC_CONCURRENCY - .get_or_init(|| quickwit_common::get_from_env_opt::("QW_INDEX_GC_CONCURRENCY")) + *INDEX_GC_CONCURRENCY.get_or_init(|| { + quickwit_common::get_from_env_opt::("QW_INDEX_GC_CONCURRENCY", false) + }) } /// Removes any splits marked for deletion which haven't been diff --git a/quickwit/quickwit-index-management/src/index.rs b/quickwit/quickwit-index-management/src/index.rs index 87cc0135332..2cfa0614e40 100644 --- a/quickwit/quickwit-index-management/src/index.rs +++ b/quickwit/quickwit-index-management/src/index.rs @@ -26,13 +26,14 @@ use quickwit_indexing::check_source_connectivity; use quickwit_metastore::{ AddSourceRequestExt, CreateIndexResponseExt, IndexMetadata, IndexMetadataResponseExt, ListIndexesMetadataResponseExt, ListSplitsQuery, ListSplitsRequestExt, - MetastoreServiceStreamSplitsExt, SplitInfo, SplitMetadata, SplitState, UpdateSourceRequestExt, + MetastoreServiceStreamSplitsExt, SplitInfo, SplitMetadata, SplitState, UpdateIndexRequestExt, + UpdateSourceRequestExt, }; use quickwit_proto::metastore::{ AddSourceRequest, CreateIndexRequest, DeleteIndexRequest, EntityKind, IndexMetadataRequest, ListIndexesMetadataRequest, ListSplitsRequest, MarkSplitsForDeletionRequest, MetastoreError, - MetastoreService, MetastoreServiceClient, ResetSourceCheckpointRequest, UpdateSourceRequest, - serde_utils, + MetastoreService, MetastoreServiceClient, ResetSourceCheckpointRequest, UpdateIndexRequest, + UpdateSourceRequest, serde_utils, }; use quickwit_proto::types::{IndexUid, SplitId}; use quickwit_proto::{ServiceError, ServiceErrorCode}; @@ -155,6 +156,41 @@ impl IndexService { Ok(index_metadata) } + /// Returns the index metadata for the given index ID if it exists. + pub async fn index_metadata_opt( + &self, + index_metadata_request: IndexMetadataRequest, + ) -> Result, IndexServiceError> { + let index_metadata_response = self.metastore.index_metadata(index_metadata_request).await; + match index_metadata_response { + Ok(index_metadata_response) => { + let index_metadata = index_metadata_response.deserialize_index_metadata()?; + Ok(Some(index_metadata)) + } + Err(MetastoreError::NotFound(_)) => Ok(None), + Err(error) => Err(IndexServiceError::Metastore(error)), + } + } + + /// Updates an index with the given index config. + pub async fn update_index( + &self, + index_uid: IndexUid, + index_config: IndexConfig, + ) -> Result { + let update_index_request = UpdateIndexRequest::try_from_updates( + index_uid, + &index_config.doc_mapping, + &index_config.indexing_settings, + &index_config.ingest_settings, + &index_config.search_settings, + &index_config.retention_policy_opt, + )?; + let update_index_response = self.metastore.update_index(update_index_request).await?; + let index_metadata = update_index_response.deserialize_index_metadata()?; + Ok(index_metadata) + } + /// Deletes the index specified with `index_id`. /// This is equivalent to running `rm -rf ` for a local index or /// `aws s3 rm --recursive ` for a remote Amazon S3 index. @@ -553,7 +589,9 @@ pub async fn validate_storage_uri( mod tests { use quickwit_common::uri::Uri; - use quickwit_config::{CLI_SOURCE_ID, INGEST_API_SOURCE_ID, INGEST_V2_SOURCE_ID, IndexConfig}; + use quickwit_config::{ + CLI_SOURCE_ID, INGEST_API_SOURCE_ID, INGEST_V2_SOURCE_ID, IndexConfig, RetentionPolicy, + }; use quickwit_metastore::{ MetastoreServiceExt, SplitMetadata, StageSplitsRequestExt, metastore_for_test, }; @@ -609,6 +647,68 @@ mod tests { assert!(index_metadata_0.index_uid != index_metadata_1.index_uid); } + #[tokio::test] + async fn test_index_metadata_opt() { + let metastore = metastore_for_test(); + let storage_resolver = StorageResolver::for_test(); + let mut index_service = IndexService::new(metastore.clone(), storage_resolver); + + let index_id = "test-index"; + let index_metadata_request = IndexMetadataRequest::for_index_id(index_id.to_string()); + let index_metadata = index_service + .index_metadata_opt(index_metadata_request) + .await + .unwrap(); + assert!(index_metadata.is_none()); + + let index_uri = "ram://indexes/test-index"; + let index_config = IndexConfig::for_test(index_id, index_uri); + let index_uid = index_service + .create_index(index_config.clone(), false) + .await + .unwrap() + .index_uid; + let index_metadata_request = IndexMetadataRequest::for_index_uid(index_uid.clone()); + let index_metadata = index_service + .index_metadata_opt(index_metadata_request) + .await + .unwrap() + .unwrap(); + assert_eq!(index_metadata.index_uid, index_uid); + } + + #[tokio::test] + async fn test_update_index() { + let metastore = metastore_for_test(); + let storage_resolver = StorageResolver::for_test(); + let mut index_service = IndexService::new(metastore.clone(), storage_resolver); + + let index_id = "test-index"; + let index_uri = "ram://indexes/test-index"; + let mut index_config = IndexConfig::for_test(index_id, index_uri); + let index_uid = index_service + .create_index(index_config.clone(), false) + .await + .unwrap() + .index_uid; + + let retention_policy = RetentionPolicy { + retention_period: "42 hours".to_string(), + evaluation_schedule: "hourly".to_string(), + }; + index_config.retention_policy_opt = Some(retention_policy.clone()); + + let updated_index_metadata = index_service + .update_index(index_uid, index_config) + .await + .unwrap(); + let updated_retention_policy = updated_index_metadata + .index_config + .retention_policy_opt + .unwrap(); + assert_eq!(updated_retention_policy, retention_policy); + } + #[tokio::test] async fn test_delete_index() { let mut metastore = metastore_for_test(); diff --git a/quickwit/quickwit-indexing/Cargo.toml b/quickwit/quickwit-indexing/Cargo.toml index 0880217576a..677b95e32aa 100644 --- a/quickwit/quickwit-indexing/Cargo.toml +++ b/quickwit/quickwit-indexing/Cargo.toml @@ -50,6 +50,8 @@ utoipa = { workspace = true } vrl = { workspace = true, optional = true } warp = { workspace = true, optional = true } +arrow = { workspace = true } + quickwit-actors = { workspace = true } quickwit-aws = { workspace = true } quickwit-cluster = { workspace = true } @@ -60,6 +62,7 @@ quickwit-doc-mapper = { workspace = true } quickwit-ingest = { workspace = true } quickwit-metastore = { workspace = true } quickwit-opentelemetry = { workspace = true } +quickwit-parquet-engine = { workspace = true } quickwit-proto = { workspace = true } quickwit-storage = { workspace = true } @@ -104,16 +107,19 @@ testsuite = [ "quickwit-storage/testsuite" ] vrl = ["dep:vrl", "quickwit-config/vrl"] +postgres = ["quickwit-metastore/postgres"] ci-test = [] [dev-dependencies] bytes = { workspace = true } criterion = { workspace = true, features = ["async_tokio"] } mockall = { workspace = true } +parquet = { workspace = true } proptest = { workspace = true } prost = { workspace = true } rand = { workspace = true } reqwest = { workspace = true } +sqlx = { workspace = true, features = ["runtime-tokio", "postgres"] } tempfile = { workspace = true } quickwit-actors = { workspace = true, features = ["testsuite"] } diff --git a/quickwit/quickwit-indexing/src/actors/doc_processor.rs b/quickwit/quickwit-indexing/src/actors/doc_processor.rs index 407c55ff526..20f9a78e409 100644 --- a/quickwit/quickwit-indexing/src/actors/doc_processor.rs +++ b/quickwit/quickwit-indexing/src/actors/doc_processor.rs @@ -239,7 +239,8 @@ impl Iterator for JsonDocIterator { } impl From> for JsonDocIterator -where E: Into +where + E: Into, { fn from(result: Result) -> Self { match result { @@ -276,7 +277,9 @@ pub struct DocProcessorCounter { impl Serialize for DocProcessorCounter { fn serialize(&self, serializer: S) -> Result - where S: serde::Serializer { + where + S: serde::Serializer, + { serializer.serialize_u64(self.get_num_docs()) } } diff --git a/quickwit/quickwit-indexing/src/actors/indexing_pipeline.rs b/quickwit/quickwit-indexing/src/actors/indexing_pipeline.rs index 8a745f9a19b..16daf18102f 100644 --- a/quickwit/quickwit-indexing/src/actors/indexing_pipeline.rs +++ b/quickwit/quickwit-indexing/src/actors/indexing_pipeline.rs @@ -15,6 +15,7 @@ use std::collections::BTreeSet; use std::path::PathBuf; use std::sync::Arc; +use std::sync::atomic::Ordering; use std::time::{Duration, Instant}; use async_trait::async_trait; @@ -22,9 +23,10 @@ use quickwit_actors::{ Actor, ActorContext, ActorExitStatus, ActorHandle, HEARTBEAT, Handler, Health, Mailbox, QueueCapacity, Supervisable, }; -use quickwit_common::KillSwitch; +use quickwit_common::metrics::OwnedGaugeGuard; use quickwit_common::pubsub::EventBroker; use quickwit_common::temp_dir::TempDirectory; +use quickwit_common::{KillSwitch, is_metrics_index}; use quickwit_config::{IndexingSettings, RetentionPolicy, SourceConfig}; use quickwit_doc_mapper::DocMapper; use quickwit_ingest::IngesterPool; @@ -42,11 +44,15 @@ use crate::actors::index_serializer::IndexSerializer; use crate::actors::publisher::PublisherType; use crate::actors::sequencer::Sequencer; use crate::actors::uploader::UploaderType; -use crate::actors::{Indexer, Packager, Publisher, Uploader}; +use crate::actors::{ + Indexer, Packager, ParquetDocProcessor, ParquetIndexer, ParquetPackager, ParquetPublisher, + ParquetUploader, Publisher, Uploader, +}; use crate::merge_policy::MergePolicy; use crate::models::IndexingStatistics; use crate::source::{ - AssignShards, Assignment, SourceActor, SourceRuntime, quickwit_supported_sources, + AssignShards, Assignment, SourceActor, SourceRuntime, quickwit_supported_parquet_sources, + quickwit_supported_sources, }; use crate::split_store::IndexingSplitStore; @@ -77,9 +83,10 @@ pub(crate) fn wait_duration_before_retry(retry_count: usize) -> Duration { /// See also . static SPAWN_PIPELINE_SEMAPHORE: Semaphore = Semaphore::const_new(10); +/// Handles for standard Tantivy-based indexing pipeline. struct IndexingPipelineHandles { - source_mailbox: Mailbox, - source_handle: ActorHandle, + source_mailbox: Mailbox>, + source_handle: ActorHandle>, doc_processor: ActorHandle, indexer: ActorHandle, index_serializer: ActorHandle, @@ -101,6 +108,42 @@ impl IndexingPipelineHandles { } } +/// Handles for metrics Parquet/DataFusion indexing pipeline. +struct ParquetIndexingPipelineHandles { + source_mailbox: Mailbox>, + source_handle: ActorHandle>, + doc_processor: ActorHandle, + indexer: ActorHandle, + packager: ActorHandle, + uploader: ActorHandle, + sequencer: ActorHandle>, + publisher: ActorHandle, + next_check_for_progress: Instant, +} + +impl ParquetIndexingPipelineHandles { + fn should_check_for_progress(&mut self) -> bool { + let now = Instant::now(); + let check_for_progress = now > self.next_check_for_progress; + if check_for_progress { + self.next_check_for_progress = now + *HEARTBEAT; + } + check_for_progress + } +} + +/// Enum representing either a standard or metrics pipeline's actor handles. +/// +/// This allows `IndexingPipeline` to properly manage both pipeline types +/// without resorting to `Option` workarounds where `None` implicitly means +/// "metrics pipeline". +enum PipelineHandles { + /// Standard Tantivy-based pipeline for logs/traces. + Standard(IndexingPipelineHandles), + /// Parquet/DataFusion pipeline for time-series data. + Parquet(ParquetIndexingPipelineHandles), +} + // Messages #[derive(Clone, Copy, Debug, Default)] @@ -112,7 +155,7 @@ pub struct IndexingPipeline { params: IndexingPipelineParams, previous_generations_statistics: IndexingStatistics, statistics: IndexingStatistics, - handles_opt: Option, + handles_opt: Option, // Killswitch used for the actors in the pipeline. This is not the supervisor killswitch. kill_switch: KillSwitch, @@ -120,6 +163,7 @@ pub struct IndexingPipeline { // requiring a respawn of the pipeline. // We keep the list of shards here however, to reassign them after a respawn. shard_ids: BTreeSet, + _indexing_pipelines_gauge_guard: OwnedGaugeGuard, } #[async_trait] @@ -154,6 +198,10 @@ impl Actor for IndexingPipeline { impl IndexingPipeline { pub fn new(params: IndexingPipelineParams) -> Self { + let indexing_pipelines_gauge = crate::metrics::INDEXER_METRICS + .indexing_pipelines + .with_label_values([¶ms.pipeline_id.index_uid.index_id]); + let indexing_pipelines_gauge_guard = OwnedGaugeGuard::from_gauge(indexing_pipelines_gauge); let params_fingerprint = params.params_fingerprint; IndexingPipeline { params, @@ -165,24 +213,36 @@ impl IndexingPipeline { ..Default::default() }, shard_ids: Default::default(), + _indexing_pipelines_gauge_guard: indexing_pipelines_gauge_guard, } } fn supervisables(&self) -> Vec<&dyn Supervisable> { - if let Some(handles) = &self.handles_opt { - let supervisables: Vec<&dyn Supervisable> = vec![ - &handles.source_handle, - &handles.doc_processor, - &handles.indexer, - &handles.index_serializer, - &handles.packager, - &handles.uploader, - &handles.sequencer, - &handles.publisher, - ]; - supervisables - } else { - Vec::new() + match &self.handles_opt { + Some(PipelineHandles::Standard(handles)) => { + vec![ + &handles.source_handle, + &handles.doc_processor, + &handles.indexer, + &handles.index_serializer, + &handles.packager, + &handles.uploader, + &handles.sequencer, + &handles.publisher, + ] + } + Some(PipelineHandles::Parquet(handles)) => { + vec![ + &handles.source_handle, + &handles.doc_processor, + &handles.indexer, + &handles.packager, + &handles.uploader, + &handles.sequencer, + &handles.publisher, + ] + } + None => Vec::new(), } } @@ -244,26 +304,60 @@ impl IndexingPipeline { } fn perform_observe(&mut self, ctx: &ActorContext) { - let Some(handles) = &self.handles_opt else { - return; - }; - handles.doc_processor.refresh_observe(); - handles.indexer.refresh_observe(); - handles.uploader.refresh_observe(); - handles.publisher.refresh_observe(); - self.statistics = self - .previous_generations_statistics - .clone() - .add_actor_counters( - &handles.doc_processor.last_observation(), - &handles.indexer.last_observation(), - &handles.uploader.last_observation(), - &handles.publisher.last_observation(), - ) - .set_generation(self.statistics.generation) - .set_num_spawn_attempts(self.statistics.num_spawn_attempts); - let pipeline_metrics_opt = handles.indexer.last_observation().pipeline_metrics_opt; - self.statistics.pipeline_metrics_opt = pipeline_metrics_opt; + // Update statistics based on pipeline type + match &self.handles_opt { + Some(PipelineHandles::Standard(handles)) => { + handles.doc_processor.refresh_observe(); + handles.indexer.refresh_observe(); + handles.uploader.refresh_observe(); + handles.publisher.refresh_observe(); + self.statistics = self + .previous_generations_statistics + .clone() + .add_actor_counters( + &handles.doc_processor.last_observation(), + &handles.indexer.last_observation(), + &handles.uploader.last_observation(), + &handles.publisher.last_observation(), + ) + .set_generation(self.statistics.generation) + .set_num_spawn_attempts(self.statistics.num_spawn_attempts); + let pipeline_metrics_opt = handles.indexer.last_observation().pipeline_metrics_opt; + self.statistics.pipeline_metrics_opt = pipeline_metrics_opt; + } + Some(PipelineHandles::Parquet(handles)) => { + handles.doc_processor.refresh_observe(); + handles.indexer.refresh_observe(); + handles.uploader.refresh_observe(); + handles.publisher.refresh_observe(); + + let doc_counters = handles.doc_processor.last_observation(); + let indexer_counters = handles.indexer.last_observation(); + let uploader_counters = handles.uploader.last_observation(); + let publisher_counters = handles.publisher.last_observation(); + + let mut stats = self.previous_generations_statistics.clone(); + stats.num_docs += doc_counters.valid_rows; + stats.num_invalid_docs += doc_counters.num_errors(); + stats.total_bytes_processed += doc_counters.bytes_total; + stats.num_local_splits += indexer_counters.batches_flushed; + stats.num_staged_splits += + uploader_counters.num_staged_splits.load(Ordering::Relaxed); + stats.num_uploaded_splits += uploader_counters + .num_uploaded_splits + .load(Ordering::Relaxed); + stats.num_published_splits += publisher_counters.num_published_splits; + stats.num_empty_splits += publisher_counters.num_empty_splits; + stats.generation = self.statistics.generation; + stats.num_spawn_attempts = self.statistics.num_spawn_attempts; + self.statistics = stats; + } + None => { + // Pipeline not yet spawned + } + } + // Always update params_fingerprint, shard_ids, and emit observation. + // This ensures shard assignments are reported to the control plane via chitchat. self.statistics.params_fingerprint = self.params.params_fingerprint; self.statistics.shard_ids.clone_from(&self.shard_ids); ctx.observe(self); @@ -274,14 +368,12 @@ impl IndexingPipeline { &mut self, ctx: &ActorContext, ) -> Result<(), ActorExitStatus> { - let Some(handles) = self.handles_opt.as_mut() else { - return Ok(()); + // Check if it's time to verify progress based on pipeline type + let check_for_progress = match &mut self.handles_opt { + Some(PipelineHandles::Standard(handles)) => handles.should_check_for_progress(), + Some(PipelineHandles::Parquet(handles)) => handles.should_check_for_progress(), + None => return Ok(()), }; - - // While we check if the actor has terminated or not, we do not check for progress - // at every single loop. Instead, we wait for the `HEARTBEAT` duration to have elapsed, - // since our last check. - let check_for_progress = handles.should_check_for_progress(); let health = self.healthcheck(check_for_progress); match health { Health::Healthy => {} @@ -316,6 +408,12 @@ impl IndexingPipeline { self.kill_switch = ctx.kill_switch().child(); let index_id = &self.params.pipeline_id.index_uid.index_id; + + // Route metrics indexes to the Parquet/DataFusion pipeline + if is_metrics_index(index_id) { + return self.spawn_parquet_pipeline(ctx).await; + } + let source_id = &self.params.pipeline_id.source_id; info!( @@ -327,7 +425,7 @@ impl IndexingPipeline { ); let (source_mailbox, source_inbox) = ctx .spawn_ctx() - .create_mailbox::("SourceActor", QueueCapacity::Unbounded); + .create_mailbox::>("SourceActor", QueueCapacity::Unbounded); // Publisher let publisher = Publisher::new( @@ -445,7 +543,7 @@ impl IndexingPipeline { .await?; let actor_source = SourceActor { source, - doc_processor_mailbox, + processor_mailbox: doc_processor_mailbox, }; let (source_mailbox, source_handle) = ctx .spawn_actor() @@ -460,7 +558,7 @@ impl IndexingPipeline { // Increment generation once we are sure there will be no spawning error. self.previous_generations_statistics = self.statistics.clone(); self.statistics.generation += 1; - self.handles_opt = Some(IndexingPipelineHandles { + self.handles_opt = Some(PipelineHandles::Standard(IndexingPipelineHandles { source_mailbox, source_handle, doc_processor: doc_processor_handle, @@ -471,20 +569,181 @@ impl IndexingPipeline { sequencer: sequencer_handle, publisher: publisher_handle, next_check_for_progress: Instant::now() + *HEARTBEAT, + })); + Ok(()) + } + + /// Spawn the parquet pipeline using ParquetDocProcessor and ParquetIndexer. + /// + /// This pipeline routes metrics data through the Parquet/DataFusion path instead of + /// the Tantivy path. The output is ParquetSplit files written to the indexing directory. + #[instrument( + name="spawn_parquet_pipeline", + level="info", + skip_all, + fields( + index=%self.params.pipeline_id.index_uid.index_id, + r#gen=self.generation() + ))] + async fn spawn_parquet_pipeline(&mut self, ctx: &ActorContext) -> anyhow::Result<()> { + let index_id = &self.params.pipeline_id.index_uid.index_id; + let source_id = &self.params.pipeline_id.source_id; + + info!( + index_id, + source_id, + pipeline_uid=%self.params.pipeline_id.pipeline_uid, + root_dir=%self.params.indexing_directory.path().display(), + "spawning parquet indexing pipeline for metrics", + ); + + let (source_mailbox, source_inbox) = ctx + .spawn_ctx() + .create_mailbox::>( + "SourceActor", + QueueCapacity::Unbounded, + ); + + // ParquetPublisher + let parquet_publisher = ParquetPublisher::new( + PublisherType::ParquetPublisher, + self.params.metastore.clone(), + None, + Some(source_mailbox.clone()), + ); + let (parquet_publisher_mailbox, parquet_publisher_handle) = ctx + .spawn_actor() + .set_kill_switch(self.kill_switch.clone()) + .spawn(parquet_publisher); + + // Sequencer for ordered delivery + let parquet_sequencer = Sequencer::new(parquet_publisher_mailbox); + let (parquet_sequencer_mailbox, parquet_sequencer_handle) = ctx + .spawn_actor() + .set_kill_switch(self.kill_switch.clone()) + .spawn(parquet_sequencer); + + // ParquetUploader + let parquet_uploader = ParquetUploader::new( + super::UploaderType::IndexUploader, + self.params.metastore.clone(), + self.params.storage.clone(), + SplitsUpdateMailbox::Sequencer(parquet_sequencer_mailbox), + self.params.max_concurrent_split_uploads_index, + ); + let (parquet_uploader_mailbox, parquet_uploader_handle) = ctx + .spawn_actor() + .set_kill_switch(self.kill_switch.clone()) + .spawn(parquet_uploader); + + // ParquetPackager + let parquet_schema = quickwit_parquet_engine::schema::ParquetSchema::new(); + let writer_config = quickwit_parquet_engine::storage::ParquetWriterConfig::default(); + let split_writer = quickwit_parquet_engine::storage::ParquetSplitWriter::new( + parquet_schema, + writer_config, + self.params.indexing_directory.path(), + ); + let parquet_packager = ParquetPackager::new(split_writer, parquet_uploader_mailbox); + let (parquet_packager_mailbox, parquet_packager_handle) = ctx + .spawn_actor() + .set_kill_switch(self.kill_switch.clone()) + .spawn(parquet_packager); + + // ParquetIndexer + let commit_timeout = + Duration::from_secs(self.params.indexing_settings.commit_timeout_secs as u64); + let parquet_indexer = ParquetIndexer::new( + self.params.pipeline_id.index_uid.clone(), + source_id.to_string(), + None, + parquet_packager_mailbox, + Some(commit_timeout), + ); + let (parquet_indexer_mailbox, parquet_indexer_handle) = ctx + .spawn_actor() + .set_kill_switch(self.kill_switch.clone()) + .spawn(parquet_indexer); + + // ParquetDocProcessor + let parquet_doc_processor = ParquetDocProcessor::new( + index_id.to_string(), + source_id.to_string(), + parquet_indexer_mailbox, + ); + let (parquet_doc_processor_mailbox, parquet_doc_processor_handle) = ctx + .spawn_actor() + .set_kill_switch(self.kill_switch.clone()) + .spawn(parquet_doc_processor); + + // Source (using parquet source loader) + let source_runtime = SourceRuntime { + pipeline_id: self.params.pipeline_id.clone(), + source_config: self.params.source_config.clone(), + metastore: self.params.metastore.clone(), + ingester_pool: self.params.ingester_pool.clone(), + queues_dir_path: self.params.queues_dir_path.clone(), + storage_resolver: self.params.source_storage_resolver.clone(), + event_broker: self.params.event_broker.clone(), + indexing_setting: self.params.indexing_settings.clone(), + }; + let source = ctx + .protect_future(quickwit_supported_parquet_sources().load_source(source_runtime)) + .await?; + let actor_source = SourceActor { + source, + processor_mailbox: parquet_doc_processor_mailbox, + }; + let (source_mailbox, source_handle) = ctx + .spawn_actor() + .set_mailboxes(source_mailbox, source_inbox) + .set_kill_switch(self.kill_switch.clone()) + .spawn(actor_source); + let assign_shards_message = AssignShards(Assignment { + shard_ids: self.shard_ids.clone(), }); + source_mailbox.send_message(assign_shards_message).await?; + + // Increment generation once we are sure there will be no spawning error. + self.previous_generations_statistics = self.statistics.clone(); + self.statistics.generation += 1; + self.handles_opt = Some(PipelineHandles::Parquet(ParquetIndexingPipelineHandles { + source_mailbox, + source_handle, + doc_processor: parquet_doc_processor_handle, + indexer: parquet_indexer_handle, + packager: parquet_packager_handle, + uploader: parquet_uploader_handle, + sequencer: parquet_sequencer_handle, + publisher: parquet_publisher_handle, + next_check_for_progress: Instant::now() + *HEARTBEAT, + })); Ok(()) } async fn terminate(&mut self) { self.kill_switch.kill(); if let Some(handles) = self.handles_opt.take() { - tokio::join!( - handles.source_handle.kill(), - handles.indexer.kill(), - handles.packager.kill(), - handles.uploader.kill(), - handles.publisher.kill(), - ); + match handles { + PipelineHandles::Standard(handles) => { + tokio::join!( + handles.source_handle.kill(), + handles.indexer.kill(), + handles.packager.kill(), + handles.uploader.kill(), + handles.publisher.kill(), + ); + } + PipelineHandles::Parquet(handles) => { + tokio::join!( + handles.source_handle.kill(), + handles.indexer.kill(), + handles.packager.kill(), + handles.uploader.kill(), + handles.publisher.kill(), + ); + } + } } } } @@ -550,15 +809,28 @@ impl Handler for IndexingPipeline { .clone_from(&assign_shards_message.0.shard_ids); // If the pipeline is running, we forward the message to its source. // If it is not, it will be respawned soon, and the shards will be assigned afterward. - if let Some(handles) = &mut self.handles_opt { - info!( - shard_ids=?assign_shards_message.0.shard_ids, - "assigning shards to indexing pipeline" - ); - handles - .source_mailbox - .send_message(assign_shards_message) - .await?; + match &mut self.handles_opt { + Some(PipelineHandles::Standard(handles)) => { + info!( + shard_ids=?assign_shards_message.0.shard_ids, + "assigning shards to indexing pipeline" + ); + handles + .source_mailbox + .send_message(assign_shards_message) + .await?; + } + Some(PipelineHandles::Parquet(handles)) => { + info!( + shard_ids=?assign_shards_message.0.shard_ids, + "assigning shards to parquet indexing pipeline" + ); + handles + .source_mailbox + .send_message(assign_shards_message) + .await?; + } + None => {} } // We perform observe to make sure the set of shard ids is up to date. self.perform_observe(ctx); @@ -726,8 +998,18 @@ mod tests { let pipeline = IndexingPipeline::new(pipeline_params); let (_pipeline_mailbox, pipeline_handle) = universe.spawn_builder().spawn(pipeline); let (pipeline_exit_status, pipeline_statistics) = pipeline_handle.join().await; - assert_eq!(pipeline_statistics.generation, 1); - assert_eq!(pipeline_statistics.num_spawn_attempts, 1 + num_fails); + assert_eq!( + pipeline_statistics.generation, 1, + "generation is {}, expected 1", + pipeline_statistics.generation + ); + assert_eq!( + pipeline_statistics.num_spawn_attempts, + 1 + num_fails, + "num spawn attempts is {}, expected 1 + {}", + pipeline_statistics.num_spawn_attempts, + 1 + num_fails + ); assert!(pipeline_exit_status.is_success()); Ok(()) } diff --git a/quickwit/quickwit-indexing/src/actors/indexing_service.rs b/quickwit/quickwit-indexing/src/actors/indexing_service.rs index 5ae991d8451..5d307fb200b 100644 --- a/quickwit/quickwit-indexing/src/actors/indexing_service.rs +++ b/quickwit/quickwit-indexing/src/actors/indexing_service.rs @@ -778,13 +778,11 @@ impl IndexingService { // If at least one ingest source has been removed, the related index has possibly been // deleted. Thus we run a garbage collect to remove queues of potentially deleted // indexes. - if should_gc_ingest_api_queues { - if let Err(error) = self.run_ingest_api_queues_gc().await { - warn!( - %error, - "failed to garbage collect ingest API queues", - ); - } + if should_gc_ingest_api_queues && let Err(error) = self.run_ingest_api_queues_gc().await { + warn!( + %error, + "failed to garbage collect ingest API queues", + ); } } @@ -1194,7 +1192,7 @@ mod tests { }; let create_index_request = CreateIndexRequest::try_from_index_and_source_configs( &index_config, - &[source_config.clone()], + std::slice::from_ref(&source_config), ) .unwrap(); metastore.create_index(create_index_request).await.unwrap(); diff --git a/quickwit/quickwit-indexing/src/actors/merge_executor.rs b/quickwit/quickwit-indexing/src/actors/merge_executor.rs index fa45192782c..eb4b9728600 100644 --- a/quickwit/quickwit-indexing/src/actors/merge_executor.rs +++ b/quickwit/quickwit-indexing/src/actors/merge_executor.rs @@ -534,7 +534,7 @@ impl MergeExecutor { parsed_query_ast ); let (query, _) = - doc_mapper.query(union_index.schema(), &parsed_query_ast, false)?; + doc_mapper.query(union_index.schema(), parsed_query_ast, false, None)?; index_writer.delete_query(query)?; } debug!("commit-delete-operations"); @@ -605,10 +605,11 @@ pub async fn merge_split_directories_standalone( let delete_query = delete_task .delete_query .expect("A delete task must have a delete query."); - let query_ast: QueryAst = serde_json::from_str(&delete_query.query_ast) - .context("invalid query_ast json")?; + let query_ast: QueryAst = + serde_json::from_str(&delete_query.query_ast).context("invalid query_ast json")?; let parsed_query_ast = query_ast.parse_user_query(&[]).context("invalid query")?; - let (query, _) = doc_mapper.query(union_index.schema(), &parsed_query_ast, false)?; + let (query, _) = + doc_mapper.query(union_index.schema(), parsed_query_ast, false, None)?; index_writer.delete_query(query)?; } debug!("commit-delete-operations"); @@ -907,7 +908,8 @@ mod tests { let documents_left = searcher .search( &tantivy::query::AllQuery, - &tantivy::collector::TopDocs::with_limit(result_docs.len() + 1), + &tantivy::collector::TopDocs::with_limit(result_docs.len() + 1) + .order_by_score(), )? .into_iter() .map(|(_, doc_address)| { diff --git a/quickwit/quickwit-indexing/src/actors/merge_planner.rs b/quickwit/quickwit-indexing/src/actors/merge_planner.rs index fbfb528be01..479cefcaa23 100644 --- a/quickwit/quickwit-indexing/src/actors/merge_planner.rs +++ b/quickwit/quickwit-indexing/src/actors/merge_planner.rs @@ -21,7 +21,6 @@ use quickwit_actors::{Actor, ActorContext, ActorExitStatus, Handler, Mailbox, Qu use quickwit_metastore::{SplitMaturity, SplitMetadata}; use quickwit_proto::indexing::MergePipelineId; use quickwit_proto::types::DocMappingUid; -use serde::Serialize; use tantivy::Inventory; use time::OffsetDateTime; use tracing::{info, warn}; @@ -248,7 +247,10 @@ impl MergePlanner { // No need to rebuild every time, we do once out of 100 times. fn recompute_known_splits_if_necessary(&mut self) { self.known_split_ids_recompute_attempt_id += 1; - if self.known_split_ids_recompute_attempt_id % 100 == 0 { + if self + .known_split_ids_recompute_attempt_id + .is_multiple_of(100) + { self.known_split_ids = self.rebuild_known_split_ids(); self.known_split_ids_recompute_attempt_id = 0; } @@ -357,11 +359,6 @@ struct PlanMerge { incarnation_started_at: Instant, } -#[derive(Clone, Debug, Serialize)] -pub struct MergePlannerState { - pub(crate) ongoing_merge_operations: Vec, -} - #[cfg(test)] mod tests { use std::sync::Arc; diff --git a/quickwit/quickwit-indexing/src/actors/mod.rs b/quickwit/quickwit-indexing/src/actors/mod.rs index 9b4ed7c7883..9c7b814d48c 100644 --- a/quickwit/quickwit-indexing/src/actors/mod.rs +++ b/quickwit/quickwit-indexing/src/actors/mod.rs @@ -24,12 +24,24 @@ mod merge_planner; mod merge_scheduler_service; mod merge_split_downloader; mod packager; +mod parquet_doc_processor; +mod parquet_indexer; +mod parquet_packager; +mod parquet_uploader; mod publisher; mod sequencer; mod uploader; #[cfg(feature = "vrl")] mod vrl_processing; +#[cfg(test)] +#[allow( + clippy::disallowed_methods, + clippy::needless_borrow, + clippy::unnecessary_map_or +)] +mod parquet_e2e_test; + pub use doc_processor::{DocProcessor, DocProcessorCounters}; pub use index_serializer::IndexSerializer; pub use indexer::{Indexer, IndexerCounters}; @@ -41,7 +53,27 @@ pub(crate) use merge_planner::{MergePlanner, RunFinalizeMergePolicyAndQuit}; pub use merge_scheduler_service::{MergePermit, MergeSchedulerService, schedule_merge}; pub use merge_split_downloader::MergeSplitDownloader; pub use packager::Packager; -pub use publisher::{Publisher, PublisherCounters, PublisherType}; +pub use parquet_doc_processor::{ + ParquetDocProcessor, ParquetDocProcessorCounters, ParquetDocProcessorError, is_arrow_ipc, +}; +pub use parquet_indexer::{ParquetIndexer, ParquetIndexerCounters, ParquetSplitBatch}; +pub use parquet_packager::{ParquetBatchForPackager, ParquetPackager, ParquetPackagerCounters}; +pub use parquet_uploader::ParquetUploader; +pub use publisher::{ParquetPublisher, Publisher, PublisherCounters, PublisherType}; +use quickwit_actors::{Actor, Handler}; pub use quickwit_proto::indexing::IndexingError; pub use sequencer::Sequencer; pub use uploader::{SplitsUpdateMailbox, Uploader, UploaderCounters, UploaderType}; + +use crate::models::{NewPublishLock, NewPublishToken, RawDocBatch}; + +/// Trait alias for actor types that can process document batches. +pub trait Processor: + Actor + Handler + Handler + Handler +{ +} + +impl Processor for T where + T: Actor + Handler + Handler + Handler +{ +} diff --git a/quickwit/quickwit-indexing/src/actors/packager.rs b/quickwit/quickwit-indexing/src/actors/packager.rs index 340979ba0e9..9dbb1fd8963 100644 --- a/quickwit/quickwit-indexing/src/actors/packager.rs +++ b/quickwit/quickwit-indexing/src/actors/packager.rs @@ -360,8 +360,8 @@ fn field_metadata_to_list_field_serialized( ListFieldsEntryResponse { field_name: field_metadata.field_name.to_string(), field_type: tantivy_type_to_list_field_type(field_metadata.typ) as i32, - searchable: field_metadata.indexed, - aggregatable: field_metadata.fast, + searchable: field_metadata.is_indexed(), + aggregatable: field_metadata.is_fast(), index_ids: Vec::new(), non_searchable_index_ids: Vec::new(), non_aggregatable_index_ids: Vec::new(), @@ -398,23 +398,29 @@ mod tests { FieldMetadata { field_name: "test".to_string(), typ: Type::Str, - indexed: true, stored: true, - fast: true, + fast_size: Some(10u64.into()), + term_dictionary_size: Some(10u64.into()), + postings_size: Some(10u64.into()), + positions_size: Some(10u64.into()), }, FieldMetadata { field_name: "test2".to_string(), typ: Type::Str, - indexed: true, stored: false, - fast: false, + fast_size: None, + term_dictionary_size: Some(10u64.into()), + postings_size: Some(10u64.into()), + positions_size: Some(10u64.into()), }, FieldMetadata { field_name: "test3".to_string(), typ: Type::U64, - indexed: true, stored: false, - fast: true, + fast_size: Some(10u64.into()), + term_dictionary_size: Some(10u64.into()), + postings_size: Some(10u64.into()), + positions_size: Some(10u64.into()), }, ]; diff --git a/quickwit/quickwit-indexing/src/actors/parquet_doc_processor.rs b/quickwit/quickwit-indexing/src/actors/parquet_doc_processor.rs new file mode 100644 index 00000000000..2a83c9610c9 --- /dev/null +++ b/quickwit/quickwit-indexing/src/actors/parquet_doc_processor.rs @@ -0,0 +1,819 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! ParquetDocProcessor actor for routing Arrow IPC batches to the metrics engine. +//! +//! This actor processes RawDocBatch messages containing Arrow IPC data and routes +//! them directly to the metrics engine, bypassing Tantivy document conversion. + +use arrow::record_batch::RecordBatch; +use async_trait::async_trait; +use quickwit_actors::{Actor, ActorContext, ActorExitStatus, Handler, Mailbox, QueueCapacity}; +use quickwit_common::rate_limited_tracing::rate_limited_warn; +use quickwit_common::runtimes::RuntimeType; +use quickwit_metastore::checkpoint::SourceCheckpointDelta; +use quickwit_parquet_engine::ingest::{IngestError, ParquetIngestProcessor}; +use quickwit_parquet_engine::schema::ParquetSchema; +use quickwit_proto::types::{IndexId, SourceId}; +use serde::Serialize; +use tokio::runtime::Handle; +use tracing::{debug, info, instrument}; + +use super::ParquetIndexer; +use crate::models::{ + NewPublishLock, NewPublishToken, ProcessedParquetBatch, PublishLock, RawDocBatch, +}; + +/// Arrow IPC stream continuation marker (4 bytes of 0xFF). +const ARROW_IPC_CONTINUATION_MARKER: [u8; 4] = [0xFF, 0xFF, 0xFF, 0xFF]; + +/// Check if the bytes look like Arrow IPC stream format. +pub fn is_arrow_ipc(bytes: &[u8]) -> bool { + bytes.len() >= 4 && bytes[0..4] == ARROW_IPC_CONTINUATION_MARKER +} + +/// Counters for ParquetDocProcessor. +#[derive(Debug, Serialize, Clone)] +pub struct ParquetDocProcessorCounters { + /// Index identifier. + pub index_id: IndexId, + /// Source identifier. + pub source_id: SourceId, + /// Number of valid batches processed. + pub valid_batches: u64, + /// Number of valid rows processed. + pub valid_rows: u64, + /// Number of batches that failed to parse. + pub parse_errors: u64, + /// Number of batches that were not Arrow IPC format. + pub format_errors: u64, + /// Total bytes processed. + pub bytes_total: u64, +} + +impl ParquetDocProcessorCounters { + /// Create new counters for the given index and source. + pub fn new(index_id: IndexId, source_id: SourceId) -> Self { + Self { + index_id, + source_id, + valid_batches: 0u64, + valid_rows: 0u64, + parse_errors: 0u64, + format_errors: 0u64, + bytes_total: 0u64, + } + } + + /// Record a successfully processed batch. + pub fn record_valid(&mut self, num_rows: usize, num_bytes: usize) { + self.valid_batches += 1; + self.valid_rows += num_rows as u64; + self.bytes_total += num_bytes as u64; + } + + /// Record a parse error. + pub fn record_parse_error(&mut self, num_bytes: usize) { + self.parse_errors += 1; + self.bytes_total += num_bytes as u64; + } + + /// Record a format error (not Arrow IPC). + pub fn record_format_error(&mut self, num_bytes: usize) { + self.format_errors += 1; + self.bytes_total += num_bytes as u64; + } + + /// Get total number of batches processed (valid or not). + pub fn num_processed_batches(&self) -> u64 { + self.valid_batches + self.parse_errors + self.format_errors + } + + /// Get total number of errors. + pub fn num_errors(&self) -> u64 { + self.parse_errors + self.format_errors + } +} + +/// Error type for ParquetDocProcessor. +#[derive(Debug, thiserror::Error)] +pub enum ParquetDocProcessorError { + /// Input was not Arrow IPC format. + #[error("Invalid format: expected Arrow IPC")] + InvalidFormat, + /// Arrow IPC parsing or schema validation failed. + #[error("Ingest error: {0}")] + Ingest(#[from] IngestError), +} + +/// ParquetDocProcessor actor that routes Arrow IPC batches to the metrics engine. +/// +/// This actor receives RawDocBatch messages containing Arrow IPC data and converts +/// them to RecordBatch using ParquetIngestProcessor. The resulting batches are +/// forwarded to ParquetIndexer for accumulation and split production. +/// +/// Unlike DocProcessor which converts to Tantivy documents, this actor works +/// exclusively with Arrow RecordBatch for high-throughput metrics ingestion. +pub struct ParquetDocProcessor { + /// Processor for converting Arrow IPC to RecordBatch. + processor: ParquetIngestProcessor, + /// Processing counters. + counters: ParquetDocProcessorCounters, + /// Publish lock for coordinating with sources. + publish_lock: PublishLock, + /// Mailbox for forwarding batches to ParquetIndexer. + indexer_mailbox: Mailbox, +} + +impl ParquetDocProcessor { + /// Creates a new ParquetDocProcessor. + pub fn new( + index_id: IndexId, + source_id: SourceId, + indexer_mailbox: Mailbox, + ) -> Self { + let schema = ParquetSchema::new(); + let processor = ParquetIngestProcessor::new(schema); + let counters = ParquetDocProcessorCounters::new(index_id.clone(), source_id.clone()); + + info!( + index_id = %index_id, + source_id = %source_id, + "ParquetDocProcessor created" + ); + + Self { + processor, + counters, + publish_lock: PublishLock::default(), + indexer_mailbox, + } + } +} + +#[async_trait] +impl Actor for ParquetDocProcessor { + type ObservableState = ParquetDocProcessorCounters; + + fn observable_state(&self) -> Self::ObservableState { + self.counters.clone() + } + + fn queue_capacity(&self) -> QueueCapacity { + QueueCapacity::Bounded(10) + } + + fn runtime_handle(&self) -> Handle { + RuntimeType::Blocking.get_runtime_handle() + } + + #[inline] + fn yield_after_each_message(&self) -> bool { + false + } + + async fn finalize( + &mut self, + exit_status: &ActorExitStatus, + ctx: &ActorContext, + ) -> anyhow::Result<()> { + match exit_status { + ActorExitStatus::DownstreamClosed + | ActorExitStatus::Killed + | ActorExitStatus::Failure(_) + | ActorExitStatus::Panicked => return Ok(()), + ActorExitStatus::Quit | ActorExitStatus::Success => { + let _ = ctx.send_exit_with_success(&self.indexer_mailbox).await; + } + } + Ok(()) + } +} + +#[async_trait] +impl Handler for ParquetDocProcessor { + type Reply = (); + + #[instrument( + skip(self, raw_doc_batch, ctx), + fields( + index_id = %self.counters.index_id, + source_id = %self.counters.source_id, + batch_len = raw_doc_batch.docs.len(), + ) + )] + async fn handle( + &mut self, + raw_doc_batch: RawDocBatch, + ctx: &ActorContext, + ) -> Result<(), ActorExitStatus> { + if self.publish_lock.is_dead() { + debug!("publish lock is dead, skipping batch"); + return Ok(()); + } + + let force_commit = raw_doc_batch.force_commit; + let mut checkpoint_delta = raw_doc_batch.checkpoint_delta; + let num_docs = raw_doc_batch.docs.len(); + let mut doc_index = 0; + + debug!( + num_docs = num_docs, + force_commit = force_commit, + "processing raw doc batch" + ); + + // Track whether the checkpoint delta has been forwarded to the indexer. + // If all docs fail, we must still forward the checkpoint so the pipeline + // can advance past this batch. + let mut checkpoint_forwarded = false; + + // Process each raw document in the batch (expecting Arrow IPC format from OTLP gRPC) + for raw_doc in &raw_doc_batch.docs { + let _protected_zone_guard = ctx.protect_zone(); + let num_bytes = raw_doc.len(); + doc_index += 1; + let is_last_doc = doc_index == num_docs; + + // Verify Arrow IPC format + if !is_arrow_ipc(raw_doc) { + rate_limited_warn!( + limit_per_min = 10, + index_id = %self.counters.index_id, + source_id = %self.counters.source_id, + "metrics pipeline only accepts Arrow IPC format (from OTLP gRPC)" + ); + self.counters.record_format_error(num_bytes); + continue; + } + + // Convert Arrow IPC to RecordBatch + match self.processor.process_ipc(raw_doc) { + Ok(batch) => { + let num_rows = batch.num_rows(); + self.counters.record_valid(num_rows, num_bytes); + + { + let should_force_commit = force_commit && is_last_doc; + // Pass checkpoint_delta only on the last doc to avoid double-counting + let batch_checkpoint = if is_last_doc { + std::mem::take(&mut checkpoint_delta) + } else { + SourceCheckpointDelta::default() + }; + + let processed_batch = ProcessedParquetBatch::new( + batch, + batch_checkpoint, + should_force_commit, + ); + + ctx.send_message(&self.indexer_mailbox, processed_batch) + .await?; + if is_last_doc { + checkpoint_forwarded = true; + } + } + } + Err(error) => { + rate_limited_warn!( + limit_per_min = 10, + index_id = %self.counters.index_id, + source_id = %self.counters.source_id, + "Arrow IPC processing failed: {error}" + ); + self.counters.record_parse_error(num_bytes); + } + } + + ctx.record_progress(); + } + + // If no successful doc forwarded the checkpoint (all docs failed), we must + // still send the checkpoint delta so the pipeline advances past this batch. + // Without this, a batch of consistently malformed data blocks offset progress + // forever. + if !checkpoint_forwarded && !checkpoint_delta.is_empty() { + let empty_batch = + RecordBatch::new_empty(self.processor.schema().arrow_schema().clone()); + let processed_batch = + ProcessedParquetBatch::new(empty_batch, checkpoint_delta, force_commit); + ctx.send_message(&self.indexer_mailbox, processed_batch) + .await?; + } + + Ok(()) + } +} + +#[async_trait] +impl Handler for ParquetDocProcessor { + type Reply = (); + + async fn handle( + &mut self, + message: NewPublishLock, + ctx: &ActorContext, + ) -> Result<(), ActorExitStatus> { + let NewPublishLock(publish_lock) = &message; + self.publish_lock = publish_lock.clone(); + + ctx.send_message(&self.indexer_mailbox, message).await?; + Ok(()) + } +} + +#[async_trait] +impl Handler for ParquetDocProcessor { + type Reply = (); + + async fn handle( + &mut self, + message: NewPublishToken, + ctx: &ActorContext, + ) -> Result<(), ActorExitStatus> { + ctx.send_message(&self.indexer_mailbox, message).await?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::Ordering; + + use quickwit_actors::Universe; + use quickwit_parquet_engine::ingest::record_batch_to_ipc; + use quickwit_proto::types::IndexUid; + + use super::*; + + #[test] + fn test_is_arrow_ipc() { + // Valid Arrow IPC marker + let valid = [0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00]; + assert!(is_arrow_ipc(&valid)); + + // Invalid - too short + let short = [0xFF, 0xFF, 0xFF]; + assert!(!is_arrow_ipc(&short)); + + // Invalid - wrong marker + let wrong = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]; + assert!(!is_arrow_ipc(&wrong)); + + // Invalid - JSON + let json = b"{\"metric\":\"test\"}"; + assert!(!is_arrow_ipc(json)); + } + + #[test] + fn test_counters() { + let mut counters = + ParquetDocProcessorCounters::new("test-index".to_string(), "test-source".to_string()); + + counters.record_valid(100, 1024); + counters.record_valid(50, 512); + counters.record_parse_error(256); + counters.record_format_error(128); + + assert_eq!(counters.valid_batches, 2); + assert_eq!(counters.valid_rows, 150); + assert_eq!(counters.parse_errors, 1); + assert_eq!(counters.format_errors, 1); + assert_eq!(counters.bytes_total, 1920); + assert_eq!(counters.num_processed_batches(), 4); + assert_eq!(counters.num_errors(), 2); + } + + #[tokio::test] + async fn test_metrics_doc_processor_valid_arrow_ipc() { + use std::sync::Arc as StdArc; + + use arrow::array::{ + ArrayRef, BinaryViewArray, DictionaryArray, Float64Array, Int32Array, StringArray, + StructArray, UInt8Array, UInt64Array, + }; + use arrow::datatypes::{DataType, Field, Int32Type}; + use arrow::record_batch::RecordBatch; + let universe = Universe::with_accelerated_time(); + + let (indexer_mailbox, _indexer_inbox) = universe.create_test_mailbox::(); + let metrics_doc_processor = ParquetDocProcessor::new( + "test-metrics-index".to_string(), + "test-source".to_string(), + indexer_mailbox, + ); + + let (metrics_doc_processor_mailbox, metrics_doc_processor_handle) = + universe.spawn_builder().spawn(metrics_doc_processor); + + // Create a test batch matching the metrics schema + let schema = ParquetSchema::new(); + let num_rows = 3; + + // Helper to create dictionary arrays + fn create_dict_array(values: &[&str]) -> ArrayRef { + let keys: Vec = (0..values.len()).map(|i| i as i32).collect(); + let string_array = StringArray::from(values.to_vec()); + StdArc::new( + DictionaryArray::::try_new( + Int32Array::from(keys), + StdArc::new(string_array), + ) + .unwrap(), + ) + } + + fn create_nullable_dict_array(values: &[Option<&str>]) -> ArrayRef { + let keys: Vec> = values + .iter() + .enumerate() + .map(|(i, v)| v.map(|_| i as i32)) + .collect(); + let string_values: Vec<&str> = values.iter().filter_map(|v| *v).collect(); + let string_array = StringArray::from(string_values); + StdArc::new( + DictionaryArray::::try_new( + Int32Array::from(keys), + StdArc::new(string_array), + ) + .unwrap(), + ) + } + + let metric_name: ArrayRef = create_dict_array(&vec!["cpu.usage"; num_rows]); + let metric_type: ArrayRef = StdArc::new(UInt8Array::from(vec![0u8; num_rows])); + let metric_unit: ArrayRef = StdArc::new(StringArray::from(vec![Some("bytes"); num_rows])); + let timestamp_secs: ArrayRef = StdArc::new(UInt64Array::from(vec![100u64, 101u64, 102u64])); + let start_timestamp_secs: ArrayRef = + StdArc::new(UInt64Array::from(vec![None::; num_rows])); + let value: ArrayRef = StdArc::new(Float64Array::from(vec![42.0, 43.0, 44.0])); + let tag_service: ArrayRef = create_nullable_dict_array(&vec![Some("web"); num_rows]); + let tag_env: ArrayRef = create_nullable_dict_array(&vec![Some("prod"); num_rows]); + let tag_datacenter: ArrayRef = + create_nullable_dict_array(&vec![Some("us-east-1"); num_rows]); + let tag_region: ArrayRef = create_nullable_dict_array(&vec![None; num_rows]); + let tag_host: ArrayRef = create_nullable_dict_array(&vec![Some("host-001"); num_rows]); + + // Create empty Variant (Struct with metadata and value BinaryView fields) + let metadata_array = StdArc::new(BinaryViewArray::from(vec![b"" as &[u8]; num_rows])); + let value_array = StdArc::new(BinaryViewArray::from(vec![b"" as &[u8]; num_rows])); + let attributes: ArrayRef = StdArc::new(StructArray::from(vec![ + ( + StdArc::new(Field::new("metadata", DataType::BinaryView, false)), + metadata_array.clone() as ArrayRef, + ), + ( + StdArc::new(Field::new("value", DataType::BinaryView, false)), + value_array.clone() as ArrayRef, + ), + ])); + + let service_name: ArrayRef = create_dict_array(&vec!["my-service"; num_rows]); + + let resource_attributes: ArrayRef = StdArc::new(StructArray::from(vec![ + ( + StdArc::new(Field::new("metadata", DataType::BinaryView, false)), + metadata_array as ArrayRef, + ), + ( + StdArc::new(Field::new("value", DataType::BinaryView, false)), + value_array as ArrayRef, + ), + ])); + + let batch = RecordBatch::try_new( + schema.arrow_schema().clone(), + vec![ + metric_name, + metric_type, + metric_unit, + timestamp_secs, + start_timestamp_secs, + value, + tag_service, + tag_env, + tag_datacenter, + tag_region, + tag_host, + attributes, + service_name, + resource_attributes, + ], + ) + .unwrap(); + + // Serialize to Arrow IPC + let ipc_bytes = record_batch_to_ipc(&batch).unwrap(); + + // Create RawDocBatch with the IPC bytes + let raw_doc_batch = RawDocBatch::for_test(&[&ipc_bytes], 0..1); + + // Send to processor + metrics_doc_processor_mailbox + .send_message(raw_doc_batch) + .await + .unwrap(); + + // Process and observe + let counters = metrics_doc_processor_handle + .process_pending_and_observe() + .await + .state; + + // Verify counters + assert_eq!(counters.valid_batches, 1); + assert_eq!(counters.valid_rows, 3); + assert_eq!(counters.parse_errors, 0); + assert_eq!(counters.format_errors, 0); + + universe.assert_quit().await; + } + + #[tokio::test] + async fn test_metrics_doc_processor_invalid_format() { + let universe = Universe::with_accelerated_time(); + + let (indexer_mailbox, _indexer_inbox) = universe.create_test_mailbox::(); + let metrics_doc_processor = ParquetDocProcessor::new( + "test-metrics-index".to_string(), + "test-source".to_string(), + indexer_mailbox, + ); + + let (metrics_doc_processor_mailbox, metrics_doc_processor_handle) = + universe.spawn_builder().spawn(metrics_doc_processor); + + // Send JSON data (not Arrow IPC) - should result in format error + let json_data = br#"{"metric":"test","value":42}"#; + let raw_doc_batch = RawDocBatch::for_test(&[json_data], 0..1); + + metrics_doc_processor_mailbox + .send_message(raw_doc_batch) + .await + .unwrap(); + + let counters = metrics_doc_processor_handle + .process_pending_and_observe() + .await + .state; + + // JSON is rejected as format error (only Arrow IPC is accepted) + assert_eq!(counters.valid_batches, 0); + assert_eq!(counters.format_errors, 1); + + universe.assert_quit().await; + } + + #[tokio::test] + async fn test_metrics_doc_processor_respects_publish_lock() { + let universe = Universe::with_accelerated_time(); + + let (indexer_mailbox, _indexer_inbox) = universe.create_test_mailbox::(); + let metrics_doc_processor = ParquetDocProcessor::new( + "test-metrics-index".to_string(), + "test-source".to_string(), + indexer_mailbox, + ); + + let (metrics_doc_processor_mailbox, metrics_doc_processor_handle) = + universe.spawn_builder().spawn(metrics_doc_processor); + + // Set up and kill publish lock + let publish_lock = PublishLock::default(); + metrics_doc_processor_mailbox + .send_message(NewPublishLock(publish_lock.clone())) + .await + .unwrap(); + metrics_doc_processor_handle + .process_pending_and_observe() + .await; + publish_lock.kill().await; + + // Send data after lock is dead + let raw_doc_batch = RawDocBatch::for_test(&[b"some data"], 0..1); + metrics_doc_processor_mailbox + .send_message(raw_doc_batch) + .await + .unwrap(); + + let counters = metrics_doc_processor_handle + .process_pending_and_observe() + .await + .state; + + // Should not process anything when publish lock is dead + assert_eq!(counters.num_processed_batches(), 0); + + universe.assert_quit().await; + } + + #[tokio::test] + async fn test_metrics_doc_processor_with_indexer() { + use std::sync::Arc as StdArc; + + use arrow::array::{ + ArrayRef, BinaryViewArray, DictionaryArray, Float64Array, Int32Array, StringArray, + StructArray, UInt8Array, UInt64Array, + }; + use arrow::datatypes::{DataType, Field, Int32Type}; + use arrow::record_batch::RecordBatch; + use quickwit_parquet_engine::storage::{ParquetSplitWriter, ParquetWriterConfig}; + use quickwit_proto::metastore::MockMetastoreService; + use quickwit_storage::RamStorage; + + use super::super::{ + ParquetIndexer, ParquetPackager, ParquetPublisher, ParquetUploader, SplitsUpdateMailbox, + }; + use crate::actors::UploaderType; + + let universe = Universe::with_accelerated_time(); + let temp_dir = tempfile::tempdir().unwrap(); + + // Create ParquetUploader + let mock_metastore = MockMetastoreService::new(); + let ram_storage = StdArc::new(RamStorage::default()); + let (publisher_mailbox, _publisher_inbox) = + universe.create_test_mailbox::(); + let uploader = ParquetUploader::new( + UploaderType::IndexUploader, + quickwit_proto::metastore::MetastoreServiceClient::from_mock(mock_metastore), + ram_storage, + SplitsUpdateMailbox::Publisher(publisher_mailbox), + 4, + ); + let (uploader_mailbox, _uploader_handle) = universe.spawn_builder().spawn(uploader); + + // Create ParquetPackager + let parquet_schema = ParquetSchema::new(); + let writer_config = ParquetWriterConfig::default(); + let split_writer = ParquetSplitWriter::new(parquet_schema, writer_config, temp_dir.path()); + let packager = ParquetPackager::new(split_writer, uploader_mailbox); + let (packager_mailbox, packager_handle) = universe.spawn_builder().spawn(packager); + + // Create ParquetIndexer + let indexer = ParquetIndexer::new( + IndexUid::for_test("test-index", 0), + "test-source".to_string(), + None, + packager_mailbox, + None, + ); + let (indexer_mailbox, indexer_handle) = universe.spawn_builder().spawn(indexer); + + let metrics_doc_processor = ParquetDocProcessor::new( + "test-index".to_string(), + "test-source".to_string(), + indexer_mailbox, + ); + let (metrics_doc_processor_mailbox, metrics_doc_processor_handle) = + universe.spawn_builder().spawn(metrics_doc_processor); + + // Create a test batch + let schema = ParquetSchema::new(); + let num_rows = 5; + + fn create_dict_array(values: &[&str]) -> ArrayRef { + let keys: Vec = (0..values.len()).map(|i| i as i32).collect(); + let string_array = StringArray::from(values.to_vec()); + StdArc::new( + DictionaryArray::::try_new( + Int32Array::from(keys), + StdArc::new(string_array), + ) + .unwrap(), + ) + } + + fn create_nullable_dict_array(values: &[Option<&str>]) -> ArrayRef { + let keys: Vec> = values + .iter() + .enumerate() + .map(|(i, v)| v.map(|_| i as i32)) + .collect(); + let string_values: Vec<&str> = values.iter().filter_map(|v| *v).collect(); + let string_array = StringArray::from(string_values); + StdArc::new( + DictionaryArray::::try_new( + Int32Array::from(keys), + StdArc::new(string_array), + ) + .unwrap(), + ) + } + + let metric_name: ArrayRef = create_dict_array(&vec!["cpu.usage"; num_rows]); + let metric_type: ArrayRef = StdArc::new(UInt8Array::from(vec![0u8; num_rows])); + let metric_unit: ArrayRef = StdArc::new(StringArray::from(vec![Some("bytes"); num_rows])); + let timestamps: Vec = (0..num_rows).map(|i| 100 + i as u64).collect(); + let timestamp_secs: ArrayRef = StdArc::new(UInt64Array::from(timestamps)); + let start_timestamp_secs: ArrayRef = + StdArc::new(UInt64Array::from(vec![None::; num_rows])); + let values: Vec = (0..num_rows).map(|i| 42.0 + i as f64).collect(); + let value: ArrayRef = StdArc::new(Float64Array::from(values)); + let tag_service: ArrayRef = create_nullable_dict_array(&vec![Some("web"); num_rows]); + let tag_env: ArrayRef = create_nullable_dict_array(&vec![Some("prod"); num_rows]); + let tag_datacenter: ArrayRef = + create_nullable_dict_array(&vec![Some("us-east-1"); num_rows]); + let tag_region: ArrayRef = create_nullable_dict_array(&vec![None; num_rows]); + let tag_host: ArrayRef = create_nullable_dict_array(&vec![Some("host-001"); num_rows]); + + // Create empty Variant (Struct with metadata and value BinaryView fields) + let metadata_array = StdArc::new(BinaryViewArray::from(vec![b"" as &[u8]; num_rows])); + let value_array = StdArc::new(BinaryViewArray::from(vec![b"" as &[u8]; num_rows])); + let attributes: ArrayRef = StdArc::new(StructArray::from(vec![ + ( + StdArc::new(Field::new("metadata", DataType::BinaryView, false)), + metadata_array.clone() as ArrayRef, + ), + ( + StdArc::new(Field::new("value", DataType::BinaryView, false)), + value_array.clone() as ArrayRef, + ), + ])); + + let service_name: ArrayRef = create_dict_array(&vec!["my-service"; num_rows]); + + let resource_attributes: ArrayRef = StdArc::new(StructArray::from(vec![ + ( + StdArc::new(Field::new("metadata", DataType::BinaryView, false)), + metadata_array as ArrayRef, + ), + ( + StdArc::new(Field::new("value", DataType::BinaryView, false)), + value_array as ArrayRef, + ), + ])); + + let batch = RecordBatch::try_new( + schema.arrow_schema().clone(), + vec![ + metric_name, + metric_type, + metric_unit, + timestamp_secs, + start_timestamp_secs, + value, + tag_service, + tag_env, + tag_datacenter, + tag_region, + tag_host, + attributes, + service_name, + resource_attributes, + ], + ) + .unwrap(); + + // Serialize to Arrow IPC + let ipc_bytes = record_batch_to_ipc(&batch).unwrap(); + + // Create RawDocBatch with force_commit to trigger split production + let mut raw_doc_batch = RawDocBatch::for_test(&[&ipc_bytes], 0..1); + raw_doc_batch.force_commit = true; + + // Send to processor + metrics_doc_processor_mailbox + .send_message(raw_doc_batch) + .await + .unwrap(); + + // Process in doc processor + let doc_counters = metrics_doc_processor_handle + .process_pending_and_observe() + .await + .state; + + // Verify doc processor counters + assert_eq!(doc_counters.valid_batches, 1); + assert_eq!(doc_counters.valid_rows, 5); + + // Process in indexer + let indexer_counters = indexer_handle.process_pending_and_observe().await.state; + + // Verify indexer received and processed the batch + assert_eq!(indexer_counters.batches_received, 1); + assert_eq!(indexer_counters.rows_indexed, 5); + // Should have flushed a batch due to force_commit + assert_eq!(indexer_counters.batches_flushed, 1); + + // Verify packager produced a split + let packager_counters = packager_handle.process_pending_and_observe().await.state; + assert_eq!(packager_counters.splits_produced.load(Ordering::Relaxed), 1); + + universe.assert_quit().await; + } +} diff --git a/quickwit/quickwit-indexing/src/actors/parquet_e2e_test.rs b/quickwit/quickwit-indexing/src/actors/parquet_e2e_test.rs new file mode 100644 index 00000000000..0823ce4ad04 --- /dev/null +++ b/quickwit/quickwit-indexing/src/actors/parquet_e2e_test.rs @@ -0,0 +1,404 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! End-to-end tests for the metrics pipeline. +//! +//! These tests wire up the full metrics pipeline: +//! ParquetDocProcessor → ParquetIndexer → ParquetUploader → ParquetPublisher + +use std::sync::Arc; +use std::sync::atomic::Ordering; +use std::time::Duration; + +use arrow::array::{ + ArrayRef, DictionaryArray, Float64Array, Int32Array, StringArray, UInt8Array, UInt64Array, +}; +use arrow::datatypes::Int32Type; +use arrow::record_batch::RecordBatch; +use bytes::Bytes; +use parquet::variant::{VariantArrayBuilder, VariantBuilderExt}; +use quickwit_actors::{ActorHandle, Universe}; +use quickwit_common::test_utils::wait_until_predicate; +use quickwit_metastore::checkpoint::SourceCheckpointDelta; +use quickwit_parquet_engine::ingest::record_batch_to_ipc; +use quickwit_parquet_engine::schema::ParquetSchema; +use quickwit_parquet_engine::storage::{ParquetSplitWriter, ParquetWriterConfig}; +use quickwit_proto::metastore::{EmptyResponse, MockMetastoreService}; +use quickwit_proto::types::IndexUid; +use quickwit_storage::RamStorage; + +use crate::actors::{ + ParquetDocProcessor, ParquetIndexer, ParquetPackager, ParquetPublisher, ParquetUploader, + PublisherType, SplitsUpdateMailbox, UploaderType, +}; +use crate::models::RawDocBatch; + +// ============================================================================= +// Helpers +// ============================================================================= + +async fn wait_for_published_splits( + publisher_handle: &ActorHandle, + expected_splits: u64, +) -> anyhow::Result<()> { + wait_until_predicate( + || async { + publisher_handle.process_pending_and_observe().await; + let counters = publisher_handle.last_observation(); + counters.num_published_splits >= expected_splits + }, + Duration::from_secs(15), + Duration::from_millis(50), + ) + .await + .map_err(|_| anyhow::anyhow!("Timeout waiting for {} published splits", expected_splits)) +} + +fn create_dict_array(values: &[&str]) -> ArrayRef { + let keys: Vec = (0..values.len()).map(|i| i as i32).collect(); + let string_array = StringArray::from(values.to_vec()); + Arc::new( + DictionaryArray::::try_new(Int32Array::from(keys), Arc::new(string_array)) + .unwrap(), + ) +} + +fn create_nullable_dict_array(values: &[Option<&str>]) -> ArrayRef { + let keys: Vec> = values + .iter() + .enumerate() + .map(|(i, v)| v.map(|_| i as i32)) + .collect(); + let string_values: Vec<&str> = values.iter().filter_map(|v| *v).collect(); + let string_array = StringArray::from(string_values); + Arc::new( + DictionaryArray::::try_new(Int32Array::from(keys), Arc::new(string_array)) + .unwrap(), + ) +} + +fn create_variant_array(num_rows: usize, fields: Option<&[(&str, &str)]>) -> ArrayRef { + let mut builder = VariantArrayBuilder::new(num_rows); + for _ in 0..num_rows { + match fields { + Some(kv_pairs) => { + let mut obj = builder.new_object(); + for &(key, value) in kv_pairs { + obj = obj.with_field(key, value); + } + obj.finish(); + } + None => { + builder.append_null(); + } + } + } + ArrayRef::from(builder.build()) +} + +fn create_test_batch( + num_rows: usize, + metric_name: &str, + service: &str, + base_timestamp: u64, + base_value: f64, +) -> RecordBatch { + let schema = ParquetSchema::new(); + + let metric_names: Vec<&str> = vec![metric_name; num_rows]; + let metric_name_arr: ArrayRef = create_dict_array(&metric_names); + let metric_type: ArrayRef = Arc::new(UInt8Array::from(vec![0u8; num_rows])); + let metric_unit: ArrayRef = Arc::new(StringArray::from(vec![Some("count"); num_rows])); + let timestamps: Vec = (0..num_rows).map(|i| base_timestamp + i as u64).collect(); + let timestamp_secs: ArrayRef = Arc::new(UInt64Array::from(timestamps)); + let start_timestamp_secs: ArrayRef = Arc::new(UInt64Array::from(vec![None::; num_rows])); + let values: Vec = (0..num_rows).map(|i| base_value + i as f64).collect(); + let value: ArrayRef = Arc::new(Float64Array::from(values)); + let tag_service: ArrayRef = create_nullable_dict_array(&vec![Some(service); num_rows]); + let tag_env: ArrayRef = create_nullable_dict_array(&vec![Some("prod"); num_rows]); + let tag_datacenter: ArrayRef = create_nullable_dict_array(&vec![Some("us-east-1"); num_rows]); + let tag_region: ArrayRef = create_nullable_dict_array(&vec![None; num_rows]); + let tag_host: ArrayRef = create_nullable_dict_array(&vec![Some("host-001"); num_rows]); + let attributes: ArrayRef = create_variant_array(num_rows, None); + let service_names: Vec<&str> = vec![service; num_rows]; + let service_name: ArrayRef = create_dict_array(&service_names); + let resource_attributes: ArrayRef = create_variant_array(num_rows, None); + + RecordBatch::try_new( + schema.arrow_schema().clone(), + vec![ + metric_name_arr, + metric_type, + metric_unit, + timestamp_secs, + start_timestamp_secs, + value, + tag_service, + tag_env, + tag_datacenter, + tag_region, + tag_host, + attributes, + service_name, + resource_attributes, + ], + ) + .unwrap() +} + +fn create_raw_doc_batch( + ipc_bytes: &[u8], + range: std::ops::Range, + force_commit: bool, +) -> RawDocBatch { + let docs = vec![Bytes::from(ipc_bytes.to_vec())]; + let checkpoint_delta = SourceCheckpointDelta::from_range(range); + RawDocBatch::new(docs, checkpoint_delta, force_commit) +} + +// ============================================================================= +// Tests +// ============================================================================= + +/// Full pipeline test: DocProcessor → Indexer → Uploader → Publisher. +/// +/// Validates: +/// - Multiple batches accumulate without producing splits +/// - force_commit triggers split production +/// - Counters are correct at each stage +/// - Splits are staged and published via metastore +#[tokio::test] +async fn test_metrics_pipeline_e2e() { + let universe = Universe::with_accelerated_time(); + let temp_dir = tempfile::tempdir().unwrap(); + + let mut mock_metastore = MockMetastoreService::new(); + mock_metastore + .expect_stage_metrics_splits() + .returning(|_| Ok(EmptyResponse {})); + mock_metastore + .expect_publish_metrics_splits() + .returning(|_| Ok(EmptyResponse {})); + + let metastore_client = + quickwit_proto::metastore::MetastoreServiceClient::from_mock(mock_metastore); + let ram_storage = Arc::new(RamStorage::default()); + + let publisher = ParquetPublisher::new( + PublisherType::ParquetPublisher, + metastore_client.clone(), + None, + None, + ); + let (publisher_mailbox, publisher_handle) = universe.spawn_builder().spawn(publisher); + + let uploader = ParquetUploader::new( + UploaderType::IndexUploader, + metastore_client.clone(), + ram_storage, + SplitsUpdateMailbox::Publisher(publisher_mailbox), + 4, + ); + let (uploader_mailbox, _uploader_handle) = universe.spawn_builder().spawn(uploader); + + // ParquetPackager between indexer and uploader + let parquet_schema = ParquetSchema::new(); + let writer_config = ParquetWriterConfig::default(); + let split_writer = ParquetSplitWriter::new(parquet_schema, writer_config, temp_dir.path()); + let packager = ParquetPackager::new(split_writer, uploader_mailbox); + let (packager_mailbox, packager_handle) = universe.spawn_builder().spawn(packager); + + let indexer = ParquetIndexer::new( + IndexUid::for_test("test-metrics-index", 0), + "test-source".to_string(), + None, + packager_mailbox, + None, + ); + let (indexer_mailbox, indexer_handle) = universe.spawn_builder().spawn(indexer); + + let doc_processor = ParquetDocProcessor::new( + "test-metrics-index".to_string(), + "test-source".to_string(), + indexer_mailbox, + ); + let (doc_processor_mailbox, doc_processor_handle) = + universe.spawn_builder().spawn(doc_processor); + + // Phase 1: Send 5 batches without force_commit — should accumulate, no splits. + let rows_per_batch: usize = 20; + for i in 0u64..5 { + let batch = create_test_batch(rows_per_batch, "cpu.usage", "web", 100 + i * 20, 0.0); + let ipc_bytes = record_batch_to_ipc(&batch).unwrap(); + let raw = create_raw_doc_batch( + &ipc_bytes, + i * rows_per_batch as u64..(i + 1) * rows_per_batch as u64, + false, + ); + doc_processor_mailbox.send_message(raw).await.unwrap(); + } + + doc_processor_handle.process_pending_and_observe().await; + let indexer_counters = indexer_handle.process_pending_and_observe().await.state; + + assert_eq!(indexer_counters.batches_received, 5); + assert_eq!(indexer_counters.rows_indexed, 100); + assert_eq!( + indexer_counters.batches_flushed, 0, + "No flushes without force_commit" + ); + + // Phase 2: Send one more batch with force_commit — triggers split. + let batch = create_test_batch(10, "cpu.usage", "web", 200, 0.0); + let ipc_bytes = record_batch_to_ipc(&batch).unwrap(); + let raw = create_raw_doc_batch(&ipc_bytes, 100..110, true); + doc_processor_mailbox.send_message(raw).await.unwrap(); + + let doc_processor_counters = doc_processor_handle + .process_pending_and_observe() + .await + .state; + let indexer_counters = indexer_handle.process_pending_and_observe().await.state; + + wait_for_published_splits(&publisher_handle, 1) + .await + .expect("Publisher should have published 1 split"); + + assert_eq!(doc_processor_counters.valid_batches, 6); + assert_eq!(doc_processor_counters.valid_rows, 110); + assert_eq!(doc_processor_counters.parse_errors, 0); + assert_eq!( + indexer_counters.batches_flushed, 1, + "force_commit should flush 1 batch" + ); + // Verify packager produced the split + let packager_counters = packager_handle.process_pending_and_observe().await.state; + assert_eq!( + packager_counters.splits_produced.load(Ordering::Relaxed), + 1, + "packager should produce 1 split" + ); + assert!(packager_counters.bytes_written.load(Ordering::Relaxed) > 0); + assert_eq!(publisher_handle.last_observation().num_published_splits, 1); + + universe.assert_quit().await; +} + +/// FileBackedMetastore metrics splits operations (no mocks). +/// +/// Validates stage, publish, list (with time range and metric name filtering). +#[tokio::test] +async fn test_file_backed_metastore_metrics_operations() { + use quickwit_config::IndexConfig; + use quickwit_metastore::{ + CreateIndexRequestExt, FileBackedMetastore, ListMetricsSplitsQuery, + ListMetricsSplitsRequestExt, ListMetricsSplitsResponseExt, StageMetricsSplitsRequestExt, + }; + use quickwit_parquet_engine::split::{MetricsSplitMetadata, MetricsSplitRecord, TimeRange}; + use quickwit_proto::metastore::{ + CreateIndexRequest, ListMetricsSplitsRequest, MetastoreService, + PublishMetricsSplitsRequest, StageMetricsSplitsRequest, + }; + + let ram_storage = Arc::new(RamStorage::default()); + let metastore = FileBackedMetastore::try_new(ram_storage.clone(), None) + .await + .unwrap(); + + let index_id = "metrics-test-index"; + let index_config = IndexConfig::for_test(index_id, "ram:///indexes/metrics-test-index"); + let create_request = CreateIndexRequest::try_from_index_config(&index_config).unwrap(); + let create_response = metastore.create_index(create_request).await.unwrap(); + let index_uid = create_response.index_uid.unwrap(); + + let split1 = MetricsSplitMetadata::builder() + .index_uid(index_uid.to_string()) + .time_range(TimeRange::new(1000, 1100)) + .num_rows(100) + .size_bytes(1024) + .add_metric_name("cpu.usage") + .build(); + + let split2 = MetricsSplitMetadata::builder() + .index_uid(index_uid.to_string()) + .time_range(TimeRange::new(2000, 2100)) + .num_rows(200) + .size_bytes(2048) + .add_metric_name("memory.used") + .build(); + + // Stage + let stage_request = StageMetricsSplitsRequest::try_from_splits_metadata( + index_uid.clone(), + &[split1.clone(), split2.clone()], + ) + .unwrap(); + metastore.stage_metrics_splits(stage_request).await.unwrap(); + + // Verify staged + let query = ListMetricsSplitsQuery::for_index(index_uid.clone()) + .with_split_states(vec!["Staged".to_string()]); + let list_request = ListMetricsSplitsRequest::try_from_query(index_uid.clone(), &query).unwrap(); + let list_response = metastore.list_metrics_splits(list_request).await.unwrap(); + let staged: Vec = list_response.deserialize_splits().unwrap(); + assert_eq!(staged.len(), 2); + + // Publish split1 + let publish_request = PublishMetricsSplitsRequest { + index_uid: Some(index_uid.clone()), + staged_split_ids: vec![split1.split_id.to_string()], + replaced_split_ids: vec![], + index_checkpoint_delta_json_opt: None, + publish_token_opt: None, + }; + metastore + .publish_metrics_splits(publish_request) + .await + .unwrap(); + + // Verify published + let query = ListMetricsSplitsQuery::for_index(index_uid.clone()) + .with_split_states(vec!["Published".to_string()]); + let list_request = ListMetricsSplitsRequest::try_from_query(index_uid.clone(), &query).unwrap(); + let list_response = metastore.list_metrics_splits(list_request).await.unwrap(); + let published: Vec = list_response.deserialize_splits().unwrap(); + assert_eq!(published.len(), 1); + assert_eq!(published[0].metadata.split_id, split1.split_id); + + // Time range filtering + let query = ListMetricsSplitsQuery::for_index(index_uid.clone()) + .with_split_states(vec!["Published".to_string()]) + .with_time_range(1000, 1100); + let list_request = ListMetricsSplitsRequest::try_from_query(index_uid.clone(), &query).unwrap(); + let list_response = metastore.list_metrics_splits(list_request).await.unwrap(); + let in_range: Vec = list_response.deserialize_splits().unwrap(); + assert_eq!(in_range.len(), 1); + + let query = ListMetricsSplitsQuery::for_index(index_uid.clone()) + .with_split_states(vec!["Published".to_string()]) + .with_time_range(5000, 5100); + let list_request = ListMetricsSplitsRequest::try_from_query(index_uid.clone(), &query).unwrap(); + let list_response = metastore.list_metrics_splits(list_request).await.unwrap(); + let out_of_range: Vec = list_response.deserialize_splits().unwrap(); + assert_eq!(out_of_range.len(), 0); + + // Metric name filtering + let query = ListMetricsSplitsQuery::for_index(index_uid.clone()) + .with_split_states(vec!["Published".to_string()]) + .with_metric_names(vec!["cpu.usage".to_string()]); + let list_request = ListMetricsSplitsRequest::try_from_query(index_uid.clone(), &query).unwrap(); + let list_response = metastore.list_metrics_splits(list_request).await.unwrap(); + let by_metric: Vec = list_response.deserialize_splits().unwrap(); + assert_eq!(by_metric.len(), 1); +} diff --git a/quickwit/quickwit-indexing/src/actors/parquet_indexer.rs b/quickwit/quickwit-indexing/src/actors/parquet_indexer.rs new file mode 100644 index 00000000000..2738a42163f --- /dev/null +++ b/quickwit/quickwit-indexing/src/actors/parquet_indexer.rs @@ -0,0 +1,1055 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! ParquetIndexer actor that accumulates RecordBatches for the metrics pipeline. +//! +//! This actor replaces the Tantivy-based Indexer for metrics workloads, accumulating +//! Arrow RecordBatches and forwarding concatenated batches to ParquetPackager for +//! Parquet encoding and file writing. + +use std::path::PathBuf; +use std::time::Duration; + +use arrow::record_batch::RecordBatch; +use async_trait::async_trait; +use quickwit_actors::{Actor, ActorContext, ActorExitStatus, Handler, Mailbox, QueueCapacity}; +use quickwit_common::runtimes::RuntimeType; +use quickwit_metastore::checkpoint::{IndexCheckpointDelta, SourceCheckpointDelta}; +use quickwit_parquet_engine::index::{ParquetBatchAccumulator, ParquetIndexingConfig}; +use quickwit_parquet_engine::split::ParquetSplit; +use quickwit_proto::types::{IndexUid, PublishToken, SourceId}; +use serde::Serialize; +use tokio::runtime::Handle; +use tracing::{debug, info, info_span, warn}; +use ulid::Ulid; + +use crate::actors::parquet_packager::{ParquetBatchForPackager, ParquetPackager}; +use crate::models::{NewPublishLock, NewPublishToken, ProcessedParquetBatch, PublishLock}; + +/// Default commit timeout for ParquetIndexer (60 seconds). +// TODO: read from index config commit_timeout_secs. +const DEFAULT_COMMIT_TIMEOUT: Duration = Duration::from_secs(60); + +/// Message to trigger a commit after timeout. +#[derive(Debug)] +struct CommitTimeout { + workbench_id: Ulid, +} + +/// Counters for ParquetIndexer observability. +#[derive(Debug, Default, Serialize, Clone)] +pub struct ParquetIndexerCounters { + /// Number of batches received. + pub batches_received: u64, + /// Number of rows indexed. + pub rows_indexed: u64, + /// Number of batches flushed to packager. + pub batches_flushed: u64, + /// Number of errors encountered. + pub errors: u64, +} + +impl ParquetIndexerCounters { + /// Record a batch received. + pub fn record_batch(&mut self, num_rows: usize) { + self.batches_received += 1u64; + self.rows_indexed += num_rows as u64; + } + + /// Record a flush (concatenated batch sent to packager). + pub fn record_flush(&mut self) { + self.batches_flushed += 1; + } + + /// Record an error. + pub fn record_error(&mut self) { + self.errors += 1; + } +} + +/// Message containing produced ParquetSplits for downstream processing. +/// +/// This is sent when the accumulator produces splits (either via threshold or force commit). +#[derive(Debug)] +pub struct ParquetSplitBatch { + /// Index unique identifier for the splits in this batch. + pub index_uid: IndexUid, + /// The splits produced. + pub splits: Vec, + /// Directory containing the Parquet files referenced by splits. + /// The uploader uses this to locate and upload the actual file content. + pub output_dir: PathBuf, + /// Checkpoint delta covering all data in these splits. + pub checkpoint_delta: IndexCheckpointDelta, + /// Publish lock for coordinating with sources. + pub publish_lock: PublishLock, + /// Optional publish token. + pub publish_token_opt: Option, +} + +/// ParquetIndexer actor that accumulates RecordBatches and forwards them to ParquetPackager. +/// +/// This actor: +/// - Receives ProcessedParquetBatch messages from ParquetDocProcessor +/// - Accumulates batches using ParquetBatchAccumulator (CPU-bound buffering + concat) +/// - Forwards concatenated RecordBatches to ParquetPackager when thresholds are exceeded or on +/// force commit +/// - Tracks checkpoint progress for reliable delivery +/// +/// Unlike the Tantivy-based Indexer, this actor works entirely with Arrow data. +/// IO-bound work (Parquet encoding, compression, file writing) is delegated to +/// the ParquetPackager actor. +pub struct ParquetIndexer { + /// Index unique identifier. + index_uid: IndexUid, + /// Source identifier. + source_id: SourceId, + /// Batch accumulator for buffering and concatenating RecordBatches. + accumulator: ParquetBatchAccumulator, + /// Accumulated checkpoint delta. + checkpoint_delta: SourceCheckpointDelta, + /// Publish lock for coordinating with sources. + publish_lock: PublishLock, + /// Optional publish token. + publish_token_opt: Option, + /// Observability counters. + counters: ParquetIndexerCounters, + /// Current workbench ID for tracing. + workbench_id: Ulid, + /// Mailbox for sending concatenated batches to packager. + packager_mailbox: Mailbox, + /// Commit timeout duration. + commit_timeout: Duration, + /// Whether a commit timeout has been scheduled for the current workbench. + commit_timeout_scheduled: bool, +} + +impl ParquetIndexer { + /// Create a new ParquetIndexer. + /// + /// # Arguments + /// * `index_uid` - The index unique identifier + /// * `source_id` - The source identifier + /// * `config` - Optional configuration for accumulation thresholds + /// * `packager_mailbox` - Mailbox for sending concatenated batches to ParquetPackager + /// * `commit_timeout` - Optional commit timeout (defaults to 60 seconds) + pub fn new( + index_uid: IndexUid, + source_id: SourceId, + config: Option, + packager_mailbox: Mailbox, + commit_timeout: Option, + ) -> Self { + let config = config.unwrap_or_default(); + let accumulator = ParquetBatchAccumulator::new(config); + let counters = ParquetIndexerCounters::default(); + let commit_timeout = commit_timeout.unwrap_or(DEFAULT_COMMIT_TIMEOUT); + + info!( + index_uid = %index_uid, + source_id = %source_id, + commit_timeout_secs = commit_timeout.as_secs(), + "ParquetIndexer created" + ); + + Self { + index_uid, + source_id, + accumulator, + checkpoint_delta: SourceCheckpointDelta::default(), + publish_lock: PublishLock::default(), + publish_token_opt: None, + counters, + workbench_id: Ulid::new(), + packager_mailbox, + commit_timeout, + commit_timeout_scheduled: false, + } + } + + /// Process a batch and potentially produce a flushed RecordBatch. + /// + /// Returns at most one concatenated RecordBatch. A flush is triggered either by + /// the accumulator exceeding its row/byte threshold, or by a force_commit request. + /// This ensures exactly one checkpoint delta per flushed batch. + fn process_batch( + &mut self, + batch: ProcessedParquetBatch, + ctx: &ActorContext, + ) -> Result, ActorExitStatus> { + let _span = info_span!( + target: "quickwit-indexing", + "metrics-indexer", + index_id = %self.index_uid, + source_id = %self.source_id, + workbench_id = %self.workbench_id, + ) + .entered(); + + // Record batch stats + let num_rows = batch.num_rows(); + self.counters.record_batch(num_rows); + + debug!( + num_rows = num_rows, + force_commit = batch.force_commit, + total_batches = self.counters.batches_received, + total_rows = self.counters.rows_indexed, + "received batch for accumulation" + ); + + // Extend checkpoint + self.checkpoint_delta + .extend(batch.checkpoint_delta) + .map_err(|e| { + warn!(error = %e, "Checkpoint delta conflict"); + ActorExitStatus::Failure(anyhow::anyhow!("{}", e).into()) + })?; + + let force_commit = batch.force_commit; + + // Add batch to accumulator (auto-flushes if threshold exceeded) + let mut flushed = match self.accumulator.add_batch(batch.batch) { + Ok(threshold_batch) => { + if let Some(ref combined) = threshold_batch { + debug!( + num_rows = combined.num_rows(), + "accumulator flushed from threshold" + ); + } + threshold_batch + } + Err(error) => { + warn!(error = %error, "Failed to add batch to accumulator"); + self.counters.record_error(); + return Err(ActorExitStatus::Failure( + anyhow::anyhow!("{}", error).into(), + )); + } + }; + + // Reset actor state if the accumulator auto-flushed. + if flushed.is_some() { + self.workbench_id = Ulid::new(); + self.commit_timeout_scheduled = false; + } + + ctx.record_progress(); + + // Force flush if requested and threshold didn't already flush + if force_commit && flushed.is_none() { + debug!("force commit requested, flushing accumulator"); + flushed = self.flush_accumulator()?; + } + + if let Some(ref combined) = flushed { + self.counters.record_flush(); + info!( + num_rows = combined.num_rows(), + "flushed concatenated batch to packager" + ); + } + + Ok(flushed) + } + + /// Flush the accumulator. + /// + /// Returns the concatenated RecordBatch, or None if no pending data. + fn flush_accumulator(&mut self) -> Result, ActorExitStatus> { + match self.accumulator.flush() { + Ok(batch_opt) => { + if batch_opt.is_some() { + // Reset workbench for next batch of work + self.workbench_id = Ulid::new(); + self.commit_timeout_scheduled = false; + } + Ok(batch_opt) + } + Err(error) => { + warn!(error = %error, "Failed to flush accumulator"); + self.counters.record_error(); + Err(ActorExitStatus::Failure( + anyhow::anyhow!("{}", error).into(), + )) + } + } + } + + /// Take the current checkpoint delta and reset it. + fn take_checkpoint_delta(&mut self) -> SourceCheckpointDelta { + std::mem::take(&mut self.checkpoint_delta) + } + + /// Create an IndexCheckpointDelta from the current source delta. + fn make_index_checkpoint_delta(&mut self) -> IndexCheckpointDelta { + IndexCheckpointDelta { + source_id: self.source_id.clone(), + source_delta: self.take_checkpoint_delta(), + } + } +} + +#[async_trait] +impl Actor for ParquetIndexer { + type ObservableState = ParquetIndexerCounters; + + fn observable_state(&self) -> Self::ObservableState { + self.counters.clone() + } + + fn queue_capacity(&self) -> QueueCapacity { + QueueCapacity::Bounded(10) + } + + fn name(&self) -> String { + "ParquetIndexer".to_string() + } + + fn runtime_handle(&self) -> Handle { + RuntimeType::Blocking.get_runtime_handle() + } + + #[inline] + fn yield_after_each_message(&self) -> bool { + false + } + + async fn finalize( + &mut self, + exit_status: &ActorExitStatus, + ctx: &ActorContext, + ) -> anyhow::Result<()> { + match exit_status { + ActorExitStatus::DownstreamClosed + | ActorExitStatus::Killed + | ActorExitStatus::Failure(_) + | ActorExitStatus::Panicked => return Ok(()), + ActorExitStatus::Quit | ActorExitStatus::Success => { + // Flush remaining data on clean shutdown. + let flushed_batch = match self.flush_accumulator() { + Ok(Some(combined)) => { + self.counters.record_flush(); + info!( + num_rows = combined.num_rows(), + "flushed final batch on shutdown" + ); + Some(combined) + } + Ok(None) => None, + Err(_) => None, + }; + + // Send even when batch is None — the checkpoint delta may contain + // EOF positions that must flow through the pipeline for graceful + // decommission to complete. + let checkpoint_delta = self.make_index_checkpoint_delta(); + if flushed_batch.is_some() || !checkpoint_delta.source_delta.is_empty() { + let batch_for_packager = ParquetBatchForPackager { + batch: flushed_batch, + index_uid: self.index_uid.clone(), + checkpoint_delta, + publish_lock: self.publish_lock.clone(), + publish_token_opt: self.publish_token_opt.clone(), + }; + let _ = ctx + .send_message(&self.packager_mailbox, batch_for_packager) + .await; + } + } + } + Ok(()) + } +} + +#[async_trait] +impl Handler for ParquetIndexer { + type Reply = (); + + async fn handle( + &mut self, + batch: ProcessedParquetBatch, + ctx: &ActorContext, + ) -> Result<(), ActorExitStatus> { + if self.publish_lock.is_dead() { + return Ok(()); + } + + // Schedule commit timeout on first batch for this workbench + if !self.commit_timeout_scheduled { + let commit_timeout_message = CommitTimeout { + workbench_id: self.workbench_id, + }; + ctx.schedule_self_msg(self.commit_timeout, commit_timeout_message); + self.commit_timeout_scheduled = true; + debug!( + workbench_id = %self.workbench_id, + commit_timeout_secs = self.commit_timeout.as_secs(), + "scheduled commit timeout" + ); + } + + let force_commit = batch.force_commit; + let flushed_batch = self.process_batch(batch, ctx)?; + + // Send downstream if we have data, or if force_commit needs to push + // the checkpoint through immediately (e.g. shard EOF). + let should_send = + flushed_batch.is_some() || (force_commit && !self.checkpoint_delta.is_empty()); + + if should_send { + let batch_for_packager = ParquetBatchForPackager { + batch: flushed_batch, + index_uid: self.index_uid.clone(), + checkpoint_delta: self.make_index_checkpoint_delta(), + publish_lock: self.publish_lock.clone(), + publish_token_opt: self.publish_token_opt.clone(), + }; + + ctx.send_message(&self.packager_mailbox, batch_for_packager) + .await + .map_err(|e| { + ActorExitStatus::Failure( + anyhow::anyhow!("failed to send to packager: {}", e).into(), + ) + })?; + } + + Ok(()) + } +} + +#[async_trait] +impl Handler for ParquetIndexer { + type Reply = (); + + async fn handle( + &mut self, + message: NewPublishLock, + _ctx: &ActorContext, + ) -> Result<(), ActorExitStatus> { + let NewPublishLock(publish_lock) = message; + + // Discard pending data — it was accumulated under the old publish lock + // and must not be published. This matches the standard Indexer behavior + // which discards its workbench on publish lock change. + self.accumulator.discard(); + + // Reset state for new lock + self.publish_lock = publish_lock; + self.checkpoint_delta = SourceCheckpointDelta::default(); + self.workbench_id = Ulid::new(); + self.commit_timeout_scheduled = false; + + Ok(()) + } +} + +#[async_trait] +impl Handler for ParquetIndexer { + type Reply = (); + + async fn handle( + &mut self, + message: NewPublishToken, + _ctx: &ActorContext, + ) -> Result<(), ActorExitStatus> { + let NewPublishToken(publish_token) = message; + self.publish_token_opt = Some(publish_token); + Ok(()) + } +} + +#[async_trait] +impl Handler for ParquetIndexer { + type Reply = (); + + async fn handle( + &mut self, + commit_timeout: CommitTimeout, + ctx: &ActorContext, + ) -> Result<(), ActorExitStatus> { + // Ignore stale timeout messages from previous workbenches + if self.workbench_id != commit_timeout.workbench_id { + return Ok(()); + } + + debug!( + workbench_id = %self.workbench_id, + "commit timeout triggered, flushing accumulator" + ); + + // Flush the accumulator + if let Some(combined) = self.flush_accumulator()? { + self.counters.record_flush(); + info!( + num_rows = combined.num_rows(), + "Flushed batch on commit timeout" + ); + + let batch_for_packager = ParquetBatchForPackager { + batch: Some(combined), + index_uid: self.index_uid.clone(), + checkpoint_delta: self.make_index_checkpoint_delta(), + publish_lock: self.publish_lock.clone(), + publish_token_opt: self.publish_token_opt.clone(), + }; + + ctx.send_message(&self.packager_mailbox, batch_for_packager) + .await + .map_err(|e| { + ActorExitStatus::Failure( + anyhow::anyhow!( + "failed to send ParquetBatchForPackager to packager on commit \ + timeout: {}", + e + ) + .into(), + ) + })?; + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::Ordering; + use std::time::Duration; + + use arrow::array::{ + ArrayRef, DictionaryArray, Float64Array, Int32Array, StringArray, UInt8Array, UInt64Array, + }; + use arrow::datatypes::Int32Type; + use arrow::record_batch::RecordBatch; + use parquet::variant::{VariantArrayBuilder, VariantBuilderExt}; + use quickwit_actors::{ActorHandle, Universe}; + use quickwit_common::test_utils::wait_until_predicate; + use quickwit_parquet_engine::schema::ParquetSchema; + use quickwit_parquet_engine::storage::{ParquetSplitWriter, ParquetWriterConfig}; + use quickwit_proto::metastore::{EmptyResponse, MockMetastoreService}; + use quickwit_storage::RamStorage; + + use super::*; + use crate::actors::{ + ParquetPackager, ParquetPublisher, ParquetUploader, SplitsUpdateMailbox, UploaderType, + }; + + /// Create a test ParquetUploader and return its mailbox. + fn create_test_uploader( + universe: &Universe, + ) -> (Mailbox, ActorHandle) { + let mock_metastore = MockMetastoreService::new(); + let ram_storage = Arc::new(RamStorage::default()); + let (publisher_mailbox, _publisher_inbox) = + universe.create_test_mailbox::(); + + let uploader = ParquetUploader::new( + UploaderType::IndexUploader, + quickwit_proto::metastore::MetastoreServiceClient::from_mock(mock_metastore), + ram_storage, + SplitsUpdateMailbox::Publisher(publisher_mailbox), + 4, + ); + universe.spawn_builder().spawn(uploader) + } + + /// Create a test ParquetUploader with a mock that expects staging calls. + fn create_test_uploader_with_staging_expectation( + universe: &Universe, + expected_index_id: &'static str, + ) -> (Mailbox, ActorHandle) { + let mut mock_metastore = MockMetastoreService::new(); + mock_metastore + .expect_stage_metrics_splits() + .withf(move |request| request.index_uid().index_id == expected_index_id) + .returning(|_| Ok(EmptyResponse {})); + + let ram_storage = Arc::new(RamStorage::default()); + let (publisher_mailbox, _publisher_inbox) = + universe.create_test_mailbox::(); + + let uploader = ParquetUploader::new( + UploaderType::IndexUploader, + quickwit_proto::metastore::MetastoreServiceClient::from_mock(mock_metastore), + ram_storage, + SplitsUpdateMailbox::Publisher(publisher_mailbox), + 4, + ); + universe.spawn_builder().spawn(uploader) + } + + /// Create a test ParquetPackager wired to an uploader. + fn create_test_packager( + universe: &Universe, + temp_dir: &std::path::Path, + uploader_mailbox: Mailbox, + ) -> (Mailbox, ActorHandle) { + let schema = ParquetSchema::new(); + let writer_config = ParquetWriterConfig::default(); + let split_writer = ParquetSplitWriter::new(schema, writer_config, temp_dir); + + let packager = ParquetPackager::new(split_writer, uploader_mailbox); + universe.spawn_builder().spawn(packager) + } + + /// Wait for the uploader to have staged the expected number of splits. + async fn wait_for_staged_splits( + uploader_handle: &ActorHandle, + expected_splits: u64, + ) -> anyhow::Result<()> { + wait_until_predicate( + || async { + uploader_handle.process_pending_and_observe().await; + let counters = uploader_handle.last_observation(); + counters.num_staged_splits.load(Ordering::Relaxed) >= expected_splits + }, + Duration::from_secs(15), + Duration::from_millis(50), + ) + .await + .map_err(|_| anyhow::anyhow!("Timeout waiting for {} staged splits", expected_splits)) + } + + /// Create dictionary array for string fields with Int32 keys. + fn create_dict_array(values: &[&str]) -> ArrayRef { + let keys: Vec = (0..values.len()).map(|i| i as i32).collect(); + let string_array = StringArray::from(values.to_vec()); + Arc::new( + DictionaryArray::::try_new(Int32Array::from(keys), Arc::new(string_array)) + .unwrap(), + ) + } + + /// Create nullable dictionary array for optional string fields. + fn create_nullable_dict_array(values: &[Option<&str>]) -> ArrayRef { + let keys: Vec> = values + .iter() + .enumerate() + .map(|(i, v)| v.map(|_| i as i32)) + .collect(); + let string_values: Vec<&str> = values.iter().filter_map(|v| *v).collect(); + let string_array = StringArray::from(string_values); + Arc::new( + DictionaryArray::::try_new(Int32Array::from(keys), Arc::new(string_array)) + .unwrap(), + ) + } + + /// Create a VARIANT array for testing with specified number of rows. + fn create_variant_array(num_rows: usize, fields: Option<&[(&str, &str)]>) -> ArrayRef { + let mut builder = VariantArrayBuilder::new(num_rows); + for _ in 0..num_rows { + match fields { + Some(kv_pairs) => { + let mut obj = builder.new_object(); + for &(key, value) in kv_pairs { + obj = obj.with_field(key, value); + } + obj.finish(); + } + None => { + builder.append_null(); + } + } + } + ArrayRef::from(builder.build()) + } + + /// Create a test batch matching the metrics schema. + fn create_test_batch(num_rows: usize) -> RecordBatch { + let schema = ParquetSchema::new(); + + let metric_names: Vec<&str> = vec!["cpu.usage"; num_rows]; + let metric_name: ArrayRef = create_dict_array(&metric_names); + let metric_type: ArrayRef = Arc::new(UInt8Array::from(vec![0u8; num_rows])); + let metric_unit: ArrayRef = Arc::new(StringArray::from(vec![Some("bytes"); num_rows])); + let timestamps: Vec = (0..num_rows).map(|i| 100 + i as u64).collect(); + let timestamp_secs: ArrayRef = Arc::new(UInt64Array::from(timestamps)); + let start_timestamp_secs: ArrayRef = + Arc::new(UInt64Array::from(vec![None::; num_rows])); + let values: Vec = (0..num_rows).map(|i| 42.0 + i as f64).collect(); + let value: ArrayRef = Arc::new(Float64Array::from(values)); + let tag_service: ArrayRef = create_nullable_dict_array(&vec![Some("web"); num_rows]); + let tag_env: ArrayRef = create_nullable_dict_array(&vec![Some("prod"); num_rows]); + let tag_datacenter: ArrayRef = + create_nullable_dict_array(&vec![Some("us-east-1"); num_rows]); + let tag_region: ArrayRef = create_nullable_dict_array(&vec![None; num_rows]); + let tag_host: ArrayRef = create_nullable_dict_array(&vec![Some("host-001"); num_rows]); + let attributes: ArrayRef = create_variant_array(num_rows, None); + let service_name: ArrayRef = create_dict_array(&vec!["my-service"; num_rows]); + let resource_attributes: ArrayRef = create_variant_array(num_rows, None); + + RecordBatch::try_new( + schema.arrow_schema().clone(), + vec![ + metric_name, + metric_type, + metric_unit, + timestamp_secs, + start_timestamp_secs, + value, + tag_service, + tag_env, + tag_datacenter, + tag_region, + tag_host, + attributes, + service_name, + resource_attributes, + ], + ) + .unwrap() + } + + #[tokio::test] + async fn test_metrics_indexer_receives_batch() { + let universe = Universe::with_accelerated_time(); + let temp_dir = tempfile::tempdir().unwrap(); + + let (uploader_mailbox, _uploader_handle) = create_test_uploader(&universe); + let (packager_mailbox, _packager_handle) = + create_test_packager(&universe, temp_dir.path(), uploader_mailbox); + + let indexer = ParquetIndexer::new( + IndexUid::for_test("test-index", 0), + "test-source".to_string(), + None, + packager_mailbox, + None, + ); + + let (indexer_mailbox, indexer_handle) = universe.spawn_builder().spawn(indexer); + + // Send a batch + let batch = create_test_batch(10); + let processed = + ProcessedParquetBatch::new(batch, SourceCheckpointDelta::from_range(0..10), false); + + indexer_mailbox.send_message(processed).await.unwrap(); + + let counters = indexer_handle.process_pending_and_observe().await.state; + + assert_eq!(counters.batches_received, 1); + assert_eq!(counters.rows_indexed, 10); + + universe.assert_quit().await; + } + + #[tokio::test] + async fn test_metrics_indexer_force_commit() { + let universe = Universe::with_accelerated_time(); + let temp_dir = tempfile::tempdir().unwrap(); + + let (uploader_mailbox, uploader_handle) = + create_test_uploader_with_staging_expectation(&universe, "test-index"); + let (packager_mailbox, packager_handle) = + create_test_packager(&universe, temp_dir.path(), uploader_mailbox); + + let indexer = ParquetIndexer::new( + IndexUid::for_test("test-index", 0), + "test-source".to_string(), + None, + packager_mailbox, + None, + ); + + let (indexer_mailbox, indexer_handle) = universe.spawn_builder().spawn(indexer); + + // Send batch with force_commit + let batch = create_test_batch(5); + let processed = ProcessedParquetBatch::new( + batch, + SourceCheckpointDelta::from_range(0..5), + true, // force_commit + ); + + indexer_mailbox.send_message(processed).await.unwrap(); + + let counters = indexer_handle.process_pending_and_observe().await.state; + + // Should have flushed a batch due to force_commit + assert_eq!(counters.batches_received, 1); + assert_eq!(counters.batches_flushed, 1); + + // Verify packager produced a split + let packager_counters = packager_handle.process_pending_and_observe().await.state; + assert_eq!(packager_counters.splits_produced.load(Ordering::Relaxed), 1); + assert!(packager_counters.bytes_written.load(Ordering::Relaxed) > 0); + + // Verify uploader received the split + uploader_handle.process_pending_and_observe().await; + + universe.assert_quit().await; + } + + #[tokio::test] + async fn test_metrics_indexer_respects_publish_lock() { + let universe = Universe::with_accelerated_time(); + let temp_dir = tempfile::tempdir().unwrap(); + + let (uploader_mailbox, _uploader_handle) = create_test_uploader(&universe); + let (packager_mailbox, _packager_handle) = + create_test_packager(&universe, temp_dir.path(), uploader_mailbox); + + let indexer = ParquetIndexer::new( + IndexUid::for_test("test-index", 0), + "test-source".to_string(), + None, + packager_mailbox, + None, + ); + + let (indexer_mailbox, indexer_handle) = universe.spawn_builder().spawn(indexer); + + // Set up and kill publish lock + let publish_lock = PublishLock::default(); + indexer_mailbox + .send_message(NewPublishLock(publish_lock.clone())) + .await + .unwrap(); + indexer_handle.process_pending_and_observe().await; + publish_lock.kill().await; + + // Send batch after lock is dead + let batch = create_test_batch(10); + let processed = + ProcessedParquetBatch::new(batch, SourceCheckpointDelta::from_range(0..10), false); + + indexer_mailbox.send_message(processed).await.unwrap(); + + let counters = indexer_handle.process_pending_and_observe().await.state; + + // Should not process anything when publish lock is dead + assert_eq!(counters.batches_received, 0); + + universe.assert_quit().await; + } + + #[tokio::test] + async fn test_metrics_indexer_threshold_flush() { + let universe = Universe::with_accelerated_time(); + let temp_dir = tempfile::tempdir().unwrap(); + + // Configure low threshold for testing + let config = ParquetIndexingConfig::default().with_max_rows(50); + + let (uploader_mailbox, uploader_handle) = + create_test_uploader_with_staging_expectation(&universe, "test-index"); + let (packager_mailbox, packager_handle) = + create_test_packager(&universe, temp_dir.path(), uploader_mailbox); + + let indexer = ParquetIndexer::new( + IndexUid::for_test("test-index", 0), + "test-source".to_string(), + Some(config), + packager_mailbox, + None, + ); + + let (indexer_mailbox, indexer_handle) = universe.spawn_builder().spawn(indexer); + + // Send batch that exceeds threshold + let batch = create_test_batch(100); + let processed = + ProcessedParquetBatch::new(batch, SourceCheckpointDelta::from_range(0..100), false); + + indexer_mailbox.send_message(processed).await.unwrap(); + + let counters = indexer_handle.process_pending_and_observe().await.state; + + // Should have flushed due to exceeding threshold + assert_eq!(counters.batches_received, 1); + assert_eq!(counters.rows_indexed, 100); + assert!(counters.batches_flushed >= 1); + + // Verify packager produced split(s) + let packager_counters = packager_handle.process_pending_and_observe().await.state; + assert!(packager_counters.splits_produced.load(Ordering::Relaxed) >= 1); + + // Verify uploader received the splits + uploader_handle.process_pending_and_observe().await; + + universe.assert_quit().await; + } + + #[tokio::test] + async fn test_metrics_indexer_flush_on_shutdown() { + let universe = Universe::with_accelerated_time(); + let temp_dir = tempfile::tempdir().unwrap(); + + let (uploader_mailbox, _uploader_handle) = + create_test_uploader_with_staging_expectation(&universe, "test-index"); + let (packager_mailbox, packager_handle) = + create_test_packager(&universe, temp_dir.path(), uploader_mailbox); + + let indexer = ParquetIndexer::new( + IndexUid::for_test("test-index", 0), + "test-source".to_string(), + None, + packager_mailbox, + None, + ); + + let (indexer_mailbox, indexer_handle) = universe.spawn_builder().spawn(indexer); + + // Send batch without force_commit + let batch = create_test_batch(10); + let processed = + ProcessedParquetBatch::new(batch, SourceCheckpointDelta::from_range(0..10), false); + + indexer_mailbox.send_message(processed).await.unwrap(); + indexer_handle.process_pending_and_observe().await; + + // Gracefully shut down - should flush remaining data + universe + .send_exit_with_success(&indexer_mailbox) + .await + .unwrap(); + let (exit_status, counters) = indexer_handle.join().await; + + assert!(exit_status.is_success()); + // Should have flushed on shutdown + assert_eq!(counters.batches_flushed, 1); + + // Verify packager produced a split + let packager_counters = packager_handle.process_pending_and_observe().await.state; + assert_eq!(packager_counters.splits_produced.load(Ordering::Relaxed), 1); + + universe.assert_quit().await; + } + + #[tokio::test] + async fn test_metrics_indexer_commit_timeout() { + let universe = Universe::with_accelerated_time(); + let temp_dir = tempfile::tempdir().unwrap(); + + let (uploader_mailbox, uploader_handle) = + create_test_uploader_with_staging_expectation(&universe, "test-index"); + let (packager_mailbox, packager_handle) = + create_test_packager(&universe, temp_dir.path(), uploader_mailbox); + + // Use a short commit timeout for testing + let commit_timeout = Duration::from_secs(2); + let indexer = ParquetIndexer::new( + IndexUid::for_test("test-index", 0), + "test-source".to_string(), + None, + packager_mailbox, + Some(commit_timeout), + ); + + let (indexer_mailbox, indexer_handle) = universe.spawn_builder().spawn(indexer); + + // Send batch without force_commit + let batch = create_test_batch(10); + let processed = + ProcessedParquetBatch::new(batch, SourceCheckpointDelta::from_range(0..10), false); + + indexer_mailbox.send_message(processed).await.unwrap(); + indexer_handle.process_pending_and_observe().await; + + // No flush yet (waiting for threshold or timeout) + let counters = indexer_handle.observe().await; + assert_eq!(counters.batches_flushed, 0); + + // Wait for commit timeout to trigger + universe + .sleep(commit_timeout + Duration::from_millis(100)) + .await; + indexer_handle.process_pending_and_observe().await; + + // Should have flushed due to commit timeout + let counters = indexer_handle.observe().await; + assert_eq!(counters.batches_flushed, 1); + + // Verify packager produced a split + let packager_counters = packager_handle.process_pending_and_observe().await.state; + assert_eq!(packager_counters.splits_produced.load(Ordering::Relaxed), 1); + + // Verify uploader received the split + uploader_handle.process_pending_and_observe().await; + + universe.assert_quit().await; + } + + #[tokio::test] + async fn test_metrics_indexer_forwards_to_packager_and_uploader() { + quickwit_common::setup_logging_for_tests(); + + let universe = Universe::with_accelerated_time(); + let temp_dir = tempfile::tempdir().unwrap(); + + // Create uploader with staging expectation + let mut mock_metastore = MockMetastoreService::new(); + mock_metastore + .expect_stage_metrics_splits() + .returning(|_| Ok(EmptyResponse {})); + + let ram_storage = Arc::new(RamStorage::default()); + let (publisher_mailbox, _publisher_inbox) = + universe.create_test_mailbox::(); + let uploader = ParquetUploader::new( + UploaderType::IndexUploader, + quickwit_proto::metastore::MetastoreServiceClient::from_mock(mock_metastore), + ram_storage, + SplitsUpdateMailbox::Publisher(publisher_mailbox), + 4, + ); + let (uploader_mailbox, uploader_handle) = universe.spawn_builder().spawn(uploader); + + let (packager_mailbox, packager_handle) = + create_test_packager(&universe, temp_dir.path(), uploader_mailbox); + + let indexer = ParquetIndexer::new( + IndexUid::for_test("test-index", 0), + "test-source".to_string(), + None, + packager_mailbox, + None, + ); + + let (indexer_mailbox, indexer_handle) = universe.spawn_builder().spawn(indexer); + + // Send batch with force_commit to trigger flush + let batch = create_test_batch(20); + let processed = ProcessedParquetBatch::new( + batch, + SourceCheckpointDelta::from_range(0..20), + true, // force_commit + ); + + indexer_mailbox.send_message(processed).await.unwrap(); + + // Process indexer messages + let indexer_counters = indexer_handle.process_pending_and_observe().await.state; + + // Verify indexer flushed a batch + assert_eq!(indexer_counters.batches_received, 1); + assert_eq!(indexer_counters.batches_flushed, 1); + + // Verify packager produced a split + let packager_counters = packager_handle.process_pending_and_observe().await.state; + assert_eq!(packager_counters.splits_produced.load(Ordering::Relaxed), 1); + + // Wait for the uploader to stage the split + wait_for_staged_splits(&uploader_handle, 1) + .await + .expect("Uploader should have staged 1 split"); + + universe.assert_quit().await; + } +} diff --git a/quickwit/quickwit-indexing/src/actors/parquet_packager.rs b/quickwit/quickwit-indexing/src/actors/parquet_packager.rs new file mode 100644 index 00000000000..a23b88789e4 --- /dev/null +++ b/quickwit/quickwit-indexing/src/actors/parquet_packager.rs @@ -0,0 +1,468 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! ParquetPackager actor that writes sorted RecordBatches to Parquet split files. +//! +//! This actor sits between ParquetIndexer and ParquetUploader. It receives +//! concatenated RecordBatches from the indexer and performs the IO-bound work: +//! Parquet encoding, compression, and file writing. This decouples CPU-bound +//! accumulation (in the indexer) from IO-bound packaging (here), allowing +//! the indexer to continue buffering while the packager writes. + +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use anyhow::Context; +use arrow::record_batch::RecordBatch; +use async_trait::async_trait; +use quickwit_actors::{Actor, ActorContext, ActorExitStatus, Handler, Mailbox, QueueCapacity}; +use quickwit_common::runtimes::RuntimeType; +use quickwit_metastore::checkpoint::IndexCheckpointDelta; +use quickwit_parquet_engine::storage::ParquetSplitWriter; +use quickwit_proto::types::{IndexUid, PublishToken}; +use serde::Serialize; +use tokio::runtime::Handle; +use tracing::{info, warn}; + +use crate::actors::ParquetUploader; +use crate::actors::parquet_indexer::ParquetSplitBatch; +use crate::models::PublishLock; + +/// A concatenated RecordBatch ready to be written to a Parquet file. +/// +/// Sent from ParquetIndexer to ParquetPackager when the accumulator flushes +/// (either from threshold or force commit). +#[derive(Debug)] +pub struct ParquetBatchForPackager { + /// The concatenated RecordBatch, ready for Parquet encoding. + /// None when the indexer is forwarding only a checkpoint delta (e.g. on finalize + /// with no pending data). + pub batch: Option, + /// Index unique identifier for split metadata. + pub index_uid: IndexUid, + /// Checkpoint delta covering all data in this batch. + pub checkpoint_delta: IndexCheckpointDelta, + /// Publish lock for coordination. + pub publish_lock: PublishLock, + /// Optional publish token. + pub publish_token_opt: Option, +} + +/// Counters for ParquetPackager observability. +#[derive(Debug, Default, Serialize)] +pub struct ParquetPackagerCounters { + /// Number of splits produced (Parquet files written). + pub splits_produced: AtomicU64, + /// Number of bytes written to Parquet files. + pub bytes_written: AtomicU64, + /// Number of errors encountered. + pub errors: AtomicU64, +} + +impl ParquetPackagerCounters { + /// Record a split produced. + pub fn record_split(&self, size_bytes: u64) { + self.splits_produced.fetch_add(1, Ordering::Relaxed); + self.bytes_written.fetch_add(size_bytes, Ordering::Relaxed); + } + + /// Record an error. + pub fn record_error(&self) { + self.errors.fetch_add(1, Ordering::Relaxed); + } +} + +/// ParquetPackager actor that writes RecordBatches to Parquet split files. +/// +/// This actor: +/// - Receives ParquetBatchForPackager messages from ParquetIndexer +/// - Writes the RecordBatch to a Parquet file via ParquetSplitWriter (sort + encode + compress) +/// - Extracts split metadata (time range, metric names, service names) +/// - Forwards the completed ParquetSplitBatch to ParquetUploader +/// +/// Runs on the blocking runtime since Parquet encoding and file IO are CPU/IO-bound. +pub struct ParquetPackager { + /// Split writer for producing Parquet files. + split_writer: ParquetSplitWriter, + /// Mailbox for sending splits to uploader. + uploader_mailbox: Mailbox, + /// Observability counters. + counters: Arc, +} + +impl ParquetPackager { + /// Create a new ParquetPackager. + /// + /// # Arguments + /// * `split_writer` - Writer for producing Parquet files with metadata + /// * `uploader_mailbox` - Mailbox for sending splits to ParquetUploader + pub fn new( + split_writer: ParquetSplitWriter, + uploader_mailbox: Mailbox, + ) -> Self { + let counters = Arc::new(ParquetPackagerCounters::default()); + + info!( + output_dir = %split_writer.base_path().display(), + "ParquetPackager created" + ); + + Self { + split_writer, + uploader_mailbox, + counters, + } + } + + /// Get a reference to the counters. + pub fn counters(&self) -> &Arc { + &self.counters + } +} + +#[async_trait] +impl Actor for ParquetPackager { + type ObservableState = Arc; + + fn observable_state(&self) -> Self::ObservableState { + self.counters.clone() + } + + fn queue_capacity(&self) -> QueueCapacity { + // Bounded(1) provides backpressure on the indexer: if the packager is busy + // writing a split, the indexer will block until the write completes. + QueueCapacity::Bounded(1) + } + + fn name(&self) -> String { + "ParquetPackager".to_string() + } + + fn runtime_handle(&self) -> Handle { + RuntimeType::Blocking.get_runtime_handle() + } + + #[inline] + fn yield_after_each_message(&self) -> bool { + false + } +} + +#[async_trait] +impl Handler for ParquetPackager { + type Reply = (); + + async fn handle( + &mut self, + batch_for_packager: ParquetBatchForPackager, + ctx: &ActorContext, + ) -> Result<(), ActorExitStatus> { + let ParquetBatchForPackager { + batch, + index_uid, + checkpoint_delta, + publish_lock, + publish_token_opt, + } = batch_for_packager; + + let output_dir = self.split_writer.base_path().clone(); + let index_uid_str = index_uid.to_string(); + + let splits = if let Some(batch) = batch { + let num_rows = batch.num_rows(); + + // Write the batch to a Parquet file + match self.split_writer.write_split(&batch, &index_uid_str) { + Ok(split) => { + let size_bytes = split.metadata.size_bytes; + self.counters.record_split(size_bytes); + + info!( + split_id = %split.metadata.split_id, + num_rows, + size_bytes, + "ParquetPackager wrote split" + ); + vec![split] + } + Err(error) => { + warn!(error = %error, num_rows, "ParquetPackager failed to write split"); + self.counters.record_error(); + return Err(ActorExitStatus::Failure( + anyhow::anyhow!("Failed to write Parquet split: {}", error).into(), + )); + } + } + } else { + // No batch data — just forwarding checkpoint delta (e.g. on finalize). + Vec::new() + }; + + ctx.record_progress(); + + // Forward to uploader + let split_batch = ParquetSplitBatch { + index_uid, + splits, + output_dir, + checkpoint_delta, + publish_lock, + publish_token_opt, + }; + + ctx.send_message(&self.uploader_mailbox, split_batch) + .await + .context("failed to send ParquetSplitBatch to uploader") + .map_err(|e| ActorExitStatus::Failure(e.into()))?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::Ordering as AtomicOrdering; + use std::time::Duration; + + use arrow::array::{ + ArrayRef, DictionaryArray, Float64Array, Int32Array, StringArray, UInt8Array, UInt64Array, + }; + use arrow::datatypes::Int32Type; + use arrow::record_batch::RecordBatch; + use parquet::variant::{VariantArrayBuilder, VariantBuilderExt}; + use quickwit_actors::{ActorHandle, Universe}; + use quickwit_common::test_utils::wait_until_predicate; + use quickwit_metastore::checkpoint::{IndexCheckpointDelta, SourceCheckpointDelta}; + use quickwit_parquet_engine::schema::ParquetSchema; + use quickwit_parquet_engine::storage::ParquetWriterConfig; + use quickwit_proto::metastore::{EmptyResponse, MockMetastoreService}; + use quickwit_storage::RamStorage; + + use super::*; + use crate::actors::{ParquetPublisher, SplitsUpdateMailbox, UploaderType}; + + fn create_dict_array(values: &[&str]) -> ArrayRef { + let keys: Vec = (0..values.len()).map(|i| i as i32).collect(); + let string_array = StringArray::from(values.to_vec()); + Arc::new( + DictionaryArray::::try_new(Int32Array::from(keys), Arc::new(string_array)) + .unwrap(), + ) + } + + fn create_nullable_dict_array(values: &[Option<&str>]) -> ArrayRef { + let keys: Vec> = values + .iter() + .enumerate() + .map(|(i, v)| v.map(|_| i as i32)) + .collect(); + let string_values: Vec<&str> = values.iter().filter_map(|v| *v).collect(); + let string_array = StringArray::from(string_values); + Arc::new( + DictionaryArray::::try_new(Int32Array::from(keys), Arc::new(string_array)) + .unwrap(), + ) + } + + fn create_variant_array(num_rows: usize, fields: Option<&[(&str, &str)]>) -> ArrayRef { + let mut builder = VariantArrayBuilder::new(num_rows); + for _ in 0..num_rows { + match fields { + Some(kv_pairs) => { + let mut obj = builder.new_object(); + for &(key, value) in kv_pairs { + obj = obj.with_field(key, value); + } + obj.finish(); + } + None => { + builder.append_null(); + } + } + } + ArrayRef::from(builder.build()) + } + + fn create_test_batch(num_rows: usize) -> RecordBatch { + let schema = ParquetSchema::new(); + + let metric_names: Vec<&str> = vec!["cpu.usage"; num_rows]; + let metric_name: ArrayRef = create_dict_array(&metric_names); + let metric_type: ArrayRef = Arc::new(UInt8Array::from(vec![0u8; num_rows])); + let metric_unit: ArrayRef = Arc::new(StringArray::from(vec![Some("bytes"); num_rows])); + let timestamps: Vec = (0..num_rows).map(|i| 100 + i as u64).collect(); + let timestamp_secs: ArrayRef = Arc::new(UInt64Array::from(timestamps)); + let start_timestamp_secs: ArrayRef = + Arc::new(UInt64Array::from(vec![None::; num_rows])); + let values: Vec = (0..num_rows).map(|i| 42.0 + i as f64).collect(); + let value: ArrayRef = Arc::new(Float64Array::from(values)); + let tag_service: ArrayRef = create_nullable_dict_array(&vec![Some("web"); num_rows]); + let tag_env: ArrayRef = create_nullable_dict_array(&vec![Some("prod"); num_rows]); + let tag_datacenter: ArrayRef = + create_nullable_dict_array(&vec![Some("us-east-1"); num_rows]); + let tag_region: ArrayRef = create_nullable_dict_array(&vec![None; num_rows]); + let tag_host: ArrayRef = create_nullable_dict_array(&vec![Some("host-001"); num_rows]); + let attributes: ArrayRef = create_variant_array(num_rows, None); + let service_name: ArrayRef = create_dict_array(&vec!["my-service"; num_rows]); + let resource_attributes: ArrayRef = create_variant_array(num_rows, None); + + RecordBatch::try_new( + schema.arrow_schema().clone(), + vec![ + metric_name, + metric_type, + metric_unit, + timestamp_secs, + start_timestamp_secs, + value, + tag_service, + tag_env, + tag_datacenter, + tag_region, + tag_host, + attributes, + service_name, + resource_attributes, + ], + ) + .unwrap() + } + + fn create_test_uploader( + universe: &Universe, + ) -> (Mailbox, ActorHandle) { + let mut mock_metastore = MockMetastoreService::new(); + mock_metastore + .expect_stage_metrics_splits() + .returning(|_| Ok(EmptyResponse {})); + + let ram_storage = Arc::new(RamStorage::default()); + let (publisher_mailbox, _publisher_inbox) = + universe.create_test_mailbox::(); + + let uploader = ParquetUploader::new( + UploaderType::IndexUploader, + quickwit_proto::metastore::MetastoreServiceClient::from_mock(mock_metastore), + ram_storage, + SplitsUpdateMailbox::Publisher(publisher_mailbox), + 4, + ); + universe.spawn_builder().spawn(uploader) + } + + fn create_test_packager( + universe: &Universe, + temp_dir: &std::path::Path, + uploader_mailbox: Mailbox, + ) -> (Mailbox, ActorHandle) { + let schema = ParquetSchema::new(); + let writer_config = ParquetWriterConfig::default(); + let split_writer = ParquetSplitWriter::new(schema, writer_config, temp_dir); + + let packager = ParquetPackager::new(split_writer, uploader_mailbox); + universe.spawn_builder().spawn(packager) + } + + async fn wait_for_staged_splits( + uploader_handle: &ActorHandle, + expected_splits: u64, + ) -> anyhow::Result<()> { + wait_until_predicate( + || async { + uploader_handle.process_pending_and_observe().await; + let counters = uploader_handle.last_observation(); + counters.num_staged_splits.load(AtomicOrdering::Relaxed) >= expected_splits + }, + Duration::from_secs(15), + Duration::from_millis(50), + ) + .await + .map_err(|_| anyhow::anyhow!("Timeout waiting for {} staged splits", expected_splits)) + } + + #[tokio::test] + async fn test_packager_writes_split() { + let universe = Universe::with_accelerated_time(); + let temp_dir = tempfile::tempdir().unwrap(); + + let (uploader_mailbox, uploader_handle) = create_test_uploader(&universe); + let (packager_mailbox, packager_handle) = + create_test_packager(&universe, temp_dir.path(), uploader_mailbox); + + // Send a batch to the packager + let batch = create_test_batch(10); + let batch_for_packager = ParquetBatchForPackager { + batch: Some(batch), + index_uid: IndexUid::for_test("test-index", 0), + checkpoint_delta: IndexCheckpointDelta { + source_id: "test-source".to_string(), + source_delta: SourceCheckpointDelta::from_range(0..10), + }, + publish_lock: PublishLock::default(), + publish_token_opt: None, + }; + + packager_mailbox + .send_message(batch_for_packager) + .await + .unwrap(); + + let counters = packager_handle.process_pending_and_observe().await.state; + + assert_eq!(counters.splits_produced.load(AtomicOrdering::Relaxed), 1); + assert!(counters.bytes_written.load(AtomicOrdering::Relaxed) > 0); + + // Verify uploader received the split + wait_for_staged_splits(&uploader_handle, 1) + .await + .expect("Uploader should have staged 1 split"); + + universe.assert_quit().await; + } + + #[tokio::test] + async fn test_packager_forwards_empty_checkpoint() { + let universe = Universe::with_accelerated_time(); + let temp_dir = tempfile::tempdir().unwrap(); + + let (uploader_mailbox, _uploader_handle) = create_test_uploader(&universe); + let (packager_mailbox, packager_handle) = + create_test_packager(&universe, temp_dir.path(), uploader_mailbox); + + // Send a batch with no data (just checkpoint delta) + let batch_for_packager = ParquetBatchForPackager { + batch: None, + index_uid: IndexUid::for_test("test-index", 0), + checkpoint_delta: IndexCheckpointDelta { + source_id: "test-source".to_string(), + source_delta: SourceCheckpointDelta::from_range(0..10), + }, + publish_lock: PublishLock::default(), + publish_token_opt: None, + }; + + packager_mailbox + .send_message(batch_for_packager) + .await + .unwrap(); + + let counters = packager_handle.process_pending_and_observe().await.state; + + // No split should be produced + assert_eq!(counters.splits_produced.load(AtomicOrdering::Relaxed), 0); + assert_eq!(counters.bytes_written.load(AtomicOrdering::Relaxed), 0); + + universe.assert_quit().await; + } +} diff --git a/quickwit/quickwit-indexing/src/actors/parquet_uploader.rs b/quickwit/quickwit-indexing/src/actors/parquet_uploader.rs new file mode 100644 index 00000000000..2f5ea943749 --- /dev/null +++ b/quickwit/quickwit-indexing/src/actors/parquet_uploader.rs @@ -0,0 +1,713 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! ParquetUploader actor for staging and uploading metrics splits. +//! +//! This actor mirrors the Uploader pattern for logs but handles ParquetSplit +//! produced by ParquetIndexer. It stages splits to the metastore and uploads +//! Parquet files to storage. + +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::Ordering; + +use anyhow::Context; +use async_trait::async_trait; +use once_cell::sync::OnceCell; +use quickwit_actors::{Actor, ActorContext, ActorExitStatus, Handler, QueueCapacity}; +use quickwit_common::spawn_named_task; +use quickwit_metastore::StageMetricsSplitsRequestExt; +use quickwit_parquet_engine::split::MetricsSplitMetadata; +use quickwit_proto::metastore::{MetastoreService, MetastoreServiceClient}; +use quickwit_storage::Storage; +use tokio::sync::{Semaphore, SemaphorePermit}; +use tracing::{Instrument, Span, debug, info, instrument, warn}; + +use crate::actors::sequencer::SequencerCommand; +use crate::actors::uploader::{SplitsUpdateMailbox, SplitsUpdateSender}; +use crate::actors::{ParquetPublisher, ParquetSplitBatch, UploaderCounters, UploaderType}; +use crate::metrics::INDEXER_METRICS; +use crate::models::ParquetSplitsUpdate; + +/// Concurrent upload permits for metrics uploader. +/// Uses same permit pool as indexer uploads. +static CONCURRENT_UPLOAD_PERMITS_METRICS: OnceCell = OnceCell::new(); + +/// ParquetUploader actor for staging and uploading metrics splits. +/// +/// Receives ParquetSplitBatch from ParquetIndexer, stages splits to the metastore, +/// uploads Parquet files to storage, and sends ParquetSplitsUpdate downstream. +#[derive(Clone)] +pub struct ParquetUploader { + uploader_type: UploaderType, + metastore: MetastoreServiceClient, + split_store: Arc, + split_update_mailbox: SplitsUpdateMailbox, + max_concurrent_uploads: usize, + counters: UploaderCounters, +} + +impl ParquetUploader { + /// Create a new ParquetUploader. + pub fn new( + uploader_type: UploaderType, + metastore: MetastoreServiceClient, + split_store: Arc, + split_update_mailbox: SplitsUpdateMailbox, + max_concurrent_uploads: usize, + ) -> Self { + Self { + uploader_type, + metastore, + split_store, + split_update_mailbox, + max_concurrent_uploads, + counters: Default::default(), + } + } + + /// Acquire a semaphore permit for upload. + async fn acquire_semaphore( + &self, + ctx: &ActorContext, + ) -> anyhow::Result> { + let _guard = ctx.protect_zone(); + let concurrent_upload_permits = CONCURRENT_UPLOAD_PERMITS_METRICS + .get_or_init(|| Semaphore::const_new(self.max_concurrent_uploads)); + let gauge = INDEXER_METRICS + .available_concurrent_upload_permits + .with_label_values(["metrics"]); + gauge.set(concurrent_upload_permits.available_permits() as i64); + concurrent_upload_permits + .acquire() + .await + .context("the metrics uploader semaphore is closed (this should never happen)") + } +} + +#[async_trait] +impl Actor for ParquetUploader { + type ObservableState = UploaderCounters; + + fn observable_state(&self) -> Self::ObservableState { + self.counters.clone() + } + + fn queue_capacity(&self) -> QueueCapacity { + // Buffer several splits to decouple ParquetIndexer from S3 upload latency. + QueueCapacity::Bounded(3) + } + + fn name(&self) -> String { + format!("ParquetUploader({:?})", self.uploader_type) + } +} + +#[async_trait] +impl Handler for ParquetUploader { + type Reply = (); + + #[instrument(name = "parquet_uploader", skip_all, fields(index_uid, num_splits))] + async fn handle( + &mut self, + batch: ParquetSplitBatch, + ctx: &ActorContext, + ) -> Result<(), ActorExitStatus> { + let index_uid = batch.index_uid.clone(); + let num_splits = batch.splits.len(); + + tracing::Span::current().record("index_uid", index_uid.to_string()); + tracing::Span::current().record("num_splits", num_splits); + + if batch.splits.is_empty() { + debug!("received empty ParquetSplitBatch, forwarding checkpoint only"); + // Even with no splits, the checkpoint delta may contain EOF positions + // that must reach the publisher for graceful decommission. + let sender = match self.split_update_mailbox.get_sender(ctx).await { + Ok(s) => s, + Err(e) => { + warn!(error = %e, "failed to reserve sequencer position for empty batch"); + return Ok(()); + } + }; + + let update = ParquetSplitsUpdate { + index_uid: index_uid.clone(), + new_splits: Vec::new(), + replaced_split_ids: Vec::new(), + checkpoint_delta_opt: Some(batch.checkpoint_delta), + publish_lock: batch.publish_lock, + publish_token_opt: batch.publish_token_opt, + parent_span: tracing::Span::current(), + }; + match sender { + SplitsUpdateSender::Publisher(mailbox) => { + if let Err(e) = ctx.send_message(&mailbox, update).await { + warn!(error = %e, "failed to send empty batch to publisher"); + } + } + SplitsUpdateSender::Sequencer(tx) => { + if tx.send(SequencerCommand::Proceed(update)).is_err() { + warn!("sequencer receiver dropped for empty batch"); + } + } + } + return Ok(()); + } + + // Reserve position in sequencer BEFORE starting async work. + // This ensures that even if uploads complete out of order, they will + // be published in the order they were submitted. + let sender = match self.split_update_mailbox.get_sender(ctx).await { + Ok(s) => s, + Err(e) => { + warn!(error = %e, "failed to reserve sequencer position"); + return Ok(()); + } + }; + + // Acquire upload permit + let permit_guard = self.acquire_semaphore(ctx).await?; + let kill_switch = ctx.kill_switch().clone(); + + if kill_switch.is_dead() { + warn!("kill switch was activated, cancelling metrics upload"); + // Discard the sequencer position since we're not proceeding + if let SplitsUpdateSender::Sequencer(tx) = sender { + let _ = tx.send(SequencerCommand::Discard); + } + return Err(ActorExitStatus::Killed); + } + + // Clone what we need for the async task + let metastore = self.metastore.clone(); + let split_store = self.split_store.clone(); + let counters = self.counters.clone(); + let ctx_clone = ctx.clone(); + + let output_dir = batch.output_dir; + let checkpoint_delta = batch.checkpoint_delta; + let publish_lock = batch.publish_lock; + let publish_token_opt = batch.publish_token_opt; + let splits = batch.splits; + + debug!( + index_uid = %index_uid, + num_splits = splits.len(), + "starting metrics split staging and upload" + ); + + spawn_named_task( + async move { + // Check publish lock before doing any work. Between the time + // the handler reserved the sequencer position and now, the lock + // may have died (e.g. source reassignment). + if publish_lock.is_dead() { + info!("splits' publish lock is dead"); + if let SplitsUpdateSender::Sequencer(tx) = sender { + let _ = tx.send(SequencerCommand::Discard); + } + return; + } + + // Collect metadata for staging + let splits_metadata: Vec = splits + .iter() + .map(|s| s.metadata.clone()) + .collect(); + + // Stage splits in metastore + let stage_request = match quickwit_proto::metastore::StageMetricsSplitsRequest::try_from_splits_metadata( + index_uid.clone(), + &splits_metadata, + ) { + Ok(req) => req, + Err(e) => { + warn!(error = %e, "failed to create stage metrics splits request"); + // Discard sequencer position on error + if let SplitsUpdateSender::Sequencer(tx) = sender { + let _ = tx.send(SequencerCommand::Discard); + } + kill_switch.kill(); + return; + } + }; + + if let Err(e) = metastore.clone().stage_metrics_splits(stage_request).await { + warn!(error = %e, "failed to stage metrics splits"); + // Discard sequencer position on error + if let SplitsUpdateSender::Sequencer(tx) = sender { + let _ = tx.send(SequencerCommand::Discard); + } + kill_switch.kill(); + return; + } + + counters.num_staged_splits.fetch_add(splits_metadata.len() as u64, Ordering::SeqCst); + info!( + index_uid = %index_uid, + num_splits = splits_metadata.len(), + "staged metrics splits in metastore" + ); + + // Upload Parquet files to storage + for split in &splits { + let parquet_file = split.metadata.parquet_filename(); + // Read the local Parquet file from output_dir + let local_path = output_dir.join(&parquet_file); + let file_content = match tokio::fs::read(&local_path).await { + Ok(content) => content, + Err(e) => { + warn!( + error = %e, + local_path = %local_path.display(), + split_id = %split.metadata.split_id, + parquet_file = %parquet_file, + "failed to read local parquet file" + ); + // Discard sequencer position on error + if let SplitsUpdateSender::Sequencer(tx) = sender { + let _ = tx.send(SequencerCommand::Discard); + } + kill_switch.kill(); + return; + } + }; + + let file_size = file_content.len(); + let payload: Box = Box::new(file_content); + + // Upload to S3 using the filename directly (matches logs pipeline) + if let Err(e) = split_store + .put(Path::new(&parquet_file), payload) + .await + { + warn!( + error = %e, + split_id = %split.metadata.split_id, + parquet_file = %parquet_file, + "failed to upload parquet file" + ); + // Discard sequencer position on error + if let SplitsUpdateSender::Sequencer(tx) = sender { + let _ = tx.send(SequencerCommand::Discard); + } + kill_switch.kill(); + return; + } + + counters.num_uploaded_splits.fetch_add(1, Ordering::SeqCst); + + // Delete the local parquet file after successful upload. + if let Err(e) = tokio::fs::remove_file(&local_path).await { + warn!( + error = %e, + local_path = %local_path.display(), + "failed to delete local parquet file after upload" + ); + } + + debug!( + split_id = %split.metadata.split_id, + parquet_file = %parquet_file, + file_size = file_size, + "uploaded parquet file to storage" + ); + } + + // Create ParquetSplitsUpdate and send downstream + let update = ParquetSplitsUpdate { + index_uid, + new_splits: splits_metadata, + replaced_split_ids: Vec::new(), // No merging yet + checkpoint_delta_opt: Some(checkpoint_delta), + publish_lock, + publish_token_opt, + parent_span: Span::current(), + }; + + // Send via the appropriate channel + match sender { + SplitsUpdateSender::Publisher(mailbox) => { + if let Err(e) = ctx_clone.send_message(&mailbox, update).await { + warn!(error = %e, "failed to send to publisher"); + } + } + SplitsUpdateSender::Sequencer(tx) => { + if tx.send(SequencerCommand::Proceed(update)).is_err() { + warn!("sequencer receiver dropped"); + } + } + } + + // Drop permit to allow next upload + drop(permit_guard); + } + .instrument(Span::current()), + "metrics_upload_task", + ); + + Ok(()) + } +} + +#[cfg(test)] +#[allow(clippy::disallowed_methods)] +mod tests { + use quickwit_actors::{ObservationType, Universe}; + use quickwit_metastore::checkpoint::{IndexCheckpointDelta, SourceCheckpointDelta}; + use quickwit_parquet_engine::split::{MetricsSplitMetadata, ParquetSplit, TimeRange}; + use quickwit_proto::metastore::{EmptyResponse, MockMetastoreService}; + use quickwit_proto::types::IndexUid; + use quickwit_storage::RamStorage; + + use super::*; + use crate::actors::Sequencer; + use crate::models::PublishLock; + + fn create_test_metrics_split(index_uid: &str, split_id: &str) -> ParquetSplit { + let metadata = MetricsSplitMetadata::builder() + .index_uid(index_uid) + .split_id(quickwit_parquet_engine::split::SplitId::new(split_id)) + .time_range(TimeRange::new(1000, 2000)) + .num_rows(100) + .size_bytes(1024) + .build(); + ParquetSplit::new(metadata) + } + + /// Create placeholder parquet files in the temp directory for testing. + /// The uploader expects to read these files from output_dir. + fn create_placeholder_parquet_files(temp_dir: &std::path::Path, splits: &[ParquetSplit]) { + for split in splits { + let parquet_filename = split.metadata.parquet_filename(); + let file_path = temp_dir.join(&parquet_filename); + // Write minimal valid content (actual parquet not needed for staging test) + std::fs::write(&file_path, b"placeholder parquet content") + .expect("Failed to create placeholder parquet file"); + } + } + + #[tokio::test] + async fn test_metrics_uploader_stages_and_uploads() { + quickwit_common::setup_logging_for_tests(); + + let universe = Universe::new(); + let temp_dir = tempfile::tempdir().unwrap(); + let (publisher_mailbox, _publisher_inbox) = + universe.create_test_mailbox::(); + + let mut mock_metastore = MockMetastoreService::new(); + mock_metastore + .expect_stage_metrics_splits() + .withf(|request| request.index_uid().index_id == "test-index") + .times(1) + .returning(|_| Ok(EmptyResponse {})); + + let ram_storage = Arc::new(RamStorage::default()); + let uploader = ParquetUploader::new( + UploaderType::IndexUploader, + MetastoreServiceClient::from_mock(mock_metastore), + ram_storage.clone(), + SplitsUpdateMailbox::Publisher(publisher_mailbox), + 4, + ); + + let (uploader_mailbox, uploader_handle) = universe.spawn_builder().spawn(uploader); + + // Create test batch with temp directory as output_dir + let splits = vec![create_test_metrics_split("test-index", "test-split-1")]; + // Create placeholder parquet files that the uploader will read + create_placeholder_parquet_files(temp_dir.path(), &splits); + let checkpoint_delta = IndexCheckpointDelta { + source_id: "test-source".to_string(), + source_delta: SourceCheckpointDelta::from_range(0..10), + }; + let batch = ParquetSplitBatch { + index_uid: IndexUid::new_with_random_ulid("test-index"), + splits, + output_dir: temp_dir.path().to_path_buf(), + checkpoint_delta, + publish_lock: PublishLock::default(), + publish_token_opt: None, + }; + + uploader_mailbox.send_message(batch).await.unwrap(); + + let observation = uploader_handle.process_pending_and_observe().await; + assert_eq!(observation.obs_type, ObservationType::Alive); + + // Verify counters + assert!(observation.state.num_staged_splits.load(Ordering::Relaxed) >= 1); + + // Wait briefly for the async upload task to complete file deletion + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + + // Verify local parquet files are deleted after successful upload + let remaining_parquet_files: Vec<_> = std::fs::read_dir(temp_dir.path()) + .unwrap() + .filter_map(|e| e.ok()) + .filter(|e| { + e.path() + .extension() + .map(|ext| ext == "parquet") + .unwrap_or(false) + }) + .collect(); + assert!( + remaining_parquet_files.is_empty(), + "expected parquet files to be deleted after upload, but found: {:?}", + remaining_parquet_files + .iter() + .map(|e| e.path()) + .collect::>() + ); + + // Verify the file was actually uploaded to storage + assert!( + ram_storage + .exists(std::path::Path::new("test-split-1.parquet")) + .await + .unwrap() + ); + + universe.assert_quit().await; + } + + #[tokio::test] + async fn test_metrics_uploader_deletes_local_files_after_upload() { + quickwit_common::setup_logging_for_tests(); + + let universe = Universe::new(); + let temp_dir = tempfile::tempdir().unwrap(); + let (publisher_mailbox, _publisher_inbox) = + universe.create_test_mailbox::(); + + let mut mock_metastore = MockMetastoreService::new(); + mock_metastore + .expect_stage_metrics_splits() + .times(1) + .returning(|_| Ok(EmptyResponse {})); + + let ram_storage = Arc::new(RamStorage::default()); + let uploader = ParquetUploader::new( + UploaderType::IndexUploader, + MetastoreServiceClient::from_mock(mock_metastore), + ram_storage.clone(), + SplitsUpdateMailbox::Publisher(publisher_mailbox), + 4, + ); + + let (uploader_mailbox, uploader_handle) = universe.spawn_builder().spawn(uploader); + + // Create multiple splits with parquet files + let splits = vec![ + create_test_metrics_split("test-index", "split-a"), + create_test_metrics_split("test-index", "split-b"), + ]; + create_placeholder_parquet_files(temp_dir.path(), &splits); + + // Verify files exist before upload + assert!(temp_dir.path().join("split-a.parquet").exists()); + assert!(temp_dir.path().join("split-b.parquet").exists()); + + let checkpoint_delta = IndexCheckpointDelta { + source_id: "test-source".to_string(), + source_delta: SourceCheckpointDelta::from_range(0..10), + }; + let batch = ParquetSplitBatch { + index_uid: IndexUid::new_with_random_ulid("test-index"), + splits, + output_dir: temp_dir.path().to_path_buf(), + checkpoint_delta, + publish_lock: PublishLock::default(), + publish_token_opt: None, + }; + + uploader_mailbox.send_message(batch).await.unwrap(); + + let observation = uploader_handle.process_pending_and_observe().await; + assert_eq!(observation.obs_type, ObservationType::Alive); + + // Wait for async upload task to complete + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + + // Both local files should be deleted + assert!( + !temp_dir.path().join("split-a.parquet").exists(), + "split-a.parquet should be deleted after upload" + ); + assert!( + !temp_dir.path().join("split-b.parquet").exists(), + "split-b.parquet should be deleted after upload" + ); + + // Both files should exist in remote storage + assert!( + ram_storage + .exists(std::path::Path::new("split-a.parquet")) + .await + .unwrap() + ); + assert!( + ram_storage + .exists(std::path::Path::new("split-b.parquet")) + .await + .unwrap() + ); + + universe.assert_quit().await; + } + + #[tokio::test] + async fn test_metrics_uploader_handles_empty_batch() { + quickwit_common::setup_logging_for_tests(); + + let universe = Universe::new(); + let temp_dir = tempfile::tempdir().unwrap(); + let (publisher_mailbox, _publisher_inbox) = + universe.create_test_mailbox::(); + + let mut mock_metastore = MockMetastoreService::new(); + // Should NOT call stage_metrics_splits for empty batch + mock_metastore.expect_stage_metrics_splits().never(); + + let ram_storage = Arc::new(RamStorage::default()); + let uploader = ParquetUploader::new( + UploaderType::IndexUploader, + MetastoreServiceClient::from_mock(mock_metastore), + ram_storage.clone(), + SplitsUpdateMailbox::Publisher(publisher_mailbox), + 4, + ); + + let (uploader_mailbox, uploader_handle) = universe.spawn_builder().spawn(uploader); + + // Create empty batch with valid checkpoint delta (1 position) + let checkpoint_delta = IndexCheckpointDelta { + source_id: "test-source".to_string(), + source_delta: SourceCheckpointDelta::from_range(0..1), + }; + let batch = ParquetSplitBatch { + index_uid: IndexUid::new_with_random_ulid("test-index"), + splits: Vec::new(), + output_dir: temp_dir.path().to_path_buf(), + checkpoint_delta, + publish_lock: PublishLock::default(), + publish_token_opt: None, + }; + + uploader_mailbox.send_message(batch).await.unwrap(); + + let observation = uploader_handle.process_pending_and_observe().await; + assert_eq!(observation.obs_type, ObservationType::Alive); + + // Should not have staged any splits + assert_eq!( + observation.state.num_staged_splits.load(Ordering::Relaxed), + 0 + ); + + universe.assert_quit().await; + } + + #[tokio::test] + async fn test_metrics_uploader_with_sequencer_ordering() { + // This test verifies that when using the Sequencer variant: + // 1. Messages flow through the sequencer to the publisher + // 2. The sequencer maintains FIFO ordering even if uploads complete out of order + quickwit_common::setup_logging_for_tests(); + + let universe = Universe::new(); + let temp_dir = tempfile::tempdir().unwrap(); + + // Create a simple receiver actor to collect ParquetSplitsUpdate messages + // We use a test mailbox for ParquetPublisher to capture what would be sent + let (publisher_mailbox, publisher_inbox) = + universe.create_test_mailbox::(); + + // Create sequencer that forwards to publisher + let sequencer = Sequencer::new(publisher_mailbox); + let (sequencer_mailbox, _sequencer_handle) = universe.spawn_builder().spawn(sequencer); + + let mut mock_metastore = MockMetastoreService::new(); + // Allow multiple stage calls for the batches + mock_metastore + .expect_stage_metrics_splits() + .returning(|_| Ok(EmptyResponse {})); + + let ram_storage = Arc::new(RamStorage::default()); + let uploader = ParquetUploader::new( + UploaderType::IndexUploader, + MetastoreServiceClient::from_mock(mock_metastore), + ram_storage.clone(), + SplitsUpdateMailbox::Sequencer(sequencer_mailbox), + 4, + ); + + let (uploader_mailbox, uploader_handle) = universe.spawn_builder().spawn(uploader); + + // Send batches with splits that can be identified by split_id + for i in 1..=3 { + let splits = vec![create_test_metrics_split( + "test-index", + &format!("split-{}", i), + )]; + // Create placeholder parquet files that the uploader will read + create_placeholder_parquet_files(temp_dir.path(), &splits); + let checkpoint_delta = IndexCheckpointDelta { + source_id: "test-source".to_string(), + source_delta: SourceCheckpointDelta::from_range((i * 10)..(i * 10 + 10)), + }; + let batch = ParquetSplitBatch { + index_uid: IndexUid::new_with_random_ulid("test-index"), + splits, + output_dir: temp_dir.path().to_path_buf(), + checkpoint_delta, + publish_lock: PublishLock::default(), + publish_token_opt: None, + }; + uploader_mailbox.send_message(batch).await.unwrap(); + } + + // Wait for all messages to be processed + // The uploader spawns background tasks, so we need to give them time + uploader_handle.process_pending_and_observe().await; + + // Give background tasks time to complete + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + + // Drain inbox and verify ordering + let mut received_split_ids = Vec::new(); + let messages = publisher_inbox.drain_for_test(); + for msg in messages { + // The inbox contains typed messages, we need to access the ParquetSplitsUpdate + if let Some(update) = msg.downcast_ref::() { + for split in &update.new_splits { + received_split_ids.push(split.split_id.as_str().to_string()); + } + } + } + + // Verify we received all 3 splits in order + // The sequencer ensures FIFO delivery: split-1, split-2, split-3 + assert_eq!( + received_split_ids, + vec!["split-1", "split-2", "split-3"], + "Sequencer should maintain FIFO ordering" + ); + + universe.assert_quit().await; + } +} diff --git a/quickwit/quickwit-indexing/src/actors/publisher.rs b/quickwit/quickwit-indexing/src/actors/publisher.rs index f5486b8757a..c4ec75b1a8a 100644 --- a/quickwit/quickwit-indexing/src/actors/publisher.rs +++ b/quickwit/quickwit-indexing/src/actors/publisher.rs @@ -16,14 +16,20 @@ use anyhow::Context; use async_trait::async_trait; use fail::fail_point; use quickwit_actors::{Actor, ActorContext, Handler, Mailbox, QueueCapacity}; -use quickwit_proto::metastore::{MetastoreService, MetastoreServiceClient, PublishSplitsRequest}; +use quickwit_metastore::checkpoint::IndexCheckpointDelta; +use quickwit_proto::metastore::{ + MetastoreService, MetastoreServiceClient, PublishMetricsSplitsRequest, PublishSplitsRequest, +}; use serde::Serialize; -use tracing::{info, instrument, warn}; +use tracing::{info, instrument}; -use crate::actors::MergePlanner; -use crate::models::{NewSplits, SplitsUpdate}; +use crate::actors::{DocProcessor, MergePlanner, ParquetDocProcessor, Processor}; +use crate::models::{NewSplits, ParquetSplitsUpdate, SplitsUpdate}; use crate::source::{SourceActor, SuggestTruncate}; +/// Type alias for the metrics publisher specialization. +pub type ParquetPublisher = Publisher; + #[derive(Clone, Debug, Default, Serialize)] pub struct PublisherCounters { pub num_published_splits: u64, @@ -35,6 +41,7 @@ pub struct PublisherCounters { pub enum PublisherType { MainPublisher, MergePublisher, + ParquetPublisher, } impl PublisherType { @@ -42,6 +49,7 @@ impl PublisherType { match self { PublisherType::MainPublisher => "Publisher", PublisherType::MergePublisher => "MergePublisher", + PublisherType::ParquetPublisher => "ParquetPublisher", } } } @@ -51,22 +59,34 @@ impl PublisherType { #[derive(Debug)] pub(crate) struct DisconnectMergePlanner; -#[derive(Clone)] -pub struct Publisher { +pub struct Publisher { publisher_type: PublisherType, metastore: MetastoreServiceClient, merge_planner_mailbox_opt: Option>, - source_mailbox_opt: Option>, + source_mailbox_opt: Option>>, counters: PublisherCounters, } -impl Publisher { +// Manual Clone impl to avoid a spurious `P: Clone` bound from #[derive(Clone)]. +impl Clone for Publisher

{ + fn clone(&self) -> Self { + Self { + publisher_type: self.publisher_type, + metastore: self.metastore.clone(), + merge_planner_mailbox_opt: self.merge_planner_mailbox_opt.clone(), + source_mailbox_opt: self.source_mailbox_opt.clone(), + counters: self.counters.clone(), + } + } +} + +impl Publisher

{ pub fn new( publisher_type: PublisherType, metastore: MetastoreServiceClient, merge_planner_mailbox_opt: Option>, - source_mailbox_opt: Option>, - ) -> Publisher { + source_mailbox_opt: Option>>, + ) -> Publisher

{ type ObservableState = PublisherCounters; fn observable_state(&self) -> Self::ObservableState { @@ -93,12 +140,13 @@ impl Actor for Publisher { match self.publisher_type { PublisherType::MainPublisher => QueueCapacity::Bounded(1), PublisherType::MergePublisher => QueueCapacity::Unbounded, + PublisherType::ParquetPublisher => QueueCapacity::Bounded(1), } } } #[async_trait] -impl Handler for Publisher { +impl Handler for Publisher

{ type Reply = (); async fn handle( @@ -113,7 +161,7 @@ impl Handler for Publisher { } #[async_trait] -impl Handler for Publisher { +impl Handler for Publisher { type Reply = (); #[instrument(name="publisher", parent=split_update.parent_span.id(), skip(self, ctx))] @@ -134,11 +182,7 @@ impl Handler for Publisher { .. } = split_update; - let index_checkpoint_delta_json_opt = checkpoint_delta_opt - .as_ref() - .map(serde_json::to_string) - .transpose() - .context("failed to serialize `IndexCheckpointDelta`")?; + let index_checkpoint_delta_json_opt = serialize_checkpoint_delta(&checkpoint_delta_opt)?; let split_ids: Vec = new_splits .iter() .map(|split| split.split_id.clone()) @@ -163,25 +207,7 @@ impl Handler for Publisher { return Ok(()); } info!("publish-new-splits"); - if let Some(source_mailbox) = self.source_mailbox_opt.as_ref() { - if let Some(checkpoint) = checkpoint_delta_opt { - // We voluntarily do not log anything here. - // - // Not being to send the truncation message is a common event and should not be - // considered an error. For instance, if the source is a - // FileSource, it will terminate upon EOF and drop its - // mailbox. - let suggest_truncate_res = ctx - .send_message( - source_mailbox, - SuggestTruncate(checkpoint.source_delta.get_source_checkpoint()), - ) - .await; - if let Err(send_truncate_err) = suggest_truncate_res { - warn!(error=?send_truncate_err, "failed to send truncate message from publisher to source"); - } - } - } + suggest_truncate(ctx, &self.source_mailbox_opt, checkpoint_delta_opt).await; if !new_splits.is_empty() { // The merge planner is not necessarily awake and this is not an error. @@ -207,6 +233,65 @@ impl Handler for Publisher { } } +#[async_trait] +impl Handler for Publisher { + type Reply = (); + + #[instrument(name = "parquet_publisher", parent = split_update.parent_span.id(), skip(self, ctx))] + async fn handle( + &mut self, + split_update: ParquetSplitsUpdate, + ctx: &ActorContext, + ) -> Result<(), quickwit_actors::ActorExitStatus> { + let ParquetSplitsUpdate { + index_uid, + new_splits, + replaced_split_ids, + checkpoint_delta_opt, + publish_lock, + publish_token_opt, + .. + } = split_update; + + let index_checkpoint_delta_json_opt = serialize_checkpoint_delta(&checkpoint_delta_opt)?; + let split_ids: Vec = new_splits + .iter() + .map(|split| split.split_id.as_str().to_string()) + .collect(); + if let Some(_guard) = publish_lock.acquire().await { + let publish_request = PublishMetricsSplitsRequest { + index_uid: Some(index_uid.clone()), + staged_split_ids: split_ids.clone(), + replaced_split_ids: replaced_split_ids.clone(), + index_checkpoint_delta_json_opt, + publish_token_opt: publish_token_opt.clone(), + }; + ctx.protect_future(self.metastore.publish_metrics_splits(publish_request)) + .await + .context("failed to publish metrics splits")?; + } else { + info!( + split_ids=?split_ids, + "Splits' publish lock is dead." + ); + return Ok(()); + } + info!("publish-metrics-splits"); + suggest_truncate(ctx, &self.source_mailbox_opt, checkpoint_delta_opt).await; + + if !split_ids.is_empty() { + if replaced_split_ids.is_empty() { + self.counters.num_published_splits += 1; + } else { + self.counters.num_replace_operations += 1; + } + } else { + self.counters.num_empty_splits += 1; + } + Ok(()) + } +} + #[cfg(test)] mod tests { use quickwit_actors::Universe; @@ -220,6 +305,7 @@ mod tests { use super::*; use crate::models::PublishLock; + use crate::source::SuggestTruncate; #[tokio::test] async fn test_publisher_publish_operation() { @@ -469,3 +555,168 @@ mod tests { universe.assert_quit().await; } } + +#[cfg(test)] +mod parquet_publisher_tests { + use quickwit_actors::Universe; + use quickwit_metastore::checkpoint::{IndexCheckpointDelta, SourceCheckpointDelta}; + use quickwit_parquet_engine::split::{MetricsSplitMetadata, SplitId, TimeRange}; + use quickwit_proto::metastore::{EmptyResponse, MetastoreServiceClient, MockMetastoreService}; + use quickwit_proto::types::IndexUid; + use tracing::Span; + + use super::*; + use crate::models::PublishLock; + + fn create_test_metrics_split_metadata(index_uid: &str, split_id: &str) -> MetricsSplitMetadata { + MetricsSplitMetadata::builder() + .index_uid(index_uid) + .split_id(SplitId::new(split_id)) + .time_range(TimeRange::new(1000, 2000)) + .num_rows(100) + .size_bytes(1024) + .build() + } + + #[tokio::test] + async fn test_metrics_publisher_publishes_splits() { + let universe = Universe::with_accelerated_time(); + + let mut mock_metastore = MockMetastoreService::new(); + mock_metastore + .expect_publish_metrics_splits() + .withf(|request| { + request.index_uid().to_string().starts_with("test-index:") + && request.staged_split_ids == vec!["split-1".to_string()] + && request.replaced_split_ids.is_empty() + && request.index_checkpoint_delta_json_opt.is_some() + && request.publish_token_opt.is_none() + }) + .times(1) + .returning(|_| Ok(EmptyResponse {})); + + let publisher = ParquetPublisher::new( + PublisherType::ParquetPublisher, + MetastoreServiceClient::from_mock(mock_metastore), + None, + None, + ); + let (publisher_mailbox, publisher_handle) = universe.spawn_builder().spawn(publisher); + + let update = ParquetSplitsUpdate { + index_uid: IndexUid::for_test("test-index", 0), + new_splits: vec![create_test_metrics_split_metadata( + "test-index:00000000000000000000000000", + "split-1", + )], + replaced_split_ids: Vec::new(), + checkpoint_delta_opt: Some(IndexCheckpointDelta { + source_id: "test-source".to_string(), + source_delta: SourceCheckpointDelta::from_range(0..10), + }), + publish_lock: PublishLock::default(), + publish_token_opt: None, + parent_span: Span::none(), + }; + + publisher_mailbox.send_message(update).await.unwrap(); + + let observation = publisher_handle.process_pending_and_observe().await.state; + assert_eq!(observation.num_published_splits, 1); + assert_eq!(observation.num_replace_operations, 0); + assert_eq!(observation.num_empty_splits, 0); + + universe.assert_quit().await; + } + + #[tokio::test] + async fn test_metrics_publisher_handles_empty_splits() { + let universe = Universe::with_accelerated_time(); + + let mut mock_metastore = MockMetastoreService::new(); + mock_metastore + .expect_publish_metrics_splits() + .withf(|request| { + request.index_uid().to_string().starts_with("test-index:") + && request.staged_split_ids.is_empty() + && request.replaced_split_ids.is_empty() + && request.index_checkpoint_delta_json_opt.is_some() + }) + .times(1) + .returning(|_| Ok(EmptyResponse {})); + + let publisher = ParquetPublisher::new( + PublisherType::ParquetPublisher, + MetastoreServiceClient::from_mock(mock_metastore), + None, + None, + ); + let (publisher_mailbox, publisher_handle) = universe.spawn_builder().spawn(publisher); + + let update = ParquetSplitsUpdate { + index_uid: IndexUid::for_test("test-index", 0), + new_splits: Vec::new(), + replaced_split_ids: Vec::new(), + checkpoint_delta_opt: Some(IndexCheckpointDelta { + source_id: "test-source".to_string(), + source_delta: SourceCheckpointDelta::from_range(0..1), + }), + publish_lock: PublishLock::default(), + publish_token_opt: None, + parent_span: Span::none(), + }; + + publisher_mailbox.send_message(update).await.unwrap(); + + let observation = publisher_handle.process_pending_and_observe().await.state; + assert_eq!(observation.num_published_splits, 0); + assert_eq!(observation.num_replace_operations, 0); + assert_eq!(observation.num_empty_splits, 1); + + universe.assert_quit().await; + } + + #[tokio::test] + async fn test_metrics_publisher_respects_publish_lock() { + let universe = Universe::with_accelerated_time(); + + let mut mock_metastore = MockMetastoreService::new(); + mock_metastore.expect_publish_metrics_splits().never(); + + let publisher = ParquetPublisher::new( + PublisherType::ParquetPublisher, + MetastoreServiceClient::from_mock(mock_metastore), + None, + None, + ); + let (publisher_mailbox, publisher_handle) = universe.spawn_builder().spawn(publisher); + + let publish_lock = PublishLock::default(); + publish_lock.kill().await; + + let update = ParquetSplitsUpdate { + index_uid: IndexUid::for_test("test-index", 0), + new_splits: vec![create_test_metrics_split_metadata( + "test-index:00000000000000000000000000", + "split-1", + )], + replaced_split_ids: Vec::new(), + checkpoint_delta_opt: Some(IndexCheckpointDelta { + source_id: "test-source".to_string(), + source_delta: SourceCheckpointDelta::from_range(0..10), + }), + publish_lock, + publish_token_opt: None, + parent_span: Span::none(), + }; + + publisher_mailbox.send_message(update).await.unwrap(); + + let observation = publisher_handle.process_pending_and_observe().await.state; + assert_eq!(observation.num_published_splits, 0); + assert_eq!(observation.num_replace_operations, 0); + assert_eq!(observation.num_empty_splits, 0); + + universe.assert_quit().await; + } +} diff --git a/quickwit/quickwit-indexing/src/actors/uploader.rs b/quickwit/quickwit-indexing/src/actors/uploader.rs index 79fd4e50e41..1827af7a153 100644 --- a/quickwit/quickwit-indexing/src/actors/uploader.rs +++ b/quickwit/quickwit-indexing/src/actors/uploader.rs @@ -63,85 +63,109 @@ pub enum UploaderType { DeleteUploader, } -/// [`SplitsUpdateMailbox`] wraps either a [`Mailbox`] or [`Mailbox`]. +/// [`SplitsUpdateMailbox`] wraps either a [`Mailbox>`] or [`Mailbox

`]. /// -/// It makes it possible to send a [`SplitsUpdate`] either to the [`Sequencer`] or directly -/// to [`Publisher`]. It is used in combination with `SplitsUpdateSender` that will do the send. +/// It makes it possible to send a splits update either to the [`Sequencer`] or directly +/// to the publisher actor `P`. It is used in combination with [`SplitsUpdateSender`] that +/// will do the send. /// /// This is useful as we have different requirements between the indexing pipeline and /// the merge/delete task pipelines. /// 1. In the indexing pipeline, we want to publish splits in the same order as they are produced by /// the indexer/packager to ensure we are publishing splits without "holes" in checkpoints. We -/// thus send [`SplitsUpdate`] to the [`Sequencer`] to keep the right ordering. -/// 2. In the merge pipeline and the delete task pipeline, we are merging splits and in in this -/// case, publishing order does not matter. In this case, we can just send [`SplitsUpdate`] -/// directly to the [`Publisher`]. -#[derive(Clone, Debug)] -pub enum SplitsUpdateMailbox { - Sequencer(Mailbox>), - Publisher(Mailbox), +/// thus send the update to the [`Sequencer`] to keep the right ordering. +/// 2. In the merge pipeline and the delete task pipeline, we are merging splits and in this case, +/// publishing order does not matter. In this case, we can just send the update directly to the +/// publisher. +/// +/// The default type parameter `P = Publisher` preserves backward compatibility with the +/// standard logs pipeline. For the metrics pipeline, use `SplitsUpdateMailbox`. +pub enum SplitsUpdateMailbox { + Sequencer(Mailbox>), + Publisher(Mailbox

), +} + +// Manual Clone impl to avoid a spurious `P: Clone` bound from #[derive(Clone)]. +impl Clone for SplitsUpdateMailbox

{ + fn clone(&self) -> Self { + match self { + Self::Sequencer(m) => Self::Sequencer(m.clone()), + Self::Publisher(m) => Self::Publisher(m.clone()), + } + } +} + +// Manual Debug impl to avoid a spurious `P: Debug` bound from #[derive(Debug)]. +impl std::fmt::Debug for SplitsUpdateMailbox

{ + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + Self::Sequencer(m) => f.debug_tuple("Sequencer").field(m).finish(), + Self::Publisher(m) => f.debug_tuple("Publisher").field(m).finish(), + } + } } -impl From> for SplitsUpdateMailbox { - fn from(publisher_mailbox: Mailbox) -> Self { +impl From> for SplitsUpdateMailbox

{ + fn from(publisher_mailbox: Mailbox

) -> Self { SplitsUpdateMailbox::Publisher(publisher_mailbox) } } -impl From>> for SplitsUpdateMailbox { - fn from(publisher_sequencer_mailbox: Mailbox>) -> Self { - SplitsUpdateMailbox::Sequencer(publisher_sequencer_mailbox) +impl From>> for SplitsUpdateMailbox

{ + fn from(sequencer_mailbox: Mailbox>) -> Self { + SplitsUpdateMailbox::Sequencer(sequencer_mailbox) } } -impl SplitsUpdateMailbox { - async fn get_split_update_sender( +impl SplitsUpdateMailbox

{ + pub(crate) async fn get_sender( &self, - ctx: &ActorContext, - ) -> anyhow::Result { + ctx: &ActorContext, + ) -> anyhow::Result> + where + P: Handler, + M: Send + Sync + std::fmt::Debug + 'static, + { match self { SplitsUpdateMailbox::Sequencer(sequencer_mailbox) => { // We send the future to the sequencer right away. // The sequencer will then resolve the future in their arrival order and ensure that // the publisher publishes splits in order. - let (split_uploaded_tx, split_uploaded_rx) = - oneshot::channel::>(); - ctx.send_message(sequencer_mailbox, split_uploaded_rx) - .await?; - Ok(SplitsUpdateSender::Sequencer(split_uploaded_tx)) + let (tx, rx) = oneshot::channel::>(); + ctx.send_message(sequencer_mailbox, rx).await?; + Ok(SplitsUpdateSender::Sequencer(tx)) } SplitsUpdateMailbox::Publisher(publisher_mailbox) => { - // We just need the publisher mailbox to send the split in this case. Ok(SplitsUpdateSender::Publisher(publisher_mailbox.clone())) } } } } -enum SplitsUpdateSender { - Sequencer(Sender>), - Publisher(Mailbox), +pub(crate) enum SplitsUpdateSender { + Sequencer(Sender>), + Publisher(Mailbox

), } -impl SplitsUpdateSender { - fn discard(self) -> anyhow::Result<()> { - if let SplitsUpdateSender::Sequencer(split_uploader_tx) = self { - if split_uploader_tx.send(SequencerCommand::Discard).is_err() { - bail!("failed to send cancel command to sequencer. the sequencer is probably dead"); - } +impl SplitsUpdateSender +where + P: Actor + Handler, + M: Send + Sync + std::fmt::Debug + 'static, +{ + pub(crate) fn discard(self) -> anyhow::Result<()> { + if let SplitsUpdateSender::Sequencer(split_uploader_tx) = self + && split_uploader_tx.send(SequencerCommand::Discard).is_err() + { + bail!("failed to send cancel command to sequencer. the sequencer is probably dead"); } Ok(()) } - async fn send( - self, - split_update: SplitsUpdate, - ctx: &ActorContext, - ) -> anyhow::Result<()> { + pub(crate) async fn send(self, msg: M, ctx: &ActorContext) -> anyhow::Result<()> { match self { SplitsUpdateSender::Sequencer(split_uploaded_tx) => { if let Err(publisher_message) = - split_uploaded_tx.send(SequencerCommand::Proceed(split_update)) + split_uploaded_tx.send(SequencerCommand::Proceed(msg)) { bail!( "failed to send upload split `{:?}`. the publisher is probably dead", @@ -150,7 +174,7 @@ impl SplitsUpdateSender { } } SplitsUpdateSender::Publisher(publisher_mailbox) => { - ctx.send_message(&publisher_mailbox, split_update).await?; + ctx.send_message(&publisher_mailbox, msg).await?; } } Ok(()) @@ -274,10 +298,7 @@ impl Handler for Uploader { ctx: &ActorContext, ) -> Result<(), ActorExitStatus> { fail_point!("uploader:before"); - let split_update_sender = self - .split_update_mailbox - .get_split_update_sender(ctx) - .await?; + let split_update_sender = self.split_update_mailbox.get_sender(ctx).await?; // The permit will be added back manually to the semaphore the task after it is finished. // This is not a valid usage of protected zone here. @@ -432,10 +453,7 @@ impl Handler for Uploader { empty_split: EmptySplit, ctx: &ActorContext, ) -> Result<(), ActorExitStatus> { - let split_update_sender = self - .split_update_mailbox - .get_split_update_sender(ctx) - .await?; + let split_update_sender = self.split_update_mailbox.get_sender(ctx).await?; let splits_update = SplitsUpdate { index_uid: empty_split.index_uid, new_splits: Vec::new(), diff --git a/quickwit/quickwit-indexing/src/controlled_directory.rs b/quickwit/quickwit-indexing/src/controlled_directory.rs index b209b4888d6..0a30118b8b3 100644 --- a/quickwit/quickwit-indexing/src/controlled_directory.rs +++ b/quickwit/quickwit-indexing/src/controlled_directory.rs @@ -93,7 +93,7 @@ impl Directory for ControlledDirectory { self.check_if_alive() .map_err(|io_err| OpenWriteError::wrap_io_error(io_err, path.to_path_buf()))?; - let underlying_wrt: Box = self + let underlying_wrt: Box = self .underlying .open_write(path)? .into_inner() @@ -147,14 +147,18 @@ impl HotswappableIoControls { impl IoControlsAccess for HotswappableIoControls { fn apply(&self, f: F) -> R - where F: Fn(&IoControls) -> R { + where + F: Fn(&IoControls) -> R, + { let guard = self.0.load(); f(&guard) } } // Wrapper to work around the orphan rule. (hence the word "Adopted"). -struct AdoptedControlledWrite(ControlledWrite>); +struct AdoptedControlledWrite( + ControlledWrite>, +); impl io::Write for AdoptedControlledWrite { fn write(&mut self, buf: &[u8]) -> io::Result { diff --git a/quickwit/quickwit-indexing/src/lib.rs b/quickwit/quickwit-indexing/src/lib.rs index b3e2b7ee921..2c88121765a 100644 --- a/quickwit/quickwit-indexing/src/lib.rs +++ b/quickwit/quickwit-indexing/src/lib.rs @@ -25,17 +25,14 @@ use quickwit_storage::StorageResolver; use tracing::info; use crate::actors::MergeSchedulerService; +pub use crate::actors::merge_executor::{ + combine_index_meta, create_shadowing_meta_json_directory, merge_split_directories_standalone, + open_index, open_split_directories, +}; pub use crate::actors::{ FinishPendingMergesAndShutdownPipeline, IndexingError, IndexingPipeline, IndexingPipelineParams, IndexingService, PublisherType, Sequencer, SplitsUpdateMailbox, }; -pub use crate::actors::merge_executor::{ - combine_index_meta, - open_split_directories, - create_shadowing_meta_json_directory, - open_index, - merge_split_directories_standalone, -}; pub use crate::controlled_directory::ControlledDirectory; use crate::models::IndexingStatistics; pub use crate::split_store::{IndexingSplitStore, get_tantivy_directory_from_split_bundle}; diff --git a/quickwit/quickwit-indexing/src/merge_policy/const_write_amplification.rs b/quickwit/quickwit-indexing/src/merge_policy/const_write_amplification.rs index 0b1f4431efb..ce3f4654de1 100644 --- a/quickwit/quickwit-indexing/src/merge_policy/const_write_amplification.rs +++ b/quickwit/quickwit-indexing/src/merge_policy/const_write_amplification.rs @@ -175,10 +175,10 @@ impl MergePolicy for ConstWriteAmplificationMergePolicy { // We first isolate mature splits. Let's not touch them. let (mature_splits, mut young_splits): (Vec, Vec) = splits.drain(..).partition(|split: &SplitMetadata| { - if let Some(max_finalize_split_num_docs) = self.config.max_finalize_split_num_docs { - if split.num_docs > max_finalize_split_num_docs { - return true; - } + if let Some(max_finalize_split_num_docs) = self.config.max_finalize_split_num_docs + && split.num_docs > max_finalize_split_num_docs + { + return true; } split.is_mature(now) }); @@ -392,7 +392,7 @@ mod tests { ..Default::default() }) .collect(); - splits.shuffle(&mut rand::thread_rng()); + splits.shuffle(&mut rand::rng()); let operations: Vec = merge_policy.operations(&mut splits); assert_eq!(operations.len(), 1); assert_eq!( diff --git a/quickwit/quickwit-indexing/src/merge_policy/mod.rs b/quickwit/quickwit-indexing/src/merge_policy/mod.rs index 02fd534c4d9..9319f8d8498 100644 --- a/quickwit/quickwit-indexing/src/merge_policy/mod.rs +++ b/quickwit/quickwit-indexing/src/merge_policy/mod.rs @@ -207,7 +207,7 @@ impl fmt::Debug for SplitShortDebug<'_> { } } -fn splits_short_debug(splits: &[SplitMetadata]) -> Vec { +fn splits_short_debug(splits: &[SplitMetadata]) -> Vec> { splits.iter().map(SplitShortDebug).collect() } @@ -336,7 +336,7 @@ pub mod tests { pub(crate) fn proptest_merge_policy(merge_policy: &dyn MergePolicy) { proptest!(|(mut splits in prop::collection::vec(split_strategy(), 0..100))| { let mut cloned_splits = splits.clone(); - cloned_splits.shuffle(&mut rand::thread_rng()); + cloned_splits.shuffle(&mut rand::rng()); let original_num_splits = splits.len(); diff --git a/quickwit/quickwit-indexing/src/merge_policy/stable_log_merge_policy.rs b/quickwit/quickwit-indexing/src/merge_policy/stable_log_merge_policy.rs index 9cd6982f847..babf49925d4 100644 --- a/quickwit/quickwit-indexing/src/merge_policy/stable_log_merge_policy.rs +++ b/quickwit/quickwit-indexing/src/merge_policy/stable_log_merge_policy.rs @@ -297,11 +297,6 @@ impl StableLogMergePolicy { } } -#[cfg(test)] -fn is_sorted(elements: &[usize]) -> bool { - elements.windows(2).all(|w| w[0] <= w[1]) -} - // Helpers which expose some internal properties of // the stable log merge policy to be tested in unit tests. #[cfg(test)] @@ -337,7 +332,8 @@ impl StableLogMergePolicy { levels: &[usize], sorted: bool, ) -> usize { - assert!(is_sorted(levels)); + assert!(levels.is_sorted()); + if num_docs == 0 { return 0; } @@ -440,7 +436,9 @@ mod tests { } #[test] - #[should_panic(expected = "All splits are expected to be smaller than `split_num_docs_target`.")] + #[should_panic( + expected = "All splits are expected to be smaller than `split_num_docs_target`." + )] fn test_stable_log_merge_policy_build_split_panics_if_exceeding_split_num_docs_target() { let merge_policy = StableLogMergePolicy::default(); let splits = create_splits(&merge_policy, vec![11_000_000]); diff --git a/quickwit/quickwit-indexing/src/metrics.rs b/quickwit/quickwit-indexing/src/metrics.rs index 9ff7336ae52..d3186616252 100644 --- a/quickwit/quickwit-indexing/src/metrics.rs +++ b/quickwit/quickwit-indexing/src/metrics.rs @@ -21,6 +21,7 @@ use quickwit_common::metrics::{ pub struct IndexerMetrics { pub processed_docs_total: IntCounterVec<2>, pub processed_bytes: IntCounterVec<2>, + pub indexing_pipelines: IntGaugeVec<1>, pub backpressure_micros: IntCounterVec<1>, pub available_concurrent_upload_permits: IntGaugeVec<1>, pub split_builders: IntGauge, @@ -51,6 +52,13 @@ impl Default for IndexerMetrics { &[], ["index", "docs_processed_status"], ), + indexing_pipelines: new_gauge_vec( + "indexing_pipelines", + "Number of running indexing pipelines", + "indexing", + &[], + ["index"], + ), backpressure_micros: new_counter_vec( "backpressure_micros", "Amount of time spent in backpressure (in micros). This time only includes the \ diff --git a/quickwit/quickwit-indexing/src/models/mod.rs b/quickwit/quickwit-indexing/src/models/mod.rs index 9dfdfde1594..fd3188e2104 100644 --- a/quickwit/quickwit-indexing/src/models/mod.rs +++ b/quickwit/quickwit-indexing/src/models/mod.rs @@ -21,7 +21,9 @@ mod merge_planner_message; mod merge_scratch; mod merge_statistics; mod packaged_split; +mod parquet_splits_update; mod processed_doc; +mod processed_parquet_batch; mod publish_lock; mod publisher_message; mod raw_doc_batch; @@ -40,7 +42,9 @@ pub use merge_planner_message::NewSplits; pub use merge_scratch::MergeScratch; pub use merge_statistics::MergeStatistics; pub use packaged_split::{PackagedSplit, PackagedSplitBatch}; +pub use parquet_splits_update::ParquetSplitsUpdate; pub use processed_doc::{ProcessedDoc, ProcessedDocBatch}; +pub use processed_parquet_batch::ProcessedParquetBatch; pub use publish_lock::{NewPublishLock, PublishLock}; pub use publisher_message::SplitsUpdate; use quickwit_proto::types::PublishToken; diff --git a/quickwit/quickwit-indexing/src/models/parquet_splits_update.rs b/quickwit/quickwit-indexing/src/models/parquet_splits_update.rs new file mode 100644 index 00000000000..5342743546a --- /dev/null +++ b/quickwit/quickwit-indexing/src/models/parquet_splits_update.rs @@ -0,0 +1,61 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Message type for metrics split updates sent by ParquetUploader to downstream actors. + +use std::fmt; + +use itertools::Itertools; +use quickwit_metastore::checkpoint::IndexCheckpointDelta; +use quickwit_parquet_engine::split::MetricsSplitMetadata; +use quickwit_proto::types::{IndexUid, PublishToken}; +use tracing::Span; + +use crate::models::PublishLock; + +/// Message sent by ParquetUploader to downstream actors after staging and uploading. +/// +/// This is analogous to `SplitsUpdate` but uses `MetricsSplitMetadata` instead of +/// `SplitMetadata`. +pub struct ParquetSplitsUpdate { + /// Index unique identifier. + pub index_uid: IndexUid, + /// The staged and uploaded splits. + pub new_splits: Vec, + /// Split IDs being replaced (for merges, typically empty for ingest). + pub replaced_split_ids: Vec, + /// Checkpoint delta covering the data in these splits. + pub checkpoint_delta_opt: Option, + /// Publish lock for coordination. + pub publish_lock: PublishLock, + /// Optional publish token. + pub publish_token_opt: Option, + /// Parent span for tracing. + pub parent_span: Span, +} + +impl fmt::Debug for ParquetSplitsUpdate { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let new_split_ids: String = self + .new_splits + .iter() + .map(|split| split.split_id.as_str()) + .join(","); + f.debug_struct("ParquetSplitsUpdate") + .field("index_uid", &self.index_uid) + .field("new_splits", &new_split_ids) + .field("checkpoint_delta", &self.checkpoint_delta_opt) + .finish() + } +} diff --git a/quickwit/quickwit-indexing/src/models/processed_parquet_batch.rs b/quickwit/quickwit-indexing/src/models/processed_parquet_batch.rs new file mode 100644 index 00000000000..0db83abcc02 --- /dev/null +++ b/quickwit/quickwit-indexing/src/models/processed_parquet_batch.rs @@ -0,0 +1,239 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! ProcessedParquetBatch message type for the parquet indexing pipeline. +//! +//! This message carries Arrow RecordBatch data from ParquetDocProcessor to ParquetIndexer, +//! bypassing the Tantivy document path used by the standard indexing pipeline. + +use std::fmt; + +use arrow::record_batch::RecordBatch; +use quickwit_common::metrics::{GaugeGuard, MEMORY_METRICS}; +use quickwit_metastore::checkpoint::SourceCheckpointDelta; + +/// Batch of parquet data as Arrow RecordBatch for the parquet indexing pipeline. +/// +/// This message type is sent from ParquetDocProcessor to ParquetIndexer, carrying +/// pre-processed Arrow data that can be directly accumulated and written to Parquet. +pub struct ProcessedParquetBatch { + /// The Arrow RecordBatch containing parquet pipeline data. + pub batch: RecordBatch, + /// Checkpoint delta for this batch. + pub checkpoint_delta: SourceCheckpointDelta, + /// Force commit flag - when true, accumulator should flush immediately. + pub force_commit: bool, + /// Memory tracking gauge guard. + _gauge_guard: GaugeGuard<'static>, +} + +impl ProcessedParquetBatch { + /// Create a new ProcessedParquetBatch. + /// + /// # Arguments + /// * `batch` - The Arrow RecordBatch containing parquet pipeline data + /// * `checkpoint_delta` - Checkpoint progress for this batch + /// * `force_commit` - Whether to force an immediate commit/flush + pub fn new( + batch: RecordBatch, + checkpoint_delta: SourceCheckpointDelta, + force_commit: bool, + ) -> Self { + // Estimate memory usage from the RecordBatch + let memory_size: i64 = batch + .columns() + .iter() + .map(|col| col.get_array_memory_size() as i64) + .sum(); + + let mut gauge_guard = GaugeGuard::from_gauge(&MEMORY_METRICS.in_flight.indexer_mailbox); + gauge_guard.add(memory_size); + + Self { + batch, + checkpoint_delta, + force_commit, + _gauge_guard: gauge_guard, + } + } + + /// Returns the number of rows in the batch. + pub fn num_rows(&self) -> usize { + self.batch.num_rows() + } + + /// Returns the estimated memory size of the batch in bytes. + pub fn memory_size(&self) -> usize { + self.batch + .columns() + .iter() + .map(|col| col.get_array_memory_size()) + .sum() + } +} + +impl fmt::Debug for ProcessedParquetBatch { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ProcessedParquetBatch") + .field("num_rows", &self.batch.num_rows()) + .field("num_columns", &self.batch.num_columns()) + .field("checkpoint_delta", &self.checkpoint_delta) + .field("force_commit", &self.force_commit) + .finish() + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow::array::{ + ArrayRef, BinaryViewArray, DictionaryArray, Float64Array, Int32Array, StringArray, + StructArray, UInt8Array, UInt64Array, + }; + use arrow::datatypes::{DataType, Field, Int32Type}; + use quickwit_parquet_engine::schema::ParquetSchema; + + use super::*; + + /// Create dictionary array for string fields with Int32 keys. + fn create_dict_array(values: &[&str]) -> ArrayRef { + let keys: Vec = (0..values.len()).map(|i| i as i32).collect(); + let string_array = StringArray::from(values.to_vec()); + Arc::new( + DictionaryArray::::try_new(Int32Array::from(keys), Arc::new(string_array)) + .unwrap(), + ) + } + + /// Create nullable dictionary array for optional string fields. + fn create_nullable_dict_array(values: &[Option<&str>]) -> ArrayRef { + let keys: Vec> = values + .iter() + .enumerate() + .map(|(i, v)| v.map(|_| i as i32)) + .collect(); + let string_values: Vec<&str> = values.iter().filter_map(|v| *v).collect(); + let string_array = StringArray::from(string_values); + Arc::new( + DictionaryArray::::try_new(Int32Array::from(keys), Arc::new(string_array)) + .unwrap(), + ) + } + + /// Create a test batch matching the metrics schema. + fn create_test_batch(num_rows: usize) -> RecordBatch { + let schema = ParquetSchema::new(); + + let metric_names: Vec<&str> = vec!["cpu.usage"; num_rows]; + let metric_name: ArrayRef = create_dict_array(&metric_names); + let metric_type: ArrayRef = Arc::new(UInt8Array::from(vec![0u8; num_rows])); + let metric_unit: ArrayRef = Arc::new(StringArray::from(vec![Some("bytes"); num_rows])); + let timestamps: Vec = (0..num_rows).map(|i| 100 + i as u64).collect(); + let timestamp_secs: ArrayRef = Arc::new(UInt64Array::from(timestamps)); + let start_timestamp_secs: ArrayRef = + Arc::new(UInt64Array::from(vec![None::; num_rows])); + let values: Vec = (0..num_rows).map(|i| 42.0 + i as f64).collect(); + let value: ArrayRef = Arc::new(Float64Array::from(values)); + let tag_service: ArrayRef = create_nullable_dict_array(&vec![Some("web"); num_rows]); + let tag_env: ArrayRef = create_nullable_dict_array(&vec![Some("prod"); num_rows]); + let tag_datacenter: ArrayRef = + create_nullable_dict_array(&vec![Some("us-east-1"); num_rows]); + let tag_region: ArrayRef = create_nullable_dict_array(&vec![None; num_rows]); + let tag_host: ArrayRef = create_nullable_dict_array(&vec![Some("host-001"); num_rows]); + + // Create empty Variant (Struct with metadata and value BinaryView fields) + let metadata_array = Arc::new(BinaryViewArray::from(vec![b"" as &[u8]; num_rows])); + let value_array = Arc::new(BinaryViewArray::from(vec![b"" as &[u8]; num_rows])); + let attributes: ArrayRef = Arc::new(StructArray::from(vec![ + ( + Arc::new(Field::new("metadata", DataType::BinaryView, false)), + metadata_array.clone() as ArrayRef, + ), + ( + Arc::new(Field::new("value", DataType::BinaryView, false)), + value_array.clone() as ArrayRef, + ), + ])); + + let service_name: ArrayRef = create_dict_array(&vec!["my-service"; num_rows]); + + let resource_attributes: ArrayRef = Arc::new(StructArray::from(vec![ + ( + Arc::new(Field::new("metadata", DataType::BinaryView, false)), + metadata_array as ArrayRef, + ), + ( + Arc::new(Field::new("value", DataType::BinaryView, false)), + value_array as ArrayRef, + ), + ])); + + RecordBatch::try_new( + schema.arrow_schema().clone(), + vec![ + metric_name, + metric_type, + metric_unit, + timestamp_secs, + start_timestamp_secs, + value, + tag_service, + tag_env, + tag_datacenter, + tag_region, + tag_host, + attributes, + service_name, + resource_attributes, + ], + ) + .unwrap() + } + + #[test] + fn test_processed_parquet_batch_new() { + let batch = create_test_batch(10); + let checkpoint_delta = SourceCheckpointDelta::from_range(0..10); + + let processed = ProcessedParquetBatch::new(batch, checkpoint_delta, false); + + assert_eq!(processed.num_rows(), 10); + assert!(!processed.force_commit); + assert!(processed.memory_size() > 0); + } + + #[test] + fn test_processed_parquet_batch_force_commit() { + let batch = create_test_batch(5); + let checkpoint_delta = SourceCheckpointDelta::from_range(0..5); + + let processed = ProcessedParquetBatch::new(batch, checkpoint_delta, true); + + assert!(processed.force_commit); + assert_eq!(processed.num_rows(), 5); + } + + #[test] + fn test_processed_parquet_batch_debug() { + let batch = create_test_batch(3); + let checkpoint_delta = SourceCheckpointDelta::from_range(0..3); + + let processed = ProcessedParquetBatch::new(batch, checkpoint_delta, false); + + let debug_str = format!("{:?}", processed); + assert!(debug_str.contains("ProcessedParquetBatch")); + assert!(debug_str.contains("num_rows: 3")); + } +} diff --git a/quickwit/quickwit-indexing/src/source/file_source.rs b/quickwit/quickwit-indexing/src/source/file_source.rs index 9271e8bb3a6..f67dd65b218 100644 --- a/quickwit/quickwit-indexing/src/source/file_source.rs +++ b/quickwit/quickwit-indexing/src/source/file_source.rs @@ -25,7 +25,7 @@ use quickwit_proto::types::SourceId; use super::doc_file_reader::ObjectUriBatchReader; #[cfg(feature = "queue-sources")] use super::queue_sources::coordinator::QueueCoordinator; -use crate::actors::DocProcessor; +use crate::actors::Processor; use crate::source::{Source, SourceContext, SourceRuntime, TypedSourceFactory}; enum FileSourceState { @@ -51,17 +51,17 @@ impl fmt::Debug for FileSource { } #[async_trait] -impl Source for FileSource { +impl Source

for FileSource { #[allow(unused_variables)] async fn initialize( &mut self, - doc_processor_mailbox: &Mailbox, - ctx: &SourceContext, + processor_mailbox: &Mailbox

, + ctx: &SourceContext

, ) -> Result<(), ActorExitStatus> { match &mut self.state { #[cfg(feature = "queue-sources")] FileSourceState::Notification(coordinator) => { - coordinator.initialize(doc_processor_mailbox, ctx).await + coordinator.initialize(processor_mailbox, ctx).await } FileSourceState::Filepath { .. } => Ok(()), } @@ -70,13 +70,13 @@ impl Source for FileSource { #[allow(unused_variables)] async fn emit_batches( &mut self, - doc_processor_mailbox: &Mailbox, - ctx: &SourceContext, + processor_mailbox: &Mailbox

, + ctx: &SourceContext

, ) -> Result { match &mut self.state { #[cfg(feature = "queue-sources")] FileSourceState::Notification(coordinator) => { - coordinator.emit_batches(doc_processor_mailbox, ctx).await?; + coordinator.emit_batches(processor_mailbox, ctx).await?; } FileSourceState::Filepath { batch_reader, @@ -88,11 +88,11 @@ impl Source for FileSource { .await?; *num_bytes_processed += batch_builder.num_bytes; *num_lines_processed += batch_builder.docs.len() as u64; - doc_processor_mailbox + processor_mailbox .send_message(batch_builder.build()) .await?; if batch_reader.is_eof() { - ctx.send_exit_with_success(doc_processor_mailbox).await?; + ctx.send_exit_with_success(processor_mailbox).await?; return Err(ActorExitStatus::Success); } } @@ -108,7 +108,7 @@ impl Source for FileSource { async fn suggest_truncate( &mut self, checkpoint: SourceCheckpoint, - ctx: &SourceContext, + ctx: &SourceContext

, ) -> anyhow::Result<()> { match &mut self.state { #[cfg(feature = "queue-sources")] @@ -209,6 +209,7 @@ mod tests { use quickwit_proto::types::{IndexUid, Position}; use super::*; + use crate::actors::DocProcessor; use crate::models::RawDocBatch; use crate::source::doc_file_reader::file_test_helpers::{ DUMMY_DOC, generate_dummy_doc_file, generate_index_doc_file, @@ -224,7 +225,7 @@ mod tests { async fn aux_test_file_source(gzip: bool) { let universe = Universe::with_accelerated_time(); - let (doc_processor_mailbox, indexer_inbox) = universe.create_test_mailbox(); + let (doc_processor_mailbox, indexer_inbox) = universe.create_test_mailbox::(); let params = if gzip { FileSourceParams::from_filepath("data/test_corpus.json.gz").unwrap() } else { @@ -245,7 +246,7 @@ mod tests { .unwrap(); let file_source_actor = SourceActor { source: Box::new(file_source), - doc_processor_mailbox, + processor_mailbox: doc_processor_mailbox, }; let (_file_source_mailbox, file_source_handle) = universe.spawn_builder().spawn(file_source_actor); @@ -276,7 +277,8 @@ mod tests { async fn aux_test_file_source_several_batch(gzip: bool) { quickwit_common::setup_logging_for_tests(); let universe = Universe::with_accelerated_time(); - let (doc_processor_mailbox, doc_processor_inbox) = universe.create_test_mailbox(); + let (doc_processor_mailbox, doc_processor_inbox) = + universe.create_test_mailbox::(); let lines = BATCH_NUM_BYTES_LIMIT as usize / DUMMY_DOC.len() + 1; let (temp_file, temp_file_size) = generate_dummy_doc_file(gzip, lines).await; let filepath = temp_file.path().to_str().unwrap(); @@ -297,7 +299,7 @@ mod tests { .unwrap(); let file_source_actor = SourceActor { source: Box::new(file_source), - doc_processor_mailbox, + processor_mailbox: doc_processor_mailbox, }; let (_file_source_mailbox, file_source_handle) = universe.spawn_builder().spawn(file_source_actor); @@ -341,7 +343,8 @@ mod tests { async fn aux_test_file_source_resume_from_checkpoint(gzip: bool) { quickwit_common::setup_logging_for_tests(); let universe = Universe::with_accelerated_time(); - let (doc_processor_mailbox, doc_processor_inbox) = universe.create_test_mailbox(); + let (doc_processor_mailbox, doc_processor_inbox) = + universe.create_test_mailbox::(); let temp_file = generate_index_doc_file(gzip, 100).await; let temp_file_path = temp_file.path().to_str().unwrap(); let uri = Uri::from_str(temp_file_path).unwrap(); @@ -373,7 +376,7 @@ mod tests { .unwrap(); let file_source_actor = SourceActor { source: Box::new(file_source), - doc_processor_mailbox, + processor_mailbox: doc_processor_mailbox, }; let (_file_source_mailbox, file_source_handle) = universe.spawn_builder().spawn(file_source_actor); @@ -407,6 +410,7 @@ mod localstack_tests { use quickwit_metastore::metastore_for_test; use super::*; + use crate::actors::DocProcessor; use crate::models::RawDocBatch; use crate::source::SourceActor; use crate::source::doc_file_reader::file_test_helpers::generate_dummy_doc_file; @@ -450,11 +454,12 @@ mod localstack_tests { // actor setup let universe = Universe::with_accelerated_time(); - let (doc_processor_mailbox, doc_processor_inbox) = universe.create_test_mailbox(); + let (doc_processor_mailbox, doc_processor_inbox) = + universe.create_test_mailbox::(); { let actor = SourceActor { source: Box::new(sqs_source), - doc_processor_mailbox: doc_processor_mailbox.clone(), + processor_mailbox: doc_processor_mailbox.clone(), }; let (_mailbox, handle) = universe.spawn_builder().spawn(actor); diff --git a/quickwit/quickwit-indexing/src/source/gcp_pubsub_source.rs b/quickwit/quickwit-indexing/src/source/gcp_pubsub_source.rs index 47b3c5b47b2..4e4a67fa4b0 100644 --- a/quickwit/quickwit-indexing/src/source/gcp_pubsub_source.rs +++ b/quickwit/quickwit-indexing/src/source/gcp_pubsub_source.rs @@ -22,7 +22,7 @@ use google_cloud_auth::credentials::CredentialsFile; use google_cloud_gax::retry::RetrySetting; use google_cloud_pubsub::client::{Client, ClientConfig}; use google_cloud_pubsub::subscription::Subscription; -use quickwit_actors::{ActorContext, ActorExitStatus, Mailbox}; +use quickwit_actors::{ActorExitStatus, Mailbox}; use quickwit_common::rand::append_random_suffix; use quickwit_config::PubSubSourceParams; use quickwit_metastore::checkpoint::{PartitionId, SourceCheckpoint}; @@ -32,8 +32,8 @@ use serde_json::{Value as JsonValue, json}; use tokio::time; use tracing::{debug, info, warn}; -use super::{BATCH_NUM_BYTES_LIMIT, EMIT_BATCHES_TIMEOUT, SourceActor}; -use crate::actors::DocProcessor; +use super::{BATCH_NUM_BYTES_LIMIT, EMIT_BATCHES_TIMEOUT}; +use crate::actors::Processor; use crate::source::{BatchBuilder, Source, SourceContext, SourceRuntime, TypedSourceFactory}; const DEFAULT_MAX_MESSAGES_PER_PULL: i32 = 1_000; @@ -153,11 +153,11 @@ impl GcpPubSubSource { } #[async_trait] -impl Source for GcpPubSubSource { +impl Source

for GcpPubSubSource { async fn emit_batches( &mut self, - doc_processor_mailbox: &Mailbox, - ctx: &SourceContext, + processor_mailbox: &Mailbox

, + ctx: &SourceContext

, ) -> Result { let now = Instant::now(); let mut batch_builder = BatchBuilder::new(SourceType::PubSub); @@ -191,7 +191,7 @@ impl Source for GcpPubSubSource { // TODO: need to wait for all the id to be ack for at_least_once if self.should_exit() { info!(subscription=%self.subscription_name, "reached end of subscription"); - ctx.send_exit_with_success(doc_processor_mailbox).await?; + ctx.send_exit_with_success(processor_mailbox).await?; return Err(ActorExitStatus::Success); } if !batch_builder.checkpoint_delta.is_empty() { @@ -201,7 +201,7 @@ impl Source for GcpPubSubSource { num_millis=%now.elapsed().as_millis(), "Sending doc batch to indexer."); let message = batch_builder.build(); - ctx.send_message(doc_processor_mailbox, message).await?; + ctx.send_message(processor_mailbox, message).await?; } Ok(Duration::default()) } @@ -209,7 +209,7 @@ impl Source for GcpPubSubSource { async fn suggest_truncate( &mut self, _checkpoint: SourceCheckpoint, - _ctx: &ActorContext, + _ctx: &SourceContext

, ) -> anyhow::Result<()> { // TODO: add ack of ids Ok(()) @@ -293,8 +293,8 @@ mod gcp_pubsub_emulator_tests { use super::*; use crate::models::RawDocBatch; - use crate::source::quickwit_supported_sources; use crate::source::tests::SourceRuntimeBuilder; + use crate::source::{SourceActor, quickwit_supported_sources}; static GCP_TEST_PROJECT: &str = "quickwit-emulator"; @@ -358,7 +358,6 @@ mod gcp_pubsub_emulator_tests { .unwrap_err(); } - #[ignore] #[tokio::test] async fn test_gcp_pubsub_source() { let universe = Universe::with_accelerated_time(); @@ -392,7 +391,7 @@ mod gcp_pubsub_emulator_tests { let (doc_processor_mailbox, doc_processor_inbox) = universe.create_test_mailbox(); let source_actor = SourceActor { source, - doc_processor_mailbox: doc_processor_mailbox.clone(), + processor_mailbox: doc_processor_mailbox.clone(), }; let (_source_mailbox, source_handle) = universe.spawn_builder().spawn(source_actor); let (exit_status, exit_state) = source_handle.join().await; diff --git a/quickwit/quickwit-indexing/src/source/ingest/mod.rs b/quickwit/quickwit-indexing/src/source/ingest/mod.rs index bd07d2f4516..b6709b2a76d 100644 --- a/quickwit/quickwit-indexing/src/source/ingest/mod.rs +++ b/quickwit/quickwit-indexing/src/source/ingest/mod.rs @@ -49,7 +49,7 @@ use super::{ BATCH_NUM_BYTES_LIMIT, BatchBuilder, EMIT_BATCHES_TIMEOUT, Source, SourceContext, SourceRuntime, TypedSourceFactory, }; -use crate::actors::DocProcessor; +use crate::actors::Processor; use crate::models::{LocalShardPositionsUpdate, NewPublishLock, NewPublishToken, PublishLock}; pub struct IngestSourceFactory; @@ -302,10 +302,10 @@ impl IngestSource { ); // Let's record all shards that have reached Eof as complete. for (shard, truncate_up_to_position_inclusive) in &truncate_up_to_positions { - if truncate_up_to_position_inclusive.is_eof() { - if let Some(assigned_shard) = self.assigned_shards.get_mut(shard) { - assigned_shard.status = IndexingStatus::Complete; - } + if truncate_up_to_position_inclusive.is_eof() + && let Some(assigned_shard) = self.assigned_shards.get_mut(shard) + { + assigned_shard.status = IndexingStatus::Complete; } } @@ -361,6 +361,7 @@ impl IngestSource { }; for num_attempts in 1..=retry_params.max_attempts { let Err(error) = ingester + .client .truncate_shards(truncate_shards_request.clone()) .await else { @@ -390,11 +391,11 @@ impl IngestSource { /// After this method has returned we are guaranteed to have the following post condition: /// - a alive publish lock / non-empty publish token /// - all currently assigned shards included in the `new_assigned_shard_ids` set. - async fn reset_if_needed( + async fn reset_if_needed( &mut self, new_assigned_shard_ids: &BTreeSet, - doc_processor_mailbox: &Mailbox, - ctx: &SourceContext, + doc_processor_mailbox: &Mailbox, + ctx: &crate::source::SourceContext, ) -> anyhow::Result<()> { // No need to do anything if the list of shards before and after are empty. if new_assigned_shard_ids.is_empty() && self.assigned_shards.is_empty() { @@ -410,9 +411,8 @@ impl IngestSource { .assigned_shards .keys() .filter(|&shard_id| !new_assigned_shard_ids.contains(shard_id)) - .cloned() .any(|removed_shard_id| { - let Some(assigned_shard) = self.assigned_shards.get(&removed_shard_id) else { + let Some(assigned_shard) = self.assigned_shards.get(removed_shard_id) else { return false; }; assigned_shard.status != IndexingStatus::Complete @@ -456,11 +456,11 @@ impl IngestSource { } #[async_trait] -impl Source for IngestSource { +impl Source

for IngestSource { async fn emit_batches( &mut self, - doc_processor_mailbox: &Mailbox, - ctx: &SourceContext, + processor_mailbox: &Mailbox

, + ctx: &SourceContext

, ) -> Result { let mut batch_builder = BatchBuilder::new(SourceType::IngestV2); @@ -502,7 +502,7 @@ impl Source for IngestSource { "Sending doc batch to indexer." ); let message = batch_builder.build(); - ctx.send_message(doc_processor_mailbox, message).await?; + ctx.send_message(processor_mailbox, message).await?; } Ok(Duration::default()) } @@ -510,10 +510,10 @@ impl Source for IngestSource { async fn assign_shards( &mut self, new_assigned_shard_ids: BTreeSet, - doc_processor_mailbox: &Mailbox, - ctx: &SourceContext, + processor_mailbox: &Mailbox

, + ctx: &SourceContext

, ) -> anyhow::Result<()> { - self.reset_if_needed(&new_assigned_shard_ids, doc_processor_mailbox, ctx) + self.reset_if_needed(&new_assigned_shard_ids, processor_mailbox, ctx) .await?; // As enforced by `reset_if_needed`, at this point, all currently assigned shards should be @@ -621,7 +621,7 @@ impl Source for IngestSource { async fn suggest_truncate( &mut self, checkpoint: SourceCheckpoint, - _ctx: &SourceContext, + _ctx: &SourceContext

, ) -> anyhow::Result<()> { let truncate_up_to_positions: Vec<(ShardId, Position)> = checkpoint .iter() @@ -673,6 +673,7 @@ mod tests { use quickwit_common::metrics::MEMORY_METRICS; use quickwit_common::stream_utils::InFlightValue; use quickwit_config::{IndexingSettings, SourceConfig, SourceParams}; + use quickwit_ingest::IngesterPoolEntry; use quickwit_proto::indexing::IndexingPipelineId; use quickwit_proto::ingest::ingester::{ FetchMessage, IngesterServiceClient, MockIngesterService, TruncateShardsResponse, @@ -685,6 +686,7 @@ mod tests { use tokio::sync::watch; use super::*; + use crate::actors::DocProcessor; use crate::models::RawDocBatch; use crate::source::SourceActor; @@ -930,7 +932,8 @@ mod tests { Ok(response) }); - let ingester_0 = IngesterServiceClient::from_mock(mock_ingester_0); + let ingester_0 = + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester_0)); ingester_pool.insert("test-ingester-0".into(), ingester_0.clone()); let event_broker = EventBroker::default(); @@ -1127,7 +1130,8 @@ mod tests { Ok(response) }); - let ingester_0 = IngesterServiceClient::from_mock(mock_ingester_0); + let ingester_0 = + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester_0)); ingester_pool.insert("test-ingester-0".into(), ingester_0.clone()); let event_broker = EventBroker::default(); @@ -1292,7 +1296,8 @@ mod tests { Ok(response) }); - let ingester_0 = IngesterServiceClient::from_mock(mock_ingester_0); + let ingester_0 = + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester_0)); ingester_pool.insert("test-ingester-0".into(), ingester_0.clone()); let event_broker = EventBroker::default(); @@ -1600,7 +1605,8 @@ mod tests { }) }); - let ingester_0 = IngesterServiceClient::from_mock(mock_ingester_0); + let ingester_0 = + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester_0)); ingester_pool.insert("test-ingester-0".into(), ingester_0.clone()); let event_broker = EventBroker::default(); @@ -1700,7 +1706,8 @@ mod tests { Ok(TruncateShardsResponse {}) }); - let ingester_0 = IngesterServiceClient::from_mock(mock_ingester_0); + let ingester_0 = + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester_0)); ingester_pool.insert("test-ingester-0".into(), ingester_0.clone()); let mut mock_ingester_1 = MockIngesterService::new(); @@ -1727,7 +1734,8 @@ mod tests { Ok(TruncateShardsResponse {}) }); - let ingester_1 = IngesterServiceClient::from_mock(mock_ingester_1); + let ingester_1 = + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester_1)); ingester_pool.insert("test-ingester-1".into(), ingester_1.clone()); let mut mock_ingester_3 = MockIngesterService::new(); @@ -1747,7 +1755,8 @@ mod tests { Ok(TruncateShardsResponse {}) }); - let ingester_3 = IngesterServiceClient::from_mock(mock_ingester_3); + let ingester_3 = + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester_3)); ingester_pool.insert("test-ingester-3".into(), ingester_3.clone()); let event_broker = EventBroker::default(); diff --git a/quickwit/quickwit-indexing/src/source/ingest_api_source.rs b/quickwit/quickwit-indexing/src/source/ingest_api_source.rs index 96feaec8fb3..a1ad35b8cc6 100644 --- a/quickwit/quickwit-indexing/src/source/ingest_api_source.rs +++ b/quickwit/quickwit-indexing/src/source/ingest_api_source.rs @@ -17,7 +17,7 @@ use std::time::Duration; use anyhow::bail; use async_trait::async_trait; -use quickwit_actors::{ActorContext, ActorExitStatus, Mailbox}; +use quickwit_actors::{ActorExitStatus, Mailbox}; use quickwit_ingest::{ CreateQueueIfNotExistsRequest, DocCommand, FetchRequest, FetchResponse, GetPartitionId, IngestApiService, SuggestTruncateRequest, get_ingest_api_service, @@ -29,8 +29,8 @@ use serde::Serialize; use serde_json::Value as JsonValue; use tracing::{error, info}; -use super::{BatchBuilder, Source, SourceActor, SourceContext, TypedSourceFactory}; -use crate::actors::DocProcessor; +use super::{BatchBuilder, Source, SourceContext, TypedSourceFactory}; +use crate::actors::Processor; use crate::source::SourceRuntime; /// Wait time for SourceActor before pooling for new documents. @@ -114,10 +114,10 @@ impl IngestApiSource { Ok(ingest_api_source) } - async fn send_suggest_truncate_to_ingest_service( + async fn send_suggest_truncate_to_ingest_service( &self, up_to_position_included: u64, - ctx: &ActorContext, + ctx: &crate::source::SourceContext, ) -> anyhow::Result<()> { let suggest_truncate_req = SuggestTruncateRequest { index_id: self.source_runtime.index_id().to_string(), @@ -136,11 +136,11 @@ impl IngestApiSource { } #[async_trait] -impl Source for IngestApiSource { +impl Source

for IngestApiSource { async fn initialize( &mut self, - _: &Mailbox, - ctx: &SourceContext, + _: &Mailbox

, + ctx: &SourceContext

, ) -> Result<(), ActorExitStatus> { if let Some(position) = self.counters.previous_offset { self.send_suggest_truncate_to_ingest_service(position, ctx) @@ -151,8 +151,8 @@ impl Source for IngestApiSource { async fn emit_batches( &mut self, - batch_sink: &Mailbox, - ctx: &SourceContext, + batch_sink: &Mailbox

, + ctx: &SourceContext

, ) -> Result { let fetch_req = FetchRequest { index_id: self.source_runtime.index_id().to_string(), @@ -209,7 +209,7 @@ impl Source for IngestApiSource { async fn suggest_truncate( &mut self, checkpoint: SourceCheckpoint, - ctx: &ActorContext, + ctx: &SourceContext

, ) -> anyhow::Result<()> { if let Some(Position::Offset(offset)) = checkpoint.position_for_partition(&self.partition_id) @@ -261,6 +261,7 @@ mod tests { use quickwit_proto::types::{IndexId, IndexUid}; use super::*; + use crate::actors::DocProcessor; use crate::models::RawDocBatch; use crate::source::SourceActor; use crate::source::tests::SourceRuntimeBuilder; @@ -311,7 +312,8 @@ mod tests { let ingest_api_service = init_ingest_api(&universe, queues_dir_path, &IngestApiConfig::default()).await?; - let (doc_processor_mailbox, doc_processor_inbox) = universe.create_test_mailbox(); + let (doc_processor_mailbox, doc_processor_inbox) = + universe.create_test_mailbox::(); let source_config = make_source_config(); let source_runtime = SourceRuntimeBuilder::new(index_uid, source_config) .with_queues_dir(queues_dir_path) @@ -319,7 +321,7 @@ mod tests { let ingest_api_source = IngestApiSource::try_new(source_runtime).await?; let ingest_api_source_actor = SourceActor { source: Box::new(ingest_api_source), - doc_processor_mailbox, + processor_mailbox: doc_processor_mailbox, }; let (_ingest_api_source_mailbox, ingest_api_source_handle) = universe.spawn_builder().spawn(ingest_api_source_actor); @@ -410,7 +412,8 @@ mod tests { .await .map_err(|err| anyhow::anyhow!(err.to_string()))?; - let (doc_processor_mailbox, doc_processor_inbox) = universe.create_test_mailbox(); + let (doc_processor_mailbox, doc_processor_inbox) = + universe.create_test_mailbox::(); let partition_id: PartitionId = ingest_api_service.ask(GetPartitionId).await?.into(); let checkpoint_delta = SourceCheckpointDelta::from_partition_delta( partition_id.clone(), @@ -428,7 +431,7 @@ mod tests { let ingest_api_source = IngestApiSource::try_new(source_runtime).await?; let ingest_api_source_actor = SourceActor { source: Box::new(ingest_api_source), - doc_processor_mailbox, + processor_mailbox: doc_processor_mailbox, }; let (_ingest_api_source_mailbox, ingest_api_source_handle) = universe.spawn_builder().spawn(ingest_api_source_actor); @@ -471,7 +474,8 @@ mod tests { let ingest_api_service = init_ingest_api(&universe, queues_dir_path, &IngestApiConfig::default()).await?; - let (doc_processor_mailbox, doc_processor_inbox) = universe.create_test_mailbox(); + let (doc_processor_mailbox, doc_processor_inbox) = + universe.create_test_mailbox::(); let source_config = make_source_config(); let source_runtime = SourceRuntimeBuilder::new(index_uid, source_config) .with_queues_dir(queues_dir_path) @@ -480,7 +484,7 @@ mod tests { let ingest_api_source = IngestApiSource::try_new(source_runtime).await?; let ingest_api_source_actor = SourceActor { source: Box::new(ingest_api_source), - doc_processor_mailbox, + processor_mailbox: doc_processor_mailbox, }; let (_ingest_api_source_mailbox, ingest_api_source_handle) = universe.spawn_builder().spawn(ingest_api_source_actor); @@ -522,7 +526,8 @@ mod tests { let ingest_api_service = init_ingest_api(&universe, queues_dir_path, &IngestApiConfig::default()).await?; - let (doc_processor_mailbox, doc_processor_inbox) = universe.create_test_mailbox(); + let (doc_processor_mailbox, doc_processor_inbox) = + universe.create_test_mailbox::(); let source_config = make_source_config(); let source_runtime = SourceRuntimeBuilder::new(index_uid, source_config) .with_queues_dir(queues_dir_path) @@ -531,7 +536,7 @@ mod tests { let ingest_api_source = IngestApiSource::try_new(source_runtime).await?; let ingest_api_source_actor = SourceActor { source: Box::new(ingest_api_source), - doc_processor_mailbox, + processor_mailbox: doc_processor_mailbox, }; let (_ingest_api_source_mailbox, ingest_api_source_handle) = universe.spawn_builder().spawn(ingest_api_source_actor); @@ -586,7 +591,8 @@ mod tests { let ingest_api_service = init_ingest_api(&universe, queues_dir_path, &IngestApiConfig::default()).await?; - let (doc_processor_mailbox, doc_processor_inbox) = universe.create_test_mailbox(); + let (doc_processor_mailbox, doc_processor_inbox) = + universe.create_test_mailbox::(); let source_config = make_source_config(); let source_runtime = SourceRuntimeBuilder::new(index_uid, source_config) .with_queues_dir(queues_dir_path) @@ -595,7 +601,7 @@ mod tests { let ingest_api_source = IngestApiSource::try_new(source_runtime).await?; let ingest_api_source_actor = SourceActor { source: Box::new(ingest_api_source), - doc_processor_mailbox, + processor_mailbox: doc_processor_mailbox, }; let (_ingest_api_source_mailbox, ingest_api_source_handle) = universe.spawn_builder().spawn(ingest_api_source_actor); @@ -646,7 +652,8 @@ mod tests { let ingest_api_service = init_ingest_api(&universe, queues_dir_path, &IngestApiConfig::default()).await?; - let (doc_processor_mailbox, _doc_processor_inbox) = universe.create_test_mailbox(); + let (doc_processor_mailbox, _doc_processor_inbox) = + universe.create_test_mailbox::(); let source_config = make_source_config(); let _source_runtime = SourceRuntimeBuilder::new(index_uid.clone(), source_config.clone()) .with_queues_dir(queues_dir_path) @@ -691,7 +698,7 @@ mod tests { let ingest_api_source = IngestApiSource::try_new(source_runtime).await?; let ingest_api_source_actor = SourceActor { source: Box::new(ingest_api_source), - doc_processor_mailbox, + processor_mailbox: doc_processor_mailbox, }; let (ingest_api_source_mailbox, ingest_api_source_handle) = universe.spawn_builder().spawn(ingest_api_source_actor); diff --git a/quickwit/quickwit-indexing/src/source/kafka_source.rs b/quickwit/quickwit-indexing/src/source/kafka_source.rs index bc5b400a9bc..f8f4c73fb4d 100644 --- a/quickwit/quickwit-indexing/src/source/kafka_source.rs +++ b/quickwit/quickwit-indexing/src/source/kafka_source.rs @@ -40,7 +40,7 @@ use tokio::task::{JoinHandle, spawn_blocking}; use tokio::time; use tracing::{debug, info, warn}; -use crate::actors::DocProcessor; +use crate::actors::Processor; use crate::models::{NewPublishLock, PublishLock}; use crate::source::{ BATCH_NUM_BYTES_LIMIT, BatchBuilder, EMIT_BATCHES_TIMEOUT, Source, SourceContext, @@ -333,9 +333,9 @@ impl KafkaSource { Ok(()) } - async fn process_assign_partitions( + async fn process_assign_partitions( &mut self, - ctx: &SourceContext, + ctx: &crate::source::SourceContext, partitions: &[i32], assignment_tx: oneshot::Sender>, ) -> anyhow::Result<()> { @@ -391,10 +391,10 @@ impl KafkaSource { Ok(()) } - async fn process_revoke_partitions( + async fn process_revoke_partitions( &mut self, - ctx: &SourceContext, - doc_processor_mailbox: &Mailbox, + ctx: &crate::source::SourceContext, + doc_processor_mailbox: &Mailbox, batch: &mut BatchBuilder, ack_tx: oneshot::Sender<()>, ) -> anyhow::Result<()> { @@ -441,22 +441,22 @@ impl KafkaSource { } #[async_trait] -impl Source for KafkaSource { +impl Source

for KafkaSource { async fn initialize( &mut self, - doc_processor_mailbox: &Mailbox, - ctx: &SourceContext, + processor_mailbox: &Mailbox

, + ctx: &SourceContext

, ) -> Result<(), ActorExitStatus> { let publish_lock = self.publish_lock.clone(); - ctx.send_message(doc_processor_mailbox, NewPublishLock(publish_lock)) + ctx.send_message(processor_mailbox, NewPublishLock(publish_lock)) .await?; Ok(()) } async fn emit_batches( &mut self, - doc_processor_mailbox: &Mailbox, - ctx: &SourceContext, + processor_mailbox: &Mailbox

, + ctx: &SourceContext

, ) -> Result { let now = Instant::now(); let mut batch_builder = BatchBuilder::new(SourceType::Kafka); @@ -470,7 +470,7 @@ impl Source for KafkaSource { match event { KafkaEvent::Message(message) => self.process_message(message, &mut batch_builder).await?, KafkaEvent::AssignPartitions { partitions, assignment_tx} => self.process_assign_partitions(ctx, &partitions, assignment_tx).await?, - KafkaEvent::RevokePartitions { ack_tx } => self.process_revoke_partitions(ctx, doc_processor_mailbox, &mut batch_builder, ack_tx).await?, + KafkaEvent::RevokePartitions { ack_tx } => self.process_revoke_partitions(ctx, processor_mailbox, &mut batch_builder, ack_tx).await?, KafkaEvent::PartitionEOF(partition) => self.process_partition_eof(partition), KafkaEvent::Error(error) => Err(ActorExitStatus::from(error))?, } @@ -492,11 +492,11 @@ impl Source for KafkaSource { "sending doc batch to indexer" ); let message = batch_builder.build(); - ctx.send_message(doc_processor_mailbox, message).await?; + ctx.send_message(processor_mailbox, message).await?; } if self.should_exit() { info!(topic = %self.topic, "reached end of topic"); - ctx.send_exit_with_success(doc_processor_mailbox).await?; + ctx.send_exit_with_success(processor_mailbox).await?; return Err(ActorExitStatus::Success); } Ok(Duration::default()) @@ -505,7 +505,7 @@ impl Source for KafkaSource { async fn suggest_truncate( &mut self, checkpoint: SourceCheckpoint, - _ctx: &SourceContext, + _ctx: &SourceContext

, ) -> anyhow::Result<()> { self.truncate(checkpoint)?; Ok(()) @@ -514,7 +514,7 @@ impl Source for KafkaSource { async fn finalize( &mut self, _exit_status: &ActorExitStatus, - _ctx: &SourceContext, + _ctx: &SourceContext

, ) -> anyhow::Result<()> { self.poll_loop_jh.abort(); Ok(()) @@ -872,6 +872,8 @@ mod kafka_broker_tests { fn get_source_config(topic: &str, auto_offset_reset: &str) -> (String, SourceConfig) { let source_id = append_random_suffix("test-kafka-source--source"); + // Setting explicitly ip v4 with `broker.address.family` is required + // because of https://github.com/fede1024/rust-rdkafka/issues/809 let source_config = SourceConfig { source_id: source_id.clone(), num_pipelines: NonZeroUsize::MIN, @@ -882,6 +884,7 @@ mod kafka_broker_tests { client_params: json!({ "auto.offset.reset": auto_offset_reset, "bootstrap.servers": "localhost:9092", + "broker.address.family": "v4", }), enable_backfill_mode: true, }), @@ -1272,7 +1275,7 @@ mod kafka_broker_tests { let (doc_processor_mailbox, doc_processor_inbox) = universe.create_test_mailbox(); let source_actor = SourceActor { source, - doc_processor_mailbox: doc_processor_mailbox.clone(), + processor_mailbox: doc_processor_mailbox.clone(), }; let (_source_mailbox, source_handle) = universe.spawn_builder().spawn(source_actor); let (exit_status, exit_state) = source_handle.join().await; @@ -1324,7 +1327,7 @@ mod kafka_broker_tests { let source = source_loader.load_source(source_runtime).await?; let source_actor = SourceActor { source, - doc_processor_mailbox: doc_processor_mailbox.clone(), + processor_mailbox: doc_processor_mailbox.clone(), }; let (_source_mailbox, source_handle) = universe.spawn_builder().spawn(source_actor); let (exit_status, exit_state) = source_handle.join().await; @@ -1398,7 +1401,7 @@ mod kafka_broker_tests { let (doc_processor_mailbox, doc_processor_inbox) = universe.create_test_mailbox(); let source_actor = SourceActor { source, - doc_processor_mailbox: doc_processor_mailbox.clone(), + processor_mailbox: doc_processor_mailbox.clone(), }; let (_source_mailbox, source_handle) = universe.spawn_builder().spawn(source_actor); let (exit_status, exit_state) = source_handle.join().await; @@ -1451,7 +1454,7 @@ mod kafka_broker_tests { let (doc_processor_mailbox, doc_processor_inbox) = universe.create_test_mailbox(); let source_actor = SourceActor { source, - doc_processor_mailbox: doc_processor_mailbox.clone(), + processor_mailbox: doc_processor_mailbox.clone(), }; let (_source_mailbox, source_handle) = universe.spawn_builder().spawn(source_actor); let (exit_status, exit_state) = source_handle.join().await; diff --git a/quickwit/quickwit-indexing/src/source/kinesis/kinesis_source.rs b/quickwit/quickwit-indexing/src/source/kinesis/kinesis_source.rs index 0889e7785df..8d6e7e662b7 100644 --- a/quickwit/quickwit-indexing/src/source/kinesis/kinesis_source.rs +++ b/quickwit/quickwit-indexing/src/source/kinesis/kinesis_source.rs @@ -35,7 +35,7 @@ use tracing::{info, warn}; use super::api::list_shards; use super::shard_consumer::{ShardConsumer, ShardConsumerHandle, ShardConsumerMessage}; -use crate::actors::DocProcessor; +use crate::actors::Processor; use crate::source::kinesis::helpers::get_kinesis_client; use crate::source::{ BATCH_NUM_BYTES_LIMIT, BatchBuilder, EMIT_BATCHES_TIMEOUT, Source, SourceContext, @@ -131,13 +131,20 @@ impl KinesisSource { Ok(kinesis_source) } - fn spawn_shard_consumer( + fn spawn_shard_consumer( &mut self, - ctx: &SourceContext, + ctx: &SourceContext, shard_id: ShardId, checkpoint: &SourceCheckpoint, ) { - assert!(!self.state.shard_consumers.contains_key(&shard_id)); + if self.state.shard_consumers.contains_key(&shard_id) { + info!( + stream_name = %self.stream_name, + shard_id = %shard_id, + "Shard consumer already exists, skipping creation." + ); + return; + } let partition_id = PartitionId::from(shard_id.as_str()); let from_position = checkpoint @@ -149,6 +156,12 @@ impl KinesisSource { Position::Offset(offset) => Some(offset.to_string()), Position::Eof(_) => panic!("position of a Kinesis shard should never be EOF"), }; + info!( + stream_name = %self.stream_name, + shard_id = %shard_id, + start_position = ?from_position, + "Spawning new shard consumer" + ); let shard_consumer = ShardConsumer::new( self.stream_name.clone(), shard_id.clone(), @@ -172,11 +185,11 @@ impl KinesisSource { } #[async_trait] -impl Source for KinesisSource { +impl Source

for KinesisSource { async fn initialize( &mut self, - _doc_processor_mailbox: &Mailbox, - ctx: &SourceContext, + _processor_mailbox: &Mailbox

, + ctx: &SourceContext

, ) -> Result<(), ActorExitStatus> { let shards = ctx .protect_future(list_shards( @@ -205,8 +218,8 @@ impl Source for KinesisSource { async fn emit_batches( &mut self, - indexer_mailbox: &Mailbox, - ctx: &SourceContext, + indexer_mailbox: &Mailbox

, + ctx: &SourceContext

, ) -> Result { let mut batch_builder = BatchBuilder::new(SourceType::Kinesis); let deadline = time::sleep(*EMIT_BATCHES_TIMEOUT); @@ -362,6 +375,7 @@ mod tests { use quickwit_proto::types::IndexUid; use super::*; + use crate::actors::DocProcessor; use crate::models::RawDocBatch; use crate::source::SourceActor; use crate::source::kinesis::helpers::tests::{ @@ -384,11 +398,68 @@ mod tests { Ok(merged_batch) } + #[ignore] + #[tokio::test] + async fn test_kinesis_source_handles_resharding_with_split() { + use crate::source::kinesis::api::tests::split_shard; + use crate::source::kinesis::helpers::tests::wait_for_active_stream; + + let universe = Universe::with_accelerated_time(); + let (doc_processor_mailbox, _doc_processor_inbox) = + universe.create_test_mailbox::(); + let (kinesis_client, stream_name) = setup("test-resharding-split", 1).await.unwrap(); + let index_id = "test-kinesis-resharding-index"; + let index_uid = IndexUid::new_with_random_ulid(index_id); + + // Split the shard (1 -> 2 shards) + let shard_id_0 = make_shard_id(0); + split_shard( + &kinesis_client, + &stream_name, + &shard_id_0, + "85070591730234615865843651857942052864", + ) + .await + .unwrap(); + + // Wait for stream to be active after split + let _ = wait_for_active_stream(&kinesis_client, &stream_name) + .await + .unwrap(); + + // Initialize source after split + let kinesis_params = KinesisSourceParams { + stream_name: stream_name.clone(), + region_or_endpoint: Some(RegionOrEndpoint::Endpoint( + "http://localhost:4566".to_string(), + )), + enable_backfill_mode: true, + }; + let source_params = SourceParams::Kinesis(kinesis_params.clone()); + let source_config = SourceConfig::for_test("test-kinesis-resharding", source_params); + let source_runtime = SourceRuntimeBuilder::new(index_uid, source_config).build(); + + let kinesis_source = KinesisSource::try_new(source_runtime, kinesis_params) + .await + .unwrap(); + + let actor = SourceActor { + source: Box::new(kinesis_source), + processor_mailbox: doc_processor_mailbox.clone(), + }; + let (_mailbox, handle) = universe.spawn_builder().spawn(actor); + let (exit_status, _exit_state) = handle.join().await; + assert!(exit_status.is_success()); + + teardown(&kinesis_client, &stream_name).await; + } + #[ignore] #[tokio::test] async fn test_kinesis_source() { let universe = Universe::with_accelerated_time(); - let (doc_processor_mailbox, doc_processor_inbox) = universe.create_test_mailbox(); + let (doc_processor_mailbox, doc_processor_inbox) = + universe.create_test_mailbox::(); let (kinesis_client, stream_name) = setup("test-kinesis-source", 3).await.unwrap(); let index_id = "test-kinesis-index"; let index_uid = IndexUid::new_with_random_ulid(index_id); @@ -409,7 +480,7 @@ mod tests { .unwrap(); let actor = SourceActor { source: Box::new(kinesis_source), - doc_processor_mailbox: doc_processor_mailbox.clone(), + processor_mailbox: doc_processor_mailbox.clone(), }; let (_mailbox, handle) = universe.spawn_builder().spawn(actor); let (exit_status, exit_state) = handle.join().await; @@ -462,7 +533,7 @@ mod tests { .unwrap(); let actor = SourceActor { source: Box::new(kinesis_source), - doc_processor_mailbox: doc_processor_mailbox.clone(), + processor_mailbox: doc_processor_mailbox.clone(), }; let (_mailbox, handle) = universe.spawn_builder().spawn(actor); let (exit_status, exit_state) = handle.join().await; @@ -532,7 +603,7 @@ mod tests { .unwrap(); let actor = SourceActor { source: Box::new(kinesis_source), - doc_processor_mailbox: doc_processor_mailbox.clone(), + processor_mailbox: doc_processor_mailbox.clone(), }; let (_mailbox, handle) = universe.spawn_builder().spawn(actor); let (exit_status, exit_state) = handle.join().await; diff --git a/quickwit/quickwit-indexing/src/source/kinesis/shard_consumer.rs b/quickwit/quickwit-indexing/src/source/kinesis/shard_consumer.rs index 45eb4c05787..df00f963588 100644 --- a/quickwit/quickwit-indexing/src/source/kinesis/shard_consumer.rs +++ b/quickwit/quickwit-indexing/src/source/kinesis/shard_consumer.rs @@ -103,7 +103,7 @@ impl ShardConsumer { } } - pub fn spawn(self, ctx: &SourceContext) -> ShardConsumerHandle { + pub fn spawn(self, ctx: &SourceContext

) -> ShardConsumerHandle { let (_mailbox, _actor_handle) = ctx.spawn_actor().spawn(self); ShardConsumerHandle { _mailbox, diff --git a/quickwit/quickwit-indexing/src/source/mod.rs b/quickwit/quickwit-indexing/src/source/mod.rs index 5601e31618d..86d40e5288f 100644 --- a/quickwit/quickwit-indexing/src/source/mod.rs +++ b/quickwit/quickwit-indexing/src/source/mod.rs @@ -116,11 +116,17 @@ pub use void_source::{VoidSource, VoidSourceFactory}; use self::doc_file_reader::dir_and_filename; use self::stdin_source::StdinSourceFactory; -use crate::actors::DocProcessor; +use crate::actors::{DocProcessor, ParquetDocProcessor, Processor}; use crate::models::RawDocBatch; use crate::source::ingest::IngestSourceFactory; use crate::source::ingest_api_source::IngestApiSourceFactory; +/// Type alias for SourceContext with ParquetDocProcessor (for metrics pipeline). +pub type ParquetSourceContext = SourceContext; + +/// Type alias for SourceLoader with ParquetDocProcessor (for metrics pipeline). +pub type ParquetSourceLoader = SourceLoader; + /// Number of bytes after which we cut a new batch. /// /// We try to emit chewable batches for the indexer. @@ -207,7 +213,9 @@ impl SourceRuntime { } } -pub type SourceContext = ActorContext; +pub type SourceContext

= ActorContext>; + +pub type LogsSourceContext = SourceContext; /// A Source is a trait that is mounted in a light wrapping Actor called `SourceActor`. /// @@ -233,12 +241,12 @@ pub type SourceContext = ActorContext; /// } /// ``` #[async_trait] -pub trait Source: Send + 'static { +pub trait Source: Send + 'static { /// This method will be called before any calls to `emit_batches`. async fn initialize( &mut self, - _doc_processor_mailbox: &Mailbox, - _ctx: &SourceContext, + _processor_mailbox: &Mailbox

, + _ctx: &SourceContext

, ) -> Result<(), ActorExitStatus> { Ok(()) } @@ -252,8 +260,8 @@ pub trait Source: Send + 'static { /// should wait before polling again. async fn emit_batches( &mut self, - doc_processor_mailbox: &Mailbox, - ctx: &SourceContext, + processor_mailbox: &Mailbox

, + ctx: &SourceContext

, ) -> Result; /// Assign shards is called when the source is assigned a new set of shards by the control @@ -261,8 +269,8 @@ pub trait Source: Send + 'static { async fn assign_shards( &mut self, _shard_ids: BTreeSet, - _doc_processor_mailbox: &Mailbox, - _ctx: &SourceContext, + _processor_mailbox: &Mailbox

, + _ctx: &SourceContext

, ) -> anyhow::Result<()> { Ok(()) } @@ -284,7 +292,7 @@ pub trait Source: Send + 'static { async fn suggest_truncate( &mut self, _checkpoint: SourceCheckpoint, - _ctx: &SourceContext, + _ctx: &SourceContext

, ) -> anyhow::Result<()> { Ok(()) } @@ -293,7 +301,7 @@ pub trait Source: Send + 'static { async fn finalize( &mut self, _exit_status: &ActorExitStatus, - _ctx: &SourceContext, + _ctx: &SourceContext

, ) -> anyhow::Result<()> { Ok(()) } @@ -311,9 +319,9 @@ pub trait Source: Send + 'static { /// The SourceActor acts as a thin wrapper over a source trait object to execute. /// /// It mostly takes care of running a loop calling `emit_batches(...)`. -pub struct SourceActor { - pub source: Box, - pub doc_processor_mailbox: Mailbox, +pub struct SourceActor { + pub source: Box>, + pub processor_mailbox: Mailbox

, } #[derive(Debug)] @@ -328,7 +336,7 @@ pub struct Assignment { pub struct AssignShards(pub Assignment); #[async_trait] -impl Actor for SourceActor { +impl Actor for SourceActor

{ type ObservableState = JsonValue; fn name(&self) -> String { @@ -347,10 +355,8 @@ impl Actor for SourceActor { false } - async fn initialize(&mut self, ctx: &SourceContext) -> Result<(), ActorExitStatus> { - self.source - .initialize(&self.doc_processor_mailbox, ctx) - .await?; + async fn initialize(&mut self, ctx: &SourceContext

) -> Result<(), ActorExitStatus> { + self.source.initialize(&self.processor_mailbox, ctx).await?; self.handle(Loop, ctx).await?; Ok(()) } @@ -358,7 +364,7 @@ impl Actor for SourceActor { async fn finalize( &mut self, exit_status: &ActorExitStatus, - ctx: &SourceContext, + ctx: &SourceContext

, ) -> anyhow::Result<()> { self.source.finalize(exit_status, ctx).await?; Ok(()) @@ -366,13 +372,17 @@ impl Actor for SourceActor { } #[async_trait] -impl Handler for SourceActor { +impl Handler for SourceActor

{ type Reply = (); - async fn handle(&mut self, _message: Loop, ctx: &SourceContext) -> Result<(), ActorExitStatus> { + async fn handle( + &mut self, + _message: Loop, + ctx: &SourceContext

, + ) -> Result<(), ActorExitStatus> { let wait_for = self .source - .emit_batches(&self.doc_processor_mailbox, ctx) + .emit_batches(&self.processor_mailbox, ctx) .await?; if wait_for.is_zero() { ctx.send_self_message(Loop).await?; @@ -384,17 +394,17 @@ impl Handler for SourceActor { } #[async_trait] -impl Handler for SourceActor { +impl Handler for SourceActor

{ type Reply = (); async fn handle( &mut self, assign_shards_message: AssignShards, - ctx: &SourceContext, + ctx: &SourceContext

, ) -> Result<(), ActorExitStatus> { let AssignShards(Assignment { shard_ids }) = assign_shards_message; self.source - .assign_shards(shard_ids, &self.doc_processor_mailbox, ctx) + .assign_shards(shard_ids, &self.processor_mailbox, ctx) .await?; Ok(()) } @@ -423,6 +433,24 @@ pub fn quickwit_supported_sources() -> &'static SourceLoader { }) } +/// Returns the source loader for parquet pipelines (ParquetDocProcessor). +/// +/// Metrics pipelines currently only support IngestV2 sources, which is the +/// production source type for metrics ingestion. +pub fn quickwit_supported_parquet_sources() -> &'static ParquetSourceLoader { + static PARQUET_SOURCE_LOADER: OnceCell = OnceCell::new(); + PARQUET_SOURCE_LOADER.get_or_init(|| { + let mut source_factory = ParquetSourceLoader::default(); + // Only IngestV2 is currently used for metrics ingestion + source_factory.add_source(SourceType::IngestV2, IngestSourceFactory); + // Add other sources for testing/development + source_factory.add_source(SourceType::File, FileSourceFactory); + source_factory.add_source(SourceType::Vec, VecSourceFactory); + source_factory.add_source(SourceType::Void, VoidSourceFactory); + source_factory + }) +} + pub async fn check_source_connectivity( storage_resolver: &StorageResolver, source_config: &SourceConfig, @@ -488,13 +516,13 @@ pub async fn check_source_connectivity( pub struct SuggestTruncate(pub SourceCheckpoint); #[async_trait] -impl Handler for SourceActor { +impl Handler for SourceActor

{ type Reply = (); async fn handle( &mut self, suggest_truncate: SuggestTruncate, - ctx: &SourceContext, + ctx: &SourceContext

, ) -> Result<(), ActorExitStatus> { let SuggestTruncate(checkpoint) = suggest_truncate; @@ -740,7 +768,11 @@ mod tests { #[cfg(all( test, - any(feature = "sqs-localstack-tests", feature = "kafka-broker-tests") + any( + feature = "sqs-localstack-tests", + feature = "kafka-broker-tests", + feature = "pulsar-broker-tests" + ) ))] mod test_setup_helper { @@ -763,7 +795,7 @@ mod test_setup_helper { let index_config = IndexConfig::for_test(index_id, &index_uri); let create_index_request = CreateIndexRequest::try_from_index_and_source_configs( &index_config, - &[source_config.clone()], + std::slice::from_ref(source_config), ) .unwrap(); let index_uid: IndexUid = metastore diff --git a/quickwit/quickwit-indexing/src/source/pulsar_source.rs b/quickwit/quickwit-indexing/src/source/pulsar_source.rs index 226a9a0a39b..ce02d857b78 100644 --- a/quickwit/quickwit-indexing/src/source/pulsar_source.rs +++ b/quickwit/quickwit-indexing/src/source/pulsar_source.rs @@ -26,7 +26,7 @@ use pulsar::message::proto::MessageIdData; use pulsar::{ Authentication, Consumer, DeserializeMessage, Payload, Pulsar, SubType, TokioExecutor, }; -use quickwit_actors::{ActorContext, ActorExitStatus, Mailbox}; +use quickwit_actors::{ActorExitStatus, Mailbox}; use quickwit_config::{PulsarSourceAuth, PulsarSourceParams}; use quickwit_metastore::checkpoint::{PartitionId, SourceCheckpoint}; use quickwit_proto::metastore::SourceType; @@ -35,9 +35,9 @@ use serde_json::{Value as JsonValue, json}; use tokio::time; use tracing::{debug, info, warn}; -use crate::actors::DocProcessor; +use crate::actors::Processor; use crate::source::{ - BATCH_NUM_BYTES_LIMIT, BatchBuilder, EMIT_BATCHES_TIMEOUT, Source, SourceActor, SourceContext, + BATCH_NUM_BYTES_LIMIT, BatchBuilder, EMIT_BATCHES_TIMEOUT, Source, SourceContext, SourceRuntime, TypedSourceFactory, }; @@ -210,11 +210,11 @@ impl PulsarSource { } #[async_trait] -impl Source for PulsarSource { +impl Source

for PulsarSource { async fn emit_batches( &mut self, - doc_processor_mailbox: &Mailbox, - ctx: &SourceContext, + processor_mailbox: &Mailbox

, + ctx: &SourceContext

, ) -> Result { let now = Instant::now(); let mut batch_builder = BatchBuilder::new(SourceType::Pulsar); @@ -252,7 +252,7 @@ impl Source for PulsarSource { "sending doc batch to indexer" ); let message = batch_builder.build(); - ctx.send_message(doc_processor_mailbox, message).await?; + ctx.send_message(processor_mailbox, message).await?; } Ok(Duration::default()) } @@ -260,7 +260,7 @@ impl Source for PulsarSource { async fn suggest_truncate( &mut self, checkpoint: SourceCheckpoint, - _ctx: &ActorContext, + _ctx: &SourceContext

, ) -> anyhow::Result<()> { self.try_ack_messages(checkpoint).await } @@ -272,7 +272,7 @@ impl Source for PulsarSource { async fn finalize( &mut self, _exit_status: &ActorExitStatus, - _ctx: &SourceContext, + _ctx: &SourceContext

, ) -> anyhow::Result<()> { self.pulsar_consumer.close().await?; Ok(()) @@ -454,10 +454,11 @@ mod pulsar_broker_tests { use reqwest::StatusCode; use super::*; + use crate::actors::DocProcessor; use crate::source::pulsar_source::{msg_id_from_position, msg_id_to_position}; use crate::source::test_setup_helper::setup_index; use crate::source::tests::SourceRuntimeBuilder; - use crate::source::{RawDocBatch, SuggestTruncate, quickwit_supported_sources}; + use crate::source::{RawDocBatch, SourceActor, SuggestTruncate, quickwit_supported_sources}; static PULSAR_URI: &str = "pulsar://localhost:6650"; static PULSAR_ADMIN_URI: &str = "http://localhost:8081"; @@ -651,7 +652,7 @@ mod pulsar_broker_tests { let (doc_processor_mailbox, doc_processor_inbox) = universe.create_test_mailbox(); let source_actor = SourceActor { source, - doc_processor_mailbox, + processor_mailbox: doc_processor_mailbox, }; let (_source_mailbox, source_handle) = universe.spawn_builder().spawn(source_actor); diff --git a/quickwit/quickwit-indexing/src/source/queue_sources/coordinator.rs b/quickwit/quickwit-indexing/src/source/queue_sources/coordinator.rs index fadb4282c37..40919879e69 100644 --- a/quickwit/quickwit-indexing/src/source/queue_sources/coordinator.rs +++ b/quickwit/quickwit-indexing/src/source/queue_sources/coordinator.rs @@ -34,7 +34,7 @@ use super::local_state::QueueLocalState; use super::message::{MessageType, PreProcessingError, ReadyMessage}; use super::shared_state::{QueueSharedState, checkpoint_messages}; use super::visibility::{VisibilitySettings, spawn_visibility_task}; -use crate::actors::DocProcessor; +use crate::actors::Processor; use crate::models::{NewPublishLock, NewPublishToken, PublishLock}; use crate::source::{SourceContext, SourceRuntime}; @@ -146,10 +146,10 @@ impl QueueCoordinator { )) } - pub async fn initialize( + pub async fn initialize( &mut self, - doc_processor_mailbox: &Mailbox, - ctx: &SourceContext, + doc_processor_mailbox: &Mailbox

, + ctx: &SourceContext

, ) -> Result<(), ActorExitStatus> { let publish_lock = self.publish_lock.clone(); ctx.send_message(doc_processor_mailbox, NewPublishLock(publish_lock)) @@ -163,7 +163,10 @@ impl QueueCoordinator { } /// Polls messages from the queue and prepares them for processing - async fn poll_messages(&mut self, ctx: &SourceContext) -> Result<(), ActorExitStatus> { + async fn poll_messages( + &mut self, + ctx: &SourceContext

, + ) -> Result<(), ActorExitStatus> { let raw_messages = self .queue_receiver .receive(1, self.visibility_settings.deadline_for_receive) @@ -249,10 +252,10 @@ impl QueueCoordinator { Ok(()) } - pub async fn emit_batches( + pub async fn emit_batches( &mut self, - doc_processor_mailbox: &Mailbox, - ctx: &SourceContext, + doc_processor_mailbox: &Mailbox

, + ctx: &SourceContext

, ) -> Result { if let Some(in_progress_ref) = self.local_state.read_in_progress_mut() { // TODO: should we kill the publish lock if the message visibility extension failed? @@ -290,10 +293,10 @@ impl QueueCoordinator { Ok(Duration::ZERO) } - pub async fn suggest_truncate( + pub async fn suggest_truncate( &mut self, checkpoint: SourceCheckpoint, - _ctx: &SourceContext, + _ctx: &SourceContext

, ) -> anyhow::Result<()> { let committed_partition_ids = checkpoint .iter() @@ -326,6 +329,7 @@ mod tests { use ulid::Ulid; use super::*; + use crate::actors::DocProcessor; use crate::models::RawDocBatch; use crate::source::doc_file_reader::file_test_helpers::{DUMMY_DOC, generate_dummy_doc_file}; use crate::source::queue_sources::memory_queue::MemoryQueueForTests; diff --git a/quickwit/quickwit-indexing/src/source/queue_sources/shared_state.rs b/quickwit/quickwit-indexing/src/source/queue_sources/shared_state.rs index b839c968043..b3551186341 100644 --- a/quickwit/quickwit-indexing/src/source/queue_sources/shared_state.rs +++ b/quickwit/quickwit-indexing/src/source/queue_sources/shared_state.rs @@ -155,7 +155,9 @@ impl QueueSharedState { info!(previous_token = shard.publish_token, "shard re-acquired"); re_acquired_shards.push(shard.shard_id().clone()); } else if is_owned && !position.is_beginning() { - bail!("Partition is owned by this indexing pipeline but is not at the beginning. This should never happen! Please, report on https://github.com/quickwit-oss/quickwit/issues.") + bail!( + "Partition is owned by this indexing pipeline but is not at the beginning. This should never happen! Please, report on https://github.com/quickwit-oss/quickwit/issues." + ) } } diff --git a/quickwit/quickwit-indexing/src/source/queue_sources/sqs_queue.rs b/quickwit/quickwit-indexing/src/source/queue_sources/sqs_queue.rs index 1e4c55142ac..8dc379d92da 100644 --- a/quickwit/quickwit-indexing/src/source/queue_sources/sqs_queue.rs +++ b/quickwit/quickwit-indexing/src/source/queue_sources/sqs_queue.rs @@ -318,21 +318,26 @@ pub mod test_helpers { /// /// Returns the queue URL to use for the source and a guard for the /// temporary mock server - pub fn start_mock_sqs_get_queue_attributes_endpoint() -> (String, oneshot::Sender<()>) { + pub async fn start_mock_sqs_get_queue_attributes_endpoint() -> (String, oneshot::Sender<()>) { let hello = warp::path!().map(|| "{}"); let (tx, rx) = oneshot::channel(); - let (addr, server) = - warp::serve(hello).bind_with_graceful_shutdown(([127, 0, 0, 1], 0), async { - rx.await.ok(); - }); - tokio::spawn(server); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("listener should bind"); + let addr = listener.local_addr().unwrap(); + + let server = warp::serve(hello).incoming(listener).graceful(async { + rx.await.ok(); + }); + tokio::spawn(server.run()); + let queue_url = format!("http://{}:{}/", addr.ip(), addr.port()); (queue_url, tx) } #[tokio::test] async fn test_mock_sqs_get_queue_attributes_endpoint() { - let (queue_url, _shutdown) = start_mock_sqs_get_queue_attributes_endpoint(); + let (queue_url, _shutdown) = start_mock_sqs_get_queue_attributes_endpoint().await; check_connectivity(&queue_url).await.unwrap(); drop(_shutdown); check_connectivity(&queue_url).await.unwrap_err(); diff --git a/quickwit/quickwit-indexing/src/source/queue_sources/visibility.rs b/quickwit/quickwit-indexing/src/source/queue_sources/visibility.rs index 3af4deaa278..8f166d96de0 100644 --- a/quickwit/quickwit-indexing/src/source/queue_sources/visibility.rs +++ b/quickwit/quickwit-indexing/src/source/queue_sources/visibility.rs @@ -23,6 +23,7 @@ use quickwit_actors::{ use serde_json::{Value as JsonValue, json}; use super::Queue; +use crate::actors::Processor; use crate::source::SourceContext; #[derive(Debug, Clone)] @@ -80,8 +81,8 @@ pub(super) struct VisibilityTaskHandle { /// (represented by its ack_id) is extended when required. We prefer applying /// ample margins in the extension process to avoid missing deadlines while also /// keeping the number of extension requests (and associated cost) small. -pub(super) fn spawn_visibility_task( - ctx: &SourceContext, +pub(super) fn spawn_visibility_task( + ctx: &SourceContext

, queue: Arc, ack_id: String, current_deadline: Instant, diff --git a/quickwit/quickwit-indexing/src/source/source_factory.rs b/quickwit/quickwit-indexing/src/source/source_factory.rs index 6530d6e5ae8..dd6ebbfc913 100644 --- a/quickwit/quickwit-indexing/src/source/source_factory.rs +++ b/quickwit/quickwit-indexing/src/source/source_factory.rs @@ -21,17 +21,20 @@ use quickwit_proto::types::SourceId; use thiserror::Error; use super::Source; +use crate::actors::{DocProcessor, ParquetDocProcessor, Processor}; use crate::source::SourceRuntime; #[async_trait] -pub trait SourceFactory: Send + Sync + 'static { - async fn create_source(&self, source_runtime: SourceRuntime) - -> anyhow::Result>; +pub trait SourceFactory: Send + Sync + 'static { + async fn create_source( + &self, + source_runtime: SourceRuntime, + ) -> anyhow::Result>>; } #[async_trait] pub trait TypedSourceFactory: Send + Sync + 'static { - type Source: Source; + type Source: Source + Source; type Params: serde::de::DeserializeOwned + Send + Sync + 'static; async fn typed_create_source( @@ -53,9 +56,29 @@ impl SourceFactory for T { } } -#[derive(Default)] -pub struct SourceLoader { - type_to_factory: HashMap>, +#[async_trait] +impl SourceFactory for T { + async fn create_source( + &self, + source_runtime: SourceRuntime, + ) -> anyhow::Result>> { + let typed_params: T::Params = + serde_json::from_value(source_runtime.source_config.params())?; + let source = Self::typed_create_source(source_runtime, typed_params).await?; + Ok(Box::new(source)) + } +} + +pub struct SourceLoader { + type_to_factory: HashMap>>, +} + +impl Default for SourceLoader

{ + fn default() -> Self { + Self { + type_to_factory: HashMap::default(), + } + } } #[derive(Error, Debug)] @@ -77,8 +100,8 @@ pub enum SourceLoaderError { }, } -impl SourceLoader { - pub fn add_source(&mut self, source_type: SourceType, source_factory: F) { +impl SourceLoader

{ + pub fn add_source>(&mut self, source_type: SourceType, source_factory: F) { self.type_to_factory .insert(source_type, Box::new(source_factory)); } @@ -86,7 +109,7 @@ impl SourceLoader { pub async fn load_source( &self, source_runtime: SourceRuntime, - ) -> Result, SourceLoaderError> { + ) -> Result>, SourceLoaderError> { let source_type = source_runtime.source_config.source_type(); let source_id = source_runtime.source_id().to_string(); let source_factory = self.type_to_factory.get(&source_type).ok_or_else(|| { diff --git a/quickwit/quickwit-indexing/src/source/stdin_source.rs b/quickwit/quickwit-indexing/src/source/stdin_source.rs index 881dca5770d..72327df18e3 100644 --- a/quickwit/quickwit-indexing/src/source/stdin_source.rs +++ b/quickwit/quickwit-indexing/src/source/stdin_source.rs @@ -22,7 +22,7 @@ use quickwit_proto::metastore::SourceType; use tokio::io::{AsyncBufReadExt, BufReader}; use super::{BATCH_NUM_BYTES_LIMIT, BatchBuilder}; -use crate::actors::DocProcessor; +use crate::actors::Processor; use crate::source::{Source, SourceContext, SourceRuntime, TypedSourceFactory}; pub struct StdinBatchReader { @@ -76,20 +76,20 @@ impl fmt::Debug for StdinSource { } #[async_trait] -impl Source for StdinSource { +impl Source

for StdinSource { async fn emit_batches( &mut self, - doc_processor_mailbox: &Mailbox, - ctx: &SourceContext, + processor_mailbox: &Mailbox

, + ctx: &SourceContext

, ) -> Result { let batch_builder = self.reader.read_batch(ctx.progress()).await?; self.num_bytes_processed += batch_builder.num_bytes; self.num_lines_processed += batch_builder.docs.len() as u64; - doc_processor_mailbox + processor_mailbox .send_message(batch_builder.build()) .await?; if self.reader.is_eof() { - ctx.send_exit_with_success(doc_processor_mailbox).await?; + ctx.send_exit_with_success(processor_mailbox).await?; return Err(ActorExitStatus::Success); } diff --git a/quickwit/quickwit-indexing/src/source/vec_source.rs b/quickwit/quickwit-indexing/src/source/vec_source.rs index a66204b72fa..aad8136d34b 100644 --- a/quickwit/quickwit-indexing/src/source/vec_source.rs +++ b/quickwit/quickwit-indexing/src/source/vec_source.rs @@ -25,7 +25,7 @@ use serde_json::Value as JsonValue; use tracing::info; use super::BatchBuilder; -use crate::actors::DocProcessor; +use crate::actors::Processor; use crate::source::{Source, SourceContext, SourceRuntime, TypedSourceFactory}; pub struct VecSource { @@ -81,11 +81,11 @@ fn position_from_offset(offset: usize) -> Position { } #[async_trait] -impl Source for VecSource { +impl Source

for VecSource { async fn emit_batches( &mut self, - batch_sink: &Mailbox, - ctx: &SourceContext, + batch_sink: &Mailbox

, + ctx: &SourceContext

, ) -> Result { let mut batch_builder = BatchBuilder::new(SourceType::Vec); @@ -138,6 +138,7 @@ mod tests { use serde_json::json; use super::*; + use crate::actors::DocProcessor; use crate::models::RawDocBatch; use crate::source::SourceActor; use crate::source::tests::SourceRuntimeBuilder; @@ -145,7 +146,8 @@ mod tests { #[tokio::test] async fn test_vec_source() -> anyhow::Result<()> { let universe = Universe::with_accelerated_time(); - let (doc_processor_mailbox, doc_processor_inbox) = universe.create_test_mailbox(); + let (doc_processor_mailbox, doc_processor_inbox) = + universe.create_test_mailbox::(); let docs = std::iter::repeat_with(|| Bytes::from_static(b"{}")) .take(100) .collect(); @@ -167,7 +169,7 @@ mod tests { let vec_source = VecSourceFactory::typed_create_source(source_runtime, params).await?; let vec_source_actor = SourceActor { source: Box::new(vec_source), - doc_processor_mailbox, + processor_mailbox: doc_processor_mailbox, }; assert_eq!( vec_source_actor.name(), @@ -195,7 +197,8 @@ mod tests { #[tokio::test] async fn test_vec_source_from_checkpoint() -> anyhow::Result<()> { let universe = Universe::with_accelerated_time(); - let (doc_processor_mailbox, doc_processor_inbox) = universe.create_test_mailbox(); + let (doc_processor_mailbox, doc_processor_inbox) = + universe.create_test_mailbox::(); let docs = (0..10).map(|i| Bytes::from(format!("{i}"))).collect(); let params = VecSourceParams { docs, @@ -218,7 +221,7 @@ mod tests { let vec_source = VecSourceFactory::typed_create_source(source_runtime, params).await?; let vec_source_actor = SourceActor { source: Box::new(vec_source), - doc_processor_mailbox, + processor_mailbox: doc_processor_mailbox, }; let (_vec_source_mailbox, vec_source_handle) = universe.spawn_builder().spawn(vec_source_actor); diff --git a/quickwit/quickwit-indexing/src/source/void_source.rs b/quickwit/quickwit-indexing/src/source/void_source.rs index 51b3b7587de..750a18c6fe5 100644 --- a/quickwit/quickwit-indexing/src/source/void_source.rs +++ b/quickwit/quickwit-indexing/src/source/void_source.rs @@ -19,17 +19,17 @@ use quickwit_actors::{ActorExitStatus, HEARTBEAT, Mailbox}; use quickwit_config::VoidSourceParams; use serde_json::Value as JsonValue; -use crate::actors::DocProcessor; +use crate::actors::Processor; use crate::source::{Source, SourceContext, SourceRuntime, TypedSourceFactory}; pub struct VoidSource; #[async_trait] -impl Source for VoidSource { +impl Source

for VoidSource { async fn emit_batches( &mut self, - _: &Mailbox, - _: &SourceContext, + _: &Mailbox

, + _: &SourceContext

, ) -> Result { tokio::time::sleep(*HEARTBEAT / 2).await; Ok(Duration::default()) @@ -71,6 +71,7 @@ mod tests { use serde_json::json; use super::*; + use crate::actors::DocProcessor; use crate::source::tests::SourceRuntimeBuilder; use crate::source::{SourceActor, SourceConfig, quickwit_supported_sources}; @@ -108,10 +109,10 @@ mod tests { let source_runtime = SourceRuntimeBuilder::new(index_uid, source_config).build(); let void_source = VoidSourceFactory::typed_create_source(source_runtime, VoidSourceParams).await?; - let (doc_processor_mailbox, _) = universe.create_test_mailbox(); + let (doc_processor_mailbox, _) = universe.create_test_mailbox::(); let void_source_actor = SourceActor { source: Box::new(void_source), - doc_processor_mailbox, + processor_mailbox: doc_processor_mailbox, }; let (_, void_source_handle) = universe.spawn_builder().spawn(void_source_actor); matches!(void_source_handle.check_health(true), Health::Healthy); diff --git a/quickwit/quickwit-indexing/src/test_utils.rs b/quickwit/quickwit-indexing/src/test_utils.rs index 57dec00aeac..82198c820f4 100644 --- a/quickwit/quickwit-indexing/src/test_utils.rs +++ b/quickwit/quickwit-indexing/src/test_utils.rs @@ -97,7 +97,7 @@ impl TestSandbox { .await?; let create_index_request = CreateIndexRequest::try_from_index_and_source_configs( &index_config, - &[source_config.clone()], + std::slice::from_ref(&source_config), )?; let index_uid: IndexUid = metastore .create_index(create_index_request) diff --git a/quickwit/quickwit-ingest/Cargo.toml b/quickwit/quickwit-ingest/Cargo.toml index 5ac233859f8..3149f2aaaf3 100644 --- a/quickwit/quickwit-ingest/Cargo.toml +++ b/quickwit/quickwit-ingest/Cargo.toml @@ -30,6 +30,7 @@ serde_json_borrow = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } tonic = { workspace = true } +tonic-prost = { workspace = true } tower = { workspace = true } tracing = { workspace = true } ulid = { workspace = true } @@ -43,7 +44,6 @@ quickwit-doc-mapper = { workspace = true, features = ["testsuite"] } quickwit-proto = { workspace = true } [dev-dependencies] -itertools = { workspace = true } mockall = { workspace = true } rand = { workspace = true } rand_distr = { workspace = true } diff --git a/quickwit/quickwit-ingest/src/codegen/ingest_service.rs b/quickwit/quickwit-ingest/src/codegen/ingest_service.rs index f08918078bd..e0faef82d76 100644 --- a/quickwit/quickwit-ingest/src/codegen/ingest_service.rs +++ b/quickwit/quickwit-ingest/src/codegen/ingest_service.rs @@ -1,53 +1,111 @@ // This file is @generated by prost-build. -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive( + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, + Clone, + PartialEq, + Eq, + Hash, + ::prost::Message, +)] pub struct QueueExistsRequest { #[prost(string, tag = "1")] pub queue_id: ::prost::alloc::string::String, } -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive( + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, + Clone, + PartialEq, + Eq, + Hash, + ::prost::Message, +)] pub struct CreateQueueRequest { #[prost(string, tag = "1")] pub queue_id: ::prost::alloc::string::String, } -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive( + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, + Clone, + PartialEq, + Eq, + Hash, + ::prost::Message, +)] pub struct CreateQueueIfNotExistsRequest { #[prost(string, tag = "1")] pub queue_id: ::prost::alloc::string::String, } -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive( + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, + Clone, + PartialEq, + Eq, + Hash, + ::prost::Message, +)] pub struct CreateQueueIfNotExistsResponse { #[prost(string, tag = "1")] pub queue_id: ::prost::alloc::string::String, #[prost(bool, tag = "2")] pub created: bool, } -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive( + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, + Clone, + PartialEq, + Eq, + Hash, + ::prost::Message, +)] pub struct DropQueueRequest { #[prost(string, tag = "1")] pub queue_id: ::prost::alloc::string::String, } -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive( + serde::Serialize, serde::Deserialize, utoipa::ToSchema, Clone, PartialEq, ::prost::Message, +)] pub struct IngestRequest { #[prost(message, repeated, tag = "1")] pub doc_batches: ::prost::alloc::vec::Vec, #[prost(enumeration = "CommitType", tag = "2")] pub commit: i32, } -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive( + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, + Clone, + Copy, + PartialEq, + Eq, + Hash, + ::prost::Message, +)] pub struct IngestResponse { #[prost(uint64, tag = "1")] pub num_docs_for_processing: u64, } /// Fetch messages with position strictly after `start_after`. -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive( + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, + Clone, + PartialEq, + Eq, + Hash, + ::prost::Message, +)] pub struct FetchRequest { #[prost(string, tag = "1")] pub index_id: ::prost::alloc::string::String, @@ -56,16 +114,32 @@ pub struct FetchRequest { #[prost(uint64, optional, tag = "3")] pub num_bytes_limit: ::core::option::Option, } -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive( + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, + Clone, + PartialEq, + Eq, + Hash, + ::prost::Message, +)] pub struct FetchResponse { #[prost(uint64, optional, tag = "1")] pub first_position: ::core::option::Option, #[prost(message, optional, tag = "2")] pub doc_batch: ::core::option::Option, } -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive( + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, + Clone, + PartialEq, + Eq, + Hash, + ::prost::Message, +)] pub struct DocBatch { #[prost(string, tag = "1")] pub index_id: ::prost::alloc::string::String, @@ -88,25 +162,58 @@ pub struct DocBatch { /// In other words, truncating from a position, and fetching records starting /// earlier than this position can yield undefined result: /// the truncated records may or may not be returned. -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive( + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, + Clone, + PartialEq, + Eq, + Hash, + ::prost::Message, +)] pub struct SuggestTruncateRequest { #[prost(string, tag = "1")] pub index_id: ::prost::alloc::string::String, #[prost(uint64, tag = "2")] pub up_to_position_included: u64, } -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive( + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, + Clone, + PartialEq, + Eq, + Hash, + ::prost::Message, +)] pub struct TailRequest { #[prost(string, tag = "1")] pub index_id: ::prost::alloc::string::String, } -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive( + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, + Clone, + Copy, + PartialEq, + Eq, + Hash, + ::prost::Message, +)] pub struct ListQueuesRequest {} -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive( + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, + Clone, + PartialEq, + Eq, + Hash, + ::prost::Message, +)] pub struct ListQueuesResponse { #[prost(string, repeated, tag = "1")] pub queues: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, @@ -147,11 +254,11 @@ impl CommitType { } } } +use quickwit_common::tower::RpcName; /// BEGIN quickwit-codegen #[allow(unused_imports)] use std::str::FromStr; use tower::{Layer, Service, ServiceExt}; -use quickwit_common::tower::RpcName; impl RpcName for IngestRequest { fn rpc_name() -> &'static str { "ingest" @@ -170,30 +277,31 @@ impl RpcName for TailRequest { #[cfg_attr(any(test, feature = "testsuite"), mockall::automock)] #[async_trait::async_trait] pub trait IngestService: std::fmt::Debug + Send + Sync + 'static { - /// Ingests document in a given queue. + ///Ingests document in a given queue. + /// + ///Upon any kind of error, the client should /// - /// Upon any kind of error, the client should - /// - retry to get at least once delivery. - /// - not retry to get at most once delivery. + ///* retry to get at least once delivery. + ///* not retry to get at most once delivery. /// - /// Exactly once delivery is not supported yet. + ///Exactly once delivery is not supported yet. async fn ingest(&self, request: IngestRequest) -> crate::Result; - /// Fetches record from a given queue. + ///Fetches record from a given queue. /// - /// Records are returned in order. + ///Records are returned in order. /// - /// The returned `FetchResponse` object is meant to be read with the - /// `crate::iter_records` function. + ///The returned `FetchResponse` object is meant to be read with the + ///`crate::iter_records` function. /// - /// Fetching does not necessarily return all of the available records. - /// If returning all records would exceed `FETCH_PAYLOAD_LIMIT` (2MB), - /// the response will be partial. + ///Fetching does not necessarily return all of the available records. + ///If returning all records would exceed `FETCH_PAYLOAD_LIMIT` (2MB), + ///the response will be partial. async fn fetch(&self, request: FetchRequest) -> crate::Result; - /// Returns a batch containing the last records. + ///Returns a batch containing the last records. /// - /// It returns the last documents, from the newest - /// to the oldest, and stops as soon as `FETCH_PAYLOAD_LIMIT` (2MB) - /// is exceeded. + ///It returns the last documents, from the newest + ///to the oldest, and stops as soon as `FETCH_PAYLOAD_LIMIT` (2MB) + ///is exceeded. async fn tail(&self, request: TailRequest) -> crate::Result; } #[derive(Debug, Clone)] @@ -209,8 +317,7 @@ impl IngestServiceClient { { #[cfg(any(test, feature = "testsuite"))] assert!( - std::any::TypeId::of:: < T > () != std::any::TypeId::of:: < MockIngestService - > (), + std::any::TypeId::of::() != std::any::TypeId::of::(), "`MockIngestService` must be wrapped in a `MockIngestServiceWrapper`: use `IngestServiceClient::from_mock(mock)` to instantiate the client" ); Self { @@ -220,9 +327,7 @@ impl IngestServiceClient { pub fn as_grpc_service( &self, max_message_size: bytesize::ByteSize, - ) -> ingest_service_grpc_server::IngestServiceGrpcServer< - IngestServiceGrpcServerAdapter, - > { + ) -> ingest_service_grpc_server::IngestServiceGrpcServer { let adapter = IngestServiceGrpcServerAdapter::new(self.clone()); ingest_service_grpc_server::IngestServiceGrpcServer::new(adapter) .accept_compressed(tonic::codec::CompressionEncoding::Gzip) @@ -238,12 +343,9 @@ impl IngestServiceClient { max_message_size: bytesize::ByteSize, compression_encoding_opt: Option, ) -> Self { - let (_, connection_keys_watcher) = tokio::sync::watch::channel( - std::collections::HashSet::from_iter([addr]), - ); - let mut client = ingest_service_grpc_client::IngestServiceGrpcClient::new( - channel, - ) + let (_, connection_keys_watcher) = + tokio::sync::watch::channel(std::collections::HashSet::from_iter([addr])); + let mut client = ingest_service_grpc_client::IngestServiceGrpcClient::new(channel) .max_decoding_message_size(max_message_size.0 as usize) .max_encoding_message_size(max_message_size.0 as usize); if let Some(compression_encoding) = compression_encoding_opt { @@ -251,10 +353,7 @@ impl IngestServiceClient { .accept_compressed(compression_encoding) .send_compressed(compression_encoding); } - let adapter = IngestServiceGrpcClientAdapter::new( - client, - connection_keys_watcher, - ); + let adapter = IngestServiceGrpcClientAdapter::new(client, connection_keys_watcher); Self::new(adapter) } pub fn from_balance_channel( @@ -263,9 +362,7 @@ impl IngestServiceClient { compression_encoding_opt: Option, ) -> IngestServiceClient { let connection_keys_watcher = balance_channel.connection_keys_watcher(); - let mut client = ingest_service_grpc_client::IngestServiceGrpcClient::new( - balance_channel, - ) + let mut client = ingest_service_grpc_client::IngestServiceGrpcClient::new(balance_channel) .max_decoding_message_size(max_message_size.0 as usize) .max_encoding_message_size(max_message_size.0 as usize); if let Some(compression_encoding) = compression_encoding_opt { @@ -273,10 +370,7 @@ impl IngestServiceClient { .accept_compressed(compression_encoding) .send_compressed(compression_encoding); } - let adapter = IngestServiceGrpcClientAdapter::new( - client, - connection_keys_watcher, - ); + let adapter = IngestServiceGrpcClientAdapter::new(client, connection_keys_watcher); Self::new(adapter) } pub fn from_mailbox(mailbox: quickwit_actors::Mailbox) -> Self @@ -328,23 +422,16 @@ pub mod mock_ingest_service { ) -> crate::Result { self.inner.lock().await.ingest(request).await } - async fn fetch( - &self, - request: super::FetchRequest, - ) -> crate::Result { + async fn fetch(&self, request: super::FetchRequest) -> crate::Result { self.inner.lock().await.fetch(request).await } - async fn tail( - &self, - request: super::TailRequest, - ) -> crate::Result { + async fn tail(&self, request: super::TailRequest) -> crate::Result { self.inner.lock().await.tail(request).await } } } -pub type BoxFuture = std::pin::Pin< - Box> + Send + 'static>, ->; +pub type BoxFuture = + std::pin::Pin> + Send + 'static>>; impl tower::Service for InnerIngestServiceClient { type Response = IngestResponse; type Error = crate::IngestServiceError; @@ -403,16 +490,10 @@ struct IngestServiceTowerServiceStack { IngestResponse, crate::IngestServiceError, >, - fetch_svc: quickwit_common::tower::BoxService< - FetchRequest, - FetchResponse, - crate::IngestServiceError, - >, - tail_svc: quickwit_common::tower::BoxService< - TailRequest, - FetchResponse, - crate::IngestServiceError, - >, + fetch_svc: + quickwit_common::tower::BoxService, + tail_svc: + quickwit_common::tower::BoxService, } #[async_trait::async_trait] impl IngestService for IngestServiceTowerServiceStack { @@ -427,31 +508,19 @@ impl IngestService for IngestServiceTowerServiceStack { } } type IngestLayer = quickwit_common::tower::BoxLayer< - quickwit_common::tower::BoxService< - IngestRequest, - IngestResponse, - crate::IngestServiceError, - >, + quickwit_common::tower::BoxService, IngestRequest, IngestResponse, crate::IngestServiceError, >; type FetchLayer = quickwit_common::tower::BoxLayer< - quickwit_common::tower::BoxService< - FetchRequest, - FetchResponse, - crate::IngestServiceError, - >, + quickwit_common::tower::BoxService, FetchRequest, FetchResponse, crate::IngestServiceError, >; type TailLayer = quickwit_common::tower::BoxLayer< - quickwit_common::tower::BoxService< - TailRequest, - FetchResponse, - crate::IngestServiceError, - >, + quickwit_common::tower::BoxService, TailRequest, FetchResponse, crate::IngestServiceError, @@ -471,7 +540,10 @@ impl IngestServiceTowerLayerStack { IngestResponse, crate::IngestServiceError, >, - > + Clone + Send + Sync + 'static, + > + Clone + + Send + + Sync + + 'static, + Clone + Send + Sync + 'static, + > + Clone + + Send + + Sync + + 'static, <, - > + Clone + Send + Sync + 'static, + > + Clone + + Send + + Sync + + 'static, + Clone + Send + Sync + 'static, + > + Clone + + Send + + Sync + + 'static, <, - > + Clone + Send + Sync + 'static, + > + Clone + + Send + + Sync + + 'static, , - >>::Service: tower::Service< - TailRequest, - Response = FetchResponse, - Error = crate::IngestServiceError, - > + Clone + Send + Sync + 'static, + >>::Service: tower::Service + + Clone + + Send + + Sync + + 'static, <, >>::Service as tower::Service>::Future: Send + 'static, { - self.ingest_layers.push(quickwit_common::tower::BoxLayer::new(layer.clone())); - self.fetch_layers.push(quickwit_common::tower::BoxLayer::new(layer.clone())); - self.tail_layers.push(quickwit_common::tower::BoxLayer::new(layer.clone())); + self.ingest_layers + .push(quickwit_common::tower::BoxLayer::new(layer.clone())); + self.fetch_layers + .push(quickwit_common::tower::BoxLayer::new(layer.clone())); + self.tail_layers + .push(quickwit_common::tower::BoxLayer::new(layer.clone())); self } pub fn stack_ingest_layer(mut self, layer: L) -> Self @@ -554,15 +641,21 @@ impl IngestServiceTowerLayerStack { IngestResponse, crate::IngestServiceError, >, - > + Send + Sync + 'static, + > + Send + + Sync + + 'static, L::Service: tower::Service< IngestRequest, Response = IngestResponse, Error = crate::IngestServiceError, - > + Clone + Send + Sync + 'static, + > + Clone + + Send + + Sync + + 'static, >::Future: Send + 'static, { - self.ingest_layers.push(quickwit_common::tower::BoxLayer::new(layer)); + self.ingest_layers + .push(quickwit_common::tower::BoxLayer::new(layer)); self } pub fn stack_fetch_layer(mut self, layer: L) -> Self @@ -573,15 +666,21 @@ impl IngestServiceTowerLayerStack { FetchResponse, crate::IngestServiceError, >, - > + Send + Sync + 'static, + > + Send + + Sync + + 'static, L::Service: tower::Service< FetchRequest, Response = FetchResponse, Error = crate::IngestServiceError, - > + Clone + Send + Sync + 'static, + > + Clone + + Send + + Sync + + 'static, >::Future: Send + 'static, { - self.fetch_layers.push(quickwit_common::tower::BoxLayer::new(layer)); + self.fetch_layers + .push(quickwit_common::tower::BoxLayer::new(layer)); self } pub fn stack_tail_layer(mut self, layer: L) -> Self @@ -592,15 +691,18 @@ impl IngestServiceTowerLayerStack { FetchResponse, crate::IngestServiceError, >, - > + Send + Sync + 'static, - L::Service: tower::Service< - TailRequest, - Response = FetchResponse, - Error = crate::IngestServiceError, - > + Clone + Send + Sync + 'static, + > + Send + + Sync + + 'static, + L::Service: tower::Service + + Clone + + Send + + Sync + + 'static, >::Future: Send + 'static, { - self.tail_layers.push(quickwit_common::tower::BoxLayer::new(layer)); + self.tail_layers + .push(quickwit_common::tower::BoxLayer::new(layer)); self } pub fn build(self, instance: T) -> IngestServiceClient @@ -640,17 +742,13 @@ impl IngestServiceTowerLayerStack { let inner_client = client.inner; self.build_from_inner_client(inner_client) } - pub fn build_from_mailbox( - self, - mailbox: quickwit_actors::Mailbox, - ) -> IngestServiceClient + pub fn build_from_mailbox(self, mailbox: quickwit_actors::Mailbox) -> IngestServiceClient where A: quickwit_actors::Actor + std::fmt::Debug + Send + 'static, IngestServiceMailbox: IngestService, { - let inner_client = InnerIngestServiceClient( - std::sync::Arc::new(IngestServiceMailbox::new(mailbox)), - ); + let inner_client = + InnerIngestServiceClient(std::sync::Arc::new(IngestServiceMailbox::new(mailbox))); self.build_from_inner_client(inner_client) } #[cfg(any(test, feature = "testsuite"))] @@ -663,30 +761,18 @@ impl IngestServiceTowerLayerStack { self, inner_client: InnerIngestServiceClient, ) -> IngestServiceClient { - let ingest_svc = self - .ingest_layers - .into_iter() - .rev() - .fold( - quickwit_common::tower::BoxService::new(inner_client.clone()), - |svc, layer| layer.layer(svc), - ); - let fetch_svc = self - .fetch_layers - .into_iter() - .rev() - .fold( - quickwit_common::tower::BoxService::new(inner_client.clone()), - |svc, layer| layer.layer(svc), - ); - let tail_svc = self - .tail_layers - .into_iter() - .rev() - .fold( - quickwit_common::tower::BoxService::new(inner_client.clone()), - |svc, layer| layer.layer(svc), - ); + let ingest_svc = self.ingest_layers.into_iter().rev().fold( + quickwit_common::tower::BoxService::new(inner_client.clone()), + |svc, layer| layer.layer(svc), + ); + let fetch_svc = self.fetch_layers.into_iter().rev().fold( + quickwit_common::tower::BoxService::new(inner_client.clone()), + |svc, layer| layer.layer(svc), + ); + let tail_svc = self.tail_layers.into_iter().rev().fold( + quickwit_common::tower::BoxService::new(inner_client.clone()), + |svc, layer| layer.layer(svc), + ); let tower_svc_stack = IngestServiceTowerServiceStack { inner: inner_client, ingest_svc, @@ -735,7 +821,8 @@ impl Clone for IngestServiceMailbox { impl tower::Service for IngestServiceMailbox where A: quickwit_actors::Actor - + quickwit_actors::DeferableReplyHandler> + Send + + quickwit_actors::DeferableReplyHandler> + + Send + 'static, M: std::fmt::Debug + Send + 'static, T: Send + 'static, @@ -757,7 +844,10 @@ where fn call(&mut self, message: M) -> Self::Future { let mailbox = self.inner.clone(); let fut = async move { - mailbox.ask_for_res(message).await.map_err(|error| error.into()) + mailbox + .ask_for_res(message) + .await + .map_err(|error| error.into()) }; Box::pin(fut) } @@ -766,21 +856,17 @@ where impl IngestService for IngestServiceMailbox where A: quickwit_actors::Actor + std::fmt::Debug, - IngestServiceMailbox< - A, - >: tower::Service< + IngestServiceMailbox: tower::Service< IngestRequest, Response = IngestResponse, Error = crate::IngestServiceError, Future = BoxFuture, - > - + tower::Service< + > + tower::Service< FetchRequest, Response = FetchResponse, Error = crate::IngestServiceError, Future = BoxFuture, - > - + tower::Service< + > + tower::Service< TailRequest, Response = FetchResponse, Error = crate::IngestServiceError, @@ -801,9 +887,8 @@ where pub struct IngestServiceGrpcClientAdapter { inner: T, #[allow(dead_code)] - connection_addrs_rx: tokio::sync::watch::Receiver< - std::collections::HashSet, - >, + connection_addrs_rx: + tokio::sync::watch::Receiver>, } impl IngestServiceGrpcClientAdapter { pub fn new( @@ -820,15 +905,16 @@ impl IngestServiceGrpcClientAdapter { } #[async_trait::async_trait] impl IngestService -for IngestServiceGrpcClientAdapter< - ingest_service_grpc_client::IngestServiceGrpcClient, -> + for IngestServiceGrpcClientAdapter> where - T: tonic::client::GrpcService + std::fmt::Debug + Clone + Send - + Sync + 'static, + T: tonic::client::GrpcService + + std::fmt::Debug + + Clone + + Send + + Sync + + 'static, T::ResponseBody: tonic::codegen::Body + Send + 'static, - ::Error: Into - + Send, + ::Error: Into + Send, T::Future: Send, { async fn ingest(&self, request: IngestRequest) -> crate::Result { @@ -837,10 +923,9 @@ where .ingest(request) .await .map(|response| response.into_inner()) - .map_err(|status| crate::error::grpc_status_to_service_error( - status, - IngestRequest::rpc_name(), - )) + .map_err(|status| { + crate::error::grpc_status_to_service_error(status, IngestRequest::rpc_name()) + }) } async fn fetch(&self, request: FetchRequest) -> crate::Result { self.inner @@ -848,10 +933,9 @@ where .fetch(request) .await .map(|response| response.into_inner()) - .map_err(|status| crate::error::grpc_status_to_service_error( - status, - FetchRequest::rpc_name(), - )) + .map_err(|status| { + crate::error::grpc_status_to_service_error(status, FetchRequest::rpc_name()) + }) } async fn tail(&self, request: TailRequest) -> crate::Result { self.inner @@ -859,10 +943,9 @@ where .tail(request) .await .map(|response| response.into_inner()) - .map_err(|status| crate::error::grpc_status_to_service_error( - status, - TailRequest::rpc_name(), - )) + .map_err(|status| { + crate::error::grpc_status_to_service_error(status, TailRequest::rpc_name()) + }) } } #[derive(Debug)] @@ -922,10 +1005,10 @@ pub mod ingest_service_grpc_client { dead_code, missing_docs, clippy::wildcard_imports, - clippy::let_unit_value, + clippy::let_unit_value )] - use tonic::codegen::*; use tonic::codegen::http::Uri; + use tonic::codegen::*; #[derive(Debug, Clone)] pub struct IngestServiceGrpcClient { inner: tonic::client::Grpc, @@ -964,14 +1047,13 @@ pub mod ingest_service_grpc_client { F: tonic::service::Interceptor, T::ResponseBody: Default, T: tonic::codegen::Service< - http::Request, - Response = http::Response< - >::ResponseBody, + http::Request, + Response = http::Response< + >::ResponseBody, + >, >, - >, - , - >>::Error: Into + std::marker::Send + std::marker::Sync, + >>::Error: + Into + std::marker::Send + std::marker::Sync, { IngestServiceGrpcClient::new(InterceptedService::new(inner, interceptor)) } @@ -1009,26 +1091,20 @@ pub mod ingest_service_grpc_client { /// Ingests document in a given queue. /// /// Upon any kind of error, the client should - /// - retry to get at least once delivery. - /// - not retry to get at most once delivery. + /// + /// * retry to get at least once delivery. + /// * not retry to get at most once delivery. /// /// Exactly once delivery is not supported yet. pub async fn ingest( &mut self, request: impl tonic::IntoRequest, ) -> std::result::Result, tonic::Status> { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/ingest_service.IngestService/Ingest", - ); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/ingest_service.IngestService/Ingest"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("ingest_service.IngestService", "Ingest")); @@ -1048,18 +1124,11 @@ pub mod ingest_service_grpc_client { &mut self, request: impl tonic::IntoRequest, ) -> std::result::Result, tonic::Status> { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/ingest_service.IngestService/Fetch", - ); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/ingest_service.IngestService/Fetch"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("ingest_service.IngestService", "Fetch")); @@ -1074,18 +1143,11 @@ pub mod ingest_service_grpc_client { &mut self, request: impl tonic::IntoRequest, ) -> std::result::Result, tonic::Status> { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/ingest_service.IngestService/Tail", - ); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/ingest_service.IngestService/Tail"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("ingest_service.IngestService", "Tail")); @@ -1100,7 +1162,7 @@ pub mod ingest_service_grpc_server { dead_code, missing_docs, clippy::wildcard_imports, - clippy::let_unit_value, + clippy::let_unit_value )] use tonic::codegen::*; /// Generated trait containing gRPC methods that should be implemented for use with IngestServiceGrpcServer. @@ -1109,8 +1171,9 @@ pub mod ingest_service_grpc_server { /// Ingests document in a given queue. /// /// Upon any kind of error, the client should - /// - retry to get at least once delivery. - /// - not retry to get at most once delivery. + /// + /// * retry to get at least once delivery. + /// * not retry to get at most once delivery. /// /// Exactly once delivery is not supported yet. async fn ingest( @@ -1162,10 +1225,7 @@ pub mod ingest_service_grpc_server { max_encoding_message_size: None, } } - pub fn with_interceptor( - inner: T, - interceptor: F, - ) -> InterceptedService + pub fn with_interceptor(inner: T, interceptor: F) -> InterceptedService where F: tonic::service::Interceptor, { @@ -1220,15 +1280,9 @@ pub mod ingest_service_grpc_server { "/ingest_service.IngestService/Ingest" => { #[allow(non_camel_case_types)] struct IngestSvc(pub Arc); - impl< - T: IngestServiceGrpc, - > tonic::server::UnaryService - for IngestSvc { + impl tonic::server::UnaryService for IngestSvc { type Response = super::IngestResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; + type Future = BoxFuture, tonic::Status>; fn call( &mut self, request: tonic::Request, @@ -1247,7 +1301,7 @@ pub mod ingest_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = IngestSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -1265,14 +1319,9 @@ pub mod ingest_service_grpc_server { "/ingest_service.IngestService/Fetch" => { #[allow(non_camel_case_types)] struct FetchSvc(pub Arc); - impl< - T: IngestServiceGrpc, - > tonic::server::UnaryService for FetchSvc { + impl tonic::server::UnaryService for FetchSvc { type Response = super::FetchResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; + type Future = BoxFuture, tonic::Status>; fn call( &mut self, request: tonic::Request, @@ -1291,7 +1340,7 @@ pub mod ingest_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = FetchSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -1309,14 +1358,9 @@ pub mod ingest_service_grpc_server { "/ingest_service.IngestService/Tail" => { #[allow(non_camel_case_types)] struct TailSvc(pub Arc); - impl< - T: IngestServiceGrpc, - > tonic::server::UnaryService for TailSvc { + impl tonic::server::UnaryService for TailSvc { type Response = super::FetchResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; + type Future = BoxFuture, tonic::Status>; fn call( &mut self, request: tonic::Request, @@ -1335,7 +1379,7 @@ pub mod ingest_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = TailSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -1350,25 +1394,19 @@ pub mod ingest_service_grpc_server { }; Box::pin(fut) } - _ => { - Box::pin(async move { - let mut response = http::Response::new( - tonic::body::Body::default(), - ); - let headers = response.headers_mut(); - headers - .insert( - tonic::Status::GRPC_STATUS, - (tonic::Code::Unimplemented as i32).into(), - ); - headers - .insert( - http::header::CONTENT_TYPE, - tonic::metadata::GRPC_CONTENT_TYPE, - ); - Ok(response) - }) - } + _ => Box::pin(async move { + let mut response = http::Response::new(tonic::body::Body::default()); + let headers = response.headers_mut(); + headers.insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers.insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) + }), } } } diff --git a/quickwit/quickwit-ingest/src/doc_batch.rs b/quickwit/quickwit-ingest/src/doc_batch.rs index bceccdd8cc0..f72151e1942 100644 --- a/quickwit/quickwit-ingest/src/doc_batch.rs +++ b/quickwit/quickwit-ingest/src/doc_batch.rs @@ -22,7 +22,8 @@ use crate::DocBatch; #[derive(Debug)] /// Represents a command that can be stored in a [`DocBatch`]. pub enum DocCommand -where T: Buf +where + T: Buf, { Ingest { payload: T }, Commit, @@ -49,7 +50,8 @@ impl From for DocCommandCode { } impl DocCommand -where T: Buf + Default +where + T: Buf + Default, { /// Returns the binary serialization code for the current version of this command. pub fn code(&self) -> DocCommandCode { @@ -131,7 +133,9 @@ impl DocBatchBuilder { /// Adds a parsed command to the batch pub fn command(&mut self, command: DocCommand) -> usize - where T: Buf + Default { + where + T: Buf + Default, + { let len = command.write(&mut self.doc_buffer); self.doc_lengths.push(len as u32); len diff --git a/quickwit/quickwit-ingest/src/ingest_api_service.rs b/quickwit/quickwit-ingest/src/ingest_api_service.rs index d669abbea99..7ee8d0b232b 100644 --- a/quickwit/quickwit-ingest/src/ingest_api_service.rs +++ b/quickwit/quickwit-ingest/src/ingest_api_service.rs @@ -185,19 +185,19 @@ impl IngestApiService { let index_id = doc_batch.index_id.clone(); let records_it = doc_batch.into_iter_raw(); let max_position = self.queues.append_batch(&index_id, records_it, ctx).await?; - if let Some(max_position) = max_position { - if commit != CommitType::Auto { - if commit == CommitType::Force { - self.queues - .append_batch( - &index_id, - iter::once(DocCommand::Commit::.into_buf()), - ctx, - ) - .await?; - } - notifications.push((index_id.clone(), max_position)); + if let Some(max_position) = max_position + && commit != CommitType::Auto + { + if commit == CommitType::Force { + self.queues + .append_batch( + &index_id, + iter::once(DocCommand::Commit::.into_buf()), + ctx, + ) + .await?; } + notifications.push((index_id.clone(), max_position)); } num_docs += batch_num_docs; diff --git a/quickwit/quickwit-ingest/src/ingest_v2/broadcast/capacity_score.rs b/quickwit/quickwit-ingest/src/ingest_v2/broadcast/capacity_score.rs new file mode 100644 index 00000000000..d2eede8cb88 --- /dev/null +++ b/quickwit/quickwit-ingest/src/ingest_v2/broadcast/capacity_score.rs @@ -0,0 +1,318 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::BTreeSet; + +use anyhow::{Context, Result}; +use bytesize::ByteSize; +use quickwit_cluster::{Cluster, ListenerHandle}; +use quickwit_common::pubsub::{Event, EventBroker}; +use quickwit_common::shared_consts::INGESTER_CAPACITY_SCORE_PREFIX; +use quickwit_proto::ingest::ingester::IngesterStatus; +use quickwit_proto::types::{NodeId, SourceUid}; +use serde::{Deserialize, Serialize}; +use tokio::task::JoinHandle; +use tracing::{info, warn}; + +use super::{BROADCAST_INTERVAL_PERIOD, make_key, parse_key}; +use crate::OpenShardCounts; +use crate::ingest_v2::state::WeakIngesterState; + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct IngesterCapacityScore { + pub capacity_score: usize, + pub open_shard_count: usize, +} + +/// Periodically snapshots the ingester's WAL memory usage and open shard counts, computes +/// a capacity score, and broadcasts it to other nodes via Chitchat. +pub struct BroadcastIngesterCapacityScoreTask { + cluster: Cluster, + weak_state: WeakIngesterState, +} + +impl BroadcastIngesterCapacityScoreTask { + pub fn spawn(cluster: Cluster, weak_state: WeakIngesterState) -> JoinHandle<()> { + let mut broadcaster = Self { + cluster, + weak_state, + }; + tokio::spawn(async move { broadcaster.run().await }) + } + + async fn snapshot(&self) -> Result> { + let state = self + .weak_state + .upgrade() + .context("ingester state has been dropped")?; + + // lock fully asserts that the ingester is ready. There's a likelihood that this task runs + // before the WAL is loaded, so we make sure that the ingester is ready just in case. + if *state.status_rx.borrow() != IngesterStatus::Ready { + return Ok(None); + } + + let mut guard = state + .lock_fully() + .await + .context("failed to acquire ingester state lock")?; + + let usage = guard.mrecordlog.resource_usage(); + let disk_used = ByteSize::b(usage.disk_used_bytes as u64); + let memory_used = ByteSize::b(usage.memory_used_bytes as u64); + let capacity_score = guard + .wal_capacity_tracker + .record_and_score(disk_used, memory_used); + let (open_shard_counts, _) = guard.get_shard_snapshot(); + + Ok(Some((capacity_score, open_shard_counts))) + } + + async fn run(&mut self) { + let mut interval = tokio::time::interval(BROADCAST_INTERVAL_PERIOD); + let mut previous_sources: BTreeSet = BTreeSet::new(); + + loop { + interval.tick().await; + + let (capacity_score, open_shard_counts) = match self.snapshot().await { + Ok(Some(snapshot)) => snapshot, + Ok(None) => continue, + Err(error) => { + info!("stopping ingester capacity broadcast: {error}"); + return; + } + }; + + previous_sources = self + .broadcast_capacity(capacity_score, &open_shard_counts, &previous_sources) + .await; + } + } + + async fn broadcast_capacity( + &self, + capacity_score: usize, + open_shard_counts: &OpenShardCounts, + previous_sources: &BTreeSet, + ) -> BTreeSet { + let mut current_sources = BTreeSet::new(); + + for (index_uid, source_id, open_shard_count) in open_shard_counts { + let source_uid = SourceUid { + index_uid: index_uid.clone(), + source_id: source_id.clone(), + }; + let key = make_key(INGESTER_CAPACITY_SCORE_PREFIX, &source_uid); + let capacity = IngesterCapacityScore { + capacity_score, + open_shard_count: *open_shard_count, + }; + let value = serde_json::to_string(&capacity) + .expect("`IngesterCapacityScore` should be JSON serializable"); + self.cluster.set_self_key_value(key, value).await; + current_sources.insert(source_uid); + } + + // When a source disappears (e.g. index deleted), broadcast open_shard_count=0 with a TTL + // so Chitchat auto-cleans the key. This is the only way routers learn to clear stale + // routing entries — Chitchat key removal doesn't reliably fire subscribers. + for removed_source in previous_sources.difference(¤t_sources) { + let key = make_key(INGESTER_CAPACITY_SCORE_PREFIX, removed_source); + let capacity = IngesterCapacityScore { + capacity_score, + open_shard_count: 0, + }; + let value = serde_json::to_string(&capacity) + .expect("`IngesterCapacityScore` should be JSON serializable"); + self.cluster + .set_self_key_value_delete_after_ttl(key, value) + .await; + } + + current_sources + } +} + +#[derive(Debug, Clone)] +pub struct IngesterCapacityScoreUpdate { + pub node_id: NodeId, + pub source_uid: SourceUid, + pub capacity_score: usize, + pub open_shard_count: usize, +} + +impl Event for IngesterCapacityScoreUpdate {} + +pub async fn setup_ingester_capacity_update_listener( + cluster: Cluster, + event_broker: EventBroker, +) -> ListenerHandle { + cluster + .subscribe(INGESTER_CAPACITY_SCORE_PREFIX, move |event| { + let Some(source_uid) = parse_key(event.key) else { + warn!("failed to parse source UID from key `{}`", event.key); + return; + }; + let Ok(ingester_capacity) = serde_json::from_str::(event.value) + else { + warn!("failed to parse ingester capacity `{}`", event.value); + return; + }; + let node_id: NodeId = event.node.node_id.clone().into(); + event_broker.publish(IngesterCapacityScoreUpdate { + node_id, + source_uid, + capacity_score: ingester_capacity.capacity_score, + open_shard_count: ingester_capacity.open_shard_count, + }); + }) + .await +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use quickwit_cluster::{ChannelTransport, create_cluster_for_test}; + use quickwit_proto::types::{IndexUid, ShardId, SourceId}; + + use super::*; + use crate::ingest_v2::models::IngesterShard; + use crate::ingest_v2::state::IngesterState; + + #[tokio::test] + async fn test_snapshot_state_dropped() { + let transport = ChannelTransport::default(); + let cluster = create_cluster_for_test(Vec::new(), &["indexer"], &transport, true) + .await + .unwrap(); + let (_temp_dir, state) = IngesterState::for_test(cluster.clone()).await; + let weak_state = state.weak(); + drop(state); + + let task = BroadcastIngesterCapacityScoreTask { + cluster, + weak_state, + }; + assert!(task.snapshot().await.is_err()); + } + + #[tokio::test] + async fn test_broadcast_ingester_capacity() { + let transport = ChannelTransport::default(); + let cluster = create_cluster_for_test(Vec::new(), &["indexer"], &transport, true) + .await + .unwrap(); + let event_broker = EventBroker::default(); + + // Use 1000 bytes disk capacity so 500 used => 50% remaining, 0 delta => score = 6 + let (_temp_dir, state) = + IngesterState::for_test_with_disk_capacity(cluster.clone(), ByteSize::b(1000)).await; + let index_uid = IndexUid::for_test("test-index", 0); + let mut state_guard = state.lock_partially().await.unwrap(); + let shard = IngesterShard::new_solo( + index_uid.clone(), + SourceId::from("test-source"), + ShardId::from(0), + ) + .advertisable() + .build(); + state_guard.shards.insert(shard.queue_id(), shard); + let (open_shard_counts, _) = state_guard.get_shard_snapshot(); + let capacity_score = state_guard + .wal_capacity_tracker + .record_and_score(ByteSize::b(500), ByteSize::b(0)); + drop(state_guard); + + assert_eq!(capacity_score, 6); + + let task = BroadcastIngesterCapacityScoreTask { + cluster: cluster.clone(), + weak_state: state.weak(), + }; + + let update_counter = Arc::new(AtomicUsize::new(0)); + let update_counter_clone = update_counter.clone(); + let index_uid_clone = index_uid.clone(); + let _sub = event_broker.subscribe(move |event: IngesterCapacityScoreUpdate| { + update_counter_clone.fetch_add(1, Ordering::Release); + assert_eq!(event.source_uid.index_uid, index_uid_clone); + assert_eq!(event.source_uid.source_id, "test-source"); + assert_eq!(event.capacity_score, 6); + assert_eq!(event.open_shard_count, 1); + }); + + let _listener = + setup_ingester_capacity_update_listener(cluster.clone(), event_broker).await; + + let previous_sources = BTreeSet::new(); + task.broadcast_capacity(capacity_score, &open_shard_counts, &previous_sources) + .await; + tokio::time::sleep(BROADCAST_INTERVAL_PERIOD * 2).await; + + assert_eq!(update_counter.load(Ordering::Acquire), 1); + + let source_uid = SourceUid { + index_uid: index_uid.clone(), + source_id: SourceId::from("test-source"), + }; + let key = make_key(INGESTER_CAPACITY_SCORE_PREFIX, &source_uid); + let value = cluster.get_self_key_value(&key).await.unwrap(); + let deserialized: IngesterCapacityScore = serde_json::from_str(&value).unwrap(); + assert_eq!(deserialized.capacity_score, 6); + assert_eq!(deserialized.open_shard_count, 1); + } + + #[tokio::test] + async fn test_removed_source_broadcasts_zero_with_ttl() { + let transport = ChannelTransport::default(); + let cluster = create_cluster_for_test(Vec::new(), &["indexer"], &transport, true) + .await + .unwrap(); + + let (_temp_dir, state) = IngesterState::for_test(cluster.clone()).await; + let task = BroadcastIngesterCapacityScoreTask { + cluster: cluster.clone(), + weak_state: state.weak(), + }; + + let index_uid = IndexUid::for_test("test-index", 0); + let source_uid = SourceUid { + index_uid: index_uid.clone(), + source_id: SourceId::from("test-source"), + }; + let key = make_key(INGESTER_CAPACITY_SCORE_PREFIX, &source_uid); + + // Cycle 1: source is alive. + let open_counts: Vec<(IndexUid, SourceId, usize)> = + vec![(index_uid.clone(), "test-source".into(), 3)]; + let current = task + .broadcast_capacity(7, &open_counts, &BTreeSet::new()) + .await; + + let value = cluster.get_self_key_value(&key).await.unwrap(); + let parsed: IngesterCapacityScore = serde_json::from_str(&value).unwrap(); + assert_eq!(parsed.open_shard_count, 3); + + // Cycle 2: source disappears. Broadcasts 0 with TTL, key still exists. + let _current = task.broadcast_capacity(7, &vec![], ¤t).await; + + let value = cluster.get_self_key_value(&key).await.unwrap(); + let parsed: IngesterCapacityScore = serde_json::from_str(&value).unwrap(); + assert_eq!(parsed.capacity_score, 7); + assert_eq!(parsed.open_shard_count, 0); + } +} diff --git a/quickwit/quickwit-ingest/src/ingest_v2/broadcast.rs b/quickwit/quickwit-ingest/src/ingest_v2/broadcast/local_shards.rs similarity index 79% rename from quickwit/quickwit-ingest/src/ingest_v2/broadcast.rs rename to quickwit/quickwit-ingest/src/ingest_v2/broadcast/local_shards.rs index ea0c28cb775..c99dbf7d74b 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/broadcast.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/broadcast/local_shards.rs @@ -18,24 +18,20 @@ use std::time::Duration; use bytesize::ByteSize; use quickwit_cluster::{Cluster, ListenerHandle}; use quickwit_common::pubsub::{Event, EventBroker}; +use quickwit_common::ring_buffer::RingBuffer; use quickwit_common::shared_consts::INGESTER_PRIMARY_SHARDS_PREFIX; use quickwit_common::sorted_iter::{KeyDiff, SortedByKeyIterator}; use quickwit_common::tower::{ConstantRate, Rate}; use quickwit_proto::ingest::ShardState; -use quickwit_proto::types::{NodeId, QueueId, ShardId, SourceUid, split_queue_id}; +use quickwit_proto::types::{NodeId, ShardId, SourceUid}; use serde::{Deserialize, Serialize, Serializer}; use tokio::task::JoinHandle; use tracing::{debug, warn}; -use super::metrics::INGEST_V2_METRICS; -use super::state::WeakIngesterState; +use super::{BROADCAST_INTERVAL_PERIOD, make_key, parse_key}; use crate::RateMibPerSec; - -const BROADCAST_INTERVAL_PERIOD: Duration = if cfg!(test) { - Duration::from_millis(50) -} else { - Duration::from_secs(5) -}; +use crate::ingest_v2::metrics::INGEST_V2_METRICS; +use crate::ingest_v2::state::WeakIngesterState; const ONE_MIB: ByteSize = ByteSize::mib(1); @@ -65,7 +61,9 @@ impl Serialize for ShardInfo { impl<'de> Deserialize<'de> for ShardInfo { fn deserialize(deserializer: D) -> Result - where D: serde::Deserializer<'de> { + where + D: serde::Deserializer<'de>, + { let value = String::deserialize(deserializer)?; let mut parts = value.split(':'); @@ -152,7 +150,7 @@ impl LocalShardsSnapshot { /// Takes a snapshot of the primary shards hosted by the ingester at regular intervals and /// broadcasts it to other nodes via Chitchat. -pub(super) struct BroadcastLocalShardsTask { +pub struct BroadcastLocalShardsTask { cluster: Cluster, weak_state: WeakIngesterState, shard_throughput_time_series_map: ShardThroughputTimeSeriesMap, @@ -229,36 +227,24 @@ impl ShardThroughputTimeSeriesMap { #[derive(Default)] struct ShardThroughputTimeSeries { shard_state: ShardState, - measurements: [ByteSize; SHARD_THROUGHPUT_LONG_TERM_WINDOW_LEN], - len: usize, + throughput: RingBuffer, } impl ShardThroughputTimeSeries { fn last(&self) -> ByteSize { - self.measurements.last().copied().unwrap_or_default() + self.throughput.last().unwrap_or_default() } fn average(&self) -> ByteSize { - if self.len == 0 { + if self.throughput.is_empty() { return ByteSize::default(); } - let sum = self - .measurements - .iter() - .rev() - .take(self.len) - .map(ByteSize::as_u64) - .sum::(); - ByteSize::b(sum / self.len as u64) + let sum = self.throughput.iter().map(ByteSize::as_u64).sum::(); + ByteSize::b(sum / self.throughput.len() as u64) } fn record(&mut self, new_throughput_measurement: ByteSize) { - self.len = (self.len + 1).min(SHARD_THROUGHPUT_LONG_TERM_WINDOW_LEN); - self.measurements.rotate_left(1); - let Some(last_measurement) = self.measurements.last_mut() else { - return; - }; - *last_measurement = new_throughput_measurement; + self.throughput.push_back(new_throughput_measurement); } } @@ -278,43 +264,24 @@ impl BroadcastLocalShardsTask { let Ok(mut state_guard) = state.lock_partially().await else { return Some(LocalShardsSnapshot::default()); }; - - let queue_ids: Vec<(QueueId, ShardState)> = state_guard - .shards - .iter() - .filter_map(|(queue_id, shard)| { - if shard.is_advertisable && !shard.is_replica() { - Some((queue_id.clone(), shard.shard_state)) - } else { - None - } - }) - .collect(); - - let mut num_open_shards = 0; - let mut num_closed_shards = 0; - #[allow(clippy::mutable_key_type)] - let ingestion_rates: HashMap<(SourceUid, ShardId), (ShardState, ConstantRate)> = queue_ids - .iter() - .flat_map(|(queue_id, shard_state)| { - let Some((_rate_limiter, rate_meter)) = state_guard.rate_trackers.get_mut(queue_id) - else { - warn!( - "rate limiter `{queue_id}` not found: this should never happen, please \ - report" - ); - return None; - }; - let (index_uid, source_id, shard_id) = split_queue_id(queue_id)?; - let source_uid = SourceUid { - index_uid, - source_id, - }; - // Shard ingestion rate in MiB/s. - Some(((source_uid, shard_id), (*shard_state, rate_meter.harvest()))) - }) - .collect(); + let ingestion_rates: HashMap<(SourceUid, ShardId), (ShardState, ConstantRate)> = + state_guard + .shards + .values_mut() + .filter(|shard| shard.is_advertisable && !shard.is_replica()) + .map(|shard| { + let source_uid = SourceUid { + index_uid: shard.index_uid.clone(), + source_id: shard.source_id.clone(), + }; + let shard_id = shard.shard_id.clone(); + let shard_state = shard.shard_state; + let rate_meter = &mut shard.rate_meter; + + ((source_uid, shard_id), (shard_state, rate_meter.harvest())) + }) + .collect(); self.shard_throughput_time_series_map .record_shard_throughputs(ingestion_rates); @@ -323,6 +290,9 @@ impl BroadcastLocalShardsTask { .shard_throughput_time_series_map .get_per_source_shard_infos(); + let mut num_open_shards = 0; + let mut num_closed_shards = 0; + for shard_infos in per_source_shard_infos.values() { for shard_info in shard_infos { match shard_info.shard_state { @@ -354,13 +324,13 @@ impl BroadcastLocalShardsTask { source_uid, shard_infos, } => { - let key = make_key(source_uid); + let key = make_key(INGESTER_PRIMARY_SHARDS_PREFIX, source_uid); let value = serde_json::to_string(&shard_infos) .expect("`ShardInfos` should be JSON serializable"); self.cluster.set_self_key_value(key, value).await; } ShardInfosChange::Removed { source_uid } => { - let key = make_key(source_uid); + let key = make_key(INGESTER_PRIMARY_SHARDS_PREFIX, source_uid); self.cluster.remove_self_key(&key).await; } } @@ -387,22 +357,6 @@ impl BroadcastLocalShardsTask { } } -fn make_key(source_uid: &SourceUid) -> String { - format!( - "{INGESTER_PRIMARY_SHARDS_PREFIX}{}:{}", - source_uid.index_uid, source_uid.source_id - ) -} - -fn parse_key(key: &str) -> Option { - let (index_uid_str, source_id_str) = key.rsplit_once(':')?; - - Some(SourceUid { - index_uid: index_uid_str.parse().ok()?, - source_id: source_id_str.to_string(), - }) -} - #[derive(Debug, Clone)] pub struct LocalShardsUpdate { pub leader_id: NodeId, @@ -440,19 +394,18 @@ pub async fn setup_local_shards_update_listener( #[cfg(test)] mod tests { - use std::str::FromStr; + use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; - use std::time::Instant; use quickwit_cluster::{ChannelTransport, create_cluster_for_test}; - use quickwit_common::rate_limiter::{RateLimiter, RateLimiterSettings}; + use quickwit_common::shared_consts::INGESTER_PRIMARY_SHARDS_PREFIX; use quickwit_proto::ingest::ShardState; - use quickwit_proto::types::{IndexUid, Position, queue_id}; + use quickwit_proto::types::{IndexUid, NodeId, ShardId, SourceId, SourceUid}; use super::*; + use crate::RateMibPerSec; use crate::ingest_v2::models::IngesterShard; - use crate::ingest_v2::rate_meter::RateMeter; use crate::ingest_v2::state::IngesterState; #[test] @@ -478,12 +431,12 @@ mod tests { assert_eq!(num_changes, 0); let previous_snapshot = LocalShardsSnapshot::default(); - let index_uid = IndexUid::from_str("test-index:00000000000000000000000000").unwrap(); + let index_uid = IndexUid::for_test("test-index", 0); let current_snapshot = LocalShardsSnapshot { per_source_shard_infos: vec![( SourceUid { index_uid: index_uid.clone(), - source_id: "test-source".to_string(), + source_id: SourceId::from("test-source"), }, vec![ShardInfo { shard_id: ShardId::from(1), @@ -524,7 +477,7 @@ mod tests { per_source_shard_infos: vec![( SourceUid { index_uid: index_uid.clone(), - source_id: "test-source".to_string(), + source_id: SourceId::from("test-source"), }, vec![ShardInfo { shard_id: ShardId::from(1), @@ -581,7 +534,7 @@ mod tests { let cluster = create_cluster_for_test(Vec::new(), &["indexer"], &transport, true) .await .unwrap(); - let (_temp_dir, state) = IngesterState::for_test().await; + let (_temp_dir, state) = IngesterState::for_test(cluster.clone()).await; let weak_state = state.weak(); let mut task = BroadcastLocalShardsTask { cluster, @@ -593,49 +546,33 @@ mod tests { let mut state_guard = state.lock_partially().await.unwrap(); - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); - let queue_id_00 = queue_id(&index_uid, "test-source", &ShardId::from(0)); + let index_uid = IndexUid::for_test("test-index", 0); let shard_00 = IngesterShard::new_solo( - ShardState::Open, - Position::Beginning, - Position::Beginning, - None, - Instant::now(), - false, - ); - state_guard.shards.insert(queue_id_00.clone(), shard_00); - - let queue_id_01 = queue_id(&index_uid, "test-source", &ShardId::from(1)); - let mut shard_01 = IngesterShard::new_solo( - ShardState::Open, - Position::Beginning, - Position::Beginning, - None, - Instant::now(), - false, - ); - shard_01.is_advertisable = true; - state_guard.shards.insert(queue_id_01.clone(), shard_01); - - let queue_id_02 = queue_id(&index_uid, "test-source", &ShardId::from(2)); - let mut shard_02 = IngesterShard::new_replica( + index_uid.clone(), + SourceId::from("test-source"), + ShardId::from(0), + ) + .build(); + state_guard.shards.insert(shard_00.queue_id(), shard_00); + + let shard_01 = IngesterShard::new_solo( + index_uid.clone(), + SourceId::from("test-source"), + ShardId::from(1), + ) + .advertisable() + .build(); + state_guard.shards.insert(shard_01.queue_id(), shard_01); + + let shard_02 = IngesterShard::new_replica( + index_uid.clone(), + SourceId::from("test-source"), + ShardId::from(2), NodeId::from("test-leader"), - ShardState::Open, - Position::Beginning, - Position::Beginning, - Instant::now(), - ); - shard_02.is_advertisable = true; - state_guard.shards.insert(queue_id_02.clone(), shard_02); - - for queue_id in [queue_id_00, queue_id_01, queue_id_02] { - let rate_limiter = RateLimiter::from_settings(RateLimiterSettings::default()); - let rate_meter = RateMeter::default(); - - state_guard - .rate_trackers - .insert(queue_id, (rate_limiter, rate_meter)); - } + ) + .advertisable() + .build(); + state_guard.shards.insert(shard_02.queue_id(), shard_02); drop(state_guard); let new_snapshot = task.snapshot_local_shards().await.unwrap(); @@ -661,30 +598,6 @@ mod tests { assert!(value_opt.is_none()); } - #[test] - fn test_make_key() { - let source_uid = SourceUid { - index_uid: IndexUid::for_test("test-index", 0), - source_id: "test-source".to_string(), - }; - let key = make_key(&source_uid); - assert_eq!( - key, - "ingester.primary_shards:test-index:00000000000000000000000000:test-source" - ); - } - - #[test] - fn test_parse_key() { - let key = "test-index:00000000000000000000000000:test-source"; - let source_uid = parse_key(key).unwrap(); - assert_eq!( - &source_uid.index_uid.to_string(), - "test-index:00000000000000000000000000" - ); - assert_eq!(source_uid.source_id, "test-source".to_string()); - } - #[tokio::test] async fn test_local_shards_update_listener() { let transport = ChannelTransport::default(); @@ -695,7 +608,7 @@ mod tests { let local_shards_update_counter = Arc::new(AtomicUsize::new(0)); let local_shards_update_counter_clone = local_shards_update_counter.clone(); - let index_uid = IndexUid::from_str("test-index:00000000000000000000000000").unwrap(); + let index_uid = IndexUid::for_test("test-index", 0); let index_uid_clone = index_uid.clone(); event_broker @@ -719,9 +632,9 @@ mod tests { let source_uid = SourceUid { index_uid: index_uid.clone(), - source_id: "test-source".to_string(), + source_id: SourceId::from("test-source"), }; - let key = make_key(&source_uid); + let key = make_key(INGESTER_PRIMARY_SHARDS_PREFIX, &source_uid); let value = serde_json::to_string(&vec![ShardInfo { shard_id: ShardId::from(1), shard_state: ShardState::Open, @@ -741,20 +654,23 @@ mod tests { let mut time_series = ShardThroughputTimeSeries::default(); assert_eq!(time_series.last(), ByteSize::mb(0)); assert_eq!(time_series.average(), ByteSize::mb(0)); + time_series.record(ByteSize::mb(2)); assert_eq!(time_series.last(), ByteSize::mb(2)); assert_eq!(time_series.average(), ByteSize::mb(2)); + time_series.record(ByteSize::mb(1)); assert_eq!(time_series.last(), ByteSize::mb(1)); assert_eq!(time_series.average(), ByteSize::kb(1500)); + time_series.record(ByteSize::mb(3)); assert_eq!(time_series.last(), ByteSize::mb(3)); assert_eq!(time_series.average(), ByteSize::mb(2)); + for _ in 0..SHARD_THROUGHPUT_LONG_TERM_WINDOW_LEN { time_series.record(ByteSize::mb(4)); assert_eq!(time_series.last(), ByteSize::mb(4)); } - assert_eq!(time_series.last(), ByteSize::mb(4)); } } diff --git a/quickwit/quickwit-ingest/src/ingest_v2/broadcast/mod.rs b/quickwit/quickwit-ingest/src/ingest_v2/broadcast/mod.rs new file mode 100644 index 00000000000..15e75311993 --- /dev/null +++ b/quickwit/quickwit-ingest/src/ingest_v2/broadcast/mod.rs @@ -0,0 +1,81 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#[allow(dead_code)] +mod capacity_score; +mod local_shards; + +use std::time::Duration; + +use quickwit_proto::types::SourceUid; + +pub(in crate::ingest_v2) const BROADCAST_INTERVAL_PERIOD: Duration = + if cfg!(any(test, feature = "testsuite")) { + Duration::from_millis(50) + } else { + Duration::from_secs(5) + }; + +pub use capacity_score::{ + BroadcastIngesterCapacityScoreTask, IngesterCapacityScoreUpdate, + setup_ingester_capacity_update_listener, +}; +pub use local_shards::{ + BroadcastLocalShardsTask, LocalShardsUpdate, ShardInfo, ShardInfos, + setup_local_shards_update_listener, +}; + +fn make_key(prefix: &str, source_uid: &SourceUid) -> String { + format!("{prefix}{}:{}", source_uid.index_uid, source_uid.source_id) +} + +fn parse_key(key: &str) -> Option { + let (index_uid_str, source_id_str) = key.rsplit_once(':')?; + Some(SourceUid { + index_uid: index_uid_str.parse().ok()?, + source_id: source_id_str.to_string(), + }) +} + +#[cfg(test)] +mod tests { + use quickwit_common::shared_consts::INGESTER_PRIMARY_SHARDS_PREFIX; + use quickwit_proto::types::{IndexUid, SourceId, SourceUid}; + + use super::*; + + #[test] + fn test_make_key() { + let source_uid = SourceUid { + index_uid: IndexUid::for_test("test-index", 0), + source_id: SourceId::from("test-source"), + }; + let key = make_key(INGESTER_PRIMARY_SHARDS_PREFIX, &source_uid); + assert_eq!( + key, + "ingester.primary_shards:test-index:00000000000000000000000000:test-source" + ); + } + + #[test] + fn test_parse_key() { + let key = "test-index:00000000000000000000000000:test-source"; + let source_uid = parse_key(key).unwrap(); + assert_eq!( + &source_uid.index_uid.to_string(), + "test-index:00000000000000000000000000" + ); + assert_eq!(source_uid.source_id, "test-source".to_string()); + } +} diff --git a/quickwit/quickwit-ingest/src/ingest_v2/doc_mapper.rs b/quickwit/quickwit-ingest/src/ingest_v2/doc_mapper.rs index f4e90931bae..0bd397b0796 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/doc_mapper.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/doc_mapper.rs @@ -22,7 +22,7 @@ use quickwit_common::thread_pool::run_cpu_intensive; use quickwit_config::{DocMapping, SearchSettings, build_doc_mapper}; use quickwit_doc_mapper::DocMapper; use quickwit_proto::ingest::{ - DocBatchV2, IngestV2Error, IngestV2Result, ParseFailure, ParseFailureReason, + DocBatchV2, DocFormat, IngestV2Error, IngestV2Result, ParseFailure, ParseFailureReason, }; use quickwit_proto::types::{DocMappingUid, DocUid}; use serde_json_borrow::Value as JsonValue; @@ -144,10 +144,19 @@ fn is_document_validation_enabled() -> bool { /// Parses the JSON documents contained in the batch and applies the doc mapper. Returns the /// original batch and a list of parse failures. +/// +/// Note: Validation is skipped for Arrow IPC format batches since they have their own schema +/// and are processed by specialized pipelines (e.g., ParquetDocProcessor). pub(super) async fn validate_doc_batch( doc_batch: DocBatchV2, doc_mapper: Arc, ) -> IngestV2Result<(DocBatchV2, Vec)> { + // Skip validation for Arrow IPC format - it has its own schema and is processed + // by specialized pipelines (e.g., metrics pipeline) + if doc_batch.doc_format() == DocFormat::ArrowIpc { + return Ok((doc_batch, Vec::new())); + } + if is_document_validation_enabled() { run_cpu_intensive(move || validate_doc_batch_impl(doc_batch, &doc_mapper)) .await diff --git a/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs b/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs index 6e8d085e35d..73c1fb2858d 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs @@ -495,7 +495,11 @@ async fn fault_tolerant_fetch_stream( shard_id: Some(shard_id.clone()), from_position_exclusive: Some(from_position_exclusive.clone()), }; - let mut fetch_stream = match ingester.open_fetch_stream(open_fetch_stream_request).await { + let mut fetch_stream = match ingester + .client + .open_fetch_stream(open_fetch_stream_request) + .await + { Ok(fetch_stream) => fetch_stream, Err(not_found_error @ IngestV2Error::ShardNotFound { .. }) => { error!( @@ -628,7 +632,7 @@ pub(super) mod tests { use tokio::time::timeout; use super::*; - use crate::MRecord; + use crate::{IngesterPoolEntry, MRecord}; pub fn into_fetch_payload(fetch_message: FetchMessage) -> FetchPayload { match fetch_message.message.unwrap() { @@ -1325,8 +1329,8 @@ pub(super) mod tests { Ok(service_stream_1) }); - let ingester_1 = IngesterServiceClient::from_mock(mock_ingester_1); - + let ingester_1 = + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester_1)); ingester_pool.insert("test-ingester-1".into(), ingester_1); let fetch_payload = FetchPayload { @@ -1425,7 +1429,8 @@ pub(super) mod tests { "open fetch stream error".to_string(), )) }); - let ingester_0 = IngesterServiceClient::from_mock(mock_ingester_0); + let ingester_0 = + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester_0)); let mut mock_ingester_1 = MockIngesterService::new(); let index_uid_clone = index_uid.clone(); @@ -1440,7 +1445,8 @@ pub(super) mod tests { Ok(service_stream_1) }); - let ingester_1 = IngesterServiceClient::from_mock(mock_ingester_1); + let ingester_1 = + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester_1)); ingester_pool.insert("test-ingester-0".into(), ingester_0); ingester_pool.insert("test-ingester-1".into(), ingester_1); @@ -1540,7 +1546,8 @@ pub(super) mod tests { Ok(service_stream_0) }); - let ingester_0 = IngesterServiceClient::from_mock(mock_ingester_0); + let ingester_0 = + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester_0)); let mut mock_ingester_1 = MockIngesterService::new(); let index_uid_clone = index_uid.clone(); @@ -1555,7 +1562,8 @@ pub(super) mod tests { Ok(service_stream_1) }); - let ingester_1 = IngesterServiceClient::from_mock(mock_ingester_1); + let ingester_1 = + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester_1)); ingester_pool.insert("test-ingester-0".into(), ingester_0); ingester_pool.insert("test-ingester-1".into(), ingester_1); @@ -1658,7 +1666,8 @@ pub(super) mod tests { shard_id: ShardId::from(1), }) }); - let ingester_0 = IngesterServiceClient::from_mock(mock_ingester_0); + let ingester_0 = + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester_0)); ingester_pool.insert("test-ingester-0".into(), ingester_0); fault_tolerant_fetch_stream( @@ -1746,7 +1755,8 @@ pub(super) mod tests { Ok(service_stream_2) }); - let ingester = IngesterServiceClient::from_mock(mock_ingester); + let ingester = + IngesterPoolEntry::ready_with_client(IngesterServiceClient::from_mock(mock_ingester)); ingester_pool.insert("test-ingester".into(), ingester); diff --git a/quickwit/quickwit-ingest/src/ingest_v2/helpers.rs b/quickwit/quickwit-ingest/src/ingest_v2/helpers.rs new file mode 100644 index 00000000000..5422d6c7ebc --- /dev/null +++ b/quickwit/quickwit-ingest/src/ingest_v2/helpers.rs @@ -0,0 +1,273 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::time::{Duration, Instant}; + +use anyhow::{Context, anyhow, bail}; +use futures::StreamExt; +use quickwit_common::pretty::PrettyDisplay; +use quickwit_proto::ingest::ingester::{ + DecommissionRequest, IngesterService, IngesterStatus, OpenObservationStreamRequest, +}; +use tracing::info; + +/// Tries to get the current status of an ingester by opening an observation stream +/// and reading the first message. +/// +/// # Errors +/// +/// Returns an error if: +/// - The observation stream fails to open +/// - The stream ends without producing a message +/// - The stream ends after returning an error +pub async fn try_get_ingester_status( + ingester: &impl IngesterService, +) -> anyhow::Result { + let mut observation_stream = ingester + .open_observation_stream(OpenObservationStreamRequest {}) + .await + .context("failed to open observation stream")?; + + let next_observation_message = observation_stream + .next() + .await + .context("observation stream ended")? + .context("observation stream failed")?; + + Ok(next_observation_message.status()) +} + +/// Waits for an ingester to reach a specific status by monitoring its observation stream. +/// +/// This function continuously polls the observation stream until the ingester reaches +/// the desired status. +/// +/// # Errors +/// +/// Returns an error if: +/// - The observation stream fails to open +/// - The stream ends without producing a message +/// - The stream ends after returning an error +/// - The timeout is exceeded +pub async fn wait_for_ingester_status( + ingester: &impl IngesterService, + status: IngesterStatus, + timeout_after: Duration, +) -> anyhow::Result<()> { + debug_assert!( + timeout_after > Duration::ZERO, + "timeout_after should be greater than zero" + ); + tokio::time::timeout( + timeout_after, + wait_for_ingester_status_inner(ingester, status), + ) + .await + .with_context(|| { + format!( + "timed out waiting for ingester to transition to status {status} after {} seconds", + timeout_after.as_secs(), + ) + })? +} + +async fn wait_for_ingester_status_inner( + ingester: &impl IngesterService, + status: IngesterStatus, +) -> anyhow::Result<()> { + let mut observation_stream = ingester + .open_observation_stream(OpenObservationStreamRequest {}) + .await + .context("failed to open observation stream")?; + + loop { + match observation_stream.next().await { + Some(Ok(observation_message)) => { + if observation_message.status() == status { + return Ok(()); + } + } + Some(Err(error)) => { + return Err(anyhow!(error).context("observation stream failed")); + } + None => { + bail!("observation stream ended"); + } + } + } +} + +/// Initiates decommission of an ingester and waits for it to complete. +/// +/// This function sends a decommission request to the ingester and then waits +/// for it to reach the `Decommissioned` status. +/// +/// # Errors +/// +/// Returns an error if: +/// - The decommission request fails +/// - The observation stream fails to open +/// - The stream ends without producing a message +/// - The stream ends after returning an error +/// - The timeout is exceeded +pub async fn wait_for_ingester_decommission( + ingester: &impl IngesterService, + timeout_after: Duration, +) -> anyhow::Result<()> { + let now = Instant::now(); + + ingester + .decommission(DecommissionRequest {}) + .await + .context("failed to initiate ingester decommission")?; + + wait_for_ingester_status( + ingester, + IngesterStatus::Decommissioned, + timeout_after.saturating_sub(now.elapsed()), + ) + .await?; + + info!( + "successfully decommissioned ingester in {}", + now.elapsed().pretty_display() + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + + use std::time::Duration; + + use quickwit_common::ServiceStream; + use quickwit_proto::ingest::ingester::{ + DecommissionResponse, IngesterServiceClient, MockIngesterService, ObservationMessage, + }; + + use super::*; + + #[tokio::test] + async fn test_try_get_ingester_status() { + let mut mock_ingester = MockIngesterService::new(); + mock_ingester + .expect_open_observation_stream() + .once() + .returning(|_| { + let (service_stream_tx, service_stream) = ServiceStream::new_bounded(1); + let message = ObservationMessage { + node_id: "test-ingester".to_string(), + status: IngesterStatus::Initializing as i32, + }; + service_stream_tx.try_send(Ok(message)).unwrap(); + Ok(service_stream) + }); + let ingester = IngesterServiceClient::from_mock(mock_ingester); + let status = try_get_ingester_status(&ingester).await.unwrap(); + assert_eq!(status, IngesterStatus::Initializing); + } + + #[tokio::test] + async fn test_wait_for_ingester_status() { + let mut mock_ingester = MockIngesterService::new(); + mock_ingester + .expect_open_observation_stream() + .once() + .returning(|_| { + let (service_stream_tx, service_stream) = ServiceStream::new_bounded(2); + let message = ObservationMessage { + node_id: "test-ingester".to_string(), + status: IngesterStatus::Initializing as i32, + }; + service_stream_tx.try_send(Ok(message)).unwrap(); + + let message = ObservationMessage { + node_id: "test-ingester".to_string(), + status: IngesterStatus::Ready as i32, + }; + service_stream_tx.try_send(Ok(message)).unwrap(); + Ok(service_stream) + }); + let ingester = IngesterServiceClient::from_mock(mock_ingester); + wait_for_ingester_status(&ingester, IngesterStatus::Ready, Duration::from_secs(1)) + .await + .unwrap(); + } + + #[tokio::test] + async fn test_wait_for_ingester_decommission_elapsed_timeout_not_zero() { + let mut mock_ingester = MockIngesterService::new(); + mock_ingester + .expect_open_observation_stream() + .once() + .returning(|_| { + let (service_stream_tx, service_stream) = ServiceStream::new_bounded(1); + // Simulate the ingester transitioning to Decommissioned after 50ms. + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(50)).await; + let message = ObservationMessage { + node_id: "test-ingester".to_string(), + status: IngesterStatus::Decommissioned as i32, + }; + service_stream_tx.try_send(Ok(message)).unwrap(); + }); + Ok(service_stream) + }); + mock_ingester + .expect_decommission() + .once() + .returning(|_| Ok(DecommissionResponse {})); + let ingester = IngesterServiceClient::from_mock(mock_ingester); + wait_for_ingester_decommission(&ingester, Duration::from_secs(1)) + .await + .unwrap(); + } + + #[tokio::test] + async fn test_wait_for_ingester_decommission() { + let mut mock_ingester = MockIngesterService::new(); + mock_ingester + .expect_open_observation_stream() + .once() + .returning(|_| { + let (service_stream_tx, service_stream) = ServiceStream::new_bounded(3); + let message = ObservationMessage { + node_id: "test-ingester".to_string(), + status: IngesterStatus::Ready as i32, + }; + service_stream_tx.try_send(Ok(message)).unwrap(); + + let message = ObservationMessage { + node_id: "test-ingester".to_string(), + status: IngesterStatus::Decommissioning as i32, + }; + service_stream_tx.try_send(Ok(message)).unwrap(); + + let message = ObservationMessage { + node_id: "test-ingester".to_string(), + status: IngesterStatus::Decommissioned as i32, + }; + service_stream_tx.try_send(Ok(message)).unwrap(); + Ok(service_stream) + }); + mock_ingester + .expect_decommission() + .once() + .returning(|_| Ok(DecommissionResponse {})); + let ingester = IngesterServiceClient::from_mock(mock_ingester); + wait_for_ingester_decommission(&ingester, Duration::from_secs(1)) + .await + .unwrap(); + } +} diff --git a/quickwit/quickwit-ingest/src/ingest_v2/idle.rs b/quickwit/quickwit-ingest/src/ingest_v2/idle.rs index 8cc20974dd0..5b91dc8025e 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/idle.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/idle.rs @@ -78,8 +78,9 @@ impl CloseIdleShardsTask { #[cfg(test)] mod tests { - use quickwit_proto::ingest::ShardState; - use quickwit_proto::types::{IndexUid, Position, ShardId, queue_id}; + use quickwit_cluster::{ChannelTransport, create_cluster_for_test}; + use quickwit_config::service::QuickwitService; + use quickwit_proto::types::{IndexUid, ShardId}; use super::*; use crate::ingest_v2::models::IngesterShard; @@ -87,9 +88,17 @@ mod tests { #[tokio::test] async fn test_close_idle_shards_run() { - let (_temp_dir, state) = IngesterState::for_test().await; + let cluster = create_cluster_for_test( + Vec::new(), + &[QuickwitService::Indexer.as_str()], + &ChannelTransport::default(), + true, + ) + .await + .unwrap(); + let (_temp_dir, state) = IngesterState::for_test(cluster).await; let weak_state = state.weak(); - let idle_shard_timeout = Duration::from_millis(200); + let idle_shard_timeout = RUN_INTERVAL_PERIOD * 4; let join_handle = CloseIdleShardsTask::spawn(weak_state, idle_shard_timeout); let mut state_guard = state.lock_partially().await.unwrap(); @@ -97,25 +106,22 @@ mod tests { let index_uid = IndexUid::for_test("test-index", 0); let shard_01 = IngesterShard::new_solo( - ShardState::Open, - Position::Beginning, - Position::Beginning, - None, - now - idle_shard_timeout, - false, - ); - let queue_id_01 = queue_id(&index_uid, "test-source", &ShardId::from(1)); + index_uid.clone(), + "test-source".to_string(), + ShardId::from(1), + ) + .with_last_write(now - idle_shard_timeout) + .build(); + let queue_id_01 = shard_01.queue_id(); state_guard.shards.insert(queue_id_01.clone(), shard_01); let shard_02 = IngesterShard::new_solo( - ShardState::Open, - Position::Beginning, - Position::Beginning, - None, - now - idle_shard_timeout / 2, - false, - ); - let queue_id_02 = queue_id(&index_uid, "test-source", &ShardId::from(2)); + index_uid.clone(), + "test-source".to_string(), + ShardId::from(2), + ) + .build(); + let queue_id_02 = shard_02.queue_id(); state_guard.shards.insert(queue_id_02.clone(), shard_02); drop(state_guard); diff --git a/quickwit/quickwit-ingest/src/ingest_v2/ingester.rs b/quickwit/quickwit-ingest/src/ingest_v2/ingester.rs index c2683af12d1..5091784b885 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/ingester.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/ingester.rs @@ -13,13 +13,12 @@ // limitations under the License. use std::collections::hash_map::Entry; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::fmt; use std::path::Path; use std::sync::Arc; use std::time::{Duration, Instant}; -use anyhow::Context; use async_trait::async_trait; use bytesize::ByteSize; use futures::StreamExt; @@ -31,25 +30,14 @@ use quickwit_common::metrics::{GaugeGuard, MEMORY_METRICS}; use quickwit_common::pretty::PrettyDisplay; use quickwit_common::pubsub::{EventBroker, EventSubscriber}; use quickwit_common::rate_limiter::{RateLimiter, RateLimiterSettings}; -use quickwit_common::tower::Pool; use quickwit_common::{ServiceStream, rate_limited_error, rate_limited_warn}; use quickwit_proto::control_plane::{ AdviseResetShardsRequest, ControlPlaneService, ControlPlaneServiceClient, }; use quickwit_proto::indexing::ShardPositionsUpdate; -use quickwit_proto::ingest::ingester::{ - AckReplicationMessage, CloseShardsRequest, CloseShardsResponse, DecommissionRequest, - DecommissionResponse, FetchMessage, IngesterService, IngesterServiceClient, - IngesterServiceStream, IngesterStatus, InitShardFailure, InitShardSuccess, InitShardsRequest, - InitShardsResponse, ObservationMessage, OpenFetchStreamRequest, OpenObservationStreamRequest, - OpenReplicationStreamRequest, OpenReplicationStreamResponse, PersistFailure, - PersistFailureReason, PersistRequest, PersistResponse, PersistSuccess, ReplicateFailureReason, - ReplicateSubrequest, RetainShardsForSource, RetainShardsRequest, RetainShardsResponse, - SynReplicationMessage, TruncateShardsRequest, TruncateShardsResponse, -}; +use quickwit_proto::ingest::ingester::*; use quickwit_proto::ingest::{ CommitTypeV2, DocBatchV2, IngestV2Error, IngestV2Result, ParseFailure, Shard, ShardIds, - ShardState, }; use quickwit_proto::types::{ IndexUid, NodeId, Position, QueueId, ShardId, SourceId, SubrequestId, queue_id, split_queue_id, @@ -60,7 +48,7 @@ use tokio::time::{sleep, timeout}; use tracing::{debug, error, info, warn}; use super::IngesterPool; -use super::broadcast::BroadcastLocalShardsTask; +use super::broadcast::{BroadcastIngesterCapacityScoreTask, BroadcastLocalShardsTask}; use super::doc_mapper::validate_doc_batch; use super::fetch::FetchStreamTask; use super::idle::CloseIdleShardsTask; @@ -101,7 +89,7 @@ const DEFAULT_BATCH_NUM_BYTES: usize = 1024 * 1024; // 1 MiB fn get_batch_num_bytes() -> usize { static BATCH_NUM_BYTES_CELL: OnceCell = OnceCell::new(); *BATCH_NUM_BYTES_CELL.get_or_init(|| { - quickwit_common::get_from_env("QW_INGEST_BATCH_NUM_BYTES", DEFAULT_BATCH_NUM_BYTES) + quickwit_common::get_from_env("QW_INGEST_BATCH_NUM_BYTES", DEFAULT_BATCH_NUM_BYTES, false) }) } @@ -133,7 +121,7 @@ impl Ingester { pub async fn try_new( cluster: Cluster, control_plane: ControlPlaneServiceClient, - ingester_pool: Pool, + ingester_pool: IngesterPool, wal_dir_path: &Path, disk_capacity: ByteSize, memory_capacity: ByteSize, @@ -142,10 +130,18 @@ impl Ingester { idle_shard_timeout: Duration, ) -> IngestV2Result { let self_node_id: NodeId = cluster.self_node_id().into(); - let state = IngesterState::load(wal_dir_path, rate_limiter_settings); + let state = IngesterState::load( + cluster.clone(), + wal_dir_path, + disk_capacity, + memory_capacity, + rate_limiter_settings, + ) + .await; let weak_state = state.weak(); - BroadcastLocalShardsTask::spawn(cluster, weak_state.clone()); + BroadcastLocalShardsTask::spawn(cluster.clone(), weak_state.clone()); + BroadcastIngesterCapacityScoreTask::spawn(cluster, weak_state.clone()); CloseIdleShardsTask::spawn(weak_state, idle_shard_timeout); let ingester = Self { @@ -165,12 +161,12 @@ impl Ingester { } /// Checks whether the ingester is fully decommissioned and updates its status accordingly. - fn check_decommissioning_status(&self, state: &mut InnerIngesterState) { + async fn check_decommissioning_status(&self, state: &mut InnerIngesterState) { if state.status() != IngesterStatus::Decommissioning { return; } if state.shards.values().all(|shard| shard.is_indexed()) { - state.set_status(IngesterStatus::Decommissioned); + state.set_status(IngesterStatus::Decommissioned).await; } } @@ -187,7 +183,7 @@ impl Ingester { shard: Shard, doc_mapping_json: &str, now: Instant, - validate: bool, + validate_docs: bool, ) -> IngestV2Result<()> { let queue_id = shard.queue_id(); info!( @@ -217,11 +213,11 @@ impl Ingester { return Err(IngestV2Error::Internal(message)); } }; + let index_uid = shard.index_uid().clone(); + let source_id = shard.source_id.clone(); + let shard_id = shard.shard_id().clone(); let rate_limiter = RateLimiter::from_settings(self.rate_limiter_settings); let rate_meter = RateMeter::default(); - state - .rate_trackers - .insert(queue_id, (rate_limiter, rate_meter)); let primary_shard = if let Some(follower_id) = &shard.follower_id { let leader_id: NodeId = shard.leader_id.clone().into(); @@ -241,24 +237,21 @@ impl Ingester { let message = format!("failed to initialize replica shard: {error}"); return Err(IngestV2Error::Internal(message)); } - IngesterShard::new_primary( - follower_id, - ShardState::Open, - Position::Beginning, - Position::Beginning, - doc_mapper, - now, - validate, - ) + IngesterShard::new_primary(index_uid, source_id, shard_id, follower_id) + .with_rate_limiter(rate_limiter) + .with_rate_meter(rate_meter) + .with_doc_mapper(doc_mapper) + .with_validate_docs(validate_docs) + .with_last_write(now) + .build() } else { - IngesterShard::new_solo( - ShardState::Open, - Position::Beginning, - Position::Beginning, - Some(doc_mapper), - now, - validate, - ) + IngesterShard::new_solo(index_uid, source_id, shard_id) + .with_rate_limiter(rate_limiter) + .with_rate_meter(rate_meter) + .with_doc_mapper(doc_mapper) + .with_validate_docs(validate_docs) + .with_last_write(now) + .build() }; entry.insert(primary_shard); Ok(()) @@ -295,6 +288,7 @@ impl Ingester { for queue_id in state_guard.mrecordlog.list_queues() { let Some((index_uid, source_id, shard_id)) = split_queue_id(queue_id) else { + // `split_queue_id` already logs an error. continue; }; per_source_shard_ids @@ -313,7 +307,10 @@ impl Ingester { }) .collect(); - let advise_reset_shards_request = AdviseResetShardsRequest { shard_ids }; + let advise_reset_shards_request = AdviseResetShardsRequest { + ingester_id: self.self_node_id.to_string(), + shard_ids, + }; let advise_reset_shards_future = self .control_plane .advise_reset_shards(advise_reset_shards_request); @@ -398,6 +395,7 @@ impl Ingester { IngestV2Error::Unavailable(message) })?; let mut ack_replication_stream = ingester + .client .open_replication_stream(syn_replication_stream) .await?; ack_replication_stream @@ -454,12 +452,12 @@ impl Ingester { let commit_type = persist_request.commit_type(); let force_commit = commit_type == CommitTypeV2::Force; - let leader_id: NodeId = persist_request.leader_id.into(); let mut state_guard = with_lock_metrics!(self.state.lock_fully().await, "persist", "write")?; + let status = state_guard.status(); - if state_guard.status() != IngesterStatus::Ready { + if !status.accepts_write_requests() { persist_failures.reserve_exact(persist_request.subrequests.len()); for subrequest in persist_request.subrequests { @@ -467,15 +465,15 @@ impl Ingester { subrequest_id: subrequest.subrequest_id, index_uid: subrequest.index_uid, source_id: subrequest.source_id, - shard_id: subrequest.shard_id, - reason: PersistFailureReason::ShardClosed as i32, + reason: PersistFailureReason::NodeUnavailable as i32, }; persist_failures.push(persist_failure); } let persist_response = PersistResponse { - leader_id: leader_id.into(), + leader_id: persist_request.leader_id, successes: Vec::new(), failures: persist_failures, + routing_update: None, }; return Ok(persist_response); } @@ -484,37 +482,32 @@ impl Ingester { let mut total_requested_capacity = ByteSize::b(0); for subrequest in persist_request.subrequests { - let queue_id = subrequest.queue_id(); - - let Some(shard) = state_guard.shards.get_mut(&queue_id) else { + let Some(shard) = state_guard + .inner + .find_most_capacity_shard_mut(subrequest.index_uid(), &subrequest.source_id) + else { + warn!( + index_uid=%subrequest.index_uid(), + source_id=%subrequest.source_id, + "no open shard found on ingester" + ); let persist_failure = PersistFailure { subrequest_id: subrequest.subrequest_id, index_uid: subrequest.index_uid, source_id: subrequest.source_id, - shard_id: subrequest.shard_id, - reason: PersistFailureReason::ShardNotFound as i32, + reason: PersistFailureReason::NoShardsAvailable as i32, }; persist_failures.push(persist_failure); continue; }; + let shard_id = shard.shard_id.clone(); + // A router can only know about a newly opened shard if it has been informed by the // control plane, which confirms that the shard was correctly opened in the // metastore. shard.is_advertisable = true; - - if shard.is_closed() { - let persist_failure = PersistFailure { - subrequest_id: subrequest.subrequest_id, - index_uid: subrequest.index_uid, - source_id: subrequest.source_id, - shard_id: subrequest.shard_id, - reason: PersistFailureReason::ShardClosed as i32, - }; - persist_failures.push(persist_failure); - continue; - } let doc_mapper = shard.doc_mapper_opt.clone().expect("shard should be open"); - let validate_shard = shard.validate; + let validate_docs = shard.validate_docs; let follower_id_opt = shard.follower_id_opt().cloned(); let from_position_exclusive = shard.replication_position_inclusive.clone(); @@ -542,26 +535,24 @@ impl Ingester { subrequest_id: subrequest.subrequest_id, index_uid: subrequest.index_uid, source_id: subrequest.source_id, - shard_id: subrequest.shard_id, reason: PersistFailureReason::WalFull as i32, }; persist_failures.push(persist_failure); continue; }; - let (rate_limiter, rate_meter) = state_guard - .rate_trackers - .get_mut(&queue_id) - .expect("rate limiter should be initialized"); - - if !rate_limiter.acquire_bytes(requested_capacity) { - debug!("failed to persist records to shard `{queue_id}`: rate limited"); + // Because we return the shard with the most available capacity, if this hits, it + // means that no shard can receive this request, and it should be retried. + if !shard.rate_limiter.acquire_bytes(requested_capacity) { + debug!( + "failed to persist records to shard `{}`: rate limited", + shard.queue_id() + ); let persist_failure = PersistFailure { subrequest_id: subrequest.subrequest_id, index_uid: subrequest.index_uid, source_id: subrequest.source_id, - shard_id: subrequest.shard_id, - reason: PersistFailureReason::ShardRateLimited as i32, + reason: PersistFailureReason::NoShardsAvailable as i32, }; persist_failures.push(persist_failure); continue; @@ -570,7 +561,7 @@ impl Ingester { // Total number of bytes (valid and invalid documents) let original_batch_num_bytes = doc_batch.num_bytes() as u64; - let (valid_doc_batch, parse_failures) = if validate_shard { + let (valid_doc_batch, parse_failures) = if validate_docs { validate_doc_batch(doc_batch, doc_mapper).await? } else { (doc_batch, Vec::new()) @@ -587,7 +578,7 @@ impl Ingester { subrequest_id: subrequest.subrequest_id, index_uid: subrequest.index_uid, source_id: subrequest.source_id, - shard_id: subrequest.shard_id, + shard_id: Some(shard_id), replication_position_inclusive: Some(from_position_exclusive), num_persisted_docs: 0, parse_failures, @@ -611,7 +602,7 @@ impl Ingester { .inc_by(original_batch_num_bytes - valid_doc_batch.num_bytes() as u64); } let valid_batch_num_bytes = valid_doc_batch.num_bytes() as u64; - rate_meter.update(valid_batch_num_bytes); + shard.rate_meter.update(valid_batch_num_bytes); total_requested_capacity += requested_capacity; let mut successfully_replicated = true; @@ -623,7 +614,7 @@ impl Ingester { subrequest_id: subrequest.subrequest_id, index_uid: subrequest.index_uid.clone(), source_id: subrequest.source_id.clone(), - shard_id: subrequest.shard_id.clone(), + shard_id: Some(shard_id.clone()), from_position_exclusive: Some(from_position_exclusive), doc_batch: Some(valid_doc_batch.clone()), }; @@ -633,11 +624,11 @@ impl Ingester { .push(replicate_subrequest); } let pending_persist_subrequest = PendingPersistSubrequest { - queue_id, + queue_id: shard.queue_id(), subrequest_id: subrequest.subrequest_id, index_uid: subrequest.index_uid, source_id: subrequest.source_id, - shard_id: subrequest.shard_id, + shard_id: Some(shard_id), doc_batch: valid_doc_batch, parse_failures, expected_position_inclusive: None, @@ -676,7 +667,7 @@ impl Ingester { // TODO: Handle replication error: // 1. Close and evict all the shards hosted by the follower. // 2. Close and evict the replication client. - // 3. Return `PersistFailureReason::ShardClosed` to router. + // 3. Return `PersistFailureReason::NodeUnavailable` to router. continue; } }; @@ -692,19 +683,12 @@ impl Ingester { for replicate_failure in replicate_response.failures { // TODO: If the replica shard is closed, close the primary shard if it is not // already. - let persist_failure_reason = match replicate_failure.reason() { - ReplicateFailureReason::Unspecified => PersistFailureReason::Unspecified, - ReplicateFailureReason::ShardNotFound => { - PersistFailureReason::ShardNotFound - } - ReplicateFailureReason::ShardClosed => PersistFailureReason::ShardClosed, - ReplicateFailureReason::WalFull => PersistFailureReason::WalFull, - }; + let persist_failure_reason: PersistFailureReason = + replicate_failure.reason().into(); let persist_failure = PersistFailure { subrequest_id: replicate_failure.subrequest_id, index_uid: replicate_failure.index_uid, source_id: replicate_failure.source_id, - shard_id: replicate_failure.shard_id, reason: persist_failure_reason as i32, }; persist_failures.push(persist_failure); @@ -739,7 +723,7 @@ impl Ingester { "failed to persist records to shard `{queue_id}`: {io_error}" ); shards_to_close.insert(queue_id); - PersistFailureReason::ShardClosed + PersistFailureReason::NodeUnavailable } AppendDocBatchError::QueueNotFound(_) => { error!( @@ -747,14 +731,13 @@ impl Ingester { not found" ); shards_to_delete.insert(queue_id); - PersistFailureReason::ShardNotFound + PersistFailureReason::NodeUnavailable } }; let persist_failure = PersistFailure { subrequest_id: subrequest.subrequest_id, index_uid: subrequest.index_uid, source_id: subrequest.source_id, - shard_id: subrequest.shard_id, reason: reason as i32, }; persist_failures.push(persist_failure); @@ -762,13 +745,13 @@ impl Ingester { } }; - if let Some(expected_position_inclusive) = subrequest.expected_position_inclusive { - if expected_position_inclusive != current_position_inclusive { - return Err(IngestV2Error::Internal(format!( - "bad replica position: expected {expected_position_inclusive:?}, got \ - {current_position_inclusive:?}" - ))); - } + if let Some(expected_position_inclusive) = subrequest.expected_position_inclusive + && expected_position_inclusive != current_position_inclusive + { + return Err(IngestV2Error::Internal(format!( + "bad replica position: expected {expected_position_inclusive:?}, got \ + {current_position_inclusive:?}" + ))); } state_guard .shards @@ -802,20 +785,39 @@ impl Ingester { if !shards_to_delete.is_empty() { for queue_id in &shards_to_delete { state_guard.shards.remove(queue_id); - state_guard.rate_trackers.remove(queue_id); warn!("deleted dangling shard `{queue_id}`"); } } let wal_usage = state_guard.mrecordlog.resource_usage(); - drop(state_guard); - let disk_used = wal_usage.disk_used_bytes as u64; + let memory_used = wal_usage.memory_used_bytes as u64; + let (open_shard_counts, closed_shards) = state_guard.get_shard_snapshot(); + let capacity_score = state_guard + .wal_capacity_tracker + .score(ByteSize::b(disk_used), ByteSize::b(memory_used)) + as u32; + drop(state_guard); if disk_used >= self.disk_capacity.as_u64() * 90 / 100 { self.background_reset_shards(); } report_wal_usage(wal_usage); + let source_shard_updates = open_shard_counts + .into_iter() + .map(|(index_uid, source_id, count)| SourceShardUpdate { + index_uid: Some(index_uid), + source_id, + open_shard_count: count as u32, + }) + .collect(); + + let routing_update = RoutingUpdate { + capacity_score, + source_shard_updates, + closed_shards, + }; + #[cfg(test)] { persist_successes.sort_by_key(|success| success.subrequest_id); @@ -826,6 +828,7 @@ impl Ingester { leader_id, successes: persist_successes, failures: persist_failures, + routing_update: Some(routing_update), }; Ok(persist_response) } @@ -849,9 +852,13 @@ impl Ingester { let follower_id: NodeId = open_replication_stream_request.follower_id.into(); let mut state_guard = self.state.lock_partially().await?; + let status = state_guard.status(); - if state_guard.status() != IngesterStatus::Ready { - return Err(IngestV2Error::Internal("node decommissioned".to_string())); + if !status.accepts_write_requests() { + let error = IngestV2Error::Unavailable(format!( + "ingester {follower_id} is not ready: {status}", + )); + return Err(error); } let Entry::Vacant(entry) = state_guard.replication_tasks.entry(leader_id.clone()) else { return Err(IngestV2Error::Internal(format!( @@ -934,9 +941,14 @@ impl Ingester { ) -> IngestV2Result { let mut state_guard = with_lock_metrics!(self.state.lock_fully().await, "init_shards", "write")?; - - if state_guard.status() != IngesterStatus::Ready { - return Err(IngestV2Error::Internal("node decommissioned".to_string())); + let status = state_guard.status(); + + if !status.accepts_write_requests() { + let error = IngestV2Error::Unavailable(format!( + "ingester {} is not ready: {status}", + self.self_node_id + )); + return Err(error); } let mut successes = Vec::with_capacity(init_shards_request.subrequests.len()); let mut failures = Vec::new(); @@ -995,17 +1007,17 @@ impl Ingester { let truncate_up_to_position_inclusive = subrequest.truncate_up_to_position_inclusive(); if truncate_up_to_position_inclusive.is_eof() { - state_guard.delete_shard(&queue_id).await; + state_guard.delete_shard(&queue_id, "indexer-rpc").await; } else { state_guard - .truncate_shard(&queue_id, truncate_up_to_position_inclusive) + .truncate_shard(&queue_id, truncate_up_to_position_inclusive, "indexer-rpc") .await; } } let wal_usage = state_guard.mrecordlog.resource_usage(); report_wal_usage(wal_usage); - self.check_decommissioning_status(&mut state_guard); + self.check_decommissioning_status(&mut state_guard).await; let truncate_response = TruncateShardsResponse {}; Ok(truncate_response) } @@ -1032,37 +1044,22 @@ impl Ingester { Ok(response) } - async fn decommission_inner( - &self, - _decommission_request: DecommissionRequest, - ) -> IngestV2Result { - info!("decommissioning ingester"); - let mut state_guard = self.state.lock_partially().await?; - - for shard in state_guard.shards.values_mut() { - shard.close(); - } - state_guard.set_status(IngesterStatus::Decommissioning); - self.check_decommissioning_status(&mut state_guard); - - Ok(DecommissionResponse {}) - } - pub async fn debug_info(&self) -> JsonValue { let state_guard = match self.state.lock_fully().await { Ok(state_guard) => state_guard, Err(_) => { return json!({ "status": "initializing", - "shards": [], + "shards": {}, "mrecordlog": {}, }); } }; - let mut per_index_shards_json: HashMap> = HashMap::new(); + let mut per_index_shards_json: BTreeMap> = BTreeMap::new(); for (queue_id, shard) in &state_guard.shards { let Some((index_uid, source_id, shard_id)) = split_queue_id(queue_id) else { + // `split_queue_id` already logs an error. continue; }; let mut shard_json = json!({ @@ -1179,9 +1176,11 @@ impl IngesterService for Ingester { .collect(); info!(queues=?remove_queue_ids, "removing queues"); for queue_id in remove_queue_ids { - state_guard.delete_shard(&queue_id).await; + state_guard + .delete_shard(&queue_id, "control-plane-retain-shards-rpc") + .await; } - self.check_decommissioning_status(&mut state_guard); + self.check_decommissioning_status(&mut state_guard).await; Ok(RetainShardsResponse {}) } @@ -1201,9 +1200,50 @@ impl IngesterService for Ingester { async fn decommission( &self, - decommission_request: DecommissionRequest, + _decommission_request: DecommissionRequest, ) -> IngestV2Result { - self.decommission_inner(decommission_request).await + // Retire the ingester immediately by setting its status to `Retiring`. + info!("retiring ingester"); + let mut state_guard = self.state.lock_partially().await?; + state_guard.set_status(IngesterStatus::Retiring).await; + drop(state_guard); // Dropping explicitly for readability. + + // Drain write requests by scheduling the decommissioning of the ingester after a delay + // allowing the propagation of the `Retiring` status to other nodes. + let self_clone = self.clone(); + tokio::spawn(async move { + const DECOMMISSION_DELAY: Duration = if cfg!(any(test, feature = "testsuite")) { + Duration::from_millis(100) + } else { + // Having to wait for 10s is not great but we can live with it. During this time, we + // still make progress towards decommissioning because we gradually receive less + // write requests and indexing is still ongoing. However, it sets a floor on the + // amount of time with which we can fully decommission an ingester. This will be + // most noticeable when using Quickwit locally. + Duration::from_secs(10) + }; + tokio::time::sleep(DECOMMISSION_DELAY).await; + + info!("decommissioning ingester"); + let mut state_guard = match self_clone.state.lock_partially().await { + Ok(state_guard) => state_guard, + Err(error) => { + error!(%error, "failed to decommission ingester"); + return; + } + }; + state_guard + .set_status(IngesterStatus::Decommissioning) + .await; + + for shard in state_guard.shards.values_mut() { + shard.close(); + } + self_clone + .check_decommissioning_status(&mut state_guard) + .await; + }); + Ok(DecommissionResponse {}) } } @@ -1226,51 +1266,16 @@ impl EventSubscriber for WeakIngesterState { for (shard_id, shard_position) in shard_positions_update.updated_shard_positions { let queue_id = queue_id(&index_uid, &source_id, &shard_id); if shard_position.is_eof() { - state_guard.delete_shard(&queue_id).await; + state_guard.delete_shard(&queue_id, "indexer-gossip").await; } else if !shard_position.is_beginning() { - state_guard.truncate_shard(&queue_id, shard_position).await; + state_guard + .truncate_shard(&queue_id, shard_position, "indexer-gossip") + .await; } } } } -pub async fn wait_for_ingester_status( - ingester: impl IngesterService, - status: IngesterStatus, -) -> anyhow::Result<()> { - let mut observation_stream = ingester - .open_observation_stream(OpenObservationStreamRequest {}) - .await - .context("failed to open observation stream")?; - - while let Some(observation_message_result) = observation_stream.next().await { - let observation_message = - observation_message_result.context("observation stream ended unexpectedly")?; - - if observation_message.status() == status { - break; - } - } - Ok(()) -} - -pub async fn wait_for_ingester_decommission(ingester: Ingester) -> anyhow::Result<()> { - let now = Instant::now(); - - ingester - .decommission(DecommissionRequest {}) - .await - .context("failed to initiate ingester decommission")?; - - wait_for_ingester_status(ingester, IngesterStatus::Decommissioned).await?; - - info!( - "successfully decommissioned ingester in {}", - now.elapsed().pretty_display() - ); - Ok(()) -} - struct PendingPersistSubrequest { queue_id: QueueId, subrequest_id: u32, @@ -1298,11 +1303,12 @@ mod tests { use quickwit_config::service::QuickwitService; use quickwit_proto::control_plane::{AdviseResetShardsResponse, MockControlPlaneService}; use quickwit_proto::ingest::ingester::{ - IngesterServiceGrpcServer, IngesterServiceGrpcServerAdapter, InitShardSubrequest, - PersistSubrequest, TruncateShardsSubrequest, + IngesterServiceClient, IngesterServiceGrpcServer, IngesterServiceGrpcServerAdapter, + IngesterStatus, InitShardSubrequest, PersistSubrequest, TruncateShardsSubrequest, }; use quickwit_proto::ingest::{ DocBatchV2, ParseFailureReason, ShardIdPosition, ShardIdPositions, ShardIds, ShardPKey, + ShardState, }; use quickwit_proto::types::{DocMappingUid, DocUid, ShardId, SourceUid, queue_id}; use tokio::task::yield_now; @@ -1310,11 +1316,12 @@ mod tests { use tonic::transport::{Endpoint, Server}; use super::*; - use crate::MRecord; use crate::ingest_v2::DEFAULT_IDLE_SHARD_TIMEOUT; use crate::ingest_v2::broadcast::ShardInfos; use crate::ingest_v2::doc_mapper::try_build_doc_mapper; use crate::ingest_v2::fetch::tests::{into_fetch_eof, into_fetch_payload}; + use crate::ingest_v2::helpers::wait_for_ingester_status; + use crate::{IngesterPoolEntry, MRecord}; const MAX_GRPC_MESSAGE_SIZE: ByteSize = ByteSize::mib(1); @@ -1430,7 +1437,7 @@ mod tests { .await .unwrap(); - wait_for_ingester_status(ingester.clone(), IngesterStatus::Ready) + wait_for_ingester_status(&ingester, IngesterStatus::Ready, Duration::from_secs(1)) .await .unwrap(); @@ -1458,10 +1465,11 @@ mod tests { let (ingester_ctx, ingester) = IngesterForTest::default().build().await; let mut state_guard = ingester.state.lock_fully().await.unwrap(); - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); - let queue_id_01 = queue_id(&index_uid, "test-source", &ShardId::from(1)); - let queue_id_02 = queue_id(&index_uid, "test-source", &ShardId::from(2)); - let queue_id_03 = queue_id(&index_uid, "test-source", &ShardId::from(3)); + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); + let queue_id_01 = queue_id(&index_uid, &source_id, &ShardId::from(1)); + let queue_id_02 = queue_id(&index_uid, &source_id, &ShardId::from(2)); + let queue_id_03 = queue_id(&index_uid, &source_id, &ShardId::from(3)); state_guard .mrecordlog @@ -1513,7 +1521,7 @@ mod tests { .await .unwrap(); - state_guard.set_status(IngesterStatus::Initializing); + state_guard.set_status(IngesterStatus::Initializing).await; drop(state_guard); @@ -1536,8 +1544,6 @@ mod tests { .mrecordlog .assert_records_eq(&queue_id_02, .., &[(1, [0, 0], "test-doc-bar")]); - state_guard.rate_trackers.contains_key(&queue_id_02); - assert_eq!(state_guard.status(), IngesterStatus::Ready); } @@ -1546,38 +1552,18 @@ mod tests { let (ingester_ctx, ingester) = IngesterForTest::default().build().await; let mut state_guard = ingester.state.lock_fully().await.unwrap(); - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); - let queue_id_00 = queue_id(&index_uid, "test-source", &ShardId::from(0)); - let shard_00 = IngesterShard::new_solo( - ShardState::Open, - Position::Beginning, - Position::Beginning, - None, - Instant::now(), - false, - ); - state_guard.shards.insert(queue_id_00.clone(), shard_00); + let shard_00 = + IngesterShard::new_solo(index_uid.clone(), source_id.clone(), ShardId::from(0)).build(); + state_guard.shards.insert(shard_00.queue_id(), shard_00); - let queue_id_01 = queue_id(&index_uid, "test-source", &ShardId::from(1)); - let mut shard_01 = IngesterShard::new_solo( - ShardState::Open, - Position::Beginning, - Position::Beginning, - None, - Instant::now(), - false, - ); - shard_01.is_advertisable = true; + let shard_01 = IngesterShard::new_solo(index_uid.clone(), source_id, ShardId::from(1)) + .advertisable() + .build(); + let queue_id_01 = shard_01.queue_id(); state_guard.shards.insert(queue_id_01.clone(), shard_01); - - for queue_id in [&queue_id_00, &queue_id_01] { - let rate_limiter = RateLimiter::from_settings(RateLimiterSettings::default()); - let rate_meter = RateMeter::default(); - state_guard - .rate_trackers - .insert(queue_id.clone(), (rate_limiter, rate_meter)); - } drop(state_guard); tokio::time::sleep(Duration::from_millis(100)).await; @@ -1628,7 +1614,8 @@ mod tests { async fn test_ingester_init_primary_shard() { let (ingester_ctx, ingester) = IngesterForTest::default().build().await; - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); let doc_mapping_uid = DocMappingUid::random(); let doc_mapping_json = format!( @@ -1642,7 +1629,7 @@ mod tests { ); let primary_shard = Shard { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(1)), shard_state: ShardState::Open as i32, leader_id: ingester_ctx.node_id.to_string(), @@ -1663,7 +1650,7 @@ mod tests { .await .unwrap(); - let queue_id = queue_id(&index_uid, "test-source", &ShardId::from(1)); + let queue_id = queue_id(&index_uid, &source_id, &ShardId::from(1)); let shard = state_guard.shards.get(&queue_id).unwrap(); shard.assert_is_solo(); shard.assert_is_open(); @@ -1676,7 +1663,8 @@ mod tests { async fn test_ingester_init_shards() { let (ingester_ctx, ingester) = IngesterForTest::default().build().await; - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); let doc_mapping_uid = DocMappingUid::random(); let doc_mapping_json = format!( @@ -1686,7 +1674,7 @@ mod tests { ); let shard = Shard { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(1)), shard_state: ShardState::Open as i32, leader_id: ingester_ctx.node_id.to_string(), @@ -1714,14 +1702,13 @@ mod tests { let state_guard = ingester.state.lock_fully().await.unwrap(); - let queue_id = queue_id(&index_uid, "test-source", &ShardId::from(1)); + let queue_id = queue_id(&index_uid, &source_id, &ShardId::from(1)); let shard = state_guard.shards.get(&queue_id).unwrap(); shard.assert_is_solo(); shard.assert_is_open(); shard.assert_replication_position(Position::Beginning); shard.assert_truncation_position(Position::Beginning); - assert!(state_guard.rate_trackers.contains_key(&queue_id)); assert!(state_guard.mrecordlog.queue_exists(&queue_id)); } @@ -1729,8 +1716,9 @@ mod tests { async fn test_ingester_persist() { let (ingester_ctx, ingester) = IngesterForTest::default().build().await; - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); - let index_uid2: IndexUid = IndexUid::for_test("test-index", 1); + let index_uid_0 = IndexUid::for_test("test-index", 0); + let index_uid_1 = IndexUid::for_test("test-index", 1); + let source_id = SourceId::from("test-source"); let doc_mapping_uid = DocMappingUid::random(); let doc_mapping_json = format!( @@ -1743,8 +1731,8 @@ mod tests { InitShardSubrequest { subrequest_id: 0, shard: Some(Shard { - index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + index_uid: Some(index_uid_0.clone()), + source_id: source_id.clone(), shard_id: Some(ShardId::from(1)), shard_state: ShardState::Open as i32, leader_id: ingester_ctx.node_id.to_string(), @@ -1757,8 +1745,8 @@ mod tests { InitShardSubrequest { subrequest_id: 1, shard: Some(Shard { - index_uid: Some(index_uid2.clone()), - source_id: "test-source".to_string(), + index_uid: Some(index_uid_1.clone()), + source_id: source_id.clone(), shard_id: Some(ShardId::from(1)), shard_state: ShardState::Open as i32, leader_id: ingester_ctx.node_id.to_string(), @@ -1778,16 +1766,14 @@ mod tests { subrequests: vec![ PersistSubrequest { subrequest_id: 0, - index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), + index_uid: Some(index_uid_0.clone()), + source_id: source_id.clone(), doc_batch: Some(DocBatchV2::for_test([r#"{"doc": "test-doc-010"}"#])), }, PersistSubrequest { subrequest_id: 1, - index_uid: Some(index_uid2.clone()), - source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), + index_uid: Some(index_uid_1.clone()), + source_id: source_id.clone(), doc_batch: Some(DocBatchV2::for_test([ r#"{"doc": "test-doc-110"}"#, r#"{"doc": "test-doc-111"}"#, @@ -1802,9 +1788,8 @@ mod tests { let persist_success_0 = &persist_response.successes[0]; assert_eq!(persist_success_0.subrequest_id, 0); - assert_eq!(persist_success_0.index_uid(), &index_uid); + assert_eq!(persist_success_0.index_uid(), &index_uid_0); assert_eq!(persist_success_0.source_id, "test-source"); - assert_eq!(persist_success_0.shard_id(), ShardId::from(1)); assert_eq!( persist_success_0.replication_position_inclusive, Some(Position::offset(1u64)) @@ -1812,9 +1797,8 @@ mod tests { let persist_success_1 = &persist_response.successes[1]; assert_eq!(persist_success_1.subrequest_id, 1); - assert_eq!(persist_success_1.index_uid(), &index_uid2); + assert_eq!(persist_success_1.index_uid(), &index_uid_1); assert_eq!(persist_success_1.source_id, "test-source"); - assert_eq!(persist_success_1.shard_id(), ShardId::from(1)); assert_eq!( persist_success_1.replication_position_inclusive, Some(Position::offset(2u64)) @@ -1823,7 +1807,7 @@ mod tests { let state_guard = ingester.state.lock_fully().await.unwrap(); assert_eq!(state_guard.shards.len(), 2); - let queue_id_01 = queue_id(&index_uid, "test-source", &ShardId::from(1)); + let queue_id_01 = queue_id(&index_uid_0, &source_id, &ShardId::from(1)); let solo_shard_01 = state_guard.shards.get(&queue_id_01).unwrap(); solo_shard_01.assert_is_solo(); solo_shard_01.assert_is_open(); @@ -1835,7 +1819,7 @@ mod tests { &[(0, [0, 0], r#"{"doc": "test-doc-010"}"#), (1, [0, 1], "")], ); - let queue_id_11 = queue_id(&index_uid2, "test-source", &ShardId::from(1)); + let queue_id_11 = queue_id(&index_uid_1, &source_id, &ShardId::from(1)); let solo_shard_11 = state_guard.shards.get(&queue_id_11).unwrap(); solo_shard_11.assert_is_solo(); solo_shard_11.assert_is_open(); @@ -1856,7 +1840,8 @@ mod tests { async fn test_ingester_persist_empty() { let (ingester_ctx, ingester) = IngesterForTest::default().build().await; - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); let doc_mapping_uid = DocMappingUid::random(); let doc_mapping_json = format!( @@ -1869,7 +1854,7 @@ mod tests { subrequest_id: 0, shard: Some(Shard { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(0)), shard_state: ShardState::Open as i32, leader_id: ingester_ctx.node_id.to_string(), @@ -1901,7 +1886,6 @@ mod tests { subrequest_id: 0, index_uid: Some(index_uid.clone()), source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(0)), doc_batch: None, }], }; @@ -1914,7 +1898,6 @@ mod tests { assert_eq!(persist_success.subrequest_id, 0); assert_eq!(persist_success.index_uid(), &index_uid); assert_eq!(persist_success.source_id, "test-source"); - assert_eq!(persist_success.shard_id(), ShardId::from(0)); assert_eq!( persist_success.replication_position_inclusive, Some(Position::Beginning) @@ -1925,7 +1908,8 @@ mod tests { async fn test_ingester_persist_validates_docs() { let (ingester_ctx, ingester) = IngesterForTest::default().build().await; - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); let doc_mapping_uid = DocMappingUid::random(); let doc_mapping_json = format!( @@ -1940,7 +1924,7 @@ mod tests { subrequest_id: 0, shard: Some(Shard { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(0)), shard_state: ShardState::Open as i32, leader_id: ingester_ctx.node_id.to_string(), @@ -1962,7 +1946,6 @@ mod tests { subrequest_id: 0, index_uid: Some(index_uid.clone()), source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(0)), doc_batch: Some(DocBatchV2::for_test([ "", // invalid "[]", // invalid @@ -2000,7 +1983,8 @@ mod tests { async fn test_ingester_persist_doesnt_validates_docs_when_requested() { let (ingester_ctx, ingester) = IngesterForTest::default().build().await; - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); let doc_mapping_uid = DocMappingUid::random(); let doc_mapping_json = format!( @@ -2015,7 +1999,7 @@ mod tests { subrequest_id: 0, shard: Some(Shard { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(0)), shard_state: ShardState::Open as i32, leader_id: ingester_ctx.node_id.to_string(), @@ -2037,7 +2021,6 @@ mod tests { subrequest_id: 0, index_uid: Some(index_uid.clone()), source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(0)), doc_batch: Some(DocBatchV2::for_test([ "", // invalid "[]", // invalid @@ -2063,7 +2046,8 @@ mod tests { .build() .await; - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); let doc_mapping_uid = DocMappingUid::random(); let doc_mapping_json = format!( @@ -2078,7 +2062,7 @@ mod tests { subrequest_id: 0, shard: Some(Shard { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(0)), shard_state: ShardState::Open as i32, leader_id: ingester_ctx.node_id.to_string(), @@ -2100,7 +2084,6 @@ mod tests { subrequest_id: 0, index_uid: Some(index_uid.clone()), source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(0)), doc_batch: Some(DocBatchV2::for_test(["", "[]", r#"{"foo": "bar"}"#])), }], }; @@ -2124,7 +2107,8 @@ mod tests { .build() .await; - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); let doc_mapping_uid = DocMappingUid::random(); let doc_mapping_json = format!( @@ -2139,7 +2123,7 @@ mod tests { subrequest_id: 0, shard: Some(Shard { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(0)), shard_state: ShardState::Open as i32, leader_id: ingester_ctx.node_id.to_string(), @@ -2161,7 +2145,6 @@ mod tests { subrequest_id: 0, index_uid: Some(index_uid.clone()), source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(0)), doc_batch: Some(DocBatchV2::for_test(["", "[]", r#"{"foo": "bar"}"#])), }], }; @@ -2173,7 +2156,7 @@ mod tests { let persist_failure = &persist_response.failures[0]; assert_eq!( persist_failure.reason(), - PersistFailureReason::ShardRateLimited + PersistFailureReason::NoShardsAvailable ); } @@ -2191,16 +2174,11 @@ mod tests { let (_ingester_ctx, ingester) = IngesterForTest::default().build().await; let mut state_guard = ingester.state.lock_fully().await.unwrap(); - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); - let queue_id = queue_id(&index_uid, "test-source", &ShardId::from(1)); - let solo_shard = IngesterShard::new_solo( - ShardState::Open, - Position::Beginning, - Position::Beginning, - None, - Instant::now(), - false, - ); + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); + let solo_shard = + IngesterShard::new_solo(index_uid.clone(), source_id, ShardId::from(1)).build(); + let queue_id = solo_shard.queue_id(); state_guard.shards.insert(queue_id.clone(), solo_shard); state_guard @@ -2224,7 +2202,6 @@ mod tests { subrequest_id: 0, index_uid: Some(index_uid.clone()), source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), doc_batch: Some(DocBatchV2::for_test([r#"test-doc-foo"#])), }], }; @@ -2237,8 +2214,10 @@ mod tests { assert_eq!(persist_failure.subrequest_id, 0); assert_eq!(persist_failure.index_uid(), &index_uid); assert_eq!(persist_failure.source_id, "test-source"); - assert_eq!(persist_failure.shard_id(), ShardId::from(1)); - assert_eq!(persist_failure.reason(), PersistFailureReason::ShardClosed,); + assert_eq!( + persist_failure.reason(), + PersistFailureReason::NodeUnavailable, + ); let state_guard = ingester.state.lock_fully().await.unwrap(); let shard = state_guard.shards.get(&queue_id).unwrap(); @@ -2252,29 +2231,17 @@ mod tests { let (_ingester_ctx, ingester) = IngesterForTest::default().build().await; let mut state_guard = ingester.state.lock_fully().await.unwrap(); - - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); - let queue_id = queue_id(&index_uid, "test-source", &ShardId::from(1)); + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); let doc_mapper = try_build_doc_mapper("{}").unwrap(); // Insert a dangling shard, i.e. a shard without a corresponding queue. - let solo_shard = IngesterShard::new_solo( - ShardState::Open, - Position::Beginning, - Position::Beginning, - Some(doc_mapper), - Instant::now(), - false, - ); - state_guard.shards.insert(queue_id.clone(), solo_shard); - - let rate_limiter = RateLimiter::from_settings(RateLimiterSettings::default()); - let rate_meter = RateMeter::default(); - state_guard - .rate_trackers - .insert(queue_id.clone(), (rate_limiter, rate_meter)); - + let solo_shard = + IngesterShard::new_solo(index_uid.clone(), source_id.clone(), ShardId::from(1)) + .with_doc_mapper(doc_mapper) + .build(); + state_guard.shards.insert(solo_shard.queue_id(), solo_shard); drop(state_guard); let persist_request = PersistRequest { @@ -2284,7 +2251,6 @@ mod tests { subrequest_id: 0, index_uid: Some(index_uid.clone()), source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), doc_batch: Some(DocBatchV2::for_test([r#"{"doc": "test-doc-foo"}"#])), }], }; @@ -2297,15 +2263,13 @@ mod tests { assert_eq!(persist_failure.subrequest_id, 0); assert_eq!(persist_failure.index_uid(), &index_uid); assert_eq!(persist_failure.source_id, "test-source"); - assert_eq!(persist_failure.shard_id(), ShardId::from(1)); assert_eq!( persist_failure.reason(), - PersistFailureReason::ShardNotFound + PersistFailureReason::NodeUnavailable ); let state_guard = ingester.state.lock_fully().await.unwrap(); assert_eq!(state_guard.shards.len(), 0); - assert_eq!(state_guard.rate_trackers.len(), 0); } #[tokio::test] @@ -2323,13 +2287,19 @@ mod tests { .build() .await; - leader_ctx.ingester_pool.insert( - follower_ctx.node_id.clone(), - IngesterServiceClient::new(follower.clone()), - ); + let ingester_pool_entry = IngesterPoolEntry { + client: IngesterServiceClient::new(follower.clone()), + status: IngesterStatus::Ready, + availability_zone: None, + }; - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); - let index_uid2: IndexUid = IndexUid::for_test("test-index", 1); + leader_ctx + .ingester_pool + .insert(follower_ctx.node_id.clone(), ingester_pool_entry); + + let index_uid = IndexUid::for_test("test-index", 0); + let index_uid2 = IndexUid::for_test("test-index", 1); + let source_id = SourceId::from("test-source"); let doc_mapping_uid = DocMappingUid::random(); let doc_mapping_json = format!( @@ -2343,7 +2313,7 @@ mod tests { subrequest_id: 0, shard: Some(Shard { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(1)), shard_state: ShardState::Open as i32, leader_id: leader_ctx.node_id.to_string(), @@ -2358,7 +2328,7 @@ mod tests { subrequest_id: 1, shard: Some(Shard { index_uid: Some(index_uid2.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(1)), shard_state: ShardState::Open as i32, leader_id: leader_ctx.node_id.to_string(), @@ -2381,14 +2351,12 @@ mod tests { subrequest_id: 0, index_uid: Some(index_uid.clone()), source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), doc_batch: Some(DocBatchV2::for_test([r#"{"doc": "test-doc-010"}"#])), }, PersistSubrequest { subrequest_id: 1, index_uid: Some(index_uid2.clone()), source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), doc_batch: Some(DocBatchV2::for_test([ r#"{"doc": "test-doc-110"}"#, r#"{"doc": "test-doc-111"}"#, @@ -2424,7 +2392,7 @@ mod tests { let leader_state_guard = leader.state.lock_fully().await.unwrap(); assert_eq!(leader_state_guard.shards.len(), 2); - let queue_id_01 = queue_id(&index_uid, "test-source", &ShardId::from(1)); + let queue_id_01 = queue_id(&index_uid, &source_id, &ShardId::from(1)); let primary_shard_01 = leader_state_guard.shards.get(&queue_id_01).unwrap(); primary_shard_01.assert_is_primary(); primary_shard_01.assert_is_open(); @@ -2436,7 +2404,7 @@ mod tests { &[(0, [0, 0], r#"{"doc": "test-doc-010"}"#), (1, [0, 1], "")], ); - let queue_id_11 = queue_id(&index_uid2, "test-source", &ShardId::from(1)); + let queue_id_11 = queue_id(&index_uid2, &source_id, &ShardId::from(1)); let primary_shard_11 = leader_state_guard.shards.get(&queue_id_11).unwrap(); primary_shard_11.assert_is_primary(); primary_shard_11.assert_is_open(); @@ -2525,19 +2493,26 @@ mod tests { } }); let follower_channel = Endpoint::from_static("http://127.0.0.1:7777").connect_lazy(); - let follower_grpc_client = IngesterServiceClient::from_channel( + let follower_client = IngesterServiceClient::from_channel( "127.0.0.1:7777".parse().unwrap(), follower_channel, MAX_GRPC_MESSAGE_SIZE, None, ); + let ingester_pool_entry = IngesterPoolEntry { + client: follower_client, + status: IngesterStatus::Ready, + availability_zone: None, + }; + leader_ctx .ingester_pool - .insert(follower_ctx.node_id.clone(), follower_grpc_client); + .insert(follower_ctx.node_id.clone(), ingester_pool_entry); - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); - let index_uid2: IndexUid = IndexUid::for_test("test-index", 1); + let index_uid = IndexUid::for_test("test-index", 0); + let index_uid2 = IndexUid::for_test("test-index", 1); + let source_id = SourceId::from("test-source"); let doc_mapping_uid = DocMappingUid::random(); let doc_mapping_json = format!( @@ -2551,7 +2526,7 @@ mod tests { subrequest_id: 0, shard: Some(Shard { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(1)), shard_state: ShardState::Open as i32, leader_id: leader_ctx.node_id.to_string(), @@ -2566,7 +2541,7 @@ mod tests { subrequest_id: 1, shard: Some(Shard { index_uid: Some(index_uid2.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(1)), shard_state: ShardState::Open as i32, leader_id: leader_ctx.node_id.to_string(), @@ -2589,14 +2564,12 @@ mod tests { subrequest_id: 0, index_uid: Some(index_uid.clone()), source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), doc_batch: Some(DocBatchV2::for_test([r#"{"doc": "test-doc-010"}"#])), }, PersistSubrequest { subrequest_id: 1, index_uid: Some(index_uid2.clone()), source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), doc_batch: Some(DocBatchV2::for_test([ r#"{"doc": "test-doc-110"}"#, r#"{"doc": "test-doc-111"}"#, @@ -2632,7 +2605,7 @@ mod tests { let leader_state_guard = leader.state.lock_fully().await.unwrap(); assert_eq!(leader_state_guard.shards.len(), 2); - let queue_id_01 = queue_id(&index_uid, "test-source", &ShardId::from(1)); + let queue_id_01 = queue_id(&index_uid, &source_id, &ShardId::from(1)); let primary_shard_01 = leader_state_guard.shards.get(&queue_id_01).unwrap(); primary_shard_01.assert_is_primary(); primary_shard_01.assert_is_open(); @@ -2644,7 +2617,7 @@ mod tests { &[(0, [0, 0], r#"{"doc": "test-doc-010"}"#)], ); - let queue_id_11 = queue_id(&index_uid2, "test-source", &ShardId::from(1)); + let queue_id_11 = queue_id(&index_uid2, &source_id, &ShardId::from(1)); let primary_shard_11 = leader_state_guard.shards.get(&queue_id_11).unwrap(); primary_shard_11.assert_is_primary(); primary_shard_11.assert_is_open(); @@ -2689,25 +2662,22 @@ mod tests { } #[tokio::test] - async fn test_ingester_persist_shard_closed() { + async fn test_ingester_persist_no_available_shards() { let (ingester_ctx, ingester) = IngesterForTest::default().build().await; - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); - let queue_id_01 = queue_id(&index_uid, "test-source", &ShardId::from(1)); - let solo_shard = IngesterShard::new_solo( - ShardState::Closed, - Position::Beginning, - Position::Beginning, - None, - Instant::now(), - false, - ); + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); + let solo_shard = + IngesterShard::new_solo(index_uid.clone(), source_id.clone(), ShardId::from(1)) + .with_state(ShardState::Closed) + .build(); + let queue_id = solo_shard.queue_id(); ingester .state .lock_fully() .await .unwrap() .shards - .insert(queue_id_01.clone(), solo_shard); + .insert(queue_id.clone(), solo_shard); let persist_request = PersistRequest { leader_id: ingester_ctx.node_id.to_string(), @@ -2716,7 +2686,6 @@ mod tests { subrequest_id: 0, index_uid: Some(index_uid.clone()), source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), doc_batch: Some(DocBatchV2::for_test([r#"{"doc": "test-doc-010"}"#])), }], }; @@ -2729,16 +2698,18 @@ mod tests { assert_eq!(persist_failure.subrequest_id, 0); assert_eq!(persist_failure.index_uid(), &index_uid); assert_eq!(persist_failure.source_id, "test-source"); - assert_eq!(persist_failure.shard_id(), ShardId::from(1)); - assert_eq!(persist_failure.reason(), PersistFailureReason::ShardClosed); + assert_eq!( + persist_failure.reason(), + PersistFailureReason::NoShardsAvailable + ); let state_guard = ingester.state.lock_fully().await.unwrap(); assert_eq!(state_guard.shards.len(), 1); - let solo_shard_01 = state_guard.shards.get(&queue_id_01).unwrap(); - solo_shard_01.assert_is_solo(); - solo_shard_01.assert_is_closed(); - solo_shard_01.assert_replication_position(Position::Beginning); + let solo_shard = state_guard.shards.get(&queue_id).unwrap(); + solo_shard.assert_is_solo(); + solo_shard.assert_is_closed(); + solo_shard.assert_replication_position(Position::Beginning); } #[tokio::test] @@ -2752,7 +2723,8 @@ mod tests { .build() .await; - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); let doc_mapping_uid = DocMappingUid::random(); let doc_mapping_json = format!( @@ -2762,7 +2734,7 @@ mod tests { ); let primary_shard = Shard { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(1)), shard_state: ShardState::Open as i32, leader_id: ingester_ctx.node_id.to_string(), @@ -2792,7 +2764,6 @@ mod tests { subrequest_id: 0, index_uid: Some(index_uid.clone()), source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), doc_batch: Some(DocBatchV2::for_test([r#"{"doc": "test-doc-010"}"#])), }], }; @@ -2805,16 +2776,15 @@ mod tests { assert_eq!(persist_failure.subrequest_id, 0); assert_eq!(persist_failure.index_uid(), &index_uid); assert_eq!(persist_failure.source_id, "test-source"); - assert_eq!(persist_failure.shard_id(), ShardId::from(1)); assert_eq!( persist_failure.reason(), - PersistFailureReason::ShardRateLimited + PersistFailureReason::NoShardsAvailable ); let state_guard = ingester.state.lock_fully().await.unwrap(); assert_eq!(state_guard.shards.len(), 1); - let queue_id_01 = queue_id(&index_uid, "test-source", &ShardId::from(1)); + let queue_id_01 = queue_id(&index_uid, &source_id, &ShardId::from(1)); let solo_shard_01 = state_guard.shards.get(&queue_id_01).unwrap(); solo_shard_01.assert_is_solo(); @@ -2833,7 +2803,8 @@ mod tests { .build() .await; - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); let doc_mapping_uid = DocMappingUid::random(); let doc_mapping_json = format!( @@ -2843,7 +2814,7 @@ mod tests { ); let primary_shard = Shard { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(1)), shard_state: ShardState::Open as i32, leader_id: ingester_ctx.node_id.to_string(), @@ -2873,7 +2844,6 @@ mod tests { subrequest_id: 0, index_uid: Some(index_uid.clone()), source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), doc_batch: Some(DocBatchV2::for_test([r#"{"doc": "test-doc-010"}"#])), }], }; @@ -2886,13 +2856,12 @@ mod tests { assert_eq!(persist_failure.subrequest_id, 0); assert_eq!(persist_failure.index_uid(), &index_uid); assert_eq!(persist_failure.source_id, "test-source"); - assert_eq!(persist_failure.shard_id(), ShardId::from(1)); assert_eq!(persist_failure.reason(), PersistFailureReason::WalFull); let state_guard = ingester.state.lock_fully().await.unwrap(); assert_eq!(state_guard.shards.len(), 1); - let queue_id_01 = queue_id(&index_uid, "test-source", &ShardId::from(1)); + let queue_id_01 = queue_id(&index_uid, &source_id, &ShardId::from(1)); let solo_shard_01 = state_guard.shards.get(&queue_id_01).unwrap(); solo_shard_01.assert_is_solo(); solo_shard_01.assert_is_open(); @@ -2903,6 +2872,98 @@ mod tests { .assert_records_eq(&queue_id_01, .., &[]); } + #[tokio::test] + async fn test_ingester_persist_returns_routing_update() { + let (ingester_ctx, ingester) = IngesterForTest::default().build().await; + + let index_uid_0 = IndexUid::for_test("test-index-0", 0); + let index_uid_1 = IndexUid::for_test("test-index-1", 0); + let source_id = SourceId::from("test-source"); + + let doc_mapping_uid = DocMappingUid::random(); + let doc_mapping_json = format!( + r#"{{ + "doc_mapping_uid": "{doc_mapping_uid}" + }}"# + ); + let init_shards_request = InitShardsRequest { + subrequests: vec![ + InitShardSubrequest { + subrequest_id: 0, + shard: Some(Shard { + index_uid: Some(index_uid_0.clone()), + source_id: source_id.clone(), + shard_id: Some(ShardId::from(1)), + shard_state: ShardState::Open as i32, + leader_id: ingester_ctx.node_id.to_string(), + doc_mapping_uid: Some(doc_mapping_uid), + ..Default::default() + }), + doc_mapping_json: doc_mapping_json.clone(), + validate_docs: false, + }, + InitShardSubrequest { + subrequest_id: 1, + shard: Some(Shard { + index_uid: Some(index_uid_1.clone()), + source_id: source_id.clone(), + shard_id: Some(ShardId::from(1)), + shard_state: ShardState::Open as i32, + leader_id: ingester_ctx.node_id.to_string(), + doc_mapping_uid: Some(doc_mapping_uid), + ..Default::default() + }), + doc_mapping_json, + validate_docs: false, + }, + ], + }; + ingester.init_shards(init_shards_request).await.unwrap(); + + let persist_request = PersistRequest { + leader_id: ingester_ctx.node_id.to_string(), + commit_type: CommitTypeV2::Force as i32, + subrequests: vec![ + PersistSubrequest { + subrequest_id: 0, + index_uid: Some(index_uid_0.clone()), + source_id: source_id.clone(), + doc_batch: Some(DocBatchV2::for_test([r#"{"doc": "test-doc-010"}"#])), + }, + PersistSubrequest { + subrequest_id: 1, + index_uid: Some(index_uid_1.clone()), + source_id: source_id.clone(), + doc_batch: Some(DocBatchV2::for_test([r#"{"doc": "test-doc-110"}"#])), + }, + ], + }; + let persist_response = ingester.persist(persist_request).await.unwrap(); + assert_eq!(persist_response.successes.len(), 2); + + let routing_update = persist_response + .routing_update + .expect("routing update should be present"); + + assert!( + routing_update.capacity_score > 0, + "capacity score should be non-zero after a small persist" + ); + + let mut source_shard_updates = routing_update.source_shard_updates; + source_shard_updates.sort_by(|a, b| a.index_uid().cmp(b.index_uid())); + + assert_eq!(source_shard_updates.len(), 2); + assert_eq!(source_shard_updates[0].index_uid(), &index_uid_0); + assert_eq!(source_shard_updates[0].source_id, source_id.as_str()); + assert_eq!(source_shard_updates[0].open_shard_count, 1); + assert_eq!(source_shard_updates[1].index_uid(), &index_uid_1); + assert_eq!(source_shard_updates[1].source_id, source_id.as_str()); + assert_eq!(source_shard_updates[1].open_shard_count, 1); + + assert!(routing_update.closed_shards.is_empty()); + } + #[tokio::test] async fn test_ingester_open_replication_stream() { let (_ingester_ctx, ingester) = IngesterForTest::default() @@ -2941,11 +3002,12 @@ mod tests { async fn test_ingester_open_fetch_stream() { let (_ingester_ctx, ingester) = IngesterForTest::default().build().await; - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); let open_fetch_stream_request = OpenFetchStreamRequest { client_id: "test-client".to_string(), index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(1337)), from_position_exclusive: Some(Position::Beginning), }; @@ -2965,13 +3027,13 @@ mod tests { ); let shard = Shard { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(1)), shard_state: ShardState::Open as i32, doc_mapping_uid: Some(doc_mapping_uid), ..Default::default() }; - let queue_id = queue_id(&index_uid, "test-source", &ShardId::from(1)); + let queue_id = queue_id(&index_uid, &source_id, &ShardId::from(1)); let mut state_guard = ingester.state.lock_fully().await.unwrap(); @@ -3000,7 +3062,7 @@ mod tests { let open_fetch_stream_request = OpenFetchStreamRequest { client_id: "test-client".to_string(), index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id, shard_id: Some(ShardId::from(1)), from_position_exclusive: Some(Position::Beginning), }; @@ -3064,9 +3126,10 @@ mod tests { async fn test_ingester_truncate_shards() { let (ingester_ctx, ingester) = IngesterForTest::default().build().await; - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); - let queue_id_01 = queue_id(&index_uid, "test-source", &ShardId::from(1)); - let queue_id_02 = queue_id(&index_uid, "test-source", &ShardId::from(2)); + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); + let queue_id_01 = queue_id(&index_uid, &source_id, &ShardId::from(1)); + let queue_id_02 = queue_id(&index_uid, &source_id, &ShardId::from(2)); let doc_mapping_uid_01 = DocMappingUid::random(); let doc_mapping_json_01 = format!( @@ -3076,7 +3139,7 @@ mod tests { ); let shard_01 = Shard { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(1)), shard_state: ShardState::Open as i32, doc_mapping_uid: Some(doc_mapping_uid_01), @@ -3091,7 +3154,7 @@ mod tests { ); let shard_02 = Shard { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(2)), shard_state: ShardState::Closed as i32, doc_mapping_uid: Some(doc_mapping_uid_02), @@ -3153,19 +3216,19 @@ mod tests { subrequests: vec![ TruncateShardsSubrequest { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(1)), truncate_up_to_position_inclusive: Some(Position::offset(0u64)), }, TruncateShardsSubrequest { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(2)), truncate_up_to_position_inclusive: Some(Position::eof(0u64)), }, TruncateShardsSubrequest { index_uid: Some(IndexUid::for_test("test-index", 1337)), - source_id: "test-source".to_string(), + source_id, shard_id: Some(ShardId::from(1337)), truncate_up_to_position_inclusive: Some(Position::offset(1337u64)), }, @@ -3199,33 +3262,20 @@ mod tests { async fn test_ingester_truncate_shards_deletes_dangling_shards() { let (ingester_ctx, ingester) = IngesterForTest::default().build().await; - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); - let queue_id = queue_id(&index_uid, "test-source", &ShardId::from(1)); + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); let mut state_guard = ingester.state.lock_fully().await.unwrap(); - let solo_shard = IngesterShard::new_solo( - ShardState::Open, - Position::Beginning, - Position::Beginning, - None, - Instant::now(), - false, - ); - state_guard.shards.insert(queue_id.clone(), solo_shard); - - let rate_limiter = RateLimiter::from_settings(RateLimiterSettings::default()); - let rate_meter = RateMeter::default(); - state_guard - .rate_trackers - .insert(queue_id.clone(), (rate_limiter, rate_meter)); - + let solo_shard = + IngesterShard::new_solo(index_uid.clone(), source_id.clone(), ShardId::from(1)).build(); + state_guard.shards.insert(solo_shard.queue_id(), solo_shard); drop(state_guard); let truncate_shards_request = TruncateShardsRequest { ingester_id: ingester_ctx.node_id.to_string(), subrequests: vec![TruncateShardsSubrequest { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id, shard_id: Some(ShardId::from(1)), truncate_up_to_position_inclusive: Some(Position::offset(0u64)), }], @@ -3237,7 +3287,6 @@ mod tests { let state_guard = ingester.state.lock_fully().await.unwrap(); assert_eq!(state_guard.shards.len(), 0); - assert_eq!(state_guard.rate_trackers.len(), 0); } #[tokio::test] @@ -3252,6 +3301,7 @@ mod tests { .expect_advise_reset_shards() .once() .returning(|mut request| { + assert_eq!(request.ingester_id, "test-ingester"); assert_eq!(request.shard_ids.len(), 1); assert_eq!(request.shard_ids[0].index_uid(), &("test-index", 0)); assert_eq!(request.shard_ids[0].source_id, "test-source"); @@ -3284,7 +3334,8 @@ mod tests { .build() .await; - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); let doc_mapping_uid = DocMappingUid::random(); let doc_mapping_json = format!( @@ -3294,7 +3345,7 @@ mod tests { ); let shard_01 = Shard { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(1)), shard_state: ShardState::Open as i32, doc_mapping_uid: Some(doc_mapping_uid), @@ -3302,13 +3353,13 @@ mod tests { }; let shard_02 = Shard { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(2)), shard_state: ShardState::Open as i32, doc_mapping_uid: Some(doc_mapping_uid), ..Default::default() }; - let queue_id_02 = queue_id(&index_uid, "test-source", &ShardId::from(2)); + let queue_id_02 = queue_id(&index_uid, &source_id, &ShardId::from(2)); let mut state_guard = ingester.state.lock_fully().await.unwrap(); let now = Instant::now(); @@ -3363,7 +3414,8 @@ mod tests { async fn test_ingester_retain_shards() { let (_ingester_ctx, ingester) = IngesterForTest::default().build().await; - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); let doc_mapping_uid = DocMappingUid::random(); let doc_mapping_json = format!( @@ -3373,7 +3425,7 @@ mod tests { ); let shard_17 = Shard { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(17)), shard_state: ShardState::Open as i32, doc_mapping_uid: Some(doc_mapping_uid), @@ -3382,7 +3434,7 @@ mod tests { let shard_18 = Shard { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(18)), shard_state: ShardState::Closed as i32, doc_mapping_uid: Some(doc_mapping_uid), @@ -3431,7 +3483,7 @@ mod tests { let retain_shards_request = RetainShardsRequest { retain_shards_for_sources: vec![RetainShardsForSource { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id, shard_ids: vec![ShardId::from(17u64)], }], }; @@ -3448,8 +3500,9 @@ mod tests { async fn test_ingester_close_shards() { let (_ingester_ctx, ingester) = IngesterForTest::default().build().await; - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); - let queue_id = queue_id(&index_uid, "test-source", &ShardId::from(1)); + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); + let queue_id = queue_id(&index_uid, &source_id, &ShardId::from(1)); let doc_mapping_uid = DocMappingUid::random(); let doc_mapping_json = format!( @@ -3459,7 +3512,7 @@ mod tests { ); let shard = Shard { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(1)), shard_state: ShardState::Open as i32, doc_mapping_uid: Some(doc_mapping_uid), @@ -3483,7 +3536,7 @@ mod tests { let open_fetch_stream_request = OpenFetchStreamRequest { client_id: "test-client".to_string(), index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(1)), from_position_exclusive: Some(Position::Beginning), }; @@ -3496,12 +3549,12 @@ mod tests { shard_pkeys: vec![ ShardPKey { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(1)), }, ShardPKey { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id, shard_id: Some(ShardId::from(1337)), }, ], @@ -3550,7 +3603,9 @@ mod tests { assert_eq!(observation.status(), IngesterStatus::Ready); let mut state_guard = ingester.state.lock_fully().await.unwrap(); - state_guard.set_status(IngesterStatus::Decommissioning); + state_guard + .set_status(IngesterStatus::Decommissioning) + .await; drop(state_guard); let observation = observation_stream.next().await.unwrap().unwrap(); @@ -3563,41 +3618,87 @@ mod tests { assert!(observation_opt.is_none()); } + #[tokio::test] + async fn test_ingester_decommission() { + let (_ingester_ctx, ingester) = IngesterForTest::default().build().await; + + let mut state_guard = ingester.state.lock_fully().await.unwrap(); + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); + + let shard = IngesterShard::new_solo(index_uid, source_id, ShardId::from(1)).build(); + let queue_id = shard.queue_id(); + + state_guard.shards.insert(queue_id.clone(), shard); + drop(state_guard); + + let mut observation_stream = ingester + .open_observation_stream(OpenObservationStreamRequest {}) + .await + .unwrap(); + + ingester.decommission(DecommissionRequest {}).await.unwrap(); + + let next_observation = observation_stream.next().await.unwrap().unwrap(); + let next_status = next_observation.status(); + assert_eq!(next_status, IngesterStatus::Retiring); + + wait_for_ingester_status( + &ingester, + IngesterStatus::Decommissioning, + Duration::from_secs(1), + ) + .await + .unwrap(); + + let state_guard = ingester.state.lock_fully().await.unwrap(); + let shard = state_guard.shards.get(&queue_id).unwrap(); + shard.assert_is_closed(); + } + #[tokio::test] async fn test_check_decommissioning_status() { let (_ingester_ctx, ingester) = IngesterForTest::default().build().await; let mut state_guard = ingester.state.lock_fully().await.unwrap(); - ingester.check_decommissioning_status(&mut state_guard); + ingester + .check_decommissioning_status(&mut state_guard) + .await; assert_eq!(state_guard.status(), IngesterStatus::Ready); - state_guard.set_status(IngesterStatus::Decommissioning); - ingester.check_decommissioning_status(&mut state_guard); + state_guard + .set_status(IngesterStatus::Decommissioning) + .await; + ingester + .check_decommissioning_status(&mut state_guard) + .await; assert_eq!(state_guard.status(), IngesterStatus::Decommissioned); - state_guard.set_status(IngesterStatus::Decommissioning); + state_guard + .set_status(IngesterStatus::Decommissioning) + .await; - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); - let queue_id_01 = queue_id(&index_uid, "test-source", &ShardId::from(1)); + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); - state_guard.shards.insert( - queue_id_01.clone(), - IngesterShard::new_solo( - ShardState::Closed, - Position::offset(12u64), - Position::Beginning, - None, - Instant::now(), - false, - ), - ); - ingester.check_decommissioning_status(&mut state_guard); + let solo_shard = IngesterShard::new_solo(index_uid.clone(), source_id, ShardId::from(1)) + .with_state(ShardState::Closed) + .with_replication_position_inclusive(Position::offset(12u64)) + .build(); + let queue_id = solo_shard.queue_id(); + + state_guard.shards.insert(queue_id.clone(), solo_shard); + ingester + .check_decommissioning_status(&mut state_guard) + .await; assert_eq!(state_guard.status(), IngesterStatus::Decommissioning); - let shard = state_guard.shards.get_mut(&queue_id_01).unwrap(); + let shard = state_guard.shards.get_mut(&queue_id).unwrap(); shard.truncation_position_inclusive = Position::Beginning.as_eof(); - ingester.check_decommissioning_status(&mut state_guard); + ingester + .check_decommissioning_status(&mut state_guard) + .await; assert_eq!(state_guard.status(), IngesterStatus::Decommissioned); } @@ -3607,7 +3708,8 @@ mod tests { let event_broker = EventBroker::default(); ingester.subscribe(&event_broker); - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); let doc_mapping_uid = DocMappingUid::random(); let doc_mapping_json = format!( @@ -3617,23 +3719,23 @@ mod tests { ); let shard_01 = Shard { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(1)), shard_state: ShardState::Open as i32, doc_mapping_uid: Some(doc_mapping_uid), ..Default::default() }; - let queue_id_01 = queue_id(&index_uid, "test-source", &ShardId::from(1)); + let queue_id_01 = queue_id(&index_uid, &source_id, &ShardId::from(1)); let shard_02 = Shard { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(2)), shard_state: ShardState::Closed as i32, doc_mapping_uid: Some(doc_mapping_uid), ..Default::default() }; - let queue_id_02 = queue_id(&index_uid, "test-source", &ShardId::from(2)); + let queue_id_02 = queue_id(&index_uid, &source_id, &ShardId::from(2)); let mut state_guard = ingester.state.lock_fully().await.unwrap(); let now = Instant::now(); @@ -3686,7 +3788,7 @@ mod tests { let shard_position_update = ShardPositionsUpdate { source_uid: SourceUid { index_uid: index_uid.clone(), - source_id: "test-source".to_string(), + source_id, }, updated_shard_positions: vec![ (ShardId::from(1), Position::offset(0u64)), @@ -3725,8 +3827,9 @@ mod tests { .build() .await; - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); - let queue_id_01 = queue_id(&index_uid, "test-source", &ShardId::from(1)); + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); + let queue_id_01 = queue_id(&index_uid, &source_id, &ShardId::from(1)); let doc_mapping_uid = DocMappingUid::random(); let doc_mapping_json = format!( @@ -3736,7 +3839,7 @@ mod tests { ); let shard_01 = Shard { index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), + source_id, shard_id: Some(ShardId::from(1)), shard_state: ShardState::Open as i32, doc_mapping_uid: Some(doc_mapping_uid), @@ -3779,6 +3882,7 @@ mod tests { let index_uid_0: IndexUid = IndexUid::for_test("test-index-0", 0); let index_uid_1: IndexUid = IndexUid::for_test("test-index-1", 0); + let source_id = SourceId::from("test-source"); let doc_mapping_uid = DocMappingUid::random(); let doc_mapping_json = format!( @@ -3788,7 +3892,7 @@ mod tests { ); let shard_01 = Shard { index_uid: Some(index_uid_0.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(1)), shard_state: ShardState::Open as i32, doc_mapping_uid: Some(doc_mapping_uid), @@ -3796,7 +3900,7 @@ mod tests { }; let shard_02 = Shard { index_uid: Some(index_uid_0.clone()), - source_id: "test-source".to_string(), + source_id: source_id.clone(), shard_id: Some(ShardId::from(2)), shard_state: ShardState::Closed as i32, doc_mapping_uid: Some(doc_mapping_uid), @@ -3804,7 +3908,7 @@ mod tests { }; let shard_03 = Shard { index_uid: Some(index_uid_1.clone()), - source_id: "test-source".to_string(), + source_id, shard_id: Some(ShardId::from(3)), shard_state: ShardState::Closed as i32, doc_mapping_uid: Some(doc_mapping_uid), diff --git a/quickwit/quickwit-ingest/src/ingest_v2/metrics.rs b/quickwit/quickwit-ingest/src/ingest_v2/metrics.rs index 1fb32c0b2fd..87975a3c462 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/metrics.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/metrics.rs @@ -82,12 +82,20 @@ pub(super) struct IngestV2Metrics { pub wal_disk_used_bytes: IntGauge, pub wal_memory_used_bytes: IntGauge, pub ingest_results: IngestResultMetrics, + pub ingest_attempts: IntCounterVec<1>, } impl Default for IngestV2Metrics { fn default() -> Self { Self { ingest_results: IngestResultMetrics::default(), + ingest_attempts: new_counter_vec::<1>( + "ingest_attempts", + "Number of routing attempts by AZ locality", + "ingest", + &[], + ["az_routing"], + ), reset_shards_operations_total: new_counter_vec( "reset_shards_operations_total", "Total number of reset shards operations performed.", diff --git a/quickwit/quickwit-ingest/src/ingest_v2/mod.rs b/quickwit/quickwit-ingest/src/ingest_v2/mod.rs index c8543faf793..0523ea27045 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/mod.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/mod.rs @@ -16,6 +16,7 @@ mod broadcast; mod debouncing; mod doc_mapper; mod fetch; +mod helpers; mod idle; mod ingester; mod metrics; @@ -28,6 +29,7 @@ mod replication; mod router; mod routing_table; mod state; +mod wal_capacity_tracker; mod workbench; use std::collections::HashMap; @@ -36,26 +38,62 @@ use std::ops::{Add, AddAssign}; use std::time::Duration; use std::{env, fmt}; -pub use broadcast::{LocalShardsUpdate, ShardInfo, ShardInfos, setup_local_shards_update_listener}; +pub use broadcast::{ + LocalShardsUpdate, ShardInfo, ShardInfos, setup_ingester_capacity_update_listener, + setup_local_shards_update_listener, +}; use bytes::buf::Writer; use bytes::{BufMut, BytesMut}; use bytesize::ByteSize; use quickwit_common::tower::Pool; -use quickwit_proto::ingest::ingester::IngesterServiceClient; +use quickwit_proto::ingest::ingester::{IngesterServiceClient, IngesterStatus}; use quickwit_proto::ingest::router::{IngestRequestV2, IngestSubrequest}; -use quickwit_proto::ingest::{CommitTypeV2, DocBatchV2}; -use quickwit_proto::types::{DocUid, DocUidGenerator, IndexId, NodeId, SubrequestId}; +use quickwit_proto::ingest::{CommitTypeV2, DocBatchV2, DocFormat}; +use quickwit_proto::types::{ + DocUid, DocUidGenerator, IndexId, IndexUid, NodeId, SourceId, SubrequestId, +}; use serde::Serialize; use tracing::{error, info}; use workbench::pending_subrequests; pub use self::fetch::{FetchStreamError, MultiFetchStream}; -pub use self::ingester::{Ingester, wait_for_ingester_decommission, wait_for_ingester_status}; +pub use self::helpers::{ + try_get_ingester_status, wait_for_ingester_decommission, wait_for_ingester_status, +}; +pub use self::ingester::Ingester; use self::mrecord::MRECORD_HEADER_LEN; pub use self::mrecord::{MRecord, decoded_mrecords}; pub use self::router::IngestRouter; -pub type IngesterPool = Pool; +/// An ingester as represented in the pool, bundling the gRPC client with node metadata. +#[derive(Debug, Clone)] +pub struct IngesterPoolEntry { + pub client: IngesterServiceClient, + pub status: IngesterStatus, + pub availability_zone: Option, +} + +impl IngesterPoolEntry { + #[cfg(any(test, feature = "testsuite"))] + pub fn ready_with_client(client: IngesterServiceClient) -> Self { + IngesterPoolEntry { + client, + status: IngesterStatus::Ready, + availability_zone: None, + } + } + + #[cfg(any(test, feature = "testsuite"))] + pub fn mocked_ingester() -> Self { + IngesterPoolEntry { + client: IngesterServiceClient::mocked(), + status: IngesterStatus::Ready, + availability_zone: None, + } + } +} + +pub type IngesterPool = Pool; /// Identifies an ingester client, typically a source, for logging and debugging purposes. pub type ClientId = String; @@ -64,6 +102,8 @@ pub type LeaderId = NodeId; pub type FollowerId = NodeId; +pub type OpenShardCounts = Vec<(IndexUid, SourceId, usize)>; + const IDLE_SHARD_TIMEOUT_ENV_KEY: &str = "QW_IDLE_SHARD_TIMEOUT_SECS"; const DEFAULT_IDLE_SHARD_TIMEOUT: Duration = Duration::from_secs(15 * 60); // 15 minutes @@ -134,6 +174,7 @@ impl DocBatchV2Builder { doc_uids: self.doc_uids, doc_buffer: self.doc_buffer.freeze(), doc_lengths: self.doc_lengths, + doc_format: DocFormat::Json as i32, }; Some(doc_batch) } @@ -172,6 +213,7 @@ impl JsonDocBatchV2Builder { doc_uids: self.doc_uids, doc_buffer: self.doc_buffer.into_inner().freeze(), doc_lengths: self.doc_lengths, + doc_format: DocFormat::Json as i32, } } @@ -432,6 +474,7 @@ mod tests { doc_buffer: Vec::new().into(), doc_lengths: Vec::new(), doc_uids: Vec::new(), + doc_format: DocFormat::Json as i32, }; assert_eq!(estimate_size(&doc_batch), ByteSize(0)); @@ -439,6 +482,7 @@ mod tests { doc_buffer: vec![0u8; 100].into(), doc_lengths: vec![10, 20, 30], doc_uids: Vec::new(), + doc_format: DocFormat::Json as i32, }; assert_eq!(estimate_size(&doc_batch), ByteSize(118)); } diff --git a/quickwit/quickwit-ingest/src/ingest_v2/models.rs b/quickwit/quickwit-ingest/src/ingest_v2/models.rs index 348c4aa3a1b..275682d2c57 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/models.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/models.rs @@ -15,11 +15,14 @@ use std::sync::Arc; use std::time::{Duration, Instant}; +use quickwit_common::rate_limiter::RateLimiter; use quickwit_doc_mapper::DocMapper; use quickwit_proto::ingest::ShardState; -use quickwit_proto::types::{NodeId, Position}; +use quickwit_proto::types::{IndexUid, NodeId, Position, QueueId, ShardId, SourceId, queue_id}; use tokio::sync::watch; +use crate::ingest_v2::rate_meter::RateMeter; + #[derive(Debug, Clone)] pub(super) enum IngesterShardType { /// A primary shard hosted on a leader and replicated on a follower. @@ -35,12 +38,19 @@ pub(super) type ShardStatus = (ShardState, Position); #[derive(Debug)] pub(super) struct IngesterShard { + pub index_uid: IndexUid, + pub source_id: SourceId, + pub shard_id: ShardId, pub shard_type: IngesterShardType, pub shard_state: ShardState, /// Position of the last record written in the shard's mrecordlog queue. pub replication_position_inclusive: Position, /// Position up to which the shard has been truncated. pub truncation_position_inclusive: Position, + // Rate limiter for the shard. Unused for replica shards. + pub rate_limiter: RateLimiter, + // Rate meter for the shard. Unused for replica shards. + pub rate_meter: RateMeter, /// Whether the shard should be advertised to other nodes (routers) via gossip. /// /// Because shards are created in multiple steps, (e.g., init shard on leader, create shard in @@ -52,85 +62,183 @@ pub(super) struct IngesterShard { pub doc_mapper_opt: Option>, /// Whether to validate documents in this shard. True if no preprocessing (VRL) will happen /// before indexing. - pub validate: bool, + pub validate_docs: bool, pub shard_status_tx: watch::Sender, pub shard_status_rx: watch::Receiver, /// Instant at which the shard was last written to. pub last_write_instant: Instant, } +/// Builder for `IngesterShard`. By default, the shard is open, is empty (i.e. the replication and +/// truncation positions are at the beginning), uses the default rate limiter and rate meter, has no +/// doc mapper, does not validate documents, and is not advertisable. +pub(super) struct IngesterShardBuilder { + index_uid: IndexUid, + source_id: SourceId, + shard_id: ShardId, + shard_type: IngesterShardType, + shard_state: ShardState, + replication_position_inclusive: Position, + truncation_position_inclusive: Position, + rate_limiter: RateLimiter, + rate_meter: RateMeter, + doc_mapper_opt: Option>, + validate_docs: bool, + is_advertisable: bool, + last_write_instant: Option, +} + +impl IngesterShardBuilder { + /// Sets the shard state. Defaults to `ShardState::Open`. + pub fn with_state(mut self, shard_state: ShardState) -> Self { + self.shard_state = shard_state; + self + } + + /// Sets the rate limiter. Defaults to `RateLimiter::default()`. + pub fn with_rate_limiter(mut self, rate_limiter: RateLimiter) -> Self { + self.rate_limiter = rate_limiter; + self + } + + /// Sets the rate meter. Defaults to `RateMeter::default()`. + pub fn with_rate_meter(mut self, rate_meter: RateMeter) -> Self { + self.rate_meter = rate_meter; + self + } + + /// Sets the doc mapper. + pub fn with_doc_mapper(mut self, doc_mapper: Arc) -> Self { + self.doc_mapper_opt = Some(doc_mapper); + self + } + + /// Sets the replication position. Defaults to `Position::Beginning`. + pub fn with_replication_position_inclusive(mut self, position: Position) -> Self { + self.replication_position_inclusive = position; + self + } + + /// Sets the truncation position. Defaults to `Position::Beginning`. + pub fn with_truncation_position_inclusive(mut self, position: Position) -> Self { + self.truncation_position_inclusive = position; + self + } + + /// Sets whether to validate documents. Defaults to `false`. + pub fn with_validate_docs(mut self, validate_docs: bool) -> Self { + self.validate_docs = validate_docs; + self + } + + /// Sets whether the shard should be advertised to other nodes via gossip. Defaults to `false`. + pub fn advertisable(mut self) -> Self { + self.is_advertisable = true; + self + } + + /// Sets the last write instant. Defaults to `Instant::now()`. + pub fn with_last_write(mut self, last_write_instant: Instant) -> Self { + self.last_write_instant = Some(last_write_instant); + self + } + + /// Builds the `IngesterShard`. Uses `Instant::now()` for last write time if not specified. + pub fn build(self) -> IngesterShard { + let shard_status = ( + self.shard_state, + self.replication_position_inclusive.clone(), + ); + let (shard_status_tx, shard_status_rx) = watch::channel(shard_status); + IngesterShard { + index_uid: self.index_uid, + source_id: self.source_id, + shard_id: self.shard_id, + shard_type: self.shard_type, + shard_state: self.shard_state, + replication_position_inclusive: self.replication_position_inclusive, + truncation_position_inclusive: self.truncation_position_inclusive, + rate_limiter: self.rate_limiter, + rate_meter: self.rate_meter, + is_advertisable: self.is_advertisable, + doc_mapper_opt: self.doc_mapper_opt, + validate_docs: self.validate_docs, + shard_status_tx, + shard_status_rx, + last_write_instant: self.last_write_instant.unwrap_or_else(Instant::now), + } + } +} + impl IngesterShard { + /// Creates a builder for a primary shard hosted on a leader and replicated on a follower. pub fn new_primary( + index_uid: IndexUid, + source_id: SourceId, + shard_id: ShardId, follower_id: NodeId, - shard_state: ShardState, - replication_position_inclusive: Position, - truncation_position_inclusive: Position, - doc_mapper: Arc, - now: Instant, - validate: bool, - ) -> Self { - let shard_status = (shard_state, replication_position_inclusive.clone()); - let (shard_status_tx, shard_status_rx) = watch::channel(shard_status); - Self { + ) -> IngesterShardBuilder { + IngesterShardBuilder { + index_uid, + source_id, + shard_id, shard_type: IngesterShardType::Primary { follower_id }, - shard_state, - replication_position_inclusive, - truncation_position_inclusive, + shard_state: ShardState::Open, + replication_position_inclusive: Position::Beginning, + truncation_position_inclusive: Position::Beginning, + rate_limiter: RateLimiter::default(), + rate_meter: RateMeter::default(), + doc_mapper_opt: None, + validate_docs: false, is_advertisable: false, - doc_mapper_opt: Some(doc_mapper), - validate, - shard_status_tx, - shard_status_rx, - last_write_instant: now, + last_write_instant: None, } } + /// Creates a builder for a replica shard hosted on a follower. pub fn new_replica( + index_uid: IndexUid, + source_id: SourceId, + shard_id: ShardId, leader_id: NodeId, - shard_state: ShardState, - replication_position_inclusive: Position, - truncation_position_inclusive: Position, - now: Instant, - ) -> Self { - let shard_status = (shard_state, replication_position_inclusive.clone()); - let (shard_status_tx, shard_status_rx) = watch::channel(shard_status); - Self { + ) -> IngesterShardBuilder { + IngesterShardBuilder { + index_uid, + source_id, + shard_id, shard_type: IngesterShardType::Replica { leader_id }, - shard_state, - replication_position_inclusive, - truncation_position_inclusive, - // This is irrelevant for replica shards since they are not advertised via gossip - // anyway. - is_advertisable: false, + shard_state: ShardState::Open, + replication_position_inclusive: Position::Beginning, + truncation_position_inclusive: Position::Beginning, + rate_limiter: RateLimiter::default(), + rate_meter: RateMeter::default(), doc_mapper_opt: None, - validate: false, - shard_status_tx, - shard_status_rx, - last_write_instant: now, + validate_docs: false, + is_advertisable: false, + last_write_instant: None, } } + /// Creates a builder for a solo shard hosted on a single node (i.e. replication factor = 1). pub fn new_solo( - shard_state: ShardState, - replication_position_inclusive: Position, - truncation_position_inclusive: Position, - doc_mapper_opt: Option>, - now: Instant, - validate: bool, - ) -> Self { - let shard_status = (shard_state, replication_position_inclusive.clone()); - let (shard_status_tx, shard_status_rx) = watch::channel(shard_status); - Self { + index_uid: IndexUid, + source_id: SourceId, + shard_id: ShardId, + ) -> IngesterShardBuilder { + IngesterShardBuilder { + index_uid, + source_id, + shard_id, shard_type: IngesterShardType::Solo, - shard_state, - replication_position_inclusive, - truncation_position_inclusive, + shard_state: ShardState::Open, + replication_position_inclusive: Position::Beginning, + truncation_position_inclusive: Position::Beginning, + rate_limiter: RateLimiter::default(), + rate_meter: RateMeter::default(), + doc_mapper_opt: None, + validate_docs: false, is_advertisable: false, - doc_mapper_opt, - validate, - shard_status_tx, - shard_status_rx, - last_write_instant: now, + last_write_instant: None, } } @@ -178,6 +286,10 @@ impl IngesterShard { .expect("channel should be open"); } + pub fn queue_id(&self) -> QueueId { + queue_id(&self.index_uid, &self.source_id, &self.shard_id) + } + pub fn set_replication_position_inclusive( &mut self, replication_position_inclusive: Position, @@ -250,14 +362,17 @@ mod tests { let doc_mapper = build_doc_mapper(&doc_mapping, &search_settings).unwrap(); let primary_shard = IngesterShard::new_primary( - "test-follower".into(), - ShardState::Closed, - Position::offset(42u64), - Position::Beginning, - doc_mapper, - Instant::now(), - true, - ); + IndexUid::for_test("test-index", 0), + SourceId::from("test-source"), + ShardId::from(1), + NodeId::from("test-follower"), + ) + .with_state(ShardState::Closed) + .with_replication_position_inclusive(Position::offset(42u64)) + .with_doc_mapper(doc_mapper) + .with_validate_docs(true) + .build(); + assert!(matches!( &primary_shard.shard_type, IngesterShardType::Primary { follower_id, .. } if *follower_id == "test-follower" @@ -278,12 +393,15 @@ mod tests { #[test] fn test_new_replica_shard() { let replica_shard = IngesterShard::new_replica( - "test-leader".into(), - ShardState::Closed, - Position::offset(42u64), - Position::Beginning, - Instant::now(), - ); + IndexUid::for_test("test-index", 0), + SourceId::from("test-source"), + ShardId::from(1), + NodeId::from("test-leader"), + ) + .with_state(ShardState::Closed) + .with_replication_position_inclusive(Position::offset(42u64)) + .build(); + assert!(matches!( &replica_shard.shard_type, IngesterShardType::Replica { leader_id } if *leader_id == "test-leader" @@ -304,13 +422,14 @@ mod tests { #[test] fn test_new_solo_shard() { let solo_shard = IngesterShard::new_solo( - ShardState::Closed, - Position::offset(42u64), - Position::Beginning, - None, - Instant::now(), - false, - ); + IndexUid::for_test("test-index", 0), + SourceId::from("test-source"), + ShardId::from(1), + ) + .with_state(ShardState::Closed) + .with_replication_position_inclusive(Position::offset(42u64)) + .build(); + solo_shard.assert_is_solo(); assert!(!solo_shard.is_replica()); assert_eq!(solo_shard.shard_state, ShardState::Closed); diff --git a/quickwit/quickwit-ingest/src/ingest_v2/publish_tracker.rs b/quickwit/quickwit-ingest/src/ingest_v2/publish_tracker.rs index f005bedf2e8..a6b514379ce 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/publish_tracker.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/publish_tracker.rs @@ -62,16 +62,6 @@ impl PublishTracker { } } - pub fn register_requested_shards<'a>( - &'a self, - shard_ids: impl IntoIterator, - ) { - let mut publish_states = self.state.lock().unwrap(); - for shard_id in shard_ids { - publish_states.shard_tracked(shard_id.clone()); - } - } - pub fn track_persisted_shard_position(&self, shard_id: ShardId, new_position: Position) { let mut publish_states = self.state.lock().unwrap(); publish_states.position_persisted(&shard_id, &new_position) @@ -91,8 +81,6 @@ impl PublishTracker { } enum PublishState { - /// The persist request for this shard has been sent - Tracked, /// The persist request for this shard success response has been received /// but the position has not yet been published AwaitingPublish(Position), @@ -108,68 +96,44 @@ struct ShardPublishStates { } impl ShardPublishStates { - fn shard_tracked(&mut self, shard_id: ShardId) { - self.states.entry(shard_id).or_insert(PublishState::Tracked); - } - fn position_published( &mut self, shard_id: &ShardId, new_position: &Position, publish_complete_notifier: &Notify, ) { - if let Some(publish_state) = self.states.get_mut(shard_id) { - match publish_state { - PublishState::AwaitingPublish(shard_position) if new_position >= shard_position => { - *publish_state = PublishState::Published(new_position.clone()); - self.awaiting_count -= 1; - if self.awaiting_count == 0 { - // The notification is only relevant once - // `self.wait_publish_complete()` is called. - // Before that, `state.awaiting_publish` might - // still be re-populated. - publish_complete_notifier.notify_waiters(); - } - } - PublishState::Published(current_position) if new_position > current_position => { - *current_position = new_position.clone(); - } - PublishState::Tracked => { - *publish_state = PublishState::Published(new_position.clone()); - } - PublishState::Published(_) => { - // looks like a duplicate or out-of-order event - } - PublishState::AwaitingPublish(_) => { - // the shard made some progress but we are waiting for more + let Some(publish_state) = self.states.get_mut(shard_id) else { + return; + }; + + match publish_state { + PublishState::AwaitingPublish(shard_position) if new_position >= shard_position => { + *publish_state = PublishState::Published(new_position.clone()); + self.awaiting_count -= 1; + if self.awaiting_count == 0 { + publish_complete_notifier.notify_waiters(); } } + PublishState::Published(current_position) if new_position > current_position => { + *current_position = new_position.clone(); + } + PublishState::Published(_) | PublishState::AwaitingPublish(_) => { + // duplicate/out-of-order or not enough progress yet + } } - // else: this shard is not being tracked here } fn position_persisted(&mut self, shard_id: &ShardId, new_position: &Position) { - if let Some(publish_state) = self.states.get_mut(shard_id) { - match publish_state { - PublishState::Published(current_position) if new_position <= current_position => { - // new position already published, no need to track it - } - PublishState::AwaitingPublish(old_position) => { - error!( - %old_position, - %new_position, - %shard_id, - "shard persisted positions should not be tracked multiple times" - ); - } - PublishState::Tracked | PublishState::Published(_) => { - *publish_state = PublishState::AwaitingPublish(new_position.clone()); - self.awaiting_count += 1; - } - } - } else { - error!(%shard_id, "requested shards should be registered before their position is tracked") + if self.states.contains_key(shard_id) { + error!(%shard_id, "shard persisted positions should not be tracked multiple times"); + return; } + + self.states.insert( + shard_id.clone(), + PublishState::AwaitingPublish(new_position.clone()), + ); + self.awaiting_count += 1; } } @@ -188,12 +152,7 @@ mod tests { let shard_id_1 = ShardId::from("test-shard-1"); let shard_id_2 = ShardId::from("test-shard-2"); - let shard_id_3 = ShardId::from("test-shard-3"); - let shard_id_4 = ShardId::from("test-shard-4"); // not tracked - - shard_publish_states.shard_tracked(shard_id_1.clone()); - shard_publish_states.shard_tracked(shard_id_2.clone()); - shard_publish_states.shard_tracked(shard_id_3.clone()); + let shard_id_3 = ShardId::from("test-shard-3"); // not tracked let notifier_receiver = notifier.clone(); let notified_subscription = notifier_receiver.notified(); @@ -215,19 +174,10 @@ mod tests { .await .unwrap(); - let notified_subscription = notifier_receiver.notified(); + // shard 3 is not tracked shard_publish_states.position_published(&shard_id_3, &Position::offset(10usize), ¬ifier); assert_eq!(shard_publish_states.awaiting_count, 0); - shard_publish_states.position_persisted(&shard_id_3, &Position::offset(10usize)); - assert_eq!(shard_publish_states.awaiting_count, 0); - // no notification expected here as the shard never becomes AwaitingPublish - tokio::time::timeout(Duration::from_millis(100), notified_subscription) - .await - .unwrap_err(); - // shard 4 is not tracked - shard_publish_states.position_published(&shard_id_4, &Position::offset(10usize), ¬ifier); - assert_eq!(shard_publish_states.awaiting_count, 0); - assert!(!shard_publish_states.states.contains_key(&shard_id_4)); + assert!(!shard_publish_states.states.contains_key(&shard_id_3)); } #[tokio::test] @@ -241,8 +191,6 @@ mod tests { let shard_id_4 = ShardId::from("test-shard-4"); let shard_id_5 = ShardId::from("test-shard-5"); // not tracked - tracker.register_requested_shards([&shard_id_1, &shard_id_2, &shard_id_3, &shard_id_4]); - tracker.track_persisted_shard_position(shard_id_1.clone(), Position::offset(42usize)); tracker.track_persisted_shard_position(shard_id_2.clone(), Position::offset(42usize)); tracker.track_persisted_shard_position(shard_id_3.clone(), Position::offset(42usize)); @@ -292,7 +240,6 @@ mod tests { { let event_broker = EventBroker::default(); let tracker = PublishTracker::new(event_broker.clone()); - tracker.register_requested_shards([&shard_id_1, &shard_id_2]); tracker.track_persisted_shard_position(shard_id_1.clone(), position.clone()); tracker.track_persisted_shard_position(shard_id_2.clone(), position.clone()); @@ -313,7 +260,6 @@ mod tests { { let event_broker = EventBroker::default(); let tracker = PublishTracker::new(event_broker.clone()); - tracker.register_requested_shards([&shard_id_1, &shard_id_2]); tracker.track_persisted_shard_position(shard_id_1.clone(), position.clone()); event_broker.publish(ShardPositionsUpdate { source_uid: SourceUid { diff --git a/quickwit/quickwit-ingest/src/ingest_v2/replication.rs b/quickwit/quickwit-ingest/src/ingest_v2/replication.rs index 5e286ec5b84..bbf0cd037c5 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/replication.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/replication.rs @@ -27,7 +27,7 @@ use quickwit_proto::ingest::ingester::{ syn_replication_message, }; use quickwit_proto::ingest::{CommitTypeV2, IngestV2Error, IngestV2Result, Shard, ShardState}; -use quickwit_proto::types::{NodeId, Position, QueueId}; +use quickwit_proto::types::{NodeId, QueueId}; use tokio::sync::mpsc::error::TryRecvError; use tokio::sync::{mpsc, oneshot}; use tokio::task::JoinHandle; @@ -465,13 +465,13 @@ impl ReplicationTask { return Err(IngestV2Error::Internal(message)); } }; - let replica_shard = IngesterShard::new_replica( - replica_shard.leader_id.into(), - ShardState::Open, - Position::Beginning, - Position::Beginning, - Instant::now(), - ); + let index_uid = replica_shard.index_uid().clone(); + let shard_id = replica_shard.shard_id().clone(); + let source_id = replica_shard.source_id; + let leader_id = NodeId::from(replica_shard.leader_id); + + let replica_shard = + IngesterShard::new_replica(index_uid, source_id, shard_id, leader_id).build(); state_guard.shards.insert(queue_id, replica_shard); let init_replica_response = InitReplicaResponse { @@ -698,7 +698,6 @@ impl ReplicationTask { if !shards_to_delete.is_empty() { for queue_id in &shards_to_delete { state_guard.shards.remove(queue_id); - state_guard.rate_trackers.remove(queue_id); warn!("deleted dangling shard `{queue_id}`"); } } @@ -762,9 +761,11 @@ impl Drop for ReplicationTaskHandle { #[cfg(test)] mod tests { + use quickwit_cluster::{ChannelTransport, create_cluster_for_test}; + use quickwit_config::service::QuickwitService; use quickwit_proto::ingest::ingester::{ReplicateSubrequest, ReplicateSuccess}; use quickwit_proto::ingest::{DocBatchV2, Shard}; - use quickwit_proto::types::{IndexUid, ShardId, queue_id}; + use quickwit_proto::types::{IndexUid, Position, ShardId, queue_id}; use super::*; @@ -851,7 +852,7 @@ mod tests { }; tokio::spawn(dummy_replication_task_future); - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); + let index_uid = IndexUid::for_test("test-index", 0); let replica_shard = Shard { index_uid: Some(index_uid.clone()), source_id: "test-source".to_string(), @@ -923,7 +924,7 @@ mod tests { }; tokio::spawn(dummy_replication_task_future); - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); + let index_uid = IndexUid::for_test("test-index", 0); let index_uid2: IndexUid = IndexUid::for_test("test-index", 1); let subrequests = vec![ @@ -1037,7 +1038,15 @@ mod tests { async fn test_replication_task_happy_path() { let leader_id: NodeId = "test-leader".into(); let follower_id: NodeId = "test-follower".into(); - let (_temp_dir, state) = IngesterState::for_test().await; + let cluster = create_cluster_for_test( + Vec::new(), + &[QuickwitService::Indexer.as_str()], + &ChannelTransport::default(), + true, + ) + .await + .unwrap(); + let (_temp_dir, state) = IngesterState::for_test(cluster).await; let (syn_replication_stream_tx, syn_replication_stream) = ServiceStream::new_bounded(SYN_REPLICATION_STREAM_CAPACITY); let (ack_replication_stream_tx, mut ack_replication_stream) = @@ -1056,7 +1065,7 @@ mod tests { memory_capacity, ); - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); + let index_uid = IndexUid::for_test("test-index", 0); let index_uid2: IndexUid = IndexUid::for_test("test-index", 1); // Init shard 01. @@ -1300,7 +1309,15 @@ mod tests { async fn test_replication_task_shard_closed() { let leader_id: NodeId = "test-leader".into(); let follower_id: NodeId = "test-follower".into(); - let (_temp_dir, state) = IngesterState::for_test().await; + let cluster = create_cluster_for_test( + Vec::new(), + &[QuickwitService::Indexer.as_str()], + &ChannelTransport::default(), + true, + ) + .await + .unwrap(); + let (_temp_dir, state) = IngesterState::for_test(cluster).await; let (syn_replication_stream_tx, syn_replication_stream) = ServiceStream::new_bounded(SYN_REPLICATION_STREAM_CAPACITY); let (ack_replication_stream_tx, mut ack_replication_stream) = @@ -1319,21 +1336,21 @@ mod tests { memory_capacity, ); - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); - let queue_id_01 = queue_id(&index_uid, "test-source", &ShardId::from(1)); + let index_uid = IndexUid::for_test("test-index", 0); let replica_shard = IngesterShard::new_replica( + index_uid.clone(), + "test-source".to_string(), + ShardId::from(1), leader_id, - ShardState::Closed, - Position::Beginning, - Position::Beginning, - Instant::now(), - ); + ) + .with_state(ShardState::Closed) + .build(); state .lock_fully() .await .unwrap() .shards - .insert(queue_id_01.clone(), replica_shard); + .insert(replica_shard.queue_id(), replica_shard); let replicate_request = ReplicateRequest { leader_id: "test-leader".to_string(), @@ -1377,7 +1394,15 @@ mod tests { async fn test_replication_task_deletes_dangling_shard() { let leader_id: NodeId = "test-leader".into(); let follower_id: NodeId = "test-follower".into(); - let (_temp_dir, state) = IngesterState::for_test().await; + let cluster = create_cluster_for_test( + Vec::new(), + &[QuickwitService::Indexer.as_str()], + &ChannelTransport::default(), + true, + ) + .await + .unwrap(); + let (_temp_dir, state) = IngesterState::for_test(cluster).await; let (syn_replication_stream_tx, syn_replication_stream) = ServiceStream::new_bounded(SYN_REPLICATION_STREAM_CAPACITY); let (ack_replication_stream_tx, mut ack_replication_stream) = @@ -1396,15 +1421,15 @@ mod tests { memory_capacity, ); - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); - let queue_id_01 = queue_id(&index_uid, "test-source", &ShardId::from(1)); + let index_uid = IndexUid::for_test("test-index", 0); let replica_shard = IngesterShard::new_replica( + index_uid.clone(), + "test-source".to_string(), + ShardId::from(1), leader_id, - ShardState::Open, - Position::Beginning, - Position::Beginning, - Instant::now(), - ); + ) + .build(); + let queue_id_01 = replica_shard.queue_id(); state .lock_fully() .await @@ -1465,7 +1490,15 @@ mod tests { let leader_id: NodeId = "test-leader".into(); let follower_id: NodeId = "test-follower".into(); - let (_temp_dir, state) = IngesterState::for_test().await; + let cluster = create_cluster_for_test( + Vec::new(), + &[QuickwitService::Indexer.as_str()], + &ChannelTransport::default(), + true, + ) + .await + .unwrap(); + let (_temp_dir, state) = IngesterState::for_test(cluster).await; let (syn_replication_stream_tx, syn_replication_stream) = ServiceStream::new_bounded(SYN_REPLICATION_STREAM_CAPACITY); let (ack_replication_stream_tx, mut ack_replication_stream) = @@ -1484,15 +1517,15 @@ mod tests { memory_capacity, ); - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); + let index_uid = IndexUid::for_test("test-index", 0); let queue_id_01 = queue_id(&index_uid, "test-source", &ShardId::from(1)); let replica_shard = IngesterShard::new_replica( + index_uid.clone(), + "test-source".to_string(), + ShardId::from(1), leader_id, - ShardState::Open, - Position::Beginning, - Position::Beginning, - Instant::now(), - ); + ) + .build(); let mut state_guard = state.lock_fully().await.unwrap(); state_guard @@ -1554,7 +1587,15 @@ mod tests { async fn test_replication_task_resource_exhausted() { let leader_id: NodeId = "test-leader".into(); let follower_id: NodeId = "test-follower".into(); - let (_temp_dir, state) = IngesterState::for_test().await; + let cluster = create_cluster_for_test( + Vec::new(), + &[QuickwitService::Indexer.as_str()], + &ChannelTransport::default(), + true, + ) + .await + .unwrap(); + let (_temp_dir, state) = IngesterState::for_test(cluster).await; let (syn_replication_stream_tx, syn_replication_stream) = ServiceStream::new_bounded(SYN_REPLICATION_STREAM_CAPACITY); let (ack_replication_stream_tx, mut ack_replication_stream) = @@ -1573,15 +1614,15 @@ mod tests { memory_capacity, ); - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); - let queue_id_01 = queue_id(&index_uid, "test-source", &ShardId::from(1)); + let index_uid = IndexUid::for_test("test-index", 0); let replica_shard = IngesterShard::new_replica( + index_uid.clone(), + "test-source".to_string(), + ShardId::from(1), leader_id, - ShardState::Open, - Position::Beginning, - Position::Beginning, - Instant::now(), - ); + ) + .build(); + let queue_id_01 = replica_shard.queue_id(); state .lock_fully() .await diff --git a/quickwit/quickwit-ingest/src/ingest_v2/router.rs b/quickwit/quickwit-ingest/src/ingest_v2/router.rs index e48cd647c04..d27c5ebfb77 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/router.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/router.rs @@ -27,32 +27,30 @@ use quickwit_proto::control_plane::{ ControlPlaneService, ControlPlaneServiceClient, GetOrCreateOpenShardsRequest, GetOrCreateOpenShardsSubrequest, }; -use quickwit_proto::indexing::ShardPositionsUpdate; use quickwit_proto::ingest::ingester::{ IngesterService, PersistFailureReason, PersistRequest, PersistResponse, PersistSubrequest, }; use quickwit_proto::ingest::router::{ IngestFailureReason, IngestRequestV2, IngestResponseV2, IngestRouterService, }; -use quickwit_proto::ingest::{ - CommitTypeV2, IngestV2Error, IngestV2Result, RateLimitingCause, ShardState, -}; -use quickwit_proto::types::{IndexUid, NodeId, ShardId, SourceId, SubrequestId}; +use quickwit_proto::ingest::{CommitTypeV2, IngestV2Error, IngestV2Result, RateLimitingCause}; +use quickwit_proto::types::{NodeId, SubrequestId}; use serde_json::{Value as JsonValue, json}; use tokio::sync::{Mutex, Semaphore}; use tokio::time::error::Elapsed; use tracing::{error, info}; -use super::broadcast::LocalShardsUpdate; +use super::broadcast::IngesterCapacityScoreUpdate; use super::debouncing::{ DebouncedGetOrCreateOpenShardsRequest, GetOrCreateOpenShardsRequestDebouncer, }; use super::ingester::PERSIST_REQUEST_TIMEOUT; use super::metrics::IngestResultMetrics; -use super::routing_table::{NextOpenShardError, RoutingTable}; +use super::routing_table::RoutingTable; use super::workbench::IngestWorkbench; use super::{IngesterPool, pending_subrequests}; -use crate::{LeaderId, get_ingest_router_buffer_size}; +use crate::get_ingest_router_buffer_size; +use crate::ingest_v2::metrics::INGEST_V2_METRICS; /// Duration after which ingest requests time out with [`IngestV2Error::Timeout`]. fn ingest_request_timeout() -> Duration { @@ -66,6 +64,7 @@ fn ingest_request_timeout() -> Duration { let duration_ms = quickwit_common::get_from_env( "QW_INGEST_REQUEST_TIMEOUT_MS", DEFAULT_INGEST_REQUEST_TIMEOUT.as_millis() as u64, + false, ); let minimum_ingest_request_timeout: Duration = PERSIST_REQUEST_TIMEOUT * (MAX_PERSIST_ATTEMPTS as u32) + Duration::from_secs(5); @@ -102,7 +101,7 @@ pub struct IngestRouter { struct RouterState { // Debounces `GetOrCreateOpenShardsRequest` requests to the control plane. debouncer: GetOrCreateOpenShardsRequestDebouncer, - // Holds the routing table mapping index and source IDs to shards. + // Routing table of nodes, their WAL capacity, and the number of open shards per source. routing_table: RoutingTable, } @@ -122,13 +121,11 @@ impl IngestRouter { ingester_pool: IngesterPool, replication_factor: usize, event_broker: EventBroker, + self_availability_zone: Option, ) -> Self { let state = Arc::new(Mutex::new(RouterState { debouncer: GetOrCreateOpenShardsRequestDebouncer::default(), - routing_table: RoutingTable { - self_node_id: self_node_id.clone(), - table: HashMap::default(), - }, + routing_table: RoutingTable::new(self_availability_zone), })); let ingest_semaphore_permits = get_ingest_router_buffer_size().as_u64() as usize; let ingest_semaphore = Arc::new(Semaphore::new(ingest_semaphore_permits)); @@ -147,10 +144,7 @@ impl IngestRouter { pub fn subscribe(&self) { let weak_router_state = WeakRouterState(Arc::downgrade(&self.state)); self.event_broker - .subscribe::(weak_router_state.clone()) - .forever(); - self.event_broker - .subscribe::(weak_router_state) + .subscribe::(weak_router_state) .forever(); } @@ -162,22 +156,19 @@ impl IngestRouter { ingester_pool: &IngesterPool, ) -> DebouncedGetOrCreateOpenShardsRequest { let mut debounced_request = DebouncedGetOrCreateOpenShardsRequest::default(); - - // `closed_shards` and `unavailable_leaders` are populated by calls to `has_open_shards` - // as we're looking for open shards to route the subrequests to. - let unavailable_leaders: &mut HashSet = &mut workbench.unavailable_leaders; + let unavailable_leaders: &HashSet = &workbench.unavailable_leaders; let mut state_guard = self.state.lock().await; for subrequest in pending_subrequests(&workbench.subworkbenches) { - if !state_guard.routing_table.has_open_shards( + if !state_guard.routing_table.has_open_nodes( &subrequest.index_id, &subrequest.source_id, ingester_pool, - &mut debounced_request.closed_shards, unavailable_leaders, ) { - // No shard available! Let's attempt to create one. + // No known nodes with open shards for this source. Ask the control + // plane to create shards so we have somewhere to route to. let acquire_result = state_guard .debouncer .acquire(&subrequest.index_id, &subrequest.source_id); @@ -199,8 +190,11 @@ impl IngestRouter { } drop(state_guard); - if !debounced_request.is_empty() && !debounced_request.closed_shards.is_empty() { - info!(closed_shards=?debounced_request.closed_shards, "reporting closed shard(s) to control plane"); + if !debounced_request.is_empty() && !workbench.closed_shards.is_empty() { + info!(closed_shards=?workbench.closed_shards, "reporting closed shard(s) to control plane"); + debounced_request + .closed_shards + .append(&mut workbench.closed_shards); } if !debounced_request.is_empty() && !unavailable_leaders.is_empty() { info!(unavailable_leaders=?unavailable_leaders, "reporting unavailable leader(s) to control plane"); @@ -258,7 +252,7 @@ impl IngestRouter { let mut state_guard = self.state.lock().await; for success in response.successes { - state_guard.routing_table.replace_shards( + state_guard.routing_table.merge_from_shards( success.index_uid().clone(), success.source_id, success.open_shards, @@ -276,12 +270,13 @@ impl IngestRouter { workbench: &mut IngestWorkbench, mut persist_futures: FuturesUnordered>, ) { - let mut closed_shards: HashMap<(IndexUid, SourceId), Vec> = HashMap::new(); - let mut deleted_shards: HashMap<(IndexUid, SourceId), Vec> = HashMap::new(); + let mut unavailable_leaders: HashSet = HashSet::new(); while let Some((persist_summary, persist_result)) = persist_futures.next().await { match persist_result { Ok(persist_response) => { + let leader_id = NodeId::from(persist_response.leader_id.clone()); + for persist_success in persist_response.successes { workbench.record_persist_success(persist_success); } @@ -289,37 +284,37 @@ impl IngestRouter { workbench.record_persist_failure(&persist_failure); match persist_failure.reason() { - PersistFailureReason::ShardClosed => { - let shard_id = persist_failure.shard_id().clone(); - let index_uid: IndexUid = persist_failure.index_uid().clone(); - let source_id: SourceId = persist_failure.source_id; - closed_shards - .entry((index_uid, source_id)) - .or_default() - .push(shard_id); + PersistFailureReason::NoShardsAvailable => { + // For non-critical failures, we don't mark the nodes unavailable; + // a routing update is piggybacked on PersistResponses, so shard + // counts and capacity scores will be fresh on the next try. } - PersistFailureReason::ShardNotFound => { - let shard_id = persist_failure.shard_id().clone(); - let index_uid: IndexUid = persist_failure.index_uid().clone(); - let source_id: SourceId = persist_failure.source_id; - deleted_shards - .entry((index_uid, source_id)) - .or_default() - .push(shard_id); - } - PersistFailureReason::WalFull - | PersistFailureReason::ShardRateLimited => { - // Let's record that the shard is rate limited or that the ingester - // that hosts has its wal full. - // - // That way we will avoid to retry the persist request on the very - // same node. - let shard_id = persist_failure.shard_id().clone(); - workbench.rate_limited_shards.insert(shard_id); + PersistFailureReason::NodeUnavailable + | PersistFailureReason::WalFull + | PersistFailureReason::Timeout => { + unavailable_leaders.insert(leader_id.clone()); } _ => {} } } + + if let Some(routing_update) = persist_response.routing_update { + // Since we just talked to the node, we take advantage and use the + // opportunity to get a fresh routing update. + let mut state_guard = self.state.lock().await; + for shard_update in routing_update.source_shard_updates { + state_guard.routing_table.apply_capacity_update( + leader_id.clone(), + shard_update.index_uid().clone(), + shard_update.source_id, + routing_update.capacity_score as usize, + shard_update.open_shard_count as usize, + ); + } + drop(state_guard); + + workbench.closed_shards.extend(routing_update.closed_shards); + } } Err(persist_error) => { if workbench.is_last_attempt() { @@ -339,20 +334,7 @@ impl IngestRouter { } }; } - if !closed_shards.is_empty() || !deleted_shards.is_empty() { - let mut state_guard = self.state.lock().await; - - for ((index_uid, source_id), shard_ids) in closed_shards { - state_guard - .routing_table - .close_shards(&index_uid, source_id, &shard_ids); - } - for ((index_uid, source_id), shard_ids) in deleted_shards { - state_guard - .routing_table - .delete_shards(&index_uid, source_id, &shard_ids); - } - } + workbench.unavailable_leaders.extend(unavailable_leaders); } async fn batch_persist(&self, workbench: &mut IngestWorkbench, commit_type: CommitTypeV2) { @@ -364,44 +346,43 @@ impl IngestRouter { self.populate_routing_table_debounced(workbench, debounced_request) .await; - // Subrequests for which no shards are available to route the subrequests to. + let unavailable_leaders = &workbench.unavailable_leaders; let mut no_shards_available_subrequest_ids: Vec = Vec::new(); - // Subrequests for which the shards are rate limited. - let mut rate_limited_subrequest_ids: Vec = Vec::new(); - - let mut per_leader_persist_subrequests: HashMap<&LeaderId, Vec> = + let mut per_leader_persist_subrequests: HashMap<&NodeId, Vec> = HashMap::new(); - let rate_limited_shards: &HashSet = &workbench.rate_limited_shards; let state_guard = self.state.lock().await; for subrequest in pending_subrequests(&workbench.subworkbenches) { - let next_open_shard_res_opt = state_guard - .routing_table - .find_entry(&subrequest.index_id, &subrequest.source_id) - .map(|entry| { - entry.next_open_shard_round_robin(&self.ingester_pool, rate_limited_shards) - }); - let next_open_shard = match next_open_shard_res_opt { - Some(Ok(next_open_shard)) => next_open_shard, - Some(Err(NextOpenShardError::RateLimited)) => { - rate_limited_subrequest_ids.push(subrequest.subrequest_id); - continue; - } - Some(Err(NextOpenShardError::NoShardsAvailable)) | None => { + let ingester_node = state_guard.routing_table.pick_node( + &subrequest.index_id, + &subrequest.source_id, + &self.ingester_pool, + unavailable_leaders, + ); + + let ingester_node = match ingester_node { + Some(node) => node, + None => { no_shards_available_subrequest_ids.push(subrequest.subrequest_id); continue; } }; + let az_locality = state_guard + .routing_table + .classify_az_locality(&ingester_node.node_id, &self.ingester_pool); + INGEST_V2_METRICS + .ingest_attempts + .with_label_values([az_locality]) + .inc(); let persist_subrequest = PersistSubrequest { subrequest_id: subrequest.subrequest_id, - index_uid: next_open_shard.index_uid.clone().into(), - source_id: next_open_shard.source_id.clone(), - shard_id: Some(next_open_shard.shard_id.clone()), + index_uid: Some(ingester_node.index_uid.clone()), + source_id: subrequest.source_id.clone(), doc_batch: subrequest.doc_batch.clone(), }; per_leader_persist_subrequests - .entry(&next_open_shard.leader_id) + .entry(&ingester_node.node_id) .or_default() .push(persist_subrequest); } @@ -413,7 +394,7 @@ impl IngestRouter { .iter() .map(|subrequest| subrequest.subrequest_id) .collect(); - let Some(ingester) = self.ingester_pool.get(&leader_id) else { + let Some(ingester) = self.ingester_pool.get(&leader_id).map(|h| h.client) else { no_shards_available_subrequest_ids.extend(subrequest_ids); continue; }; @@ -426,7 +407,6 @@ impl IngestRouter { subrequests, commit_type: commit_type as i32, }; - workbench.record_persist_request(&persist_request); let persist_future = async move { let persist_result = tokio::time::timeout( @@ -450,9 +430,6 @@ impl IngestRouter { for subrequest_id in no_shards_available_subrequest_ids { workbench.record_no_shards_available(subrequest_id); } - for subrequest_id in rate_limited_subrequest_ids { - workbench.record_rate_limited(subrequest_id); - } self.process_persist_results(workbench, persist_futures) .await; } @@ -505,7 +482,7 @@ impl IngestRouter { pub async fn debug_info(&self) -> JsonValue { let state_guard = self.state.lock().await; - let routing_table_json = state_guard.routing_table.debug_info(); + let routing_table_json = state_guard.routing_table.debug_info(&self.ingester_pool); json!({ "routing_table": routing_table_json, @@ -515,8 +492,7 @@ impl IngestRouter { fn update_ingest_metrics(ingest_result: &IngestV2Result, num_subrequests: usize) { let num_subrequests = num_subrequests as u64; - let ingest_results_metrics: &IngestResultMetrics = - &crate::ingest_v2::metrics::INGEST_V2_METRICS.ingest_results; + let ingest_results_metrics: &IngestResultMetrics = &INGEST_V2_METRICS.ingest_results; match ingest_result { Ok(ingest_response) => { ingest_results_metrics @@ -630,67 +606,22 @@ impl IngestRouterService for IngestRouter { struct WeakRouterState(Weak>); #[async_trait] -impl EventSubscriber for WeakRouterState { - async fn handle_event(&mut self, local_shards_update: LocalShardsUpdate) { +impl EventSubscriber for WeakRouterState { + async fn handle_event(&mut self, update: IngesterCapacityScoreUpdate) { let Some(state) = self.0.upgrade() else { return; }; - let leader_id = local_shards_update.leader_id; - let index_uid = local_shards_update.source_uid.index_uid; - let source_id = local_shards_update.source_uid.source_id; - - let mut open_shard_ids: Vec = Vec::new(); - let mut closed_shard_ids: Vec = Vec::new(); - - for shard_info in local_shards_update.shard_infos { - match shard_info.shard_state { - ShardState::Open => open_shard_ids.push(shard_info.shard_id), - ShardState::Closed => closed_shard_ids.push(shard_info.shard_id), - ShardState::Unavailable | ShardState::Unspecified => { - // Ingesters never broadcast the `Unavailable`` state because, from their point - // of view, they are never unavailable. - } - } - } let mut state_guard = state.lock().await; - - state_guard - .routing_table - .close_shards(&index_uid, &source_id, &closed_shard_ids); - - state_guard.routing_table.insert_open_shards( - &leader_id, - index_uid, - source_id, - &open_shard_ids, + state_guard.routing_table.apply_capacity_update( + update.node_id, + update.source_uid.index_uid, + update.source_uid.source_id, + update.capacity_score, + update.open_shard_count, ); } } -#[async_trait] -impl EventSubscriber for WeakRouterState { - async fn handle_event(&mut self, shard_positions_update: ShardPositionsUpdate) { - let Some(state) = self.0.upgrade() else { - return; - }; - let mut deleted_shard_ids: Vec = Vec::new(); - - for (shard_id, shard_position) in shard_positions_update.updated_shard_positions { - if shard_position.is_eof() { - deleted_shard_ids.push(shard_id); - } - } - let mut state_guard = state.lock().await; - - let index_uid = shard_positions_update.source_uid.index_uid; - let source_id = shard_positions_update.source_uid.source_id; - - state_guard - .routing_table - .delete_shards(&index_uid, &source_id, &deleted_shard_ids); - } -} - pub(super) struct PersistRequestSummary { pub leader_id: NodeId, pub subrequest_ids: Vec, @@ -698,27 +629,22 @@ pub(super) struct PersistRequestSummary { #[cfg(test)] mod tests { - use std::collections::BTreeSet; - - use mockall::Sequence; use quickwit_proto::control_plane::{ GetOrCreateOpenShardsFailure, GetOrCreateOpenShardsFailureReason, GetOrCreateOpenShardsResponse, GetOrCreateOpenShardsSuccess, MockControlPlaneService, }; use quickwit_proto::ingest::ingester::{ - IngesterServiceClient, MockIngesterService, PersistFailure, PersistResponse, PersistSuccess, + IngesterServiceClient, IngesterStatus, MockIngesterService, PersistFailure, + PersistResponse, PersistSuccess, RoutingUpdate, SourceShardUpdate, }; use quickwit_proto::ingest::router::IngestSubrequest; use quickwit_proto::ingest::{ - CommitTypeV2, DocBatchV2, ParseFailure, ParseFailureReason, Shard, ShardIds, ShardState, + CommitTypeV2, DocBatchV2, ParseFailure, ParseFailureReason, Shard, ShardState, }; - use quickwit_proto::types::{DocUid, Position, SourceUid}; - use tokio::task::yield_now; + use quickwit_proto::types::{DocUid, IndexUid, Position, ShardId, SourceUid}; use super::*; - use crate::RateMibPerSec; - use crate::ingest_v2::broadcast::ShardInfo; - use crate::ingest_v2::routing_table::{RoutingEntry, RoutingTableEntry}; + use crate::IngesterPoolEntry; use crate::ingest_v2::workbench::SubworkbenchFailure; #[tokio::test] @@ -734,6 +660,7 @@ mod tests { ingester_pool.clone(), replication_factor, EventBroker::default(), + Some("test-az".to_string()), ); let mut workbench = IngestWorkbench::default(); let (get_or_create_open_shard_request_opt, rendezvous) = router @@ -743,34 +670,21 @@ mod tests { assert!(get_or_create_open_shard_request_opt.is_none()); assert!(rendezvous.is_empty()); - let mut state_guard = router.state.lock().await; - - let index_uid: IndexUid = IndexUid::for_test("test-index-0", 0); - state_guard.routing_table.table.insert( - ("test-index-0".into(), "test-source".into()), - RoutingTableEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - local_shards: vec![ - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(1), - shard_state: ShardState::Closed, - leader_id: "test-ingester-0".into(), - }, - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(2), - shard_state: ShardState::Open, - leader_id: "test-ingester-0".into(), - }, - ], - ..Default::default() - }, - ); - drop(state_guard); + { + let mut state_guard = router.state.lock().await; + state_guard.routing_table.merge_from_shards( + IndexUid::for_test("test-index-0", 0), + "test-source".to_string(), + vec![Shard { + index_uid: Some(IndexUid::for_test("test-index-0", 0)), + source_id: "test-source".to_string(), + shard_id: Some(ShardId::from(1u64)), + shard_state: ShardState::Open as i32, + leader_id: "test-ingester-0".to_string(), + ..Default::default() + }], + ); + } let ingest_subrequests: Vec = vec![ IngestSubrequest { @@ -806,24 +720,12 @@ mod tests { assert_eq!(subrequest.index_id, "test-index-1"); assert_eq!(subrequest.source_id, "test-source"); - assert_eq!(get_or_create_open_shard_request.closed_shards.len(), 1); - assert_eq!( - get_or_create_open_shard_request.closed_shards[0], - ShardIds { - index_uid: Some(IndexUid::for_test("test-index-0", 0)), - source_id: "test-source".to_string(), - shard_ids: vec![ShardId::from(1)], - } - ); - assert_eq!( - get_or_create_open_shard_request.unavailable_leaders.len(), - 1 - ); - assert_eq!( - get_or_create_open_shard_request.unavailable_leaders[0], - "test-ingester-0" + assert!( + get_or_create_open_shard_request + .unavailable_leaders + .is_empty() ); - assert_eq!(workbench.unavailable_leaders.len(), 1); + assert!(workbench.unavailable_leaders.is_empty()); let (get_or_create_open_shard_request_opt, rendezvous_2) = router .make_get_or_create_open_shard_request(&mut workbench, &ingester_pool) @@ -838,29 +740,31 @@ mod tests { drop(rendezvous_1); drop(rendezvous_2); - ingester_pool.insert("test-ingester-0".into(), IngesterServiceClient::mocked()); + ingester_pool.insert( + "test-ingester-0".into(), + IngesterPoolEntry::mocked_ingester(), + ); { - // Ingester-0 has been marked as unavailable due to the previous requests. + // Ingester-0 is in pool and in table, but marked unavailable on the workbench + // (simulating a prior transport error). has_open_nodes returns false → both + // subrequests trigger CP request. + workbench + .unavailable_leaders + .insert("test-ingester-0".into()); let (get_or_create_open_shard_request_opt, _rendezvous) = router .make_get_or_create_open_shard_request(&mut workbench, &ingester_pool) .await .take(); let get_or_create_open_shard_request = get_or_create_open_shard_request_opt.unwrap(); assert_eq!(get_or_create_open_shard_request.subrequests.len(), 2); - assert_eq!(workbench.unavailable_leaders.len(), 1); assert_eq!( - workbench - .unavailable_leaders - .iter() - .next() - .unwrap() - .to_string(), - "test-ingester-0" + get_or_create_open_shard_request.unavailable_leaders.len(), + 1 ); } { - // With a fresh workbench, the ingester is not marked as unavailable, and present in the - // pool. + // Fresh workbench: ingester-0 is in pool, in table, and NOT unavailable. + // has_open_nodes returns true for index-0 → only index-1 triggers request. let mut workbench = IngestWorkbench::new(ingest_subrequests, 3); let (get_or_create_open_shard_request_opt, _rendezvous) = router .make_get_or_create_open_shard_request(&mut workbench, &ingester_pool) @@ -873,9 +777,10 @@ mod tests { assert_eq!(subrequest.index_id, "test-index-1"); assert_eq!(subrequest.source_id, "test-source"); - assert_eq!( - get_or_create_open_shard_request.unavailable_leaders.len(), - 0 + assert!( + get_or_create_open_shard_request + .unavailable_leaders + .is_empty() ); } } @@ -920,6 +825,7 @@ mod tests { source_id: "test-source".to_string(), shard_id: Some(ShardId::from(1)), shard_state: ShardState::Open as i32, + leader_id: "test-ingester-0".to_string(), ..Default::default() }], }, @@ -933,6 +839,7 @@ mod tests { source_id: "test-source".to_string(), shard_id: Some(ShardId::from(1)), shard_state: ShardState::Open as i32, + leader_id: "test-ingester-1".to_string(), ..Default::default() }, Shard { @@ -940,6 +847,7 @@ mod tests { source_id: "test-source".to_string(), shard_id: Some(ShardId::from(2)), shard_state: ShardState::Open as i32, + leader_id: "test-ingester-1".to_string(), ..Default::default() }, ], @@ -971,6 +879,7 @@ mod tests { ingester_pool.clone(), replication_factor, EventBroker::default(), + Some("test-az".to_string()), ); let ingest_subrequests = vec![ IngestSubrequest { @@ -1030,23 +939,6 @@ mod tests { .populate_routing_table(&mut workbench, get_or_create_open_shards_request) .await; - let state_guard = router.state.lock().await; - let routing_table = &state_guard.routing_table; - assert_eq!(routing_table.len(), 2); - - let routing_entry_0 = routing_table - .find_entry("test-index-0", "test-source") - .unwrap(); - assert_eq!(routing_entry_0.len(), 1); - assert_eq!(routing_entry_0.all_shards()[0].shard_id, ShardId::from(1)); - - let routing_entry_1 = routing_table - .find_entry("test-index-1", "test-source") - .unwrap(); - assert_eq!(routing_entry_1.len(), 2); - assert_eq!(routing_entry_1.all_shards()[0].shard_id, ShardId::from(1)); - assert_eq!(routing_entry_1.all_shards()[1].shard_id, ShardId::from(2)); - let subworkbench = workbench.subworkbenches.get(&2).unwrap(); assert!(matches!( subworkbench.last_failure_opt, @@ -1086,6 +978,7 @@ mod tests { ingester_pool.clone(), replication_factor, EventBroker::default(), + Some("test-az".to_string()), ); let ingest_subrequests = vec![IngestSubrequest { subrequest_id: 0, @@ -1145,6 +1038,7 @@ mod tests { ingester_pool.clone(), replication_factor, EventBroker::default(), + Some("test-az".to_string()), ); let ingest_subrequests = vec![IngestSubrequest { subrequest_id: 0, @@ -1175,6 +1069,7 @@ mod tests { ingester_pool.clone(), replication_factor, EventBroker::default(), + Some("test-az".to_string()), ); let ingest_subrequests = vec![IngestSubrequest { subrequest_id: 0, @@ -1201,6 +1096,11 @@ mod tests { ..Default::default() }], failures: Vec::new(), + routing_update: Some(RoutingUpdate { + capacity_score: 6, + source_shard_updates: Vec::new(), + ..Default::default() + }), }); (persist_summary, persist_result) }); @@ -1227,6 +1127,7 @@ mod tests { ingester_pool.clone(), replication_factor, EventBroker::default(), + Some("test-az".to_string()), ); let ingest_subrequests = vec![IngestSubrequest { subrequest_id: 0, @@ -1250,9 +1151,13 @@ mod tests { subrequest_id: 0, index_uid: Some(index_uid.clone()), source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), - reason: PersistFailureReason::ShardRateLimited as i32, + reason: PersistFailureReason::NoShardsAvailable as i32, }], + routing_update: Some(RoutingUpdate { + capacity_score: 6, + source_shard_updates: Vec::new(), + ..Default::default() + }), }); (persist_summary, persist_result) }); @@ -1267,97 +1172,20 @@ mod tests { )); } - #[tokio::test] - async fn test_router_process_persist_results_closes_and_deletes_shards() { - let self_node_id = "test-router".into(); - let control_plane = ControlPlaneServiceClient::from_mock(MockControlPlaneService::new()); - let ingester_pool = IngesterPool::default(); - let replication_factor = 1; - let router = IngestRouter::new( - self_node_id, - control_plane, - ingester_pool.clone(), - replication_factor, - EventBroker::default(), - ); - let index_uid: IndexUid = IndexUid::for_test("test-index-0", 0); - let mut state_guard = router.state.lock().await; - state_guard.routing_table.replace_shards( - index_uid.clone(), - "test-source", - vec![ - Shard { - index_uid: Some(index_uid.clone()), - shard_id: Some(ShardId::from(1)), - shard_state: ShardState::Open as i32, - leader_id: "test-ingester-0".to_string(), - ..Default::default() - }, - Shard { - index_uid: Some(index_uid.clone()), - shard_id: Some(ShardId::from(2)), - shard_state: ShardState::Open as i32, - leader_id: "test-ingester-0".to_string(), - ..Default::default() - }, - ], - ); - drop(state_guard); - - let mut workbench = IngestWorkbench::new(Vec::new(), 2); - let persist_futures = FuturesUnordered::new(); - - persist_futures.push(async { - let persist_summary = PersistRequestSummary { - leader_id: "test-ingester-0".into(), - subrequest_ids: vec![0], - }; - let persist_result = Ok::<_, IngestV2Error>(PersistResponse { - leader_id: "test-ingester-0".to_string(), - successes: Vec::new(), - failures: vec![ - PersistFailure { - subrequest_id: 0, - index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), - reason: PersistFailureReason::ShardNotFound as i32, - }, - PersistFailure { - subrequest_id: 1, - index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(2)), - reason: PersistFailureReason::ShardClosed as i32, - }, - ], - }); - (persist_summary, persist_result) - }); - router - .process_persist_results(&mut workbench, persist_futures) - .await; - - let state_guard = router.state.lock().await; - let routing_table_entry = state_guard - .routing_table - .find_entry("test-index-0", "test-source") - .unwrap(); - assert_eq!(routing_table_entry.len(), 1); - - let shard = routing_table_entry.all_shards()[0]; - assert_eq!(shard.shard_id, ShardId::from(2)); - assert_eq!(shard.shard_state, ShardState::Closed); - } - #[tokio::test] async fn test_router_process_persist_results_does_not_remove_unavailable_leaders() { let self_node_id = "test-router".into(); let control_plane = ControlPlaneServiceClient::from_mock(MockControlPlaneService::new()); let ingester_pool = IngesterPool::default(); - ingester_pool.insert("test-ingester-0".into(), IngesterServiceClient::mocked()); - ingester_pool.insert("test-ingester-1".into(), IngesterServiceClient::mocked()); + ingester_pool.insert( + "test-ingester-0".into(), + IngesterPoolEntry::mocked_ingester(), + ); + ingester_pool.insert( + "test-ingester-1".into(), + IngesterPoolEntry::mocked_ingester(), + ); let replication_factor = 1; let router = IngestRouter::new( @@ -1366,6 +1194,7 @@ mod tests { ingester_pool.clone(), replication_factor, EventBroker::default(), + Some("test-az".to_string()), ); let ingest_subrequests = vec![ IngestSubrequest { @@ -1443,151 +1272,85 @@ mod tests { let self_node_id = "test-router".into(); let control_plane = ControlPlaneServiceClient::from_mock(MockControlPlaneService::new()); let ingester_pool = IngesterPool::default(); - let replication_factor = 1; let router = IngestRouter::new( self_node_id, control_plane, ingester_pool.clone(), - replication_factor, + 1, EventBroker::default(), + Some("test-az".to_string()), ); - let index_uid: IndexUid = IndexUid::for_test("test-index-0", 0); - let index_uid2: IndexUid = IndexUid::for_test("test-index-1", 0); - let mut state_guard = router.state.lock().await; - state_guard.routing_table.replace_shards( - index_uid.clone(), - "test-source", - vec![Shard { - index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), - shard_state: ShardState::Open as i32, - leader_id: "test-ingester-0".to_string(), - ..Default::default() - }], - ); - state_guard.routing_table.replace_shards( - index_uid2.clone(), - "test-source", - vec![ - Shard { - index_uid: Some(index_uid2.clone()), + + let index_uid_0: IndexUid = IndexUid::for_test("test-index-0", 0); + let index_uid_1: IndexUid = IndexUid::for_test("test-index-1", 0); + { + let mut state_guard = router.state.lock().await; + state_guard.routing_table.merge_from_shards( + index_uid_0.clone(), + "test-source".to_string(), + vec![Shard { + index_uid: Some(index_uid_0.clone()), source_id: "test-source".to_string(), shard_id: Some(ShardId::from(1)), shard_state: ShardState::Open as i32, leader_id: "test-ingester-0".to_string(), - follower_id: Some("test-ingester-1".to_string()), ..Default::default() - }, - Shard { - index_uid: Some(index_uid2.clone()), + }], + ); + state_guard.routing_table.merge_from_shards( + index_uid_1.clone(), + "test-source".to_string(), + vec![Shard { + index_uid: Some(index_uid_1.clone()), source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(2)), + shard_id: Some(ShardId::from(1)), shard_state: ShardState::Open as i32, leader_id: "test-ingester-1".to_string(), - follower_id: Some("test-ingester-2".to_string()), ..Default::default() - }, - ], - ); - drop(state_guard); + }], + ); + } + let index_uid_0_clone = index_uid_0.clone(); let mut mock_ingester_0 = MockIngesterService::new(); - let index_uid_clone = index_uid.clone(); - let index_uid2_clone = index_uid2.clone(); - mock_ingester_0 - .expect_persist() - .once() - .returning(move |request| { - assert_eq!(request.leader_id, "test-ingester-0"); - assert_eq!(request.subrequests.len(), 2); - assert_eq!(request.commit_type(), CommitTypeV2::Auto); - - let subrequest = &request.subrequests[0]; - assert_eq!(subrequest.subrequest_id, 0); - assert_eq!(subrequest.index_uid(), &index_uid_clone); - assert_eq!(subrequest.source_id, "test-source"); - assert_eq!(subrequest.shard_id(), ShardId::from(1)); - assert_eq!( - subrequest.doc_batch, - Some(DocBatchV2::for_test(["", "test-doc-foo", "test-doc-bar"])) - ); - - let subrequest = &request.subrequests[1]; - assert_eq!(subrequest.subrequest_id, 1); - assert_eq!(subrequest.index_uid(), &index_uid2_clone); - assert_eq!(subrequest.source_id, "test-source"); - assert_eq!(subrequest.shard_id(), ShardId::from(1)); - assert_eq!( - subrequest.doc_batch, - Some(DocBatchV2::for_test(["test-doc-qux"])) - ); - - let response = PersistResponse { - leader_id: request.leader_id, - successes: vec![ - PersistSuccess { - subrequest_id: 0, - index_uid: Some(index_uid_clone.clone()), - source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), - replication_position_inclusive: Some(Position::offset(1u64)), - num_persisted_docs: 2, - parse_failures: vec![ParseFailure { - doc_uid: Some(DocUid::for_test(0)), - reason: ParseFailureReason::InvalidJson as i32, - message: "invalid JSON".to_string(), - }], - }, - PersistSuccess { - subrequest_id: 1, - index_uid: Some(index_uid2_clone.clone()), - source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), - replication_position_inclusive: Some(Position::offset(0u64)), - num_persisted_docs: 1, - parse_failures: Vec::new(), - }, - ], - failures: Vec::new(), - }; - Ok(response) - }); mock_ingester_0 .expect_persist() .once() .returning(move |request| { assert_eq!(request.leader_id, "test-ingester-0"); assert_eq!(request.subrequests.len(), 1); - assert_eq!(request.commit_type(), CommitTypeV2::Auto); - - let subrequest = &request.subrequests[0]; - assert_eq!(subrequest.subrequest_id, 0); - assert_eq!(subrequest.index_uid(), &index_uid); - assert_eq!(subrequest.source_id, "test-source"); - assert_eq!(subrequest.shard_id(), ShardId::from(1)); - assert_eq!( - subrequest.doc_batch, - Some(DocBatchV2::for_test(["test-doc-moo", "test-doc-baz"])) - ); - let response = PersistResponse { + Ok(PersistResponse { leader_id: request.leader_id, successes: vec![PersistSuccess { subrequest_id: 0, - index_uid: Some(index_uid.clone()), + index_uid: Some(index_uid_0_clone.clone()), source_id: "test-source".to_string(), shard_id: Some(ShardId::from(1)), - replication_position_inclusive: Some(Position::offset(3u64)), - num_persisted_docs: 4, - parse_failures: Vec::new(), + replication_position_inclusive: Some(Position::offset(1u64)), + num_persisted_docs: 2, + parse_failures: vec![ParseFailure { + doc_uid: Some(DocUid::for_test(0)), + reason: ParseFailureReason::InvalidJson as i32, + message: "invalid JSON".to_string(), + }], }], failures: Vec::new(), - }; - Ok(response) + routing_update: Some(RoutingUpdate { + capacity_score: 6, + source_shard_updates: Vec::new(), + ..Default::default() + }), + }) }); - let ingester_0 = IngesterServiceClient::from_mock(mock_ingester_0); - ingester_pool.insert("test-ingester-0".into(), ingester_0.clone()); + ingester_pool.insert( + "test-ingester-0".into(), + IngesterPoolEntry { + client: IngesterServiceClient::from_mock(mock_ingester_0), + status: IngesterStatus::Ready, + availability_zone: None, + }, + ); let mut mock_ingester_1 = MockIngesterService::new(); mock_ingester_1 @@ -1596,167 +1359,129 @@ mod tests { .returning(move |request| { assert_eq!(request.leader_id, "test-ingester-1"); assert_eq!(request.subrequests.len(), 1); - assert_eq!(request.commit_type(), CommitTypeV2::Auto); - - let subrequest = &request.subrequests[0]; - assert_eq!(subrequest.subrequest_id, 1); - assert_eq!(subrequest.index_uid(), &index_uid2); - assert_eq!(subrequest.source_id, "test-source"); - assert_eq!(subrequest.shard_id(), ShardId::from(2)); - assert_eq!( - subrequest.doc_batch, - Some(DocBatchV2::for_test(["test-doc-tux"])) - ); - let response = PersistResponse { + Ok(PersistResponse { leader_id: request.leader_id, successes: vec![PersistSuccess { subrequest_id: 1, - index_uid: Some(index_uid2.clone()), + index_uid: Some(index_uid_1.clone()), source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(2)), + shard_id: Some(ShardId::from(1)), replication_position_inclusive: Some(Position::offset(0u64)), num_persisted_docs: 1, parse_failures: Vec::new(), }], failures: Vec::new(), - }; - Ok(response) + routing_update: Some(RoutingUpdate { + capacity_score: 6, + source_shard_updates: Vec::new(), + ..Default::default() + }), + }) }); - let ingester_1 = IngesterServiceClient::from_mock(mock_ingester_1); - ingester_pool.insert("test-ingester-1".into(), ingester_1); + ingester_pool.insert( + "test-ingester-1".into(), + IngesterPoolEntry { + client: IngesterServiceClient::from_mock(mock_ingester_1), + availability_zone: None, + status: IngesterStatus::Ready, + }, + ); + + let response = router + .ingest(IngestRequestV2 { + subrequests: vec![ + IngestSubrequest { + subrequest_id: 0, + index_id: "test-index-0".to_string(), + source_id: "test-source".to_string(), + doc_batch: Some(DocBatchV2::for_test(["", "test-doc-foo", "test-doc-bar"])), + }, + IngestSubrequest { + subrequest_id: 1, + index_id: "test-index-1".to_string(), + source_id: "test-source".to_string(), + doc_batch: Some(DocBatchV2::for_test(["test-doc-qux"])), + }, + ], + commit_type: CommitTypeV2::Auto as i32, + }) + .await + .unwrap(); - let ingest_request = IngestRequestV2 { - subrequests: vec![ - IngestSubrequest { - subrequest_id: 0, - index_id: "test-index-0".to_string(), - source_id: "test-source".to_string(), - doc_batch: Some(DocBatchV2::for_test(["", "test-doc-foo", "test-doc-bar"])), - }, - IngestSubrequest { - subrequest_id: 1, - index_id: "test-index-1".to_string(), - source_id: "test-source".to_string(), - doc_batch: Some(DocBatchV2::for_test(["test-doc-qux"])), - }, - ], - commit_type: CommitTypeV2::Auto as i32, - }; - let response = router.ingest(ingest_request).await.unwrap(); assert_eq!(response.successes.len(), 2); assert_eq!(response.failures.len(), 0); let parse_failures = &response.successes[0].parse_failures; assert_eq!(parse_failures.len(), 1); - - let parse_failure = &parse_failures[0]; - assert_eq!(parse_failure.doc_uid(), DocUid::for_test(0)); - assert_eq!(parse_failure.reason(), ParseFailureReason::InvalidJson); - - let ingest_request = IngestRequestV2 { - subrequests: vec![ - IngestSubrequest { - subrequest_id: 0, - index_id: "test-index-0".to_string(), - source_id: "test-source".to_string(), - doc_batch: Some(DocBatchV2::for_test(["test-doc-moo", "test-doc-baz"])), - }, - IngestSubrequest { - subrequest_id: 1, - index_id: "test-index-1".to_string(), - source_id: "test-source".to_string(), - doc_batch: Some(DocBatchV2::for_test(["test-doc-tux"])), - }, - ], - commit_type: CommitTypeV2::Auto as i32, - }; - let response = router.ingest(ingest_request).await.unwrap(); - assert_eq!(response.successes.len(), 2); - assert_eq!(response.failures.len(), 0); - } + assert_eq!(parse_failures[0].doc_uid(), DocUid::for_test(0)); + assert_eq!(parse_failures[0].reason(), ParseFailureReason::InvalidJson); + } #[tokio::test] async fn test_router_ingest_retry() { let self_node_id = "test-router".into(); let control_plane = ControlPlaneServiceClient::from_mock(MockControlPlaneService::new()); let ingester_pool = IngesterPool::default(); - let replication_factor = 1; let router = IngestRouter::new( self_node_id, control_plane, ingester_pool.clone(), - replication_factor, + 1, EventBroker::default(), + Some("test-az".to_string()), ); - let mut state_guard = router.state.lock().await; let index_uid: IndexUid = IndexUid::for_test("test-index-0", 0); - state_guard.routing_table.replace_shards( - index_uid.clone(), - "test-source", - vec![Shard { - index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), - shard_state: ShardState::Open as i32, - leader_id: "test-ingester-0".to_string(), - ..Default::default() - }], - ); - drop(state_guard); + { + let mut state_guard = router.state.lock().await; + state_guard.routing_table.merge_from_shards( + index_uid.clone(), + "test-source".to_string(), + vec![Shard { + index_uid: Some(index_uid.clone()), + source_id: "test-source".to_string(), + shard_id: Some(ShardId::from(1)), + shard_state: ShardState::Open as i32, + leader_id: "test-ingester-0".to_string(), + ..Default::default() + }], + ); + } let mut mock_ingester_0 = MockIngesterService::new(); + // First attempt: returns NoShardsAvailable (transient, doesn't mark leader unavailable). + // The response still reports capacity_score=6 and 1 open shard so the node stays routable. let index_uid_clone = index_uid.clone(); mock_ingester_0 .expect_persist() .once() .returning(move |request| { - assert_eq!(request.leader_id, "test-ingester-0"); - assert_eq!(request.subrequests.len(), 1); - assert_eq!(request.commit_type(), CommitTypeV2::Auto); - - let subrequest = &request.subrequests[0]; - assert_eq!(subrequest.subrequest_id, 0); - assert_eq!(subrequest.index_uid(), &index_uid_clone); - assert_eq!(subrequest.source_id, "test-source"); - assert_eq!(subrequest.shard_id(), ShardId::from(1)); - assert_eq!( - subrequest.doc_batch, - Some(DocBatchV2::for_test(["test-doc-foo"])) - ); - - let response = PersistResponse { + Ok(PersistResponse { leader_id: request.leader_id, successes: Vec::new(), failures: vec![PersistFailure { subrequest_id: 0, index_uid: Some(index_uid_clone.clone()), source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), - reason: PersistFailureReason::Timeout as i32, + reason: PersistFailureReason::NoShardsAvailable as i32, }], - }; - Ok(response) + routing_update: Some(RoutingUpdate { + capacity_score: 6, + source_shard_updates: vec![SourceShardUpdate { + index_uid: Some(index_uid_clone.clone()), + source_id: "test-source".to_string(), + open_shard_count: 1, + }], + ..Default::default() + }), + }) }); + // Second attempt: succeeds. mock_ingester_0 .expect_persist() .once() .returning(move |request| { - assert_eq!(request.leader_id, "test-ingester-0"); - assert_eq!(request.subrequests.len(), 1); - assert_eq!(request.commit_type(), CommitTypeV2::Auto); - - let subrequest = &request.subrequests[0]; - assert_eq!(subrequest.subrequest_id, 0); - assert_eq!(subrequest.index_uid(), &index_uid); - assert_eq!(subrequest.source_id, "test-source"); - assert_eq!(subrequest.shard_id(), ShardId::from(1)); - assert_eq!( - subrequest.doc_batch, - Some(DocBatchV2::for_test(["test-doc-foo"])) - ); - - let response = PersistResponse { + Ok(PersistResponse { leader_id: request.leader_id, successes: vec![PersistSuccess { subrequest_id: 0, @@ -1768,115 +1493,36 @@ mod tests { parse_failures: Vec::new(), }], failures: Vec::new(), - }; - Ok(response) + routing_update: Some(RoutingUpdate { + capacity_score: 6, + source_shard_updates: Vec::new(), + ..Default::default() + }), + }) }); - let ingester_0 = IngesterServiceClient::from_mock(mock_ingester_0); - ingester_pool.insert("test-ingester-0".into(), ingester_0.clone()); - - let ingest_request = IngestRequestV2 { - subrequests: vec![IngestSubrequest { - subrequest_id: 0, - index_id: "test-index-0".to_string(), - source_id: "test-source".to_string(), - doc_batch: Some(DocBatchV2::for_test(["test-doc-foo"])), - }], - commit_type: CommitTypeV2::Auto as i32, - }; - router.ingest(ingest_request).await.unwrap(); - } - - #[tokio::test] - async fn test_router_updates_routing_table_on_chitchat_events() { - let self_node_id = "test-router".into(); - let control_plane = ControlPlaneServiceClient::from_mock(MockControlPlaneService::new()); - let ingester_pool = IngesterPool::default(); - let replication_factor = 1; - let event_broker = EventBroker::default(); - let router = IngestRouter::new( - self_node_id, - control_plane, - ingester_pool.clone(), - replication_factor, - event_broker.clone(), - ); - router.subscribe(); - let index_uid: IndexUid = IndexUid::for_test("test-index-0", 0); - - let mut state_guard = router.state.lock().await; - state_guard.routing_table.replace_shards( - index_uid.clone(), - "test-source", - vec![Shard { - index_uid: Some(index_uid.clone()), - shard_id: Some(ShardId::from(1)), - shard_state: ShardState::Open as i32, - leader_id: "test-ingester".to_string(), - ..Default::default() - }], - ); - drop(state_guard); - - let local_shards_update = LocalShardsUpdate { - leader_id: "test-ingester".into(), - source_uid: SourceUid { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - }, - shard_infos: BTreeSet::from_iter([ - ShardInfo { - shard_id: ShardId::from(1), - shard_state: ShardState::Closed, - short_term_ingestion_rate: RateMibPerSec(0), - long_term_ingestion_rate: RateMibPerSec(0), - }, - ShardInfo { - shard_id: ShardId::from(2), - shard_state: ShardState::Open, - short_term_ingestion_rate: RateMibPerSec(0), - long_term_ingestion_rate: RateMibPerSec(0), - }, - ]), - }; - event_broker.publish(local_shards_update); - - // Yield so that the event is processed. - yield_now().await; - - let state_guard = router.state.lock().await; - let shards = state_guard - .routing_table - .find_entry("test-index-0", "test-source") - .unwrap() - .all_shards(); - assert_eq!(shards.len(), 2); - assert_eq!(shards[0].shard_id, ShardId::from(1)); - assert_eq!(shards[0].shard_state, ShardState::Closed); - assert_eq!(shards[1].shard_id, ShardId::from(2)); - assert_eq!(shards[1].shard_state, ShardState::Open); - drop(state_guard); - - let shard_positions_update = ShardPositionsUpdate { - source_uid: SourceUid { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), + ingester_pool.insert( + "test-ingester-0".into(), + IngesterPoolEntry { + client: IngesterServiceClient::from_mock(mock_ingester_0), + status: IngesterStatus::Ready, + availability_zone: None, }, - updated_shard_positions: vec![(ShardId::from(1), Position::eof(0u64))], - }; - event_broker.publish(shard_positions_update); - - // Yield so that the event is processed. - yield_now().await; + ); - let state_guard = router.state.lock().await; - let shards = state_guard - .routing_table - .find_entry("test-index-0", "test-source") - .unwrap() - .all_shards(); - assert_eq!(shards.len(), 1); - assert_eq!(shards[0].shard_id, ShardId::from(2)); - drop(state_guard); + let response = router + .ingest(IngestRequestV2 { + subrequests: vec![IngestSubrequest { + subrequest_id: 0, + index_id: "test-index-0".to_string(), + source_id: "test-source".to_string(), + doc_batch: Some(DocBatchV2::for_test(["test-doc-foo"])), + }], + commit_type: CommitTypeV2::Auto as i32, + }) + .await + .unwrap(); + assert_eq!(response.successes.len(), 1); + assert_eq!(response.failures.len(), 0); } #[tokio::test] @@ -1891,46 +1537,53 @@ mod tests { ingester_pool.clone(), replication_factor, EventBroker::default(), + Some("test-az".to_string()), ); let index_uid_0: IndexUid = IndexUid::for_test("test-index-0", 0); let index_uid_1: IndexUid = IndexUid::for_test("test-index-1", 0); - let mut state_guard = router.state.lock().await; - state_guard.routing_table.replace_shards( - index_uid_0.clone(), - "test-source", - vec![Shard { - index_uid: Some(index_uid_0.clone()), - shard_id: Some(ShardId::from(1)), - shard_state: ShardState::Open as i32, - leader_id: "test-ingester".to_string(), - ..Default::default() - }], - ); - state_guard.routing_table.replace_shards( - index_uid_1.clone(), - "test-source", - vec![Shard { - index_uid: Some(index_uid_1.clone()), - shard_id: Some(ShardId::from(2)), - shard_state: ShardState::Open as i32, - leader_id: "test-ingester".to_string(), - ..Default::default() - }], - ); - drop(state_guard); + { + let mut state_guard = router.state.lock().await; + state_guard.routing_table.merge_from_shards( + index_uid_0.clone(), + "test-source".to_string(), + vec![Shard { + index_uid: Some(index_uid_0.clone()), + shard_id: Some(ShardId::from(1)), + shard_state: ShardState::Open as i32, + leader_id: "test-ingester-0".to_string(), + ..Default::default() + }], + ); + state_guard.routing_table.merge_from_shards( + index_uid_1.clone(), + "test-source".to_string(), + vec![Shard { + index_uid: Some(index_uid_1.clone()), + shard_id: Some(ShardId::from(2)), + shard_state: ShardState::Open as i32, + leader_id: "test-ingester-1".to_string(), + ..Default::default() + }], + ); + } let debug_info = router.debug_info().await; let routing_table = &debug_info["routing_table"]; assert_eq!(routing_table.as_object().unwrap().len(), 2); - assert_eq!(routing_table["test-index-0"].as_array().unwrap().len(), 1); - assert_eq!(routing_table["test-index-1"].as_array().unwrap().len(), 1); + let index_0_entries = routing_table["test-index-0"].as_array().unwrap(); + assert_eq!(index_0_entries.len(), 1); + assert_eq!(index_0_entries[0]["node_id"], "test-ingester-0"); + assert_eq!(index_0_entries[0]["capacity_score"], 5); + + let index_1_entries = routing_table["test-index-1"].as_array().unwrap(); + assert_eq!(index_1_entries.len(), 1); + assert_eq!(index_1_entries[0]["node_id"], "test-ingester-1"); } #[tokio::test] - async fn test_router_does_not_retry_rate_limited_shards() { - // We avoid retrying a shard limited shard at the scale of a workbench. + async fn test_router_returns_rate_limited_failure() { let self_node_id = "test-router".into(); let control_plane = ControlPlaneServiceClient::from_mock(MockControlPlaneService::new()); let ingester_pool = IngesterPool::default(); @@ -1941,141 +1594,69 @@ mod tests { ingester_pool.clone(), replication_factor, EventBroker::default(), + Some("test-az".to_string()), ); - let mut state_guard = router.state.lock().await; let index_uid: IndexUid = IndexUid::for_test("test-index-0", 0); - - state_guard.routing_table.replace_shards( - index_uid.clone(), - "test-source", - vec![ - Shard { + { + let mut state_guard = router.state.lock().await; + state_guard.routing_table.merge_from_shards( + index_uid.clone(), + "test-source".to_string(), + vec![Shard { index_uid: Some(index_uid.clone()), source_id: "test-source".to_string(), shard_id: Some(ShardId::from(1)), shard_state: ShardState::Open as i32, leader_id: "test-ingester-0".to_string(), ..Default::default() - }, - Shard { - index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(2)), - shard_state: ShardState::Open as i32, - leader_id: "test-ingester-0".to_string(), - ..Default::default() - }, - ], - ); - drop(state_guard); - - // We have two shards. - // - shard 1 is rate limited - // - shard 2 is timeout. - // We expect a retry on shard 2 that is then successful. - let mut seq = Sequence::new(); + }], + ); + } let mut mock_ingester_0 = MockIngesterService::new(); - mock_ingester_0 - .expect_persist() - .times(1) - .returning(move |request| { - assert_eq!(request.leader_id, "test-ingester-0"); - assert_eq!(request.commit_type(), CommitTypeV2::Auto); - assert_eq!(request.subrequests.len(), 1); - let subrequest = &request.subrequests[0]; - assert_eq!(subrequest.subrequest_id, 0); - let index_uid = subrequest.index_uid().clone(); - assert_eq!(subrequest.source_id, "test-source"); - assert_eq!(subrequest.shard_id(), ShardId::from(1)); - assert_eq!( - subrequest.doc_batch, - Some(DocBatchV2::for_test(["test-doc-foo"])) - ); - - let response = PersistResponse { - leader_id: request.leader_id, - successes: Vec::new(), - failures: vec![PersistFailure { - subrequest_id: 0, - index_uid: Some(index_uid), - source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), - reason: PersistFailureReason::ShardRateLimited as i32, - }], - }; - Ok(response) - }) - .in_sequence(&mut seq); - - mock_ingester_0 - .expect_persist() - .times(1) - .returning(move |request| { - assert_eq!(request.leader_id, "test-ingester-0"); - assert_eq!(request.commit_type(), CommitTypeV2::Auto); - assert_eq!(request.subrequests.len(), 1); - let subrequest = &request.subrequests[0]; - assert_eq!(subrequest.subrequest_id, 0); - let index_uid = subrequest.index_uid().clone(); - assert_eq!(subrequest.source_id, "test-source"); - assert_eq!(subrequest.shard_id(), ShardId::from(2)); - assert_eq!( - subrequest.doc_batch, - Some(DocBatchV2::for_test(["test-doc-foo"])) - ); - - let response = PersistResponse { - leader_id: request.leader_id, - successes: Vec::new(), - failures: vec![PersistFailure { - subrequest_id: 0, - index_uid: Some(index_uid), - source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), - reason: PersistFailureReason::Timeout as i32, - }], - }; - Ok(response) - }) - .in_sequence(&mut seq); - - mock_ingester_0 - .expect_persist() - .times(1) - .returning(move |request| { - assert_eq!(request.leader_id, "test-ingester-0"); - assert_eq!(request.commit_type(), CommitTypeV2::Auto); - assert_eq!(request.subrequests.len(), 1); - let subrequest = &request.subrequests[0]; - assert_eq!(subrequest.subrequest_id, 0); - let index_uid = subrequest.index_uid().clone(); - assert_eq!(subrequest.source_id, "test-source"); - assert_eq!(subrequest.shard_id(), ShardId::from(2)); - assert_eq!( - subrequest.doc_batch, - Some(DocBatchV2::for_test(["test-doc-foo"])) - ); + mock_ingester_0.expect_persist().returning(move |request| { + assert_eq!(request.leader_id, "test-ingester-0"); + assert_eq!(request.commit_type(), CommitTypeV2::Auto); + assert_eq!(request.subrequests.len(), 1); + let subrequest = &request.subrequests[0]; + assert_eq!(subrequest.subrequest_id, 0); + let index_uid = subrequest.index_uid().clone(); + assert_eq!(subrequest.source_id, "test-source"); + assert_eq!( + subrequest.doc_batch, + Some(DocBatchV2::for_test(["test-doc-foo"])) + ); - let response = PersistResponse { - leader_id: request.leader_id, - successes: vec![PersistSuccess { - subrequest_id: 0, + let response = PersistResponse { + leader_id: request.leader_id, + successes: Vec::new(), + failures: vec![PersistFailure { + subrequest_id: 0, + index_uid: Some(index_uid.clone()), + source_id: "test-source".to_string(), + reason: PersistFailureReason::NoShardsAvailable as i32, + }], + routing_update: Some(RoutingUpdate { + capacity_score: 6, + source_shard_updates: vec![SourceShardUpdate { index_uid: Some(index_uid), source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), - num_persisted_docs: 1, - replication_position_inclusive: Some(Position::offset(0u64)), - parse_failures: Vec::new(), + open_shard_count: 1, }], - failures: Vec::new(), - }; - Ok(response) - }) - .in_sequence(&mut seq); - + ..Default::default() + }), + }; + Ok(response) + }); let ingester_0 = IngesterServiceClient::from_mock(mock_ingester_0); - ingester_pool.insert("test-ingester-0".into(), ingester_0.clone()); + ingester_pool.insert( + "test-ingester-0".into(), + IngesterPoolEntry { + client: ingester_0.clone(), + availability_zone: None, + status: IngesterStatus::Ready, + }, + ); let ingest_request = IngestRequestV2 { subrequests: vec![IngestSubrequest { @@ -2086,88 +1667,200 @@ mod tests { }], commit_type: CommitTypeV2::Auto as i32, }; - router.ingest(ingest_request).await.unwrap(); + let ingest_response = router.ingest(ingest_request).await.unwrap(); + assert_eq!(ingest_response.successes.len(), 0); + assert_eq!(ingest_response.failures.len(), 1); + assert_eq!( + ingest_response.failures[0].reason(), + IngestFailureReason::NoShardsAvailable + ); } #[tokio::test] - async fn test_router_returns_rate_limited_failure() { - let self_node_id = "test-router".into(); - let control_plane = ControlPlaneServiceClient::from_mock(MockControlPlaneService::new()); + async fn test_router_updates_node_routing_table_on_capacity_update() { + let event_broker = EventBroker::default(); let ingester_pool = IngesterPool::default(); - let replication_factor = 1; let router = IngestRouter::new( - self_node_id, - control_plane, + "test-router".into(), + ControlPlaneServiceClient::from_mock(MockControlPlaneService::new()), ingester_pool.clone(), - replication_factor, - EventBroker::default(), + 1, + event_broker.clone(), + Some("test-az".to_string()), ); - let mut state_guard = router.state.lock().await; - let index_uid: IndexUid = IndexUid::for_test("test-index-0", 0); + router.subscribe(); - state_guard.routing_table.replace_shards( - index_uid.clone(), - "test-source", - vec![Shard { - index_uid: Some(index_uid.clone()), + event_broker.publish(IngesterCapacityScoreUpdate { + node_id: "test-ingester-0".into(), + source_uid: SourceUid { + index_uid: IndexUid::for_test("test-index", 0), + source_id: "test-source".to_string(), + }, + capacity_score: 7, + open_shard_count: 3, + }); + // Give the async subscriber a moment to process. + tokio::time::sleep(Duration::from_millis(10)).await; + + ingester_pool.insert( + "test-ingester-0".into(), + IngesterPoolEntry::mocked_ingester(), + ); + let state_guard = router.state.lock().await; + let node = state_guard + .routing_table + .pick_node("test-index", "test-source", &ingester_pool, &HashSet::new()) + .unwrap(); + assert_eq!(node.node_id, NodeId::from("test-ingester-0")); + } + + #[tokio::test] + async fn test_router_process_persist_results_marks_unavailable_on_persist_failure() { + let router = IngestRouter::new( + "test-router".into(), + ControlPlaneServiceClient::from_mock(MockControlPlaneService::new()), + IngesterPool::default(), + 1, + EventBroker::default(), + Some("test-az".to_string()), + ); + let ingest_subrequests = vec![ + IngestSubrequest { + subrequest_id: 0, + index_id: "test-index-0".to_string(), source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), - shard_state: ShardState::Open as i32, - leader_id: "test-ingester-0".to_string(), ..Default::default() - }], + }, + IngestSubrequest { + subrequest_id: 1, + index_id: "test-index-1".to_string(), + source_id: "test-source".to_string(), + ..Default::default() + }, + ]; + let mut workbench = IngestWorkbench::new(ingest_subrequests, 2); + + // NoShardsAvailable does NOT mark the leader as unavailable. + let persist_futures = FuturesUnordered::new(); + persist_futures.push(async { + let summary = PersistRequestSummary { + leader_id: "test-ingester-0".into(), + subrequest_ids: vec![0], + }; + let result = Ok::<_, IngestV2Error>(PersistResponse { + leader_id: "test-ingester-0".to_string(), + successes: Vec::new(), + failures: vec![PersistFailure { + subrequest_id: 0, + index_uid: Some(IndexUid::for_test("test-index-0", 0)), + source_id: "test-source".to_string(), + reason: PersistFailureReason::NoShardsAvailable as i32, + }], + routing_update: Some(RoutingUpdate { + capacity_score: 6, + source_shard_updates: Vec::new(), + ..Default::default() + }), + }); + (summary, result) + }); + router + .process_persist_results(&mut workbench, persist_futures) + .await; + assert!( + !workbench + .unavailable_leaders + .contains(&NodeId::from("test-ingester-0")) ); - drop(state_guard); - let mut mock_ingester_0 = MockIngesterService::new(); - mock_ingester_0 - .expect_persist() - .times(1) - .returning(move |request| { - assert_eq!(request.leader_id, "test-ingester-0"); - assert_eq!(request.commit_type(), CommitTypeV2::Auto); - assert_eq!(request.subrequests.len(), 1); - let subrequest = &request.subrequests[0]; - assert_eq!(subrequest.subrequest_id, 0); - let index_uid = subrequest.index_uid().clone(); - assert_eq!(subrequest.source_id, "test-source"); - assert_eq!(subrequest.shard_id(), ShardId::from(1)); - assert_eq!( - subrequest.doc_batch, - Some(DocBatchV2::for_test(["test-doc-foo"])) - ); + // NodeUnavailable DOES mark the leader as unavailable. + let persist_futures = FuturesUnordered::new(); + persist_futures.push(async { + let summary = PersistRequestSummary { + leader_id: "test-ingester-1".into(), + subrequest_ids: vec![1], + }; + let result = Ok::<_, IngestV2Error>(PersistResponse { + leader_id: "test-ingester-1".to_string(), + successes: Vec::new(), + failures: vec![PersistFailure { + subrequest_id: 1, + index_uid: Some(IndexUid::for_test("test-index-1", 0)), + source_id: "test-source".to_string(), + reason: PersistFailureReason::NodeUnavailable as i32, + }], + routing_update: Some(RoutingUpdate { + capacity_score: 6, + source_shard_updates: Vec::new(), + ..Default::default() + }), + }); + (summary, result) + }); + router + .process_persist_results(&mut workbench, persist_futures) + .await; + assert!( + workbench + .unavailable_leaders + .contains(&NodeId::from("test-ingester-1")) + ); + } - let response = PersistResponse { - leader_id: request.leader_id, - successes: Vec::new(), - failures: vec![PersistFailure { - subrequest_id: 0, - index_uid: Some(index_uid), + #[tokio::test] + async fn test_router_process_persist_results_applies_piggybacked_routing_updates() { + let ingester_pool = IngesterPool::default(); + let router = IngestRouter::new( + "test-router".into(), + ControlPlaneServiceClient::from_mock(MockControlPlaneService::new()), + ingester_pool.clone(), + 1, + EventBroker::default(), + Some("test-az".to_string()), + ); + let ingest_subrequests = vec![IngestSubrequest { + subrequest_id: 0, + index_id: "test-index".to_string(), + source_id: "test-source".to_string(), + ..Default::default() + }]; + let mut workbench = IngestWorkbench::new(ingest_subrequests, 2); + + let persist_futures = FuturesUnordered::new(); + persist_futures.push(async { + let summary = PersistRequestSummary { + leader_id: "test-ingester-0".into(), + subrequest_ids: vec![0], + }; + let result = Ok::<_, IngestV2Error>(PersistResponse { + leader_id: "test-ingester-0".to_string(), + successes: Vec::new(), + failures: Vec::new(), + routing_update: Some(RoutingUpdate { + capacity_score: 3, + source_shard_updates: vec![SourceShardUpdate { + index_uid: Some(IndexUid::for_test("test-index", 0)), source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), - reason: PersistFailureReason::ShardRateLimited as i32, + open_shard_count: 2, }], - }; - Ok(response) + ..Default::default() + }), }); - let ingester_0 = IngesterServiceClient::from_mock(mock_ingester_0); - ingester_pool.insert("test-ingester-0".into(), ingester_0.clone()); + (summary, result) + }); + router + .process_persist_results(&mut workbench, persist_futures) + .await; - let ingest_request = IngestRequestV2 { - subrequests: vec![IngestSubrequest { - subrequest_id: 0, - index_id: "test-index-0".to_string(), - source_id: "test-source".to_string(), - doc_batch: Some(DocBatchV2::for_test(["test-doc-foo"])), - }], - commit_type: CommitTypeV2::Auto as i32, - }; - let ingest_response = router.ingest(ingest_request).await.unwrap(); - assert_eq!(ingest_response.successes.len(), 0); - assert_eq!(ingest_response.failures.len(), 1); - assert_eq!( - ingest_response.failures[0].reason(), - IngestFailureReason::ShardRateLimited + ingester_pool.insert( + "test-ingester-0".into(), + IngesterPoolEntry::mocked_ingester(), ); + let state_guard = router.state.lock().await; + let node = state_guard + .routing_table + .pick_node("test-index", "test-source", &ingester_pool, &HashSet::new()) + .unwrap(); + assert_eq!(node.node_id, NodeId::from("test-ingester-0")); } } diff --git a/quickwit/quickwit-ingest/src/ingest_v2/routing_table.rs b/quickwit/quickwit-ingest/src/ingest_v2/routing_table.rs index 987d754ed69..edd1b4a36aa 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/routing_table.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/routing_table.rs @@ -12,1080 +12,705 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; +use std::cmp::Ordering; use std::collections::{HashMap, HashSet}; -use std::sync::atomic::{AtomicUsize, Ordering}; -use quickwit_proto::ingest::{Shard, ShardIds, ShardState}; -use quickwit_proto::types::{IndexId, IndexUid, NodeId, ShardId, SourceId}; -use serde_json::{Value as JsonValue, json}; -use tracing::{info, warn}; +use itertools::Itertools; +use quickwit_proto::ingest::Shard; +use quickwit_proto::types::{IndexId, IndexUid, NodeId, SourceId}; +use rand::rng; +use rand::seq::IndexedRandom; use crate::IngesterPool; +/// A single ingester node's routing-relevant data for a specific (index, source) pair. +/// Each entry is self-describing: it carries its own node_id, index_uid, and source_id +/// so it can always be attributed back to a specific source on a specific node. +#[derive(Debug, Clone)] +pub(super) struct IngesterNode { + pub node_id: NodeId, + pub index_uid: IndexUid, + /// Score from 0-10. Higher means more available capacity. + pub capacity_score: usize, + /// Number of open shards on this node for this (index, source) pair. Tiebreaker for power of + /// two choices comparison - we favor a node with more open shards. + pub open_shard_count: usize, +} + #[derive(Debug)] pub(super) struct RoutingEntry { pub index_uid: IndexUid, - pub source_id: SourceId, - pub shard_id: ShardId, - pub shard_state: ShardState, - pub leader_id: NodeId, + pub nodes: HashMap, + /// Whether this entry has been seeded from a control plane response. During a rolling + /// deployment, Chitchat broadcasts from already-upgraded nodes may populate the table + /// before the router ever asks the CP, causing it to miss old-version nodes. This flag + /// ensures the router asks the CP at least once per (index, source) pair. + seeded_from_cp: bool, } -impl From for RoutingEntry { - fn from(shard: Shard) -> Self { - let shard_id = shard.shard_id().clone(); - let shard_state = shard.shard_state(); +impl RoutingEntry { + fn new(index_uid: IndexUid) -> Self { Self { - index_uid: shard.index_uid().clone(), - source_id: shard.source_id, - shard_id, - shard_state, - leader_id: shard.leader_id.into(), + index_uid, + nodes: HashMap::new(), + seeded_from_cp: false, } } } -/// The set of shards the router is aware of for the given index and source. -#[derive(Debug, Default)] -pub(super) struct RoutingTableEntry { - /// Index UID of the shards. - pub index_uid: IndexUid, - /// Source ID of the shards. - pub source_id: SourceId, - /// Shards located on this node. - pub local_shards: Vec, - pub local_round_robin_idx: AtomicUsize, - /// Shards located on remote nodes. - pub remote_shards: Vec, - pub remote_round_robin_idx: AtomicUsize, +/// Given a slice of candidates, picks the better of two random choices. +/// Higher capacity_score wins; tiebreak on more open_shard_count (more landing spots). +fn power_of_two_choices<'a>(candidates: &[&'a IngesterNode]) -> &'a IngesterNode { + debug_assert!(candidates.len() >= 2); + let mut iter = candidates.choose_multiple(&mut rng(), 2); + let (&a, &b) = (iter.next().unwrap(), iter.next().unwrap()); + + if (a.capacity_score, a.open_shard_count) >= (b.capacity_score, b.open_shard_count) { + a + } else { + b + } } -impl RoutingTableEntry { - /// Creates a new entry and ensures that the shards are open, unique, and sorted by shard ID. - fn new( - self_node_id: &NodeId, - index_uid: IndexUid, - source_id: SourceId, - mut shards: Vec, - ) -> Self { - let num_shards = shards.len(); - - shards.sort_unstable_by(|left, right| left.shard_id.cmp(&right.shard_id)); - shards.dedup_by(|left, right| left.shard_id == right.shard_id); - - let (local_shards, remote_shards): (Vec<_>, Vec<_>) = shards - .into_iter() - .filter(|shard| shard.is_open()) - .map(RoutingEntry::from) - .partition(|shard| *self_node_id == shard.leader_id); +fn pick_from(candidates: Vec<&IngesterNode>) -> Option<&IngesterNode> { + match candidates.len() { + 0 => None, + 1 => Some(candidates[0]), + _ => Some(power_of_two_choices(&candidates)), + } +} - if num_shards > local_shards.len() + remote_shards.len() { - warn!("input shards should not contain closed shards or duplicates"); - } +impl RoutingEntry { + /// Pick an ingester node to persist the request to. Uses power of two choices based on reported + /// ingester capacity, if more than one eligible node exists. Prefers nodes in the same + /// availability zone, falling back to remote nodes. + fn pick_node( + &self, + ingester_pool: &IngesterPool, + unavailable_leaders: &HashSet, + self_availability_zone: &Option, + ) -> Option<&IngesterNode> { + let (local_ingesters, remote_ingesters): (Vec<&IngesterNode>, Vec<&IngesterNode>) = self + .nodes + .values() + .filter(|node| { + node.capacity_score > 0 + && node.open_shard_count > 0 + && ingester_pool + .get(&node.node_id) + .map(|entry| entry.status.is_ready()) + .unwrap_or(false) + && !unavailable_leaders.contains(&node.node_id) + }) + .partition(|node| { + let node_az = ingester_pool + .get(&node.node_id) + .and_then(|h| h.availability_zone); + node_az == *self_availability_zone + }); - Self { - index_uid, - source_id, - local_shards, - remote_shards, - ..Default::default() - } + pick_from(local_ingesters).or_else(|| pick_from(remote_ingesters)) } +} - fn empty(index_uid: IndexUid, source_id: SourceId) -> Self { +#[derive(Debug, Default)] +pub(super) struct RoutingTable { + table: HashMap<(IndexId, SourceId), RoutingEntry>, + self_availability_zone: Option, +} + +impl RoutingTable { + pub fn new(self_availability_zone: Option) -> Self { Self { - index_uid, - source_id, + self_availability_zone, ..Default::default() } } - /// Returns `true` if at least one shard in the table entry is open and has a leader available. - /// As it goes through the list of shards in the entry, it populates `closed_shard_ids` and - /// `unavailable_leaders` with the shard IDs of the closed shards and the node ID of the - /// unavailable ingesters encountered along the way. - pub fn has_open_shards( + pub fn pick_node( &self, + index_id: &str, + source_id: &str, ingester_pool: &IngesterPool, - closed_shard_ids: &mut Vec, - unavailable_leaders: &mut HashSet, - ) -> bool { - let shards = self.local_shards.iter().chain(self.remote_shards.iter()); - - for shard in shards { - match shard.shard_state { - ShardState::Closed => { - closed_shard_ids.push(shard.shard_id.clone()); - continue; - } - ShardState::Unavailable | ShardState::Unspecified => { - continue; - } - ShardState::Open => { - if unavailable_leaders.contains(&shard.leader_id) { - continue; - } - if ingester_pool.contains_key(&shard.leader_id) { - return true; - } else { - let leader_id: NodeId = shard.leader_id.clone(); - unavailable_leaders.insert(leader_id); - } - } - } - } - false + unavailable_leaders: &HashSet, + ) -> Option<&IngesterNode> { + let key = (index_id.to_string(), source_id.to_string()); + let entry = self.table.get(&key)?; + entry.pick_node( + ingester_pool, + unavailable_leaders, + &self.self_availability_zone, + ) } - /// Returns the next open and available shard in the table entry in a round-robin fashion. - pub fn next_open_shard_round_robin( + pub fn classify_az_locality( &self, + target_node_id: &NodeId, ingester_pool: &IngesterPool, - rate_limited_shards: &HashSet, - ) -> Result<&RoutingEntry, NextOpenShardError> { - let mut error = NextOpenShardError::NoShardsAvailable; - - for (shards, round_robin_idx) in [ - (&self.local_shards, &self.local_round_robin_idx), - (&self.remote_shards, &self.remote_round_robin_idx), - ] { - if shards.is_empty() { - continue; - } - for _attempt in 0..shards.len() { - let shard_idx = round_robin_idx.fetch_add(1, Ordering::Relaxed); - let shard_routing_entry: &RoutingEntry = &shards[shard_idx % shards.len()]; - - if !shard_routing_entry.shard_state.is_open() { - continue; - } - if rate_limited_shards.contains(&shard_routing_entry.shard_id) { - error = NextOpenShardError::RateLimited; - continue; - } - if ingester_pool.contains_key(&shard_routing_entry.leader_id) { - return Ok(shard_routing_entry); - } - } - } - Err(error) - } - - /// Inserts the open shards the routing table is not aware of. - fn insert_open_shards( - &mut self, - self_node_id: &NodeId, - leader_id: &NodeId, - index_uid: &IndexUid, - shard_ids: &[ShardId], - ) { - match self.index_uid.cmp(index_uid) { - // If we receive an update for a new incarnation of the index, then we clear the entry - // and insert all the shards. - std::cmp::Ordering::Less => { - self.index_uid = index_uid.clone(); - self.clear_shards(); - } - // If we receive an update for a previous incarnation of the index, then we ignore it. - std::cmp::Ordering::Greater => { - return; - } - std::cmp::Ordering::Equal => {} - }; - let target_shards = if self_node_id == leader_id { - &mut self.local_shards - } else { - &mut self.remote_shards + ) -> &'static str { + let Some(self_az) = &self.self_availability_zone else { + return "az_unaware"; }; - let mut num_inserted_shards = 0; - let num_target_shards = target_shards.len(); - - if num_target_shards == 0 { - target_shards.reserve(num_target_shards); - target_shards.extend(shard_ids.iter().map(|shard_id| RoutingEntry { - index_uid: self.index_uid.clone(), - source_id: self.source_id.clone(), - shard_id: shard_id.clone(), - shard_state: ShardState::Open, - leader_id: leader_id.clone(), - })); - num_inserted_shards = target_shards.len(); - } else { - let shard_ids_range = target_shards[0].shard_id.clone() - ..=target_shards[num_target_shards - 1].shard_id.clone(); - - for shard_id in shard_ids { - // If we can't find the shard, then we insert it. - if shard_ids_range.contains(shard_id) { - continue; - } - if target_shards[..num_target_shards] - .binary_search_by(|shard| shard.shard_id.cmp(shard_id)) - .is_err() - { - target_shards.push(RoutingEntry { - index_uid: self.index_uid.clone(), - source_id: self.source_id.clone(), - shard_id: shard_id.clone(), - shard_state: ShardState::Open, - leader_id: leader_id.clone(), - }); - num_inserted_shards += 1; - } - } - } - if num_inserted_shards > 0 { - target_shards.sort_unstable_by(|left, right| left.shard_id.cmp(&right.shard_id)); - - info!( - index_uid=%self.index_uid, - source_id=%self.source_id, - "inserted {num_inserted_shards} shards into routing table" - ); - } - } - - /// Clears local and remote shards. - fn clear_shards(&mut self) { - self.local_shards.clear(); - self.local_round_robin_idx = AtomicUsize::default(); - self.remote_shards.clear(); - self.remote_round_robin_idx = AtomicUsize::default(); - } - - /// Closes the shards identified by their shard IDs. - fn close_shards(&mut self, index_uid: &IndexUid, shard_ids: &[ShardId]) { - // If the shard table was just recently updated with shards for a new index UID, then we can - // safely discard this request. - if self.index_uid != *index_uid { - return; - } - for shards in [&mut self.local_shards, &mut self.remote_shards] { - if shards.is_empty() { - continue; - } - let num_shards = shards.len(); - let shard_ids_range = - shards[0].shard_id.clone()..=shards[num_shards - 1].shard_id.clone(); - - for shard_id in shard_ids { - if !shard_ids_range.contains(shard_id) { - continue; - } - if let Ok(shard_idx) = shards.binary_search_by(|shard| shard.shard_id.cmp(shard_id)) - { - shards[shard_idx].shard_state = ShardState::Closed; - } - } + let target_az = ingester_pool + .get(target_node_id) + .and_then(|entry| entry.availability_zone); + match target_az { + Some(ref az) if az == self_az => "same_az", + Some(_) => "cross_az", + None => "az_unaware", } } - /// Shards the shards identified by their shard IDs. - fn delete_shards(&mut self, index_uid: &IndexUid, shard_ids: &[ShardId]) { - // If the shard table was just recently updated with shards for a new index UID, then we can - // safely discard this request. - if self.index_uid != *index_uid { - return; - } - for shards in [&mut self.local_shards, &mut self.remote_shards] { - if shards.is_empty() { - continue; - } - let num_shards = shards.len(); - let shard_ids_range = - shards[0].shard_id.clone()..=shards[num_shards - 1].shard_id.clone(); - let mut deleted_any = false; - - for shard_id in shard_ids { - if !shard_ids_range.contains(shard_id) { - continue; - } - if let Ok(shard_idx) = shards.binary_search_by(|shard| shard.shard_id.cmp(shard_id)) - { - // We use `Unspecified` as a tombstone. - shards[shard_idx].shard_state = ShardState::Unspecified; - deleted_any = true; - } - } - if deleted_any { - shards.retain(|shard| shard.shard_state != ShardState::Unspecified); + pub fn debug_info( + &self, + ingester_pool: &IngesterPool, + ) -> HashMap> { + let mut per_index: HashMap> = HashMap::new(); + for ((index_id, source_id), entry) in &self.table { + for (node_id, node) in &entry.nodes { + let az = ingester_pool.get(node_id).and_then(|h| h.availability_zone); + per_index + .entry(index_id.clone()) + .or_default() + .push(serde_json::json!({ + "source_id": source_id, + "node_id": node_id, + "capacity_score": node.capacity_score, + "open_shard_count": node.open_shard_count, + "availability_zone": az, + })); } } + per_index } - #[cfg(test)] - pub fn len(&self) -> usize { - self.local_shards.len() + self.remote_shards.len() - } - - #[cfg(test)] - pub fn all_shards(&self) -> Vec<&RoutingEntry> { - let mut shards = Vec::with_capacity(self.len()); - shards.extend(&self.local_shards); - shards.extend(&self.remote_shards); - shards - } -} - -#[derive(Debug, PartialEq, Eq)] -pub(super) enum NextOpenShardError { - NoShardsAvailable, - RateLimited, -} - -/// Stores the list of shards the router is aware of for each index and source. The resolution from -/// index and source to shards is performed using index ID (not index UID) and source ID. -#[derive(Debug)] -pub(super) struct RoutingTable { - pub self_node_id: NodeId, - pub table: HashMap<(IndexId, SourceId), RoutingTableEntry>, -} - -impl RoutingTable { - pub fn find_entry( - &self, - index_id: impl Into, - source_id: impl Into, - ) -> Option<&RoutingTableEntry> { - let key = (index_id.into(), source_id.into()); - self.table.get(&key) - } - - /// Returns `true` if the router already knows about a shard for a given source that has - /// an available `leader`. - /// - /// If this function returns false, it populates the set of unavailable leaders and closed - /// shards. These will be joined to the GetOrCreate shard request emitted to the control - /// plane. - pub fn has_open_shards( + pub fn has_open_nodes( &self, - index_id: impl Into, - source_id: impl Into, + index_id: &str, + source_id: &str, ingester_pool: &IngesterPool, - closed_shards: &mut Vec, - unavailable_leaders: &mut HashSet, + unavailable_leaders: &HashSet, ) -> bool { - let Some(entry) = self.find_entry(index_id, source_id) else { + let key = (index_id.to_string(), source_id.to_string()); + let Some(entry) = self.table.get(&key) else { return false; }; - let mut closed_shard_ids: Vec = Vec::new(); - - let result = - entry.has_open_shards(ingester_pool, &mut closed_shard_ids, unavailable_leaders); - - if !closed_shard_ids.is_empty() { - closed_shards.push(ShardIds { - index_uid: entry.index_uid.clone().into(), - source_id: entry.source_id.clone(), - shard_ids: closed_shard_ids, - }); + // Routers must sync with the control plane at least once per (index, source). + if !entry.seeded_from_cp { + return false; } - result + entry.nodes.values().any(|node| { + node.capacity_score > 0 + && node.open_shard_count > 0 + && ingester_pool + .get(&node.node_id) + .map(|entry| entry.status.is_ready()) + .unwrap_or(false) + && !unavailable_leaders.contains(&node.node_id) + }) } - /// Replaces the routing table entry for the source with the provided shards. - pub fn replace_shards( + /// Applies a capacity update from the IngesterCapacityScoreUpdate broadcast. This is the + /// primary way the table learns about node availability and capacity. + pub fn apply_capacity_update( &mut self, + node_id: NodeId, index_uid: IndexUid, - source_id: impl Into, - shards: Vec, + source_id: SourceId, + capacity_score: usize, + open_shard_count: usize, ) { - let index_id: IndexId = index_uid.index_id.to_string(); - let source_id: SourceId = source_id.into(); - let key = (index_id, source_id.clone()); - - match self.table.entry(key) { - Entry::Vacant(entry) => { - entry.insert(RoutingTableEntry::new( - &self.self_node_id, - index_uid, - source_id, - shards, - )); - } - Entry::Occupied(mut entry) => { - assert!( - entry.get().index_uid <= index_uid, - "new index incarnation should be greater or equal" - ); - - entry.insert(RoutingTableEntry::new( - &self.self_node_id, - index_uid, - source_id, - shards, - )); + let key = (index_uid.index_id.to_string(), source_id); + + let entry = self + .table + .entry(key) + .or_insert_with(|| RoutingEntry::new(index_uid.clone())); + match entry.index_uid.cmp(&index_uid) { + // If we receive an update for a new incarnation of the index, then we clear the entry. + Ordering::Less => { + entry.index_uid = index_uid.clone(); + entry.nodes.clear(); + entry.seeded_from_cp = false; } + // If we receive an update for a previous incarnation of the index, then we ignore it. + Ordering::Greater => return, + Ordering::Equal => {} + } + let ingester_node = IngesterNode { + node_id: node_id.clone(), + index_uid, + capacity_score, + open_shard_count, }; + entry.nodes.insert(node_id, ingester_node); } - /// Inserts the shards the routing table is not aware of. - pub fn insert_open_shards( + /// Merges routing updates from a GetOrCreateOpenShards control plane response into the + /// table. For existing nodes, updates their open shard count, including if the count is 0, from + /// the CP response while preserving capacity scores if they already exist. + /// New nodes get a default capacity_score of 5. + pub fn merge_from_shards( &mut self, - leader_id: &NodeId, index_uid: IndexUid, - source_id: impl Into, - shard_ids: &[ShardId], - ) { - let index_id: IndexId = index_uid.index_id.to_string(); - let source_id: SourceId = source_id.into(); - let key = (index_id, source_id.clone()); - - self.table - .entry(key.clone()) - .or_insert_with(|| RoutingTableEntry::empty(index_uid.clone(), source_id)) - .insert_open_shards(&self.self_node_id, leader_id, &index_uid, shard_ids); - } - - /// Closes the targeted shards. - pub fn close_shards( - &mut self, - index_uid: &IndexUid, - source_id: impl Into, - shard_ids: &[ShardId], - ) { - let key = (index_uid.index_id.clone(), source_id.into()); - if let Some(entry) = self.table.get_mut(&key) { - entry.close_shards(index_uid, shard_ids); - } - } - - /// Deletes the targeted shards. - pub fn delete_shards( - &mut self, - index_uid: &IndexUid, - source_id: impl Into, - shard_ids: &[ShardId], + source_id: SourceId, + shards: Vec, ) { - let key = (index_uid.index_id.clone(), source_id.into()); - if let Some(entry) = self.table.get_mut(&key) { - entry.delete_shards(index_uid, shard_ids); + let key = (index_uid.index_id.to_string(), source_id); + let entry = self + .table + .entry(key) + .or_insert_with(|| RoutingEntry::new(index_uid.clone())); + match entry.index_uid.cmp(&index_uid) { + // If we receive an update for a new incarnation of the index, then we clear the entry. + Ordering::Less => { + entry.index_uid = index_uid.clone(); + entry.nodes.clear(); + } + // If we receive an update for a previous incarnation of the index, then we ignore it. + Ordering::Greater => return, + Ordering::Equal => {} } - } - pub fn debug_info(&self) -> HashMap> { - let mut per_index_shards_json: HashMap> = HashMap::new(); - - for ((index_id, source_id), entry) in &self.table { - for (shards, is_local) in &[(&entry.local_shards, true), (&entry.remote_shards, false)] - { - let shards_json = shards.iter().map(|shard| { - json!({ - "index_uid": shard.index_uid, - "source_id": source_id, - "shard_id": shard.shard_id, - "shard_state": shard.shard_state.as_json_str_name(), - "is_local": is_local, - }) + let per_leader_count: HashMap = shards + .iter() + .map(|shard| { + let num_open_shards = shard.is_open() as usize; + let leader_id = NodeId::from(shard.leader_id.clone()); + (leader_id, num_open_shards) + }) + .into_grouping_map() + .sum(); + + for (node_id, open_shard_count) in per_leader_count { + entry + .nodes + .entry(node_id.clone()) + .and_modify(|node| node.open_shard_count = open_shard_count) + .or_insert_with(|| IngesterNode { + node_id, + index_uid: index_uid.clone(), + capacity_score: 5, + open_shard_count, }); - per_index_shards_json - .entry(index_id.clone()) - .or_default() - .extend(shards_json); - } } - per_index_shards_json - } - - #[cfg(test)] - pub fn len(&self) -> usize { - self.table.len() + entry.seeded_from_cp = true; } } #[cfg(test)] mod tests { use quickwit_proto::ingest::ShardState; - use quickwit_proto::ingest::ingester::IngesterServiceClient; + use quickwit_proto::ingest::ingester::{IngesterServiceClient, IngesterStatus}; + use quickwit_proto::types::ShardId; use super::*; + use crate::IngesterPoolEntry; + + fn mocked_ingester(availability_zone: Option<&str>) -> IngesterPoolEntry { + IngesterPoolEntry { + client: IngesterServiceClient::mocked(), + status: IngesterStatus::Ready, + availability_zone: availability_zone.map(|s| s.to_string()), + } + } #[test] - fn test_routing_table_entry_new() { - let self_node_id: NodeId = "test-node-0".into(); - let index_uid = IndexUid::for_test("test-index", 0); - let source_id: SourceId = "test-source".into(); - let table_entry = RoutingTableEntry::new( - &self_node_id, - index_uid.clone(), - source_id.clone(), - Vec::new(), + fn test_apply_capacity_update() { + let mut table = RoutingTable::default(); + let key = ("test-index".to_string(), "test-source".into()); + + // Insert first node. + table.apply_capacity_update( + "node-1".into(), + IndexUid::for_test("test-index", 0), + "test-source".into(), + 8, + 3, ); - assert_eq!(table_entry.len(), 0); + let entry = table.table.get(&key).unwrap(); + assert_eq!(entry.nodes.len(), 1); + assert_eq!(entry.nodes.get("node-1").unwrap().capacity_score, 8); + + // Update existing node. + table.apply_capacity_update( + "node-1".into(), + IndexUid::for_test("test-index", 0), + "test-source".into(), + 4, + 5, + ); + let node = table.table.get(&key).unwrap().nodes.get("node-1").unwrap(); + assert_eq!(node.capacity_score, 4); + assert_eq!(node.open_shard_count, 5); + + // Add second node. + table.apply_capacity_update( + "node-2".into(), + IndexUid::for_test("test-index", 0), + "test-source".into(), + 6, + 2, + ); + assert_eq!(table.table.get(&key).unwrap().nodes.len(), 2); + + // Zero shards: node stays in table but becomes ineligible for routing. + table.apply_capacity_update( + "node-1".into(), + IndexUid::for_test("test-index", 0), + "test-source".into(), + 0, + 0, + ); + let entry = table.table.get(&key).unwrap(); + assert_eq!(entry.nodes.len(), 2); + assert_eq!(entry.nodes.get("node-1").unwrap().open_shard_count, 0); + assert_eq!(entry.nodes.get("node-1").unwrap().capacity_score, 0); + } + + #[test] + fn test_has_open_nodes() { + let mut table = RoutingTable::default(); + let pool = IngesterPool::default(); + let index_uid = IndexUid::for_test("test-index", 0); + + // Empty table. + assert!(!table.has_open_nodes("test-index", "test-source", &pool, &HashSet::new())); - let index_uid: IndexUid = IndexUid::for_test("test-index", 0); + // Seed from CP so has_open_nodes can return true. let shards = vec![ Shard { index_uid: Some(index_uid.clone()), source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(3)), + shard_id: Some(ShardId::from(1u64)), shard_state: ShardState::Open as i32, - leader_id: "test-node-0".to_string(), + leader_id: "node-1".to_string(), ..Default::default() }, Shard { index_uid: Some(index_uid.clone()), source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), + shard_id: Some(ShardId::from(2u64)), shard_state: ShardState::Open as i32, - leader_id: "test-node-0".to_string(), - ..Default::default() - }, - Shard { - index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(2)), - shard_state: ShardState::Open as i32, - leader_id: "test-node-1".to_string(), - ..Default::default() - }, - Shard { - index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(1)), - shard_state: ShardState::Open as i32, - leader_id: "test-node-0".to_string(), - ..Default::default() - }, - Shard { - index_uid: Some(index_uid.clone()), - source_id: "test-source".to_string(), - shard_id: Some(ShardId::from(4)), - shard_state: ShardState::Closed as i32, - leader_id: "test-node-0".to_string(), + leader_id: "node-2".to_string(), ..Default::default() }, ]; - let table_entry = RoutingTableEntry::new(&self_node_id, index_uid, source_id, shards); - assert_eq!(table_entry.local_shards.len(), 2); - assert_eq!(table_entry.local_shards[0].shard_id, ShardId::from(1)); - assert_eq!(table_entry.local_shards[1].shard_id, ShardId::from(3)); + table.merge_from_shards(index_uid.clone(), "test-source".into(), shards); + + // Node exists but is not in pool. + assert!(!table.has_open_nodes("test-index", "test-source", &pool, &HashSet::new())); + + // Node is in pool → true. + pool.insert("node-1".into(), mocked_ingester(None)); + assert!(table.has_open_nodes("test-index", "test-source", &pool, &HashSet::new())); + + // Node is unavailable → false. + let unavailable: HashSet = HashSet::from(["node-1".into()]); + assert!(!table.has_open_nodes("test-index", "test-source", &pool, &unavailable)); + + // Second node available → true despite first being unavailable. + pool.insert("node-2".into(), mocked_ingester(None)); + assert!(table.has_open_nodes("test-index", "test-source", &pool, &unavailable)); - assert_eq!(table_entry.remote_shards.len(), 1); - assert_eq!(table_entry.remote_shards[0].shard_id, ShardId::from(2)); + // Node with capacity_score=0 is not eligible. + table.apply_capacity_update( + "node-2".into(), + index_uid.clone(), + "test-source".into(), + 0, + 2, + ); + assert!(!table.has_open_nodes("test-index", "test-source", &pool, &unavailable)); } #[test] - fn test_routing_table_entry_has_open_shards() { - let index_uid = IndexUid::for_test("test-index", 0); - let source_id: SourceId = "test-source".into(); - let table_entry = RoutingTableEntry::empty(index_uid.clone(), source_id.clone()); - - let mut closed_shard_ids = Vec::new(); - let ingester_pool = IngesterPool::default(); - let mut unavailable_leaders = HashSet::new(); - - assert!(!table_entry.has_open_shards( - &ingester_pool, - &mut closed_shard_ids, - &mut unavailable_leaders - )); - assert!(closed_shard_ids.is_empty()); - assert!(unavailable_leaders.is_empty()); - - ingester_pool.insert("test-ingester-0".into(), IngesterServiceClient::mocked()); - ingester_pool.insert("test-ingester-1".into(), IngesterServiceClient::mocked()); - - let table_entry = RoutingTableEntry { - index_uid: index_uid.clone(), - source_id: source_id.clone(), - local_shards: vec![ - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(1), - shard_state: ShardState::Closed, - leader_id: "test-ingester-0".into(), - }, - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(2), - shard_state: ShardState::Open, - leader_id: "test-ingester-0".into(), - }, - ], - local_round_robin_idx: AtomicUsize::default(), - remote_shards: Vec::new(), - remote_round_robin_idx: AtomicUsize::default(), - }; - assert!(table_entry.has_open_shards( - &ingester_pool, - &mut closed_shard_ids, - &mut unavailable_leaders - )); - assert_eq!(closed_shard_ids.len(), 1); - assert_eq!(closed_shard_ids[0], ShardId::from(1)); - assert!(unavailable_leaders.is_empty()); - - closed_shard_ids.clear(); - - let table_entry = RoutingTableEntry { - index_uid: index_uid.clone(), - source_id, - local_shards: Vec::new(), - local_round_robin_idx: AtomicUsize::default(), - remote_shards: vec![ - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(1), - shard_state: ShardState::Closed, - leader_id: "test-ingester-1".into(), - }, - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(2), - shard_state: ShardState::Open, - leader_id: "test-ingester-2".into(), - }, - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(3), - shard_state: ShardState::Open, - leader_id: "test-ingester-1".into(), - }, - ], - remote_round_robin_idx: AtomicUsize::default(), - }; - assert!(table_entry.has_open_shards( - &ingester_pool, - &mut closed_shard_ids, - &mut unavailable_leaders - )); - assert_eq!(closed_shard_ids.len(), 1); - assert_eq!(closed_shard_ids[0], ShardId::from(1)); - assert_eq!(unavailable_leaders.len(), 1); - assert!(unavailable_leaders.contains("test-ingester-2")); + fn test_has_open_nodes_requires_cp_seed() { + let mut table = RoutingTable::default(); + let pool = IngesterPool::default(); + pool.insert("node-1".into(), mocked_ingester(None)); + + // Chitchat broadcast populates the entry, but has_open_nodes still returns false + // because the entry hasn't been seeded from the control plane yet. + table.apply_capacity_update( + "node-1".into(), + IndexUid::for_test("test-index", 0), + "test-source".into(), + 8, + 3, + ); + assert!(!table.has_open_nodes("test-index", "test-source", &pool, &HashSet::new())); + + // After merge_from_shards (CP response), has_open_nodes returns true. + let shards = vec![Shard { + index_uid: Some(IndexUid::for_test("test-index", 0)), + source_id: "test-source".to_string(), + shard_id: Some(ShardId::from(1u64)), + shard_state: ShardState::Open as i32, + leader_id: "node-1".to_string(), + ..Default::default() + }]; + table.merge_from_shards( + IndexUid::for_test("test-index", 0), + "test-source".into(), + shards, + ); + assert!(table.has_open_nodes("test-index", "test-source", &pool, &HashSet::new())); } #[test] - fn test_routing_table_entry_next_open_shard_round_robin() { - let index_uid = IndexUid::for_test("test-index", 0); - let source_id: SourceId = "test-source".into(); - let table_entry = RoutingTableEntry::empty(index_uid.clone(), source_id.clone()); - let ingester_pool = IngesterPool::default(); - let mut rate_limited_shards = HashSet::new(); - - let error = table_entry - .next_open_shard_round_robin(&ingester_pool, &rate_limited_shards) - .unwrap_err(); - assert_eq!(error, NextOpenShardError::NoShardsAvailable); - - ingester_pool.insert("test-ingester-0".into(), IngesterServiceClient::mocked()); - ingester_pool.insert("test-ingester-1".into(), IngesterServiceClient::mocked()); - - let table_entry = RoutingTableEntry { - index_uid: index_uid.clone(), - source_id: source_id.clone(), - local_shards: vec![ - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(1), - shard_state: ShardState::Closed, - leader_id: "test-ingester-0".into(), - }, - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(2), - shard_state: ShardState::Open, - leader_id: "test-ingester-0".into(), - }, - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(3), - shard_state: ShardState::Open, - leader_id: "test-ingester-0".into(), - }, - ], - local_round_robin_idx: AtomicUsize::default(), - remote_shards: Vec::new(), - remote_round_robin_idx: AtomicUsize::default(), - }; - let shard = table_entry - .next_open_shard_round_robin(&ingester_pool, &rate_limited_shards) - .unwrap(); - assert_eq!(shard.shard_id, ShardId::from(2)); + fn test_pick_node_prefers_same_az() { + let mut table = RoutingTable::new(Some("az-1".to_string())); + let pool = IngesterPool::default(); + + table.apply_capacity_update( + "node-1".into(), + IndexUid::for_test("test-index", 0), + "test-source".into(), + 5, + 1, + ); + table.apply_capacity_update( + "node-2".into(), + IndexUid::for_test("test-index", 0), + "test-source".into(), + 5, + 1, + ); + pool.insert("node-1".into(), mocked_ingester(Some("az-1"))); + pool.insert("node-2".into(), mocked_ingester(Some("az-2"))); - let shard = table_entry - .next_open_shard_round_robin(&ingester_pool, &rate_limited_shards) + let picked = table + .pick_node("test-index", "test-source", &pool, &HashSet::new()) .unwrap(); - assert_eq!(shard.shard_id, ShardId::from(3)); + assert_eq!(picked.node_id, NodeId::from("node-1")); + } - let shard = table_entry - .next_open_shard_round_robin(&ingester_pool, &rate_limited_shards) - .unwrap(); - assert_eq!(shard.shard_id, ShardId::from(2)); + #[test] + fn test_pick_node_falls_back_to_cross_az() { + let mut table = RoutingTable::new(Some("az-1".to_string())); + let pool = IngesterPool::default(); + + table.apply_capacity_update( + "node-2".into(), + IndexUid::for_test("test-index", 0), + "test-source".into(), + 5, + 1, + ); + pool.insert("node-2".into(), mocked_ingester(Some("az-2"))); - let table_entry = RoutingTableEntry { - index_uid: index_uid.clone(), - source_id: source_id.clone(), - local_shards: vec![RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(1), - shard_state: ShardState::Closed, - leader_id: "test-ingester-0".into(), - }], - local_round_robin_idx: AtomicUsize::default(), - remote_shards: vec![ - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(2), - shard_state: ShardState::Open, - leader_id: "test-ingester-1".into(), - }, - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(3), - shard_state: ShardState::Closed, - leader_id: "test-ingester-1".into(), - }, - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(4), - shard_state: ShardState::Open, - leader_id: "test-ingester-2".into(), - }, - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(5), - shard_state: ShardState::Open, - leader_id: "test-ingester-1".into(), - }, - ], - remote_round_robin_idx: AtomicUsize::default(), - }; - let shard = table_entry - .next_open_shard_round_robin(&ingester_pool, &rate_limited_shards) + let picked = table + .pick_node("test-index", "test-source", &pool, &HashSet::new()) .unwrap(); - assert_eq!(shard.shard_id, ShardId::from(2)); + assert_eq!(picked.node_id, NodeId::from("node-2")); + } - let shard = table_entry - .next_open_shard_round_robin(&ingester_pool, &rate_limited_shards) - .unwrap(); - assert_eq!(shard.shard_id, ShardId::from(5)); + #[test] + fn test_pick_node_no_az_awareness() { + let mut table = RoutingTable::default(); + let pool = IngesterPool::default(); + + table.apply_capacity_update( + "node-1".into(), + IndexUid::for_test("test-index", 0), + "test-source".into(), + 5, + 1, + ); + pool.insert("node-1".into(), mocked_ingester(Some("az-1"))); - let shard = table_entry - .next_open_shard_round_robin(&ingester_pool, &rate_limited_shards) + let picked = table + .pick_node("test-index", "test-source", &pool, &HashSet::new()) .unwrap(); - assert_eq!(shard.shard_id, ShardId::from(2)); + assert_eq!(picked.node_id, NodeId::from("node-1")); + } - rate_limited_shards.insert(ShardId::from(5)); + #[test] + fn test_pick_node_missing_entry() { + let table = RoutingTable::new(Some("az-1".to_string())); + let pool = IngesterPool::default(); + + assert!( + table + .pick_node("nonexistent", "source", &pool, &HashSet::new()) + .is_none() + ); + } - let shard = table_entry - .next_open_shard_round_robin(&ingester_pool, &rate_limited_shards) - .unwrap(); - assert_eq!(shard.shard_id, ShardId::from(2)); + #[test] + fn test_power_of_two_choices() { + // 3 candidates: best appears in the random pair 2/3 of the time and always + // wins when it does, so it should win ~67% of 1000 runs. Asserting > 550 + // is ~7.5 standard deviations from the mean — effectively impossible to flake. + let high = IngesterNode { + node_id: "high".into(), + index_uid: IndexUid::for_test("idx", 0), + capacity_score: 9, + open_shard_count: 2, + }; + let mid = IngesterNode { + node_id: "mid".into(), + index_uid: IndexUid::for_test("idx", 0), + capacity_score: 5, + open_shard_count: 2, + }; + let low = IngesterNode { + node_id: "low".into(), + index_uid: IndexUid::for_test("idx", 0), + capacity_score: 1, + open_shard_count: 2, + }; + let candidates: Vec<&IngesterNode> = vec![&high, &mid, &low]; + + let mut high_wins = 0; + for _ in 0..1000 { + if power_of_two_choices(&candidates).node_id == "high" { + high_wins += 1; + } + } + assert!(high_wins > 550, "high won only {high_wins}/1000 times"); } #[test] - fn test_routing_table_entry_next_open_shard_round_robin_rate_limited_error() { + fn test_merge_from_shards() { + let mut table = RoutingTable::default(); let index_uid = IndexUid::for_test("test-index", 0); - let source_id: SourceId = "test-source".into(); + let key = ("test-index".to_string(), "test-source".to_string()); + + let make_shard = |id: u64, leader: &str, open: bool| Shard { + index_uid: Some(index_uid.clone()), + source_id: "test-source".to_string(), + shard_id: Some(ShardId::from(id)), + shard_state: if open { + ShardState::Open as i32 + } else { + ShardState::Closed as i32 + }, + leader_id: leader.to_string(), + ..Default::default() + }; - let ingester_pool = IngesterPool::default(); - ingester_pool.insert("test-ingester-0".into(), IngesterServiceClient::mocked()); + // Two open shards on node-1, one open + one closed on node-2, only closed on node-3. + let shards = vec![ + make_shard(1, "node-1", true), + make_shard(2, "node-1", true), + make_shard(3, "node-2", true), + make_shard(4, "node-2", false), + make_shard(5, "node-3", false), + ]; + table.merge_from_shards(index_uid.clone(), "test-source".into(), shards); - let rate_limited_shards = HashSet::from_iter([ShardId::from(1)]); + let entry = table.table.get(&key).unwrap(); + assert_eq!(entry.nodes.len(), 3); - let table_entry = RoutingTableEntry { - index_uid: index_uid.clone(), - source_id: source_id.clone(), - local_shards: vec![RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(1), - shard_state: ShardState::Open, - leader_id: "test-ingester-0".into(), - }], - local_round_robin_idx: AtomicUsize::default(), - remote_shards: Vec::new(), - remote_round_robin_idx: AtomicUsize::default(), - }; - let error = table_entry - .next_open_shard_round_robin(&ingester_pool, &rate_limited_shards) - .unwrap_err(); - assert_eq!(error, NextOpenShardError::RateLimited); - } + let n1 = entry.nodes.get("node-1").unwrap(); + assert_eq!(n1.open_shard_count, 2); + assert_eq!(n1.capacity_score, 5); - #[test] - fn test_routing_table_entry_insert_open_shards() { - let index_uid_0 = IndexUid::for_test("test-index", 0); - let source_id: SourceId = "test-source".into(); - let mut table_entry = RoutingTableEntry::empty(index_uid_0.clone(), source_id.clone()); - - let local_node_id: NodeId = "test-ingester-0".into(); - let remote_node_id: NodeId = "test-ingester-1".into(); - table_entry.insert_open_shards(&local_node_id, &local_node_id, &index_uid_0, &[]); - - assert_eq!(table_entry.local_shards.len(), 0); - assert_eq!(table_entry.remote_shards.len(), 0); - - table_entry.insert_open_shards( - &local_node_id, - &local_node_id, - &index_uid_0, - &[ShardId::from(2)], - ); + let n2 = entry.nodes.get("node-2").unwrap(); + assert_eq!(n2.open_shard_count, 1); - assert_eq!(table_entry.local_shards.len(), 1); - assert_eq!(table_entry.remote_shards.len(), 0); - - assert_eq!(table_entry.local_shards[0].index_uid, index_uid_0); - assert_eq!(table_entry.local_shards[0].source_id, source_id); - assert_eq!(table_entry.local_shards[0].shard_id, ShardId::from(2)); - assert_eq!(table_entry.local_shards[0].shard_state, ShardState::Open); - assert_eq!(table_entry.local_shards[0].leader_id, local_node_id); - - table_entry.local_shards[0].shard_state = ShardState::Closed; - table_entry.insert_open_shards( - &local_node_id, - &local_node_id, - &index_uid_0, - &[ShardId::from(1), ShardId::from(2)], - ); + let n3 = entry.nodes.get("node-3").unwrap(); + assert_eq!(n3.open_shard_count, 0); - assert_eq!(table_entry.local_shards.len(), 2); - assert_eq!(table_entry.remote_shards.len(), 0); + // Merging again adds new nodes but preserves existing ones. + let shards = vec![make_shard(10, "node-4", true)]; + table.merge_from_shards(index_uid, "test-source".into(), shards); - assert_eq!(table_entry.local_shards[0].shard_id, ShardId::from(1)); - assert_eq!(table_entry.local_shards[0].shard_state, ShardState::Open); - assert_eq!(table_entry.local_shards[1].shard_id, ShardId::from(2)); - assert_eq!(table_entry.local_shards[1].shard_state, ShardState::Closed); + let entry = table.table.get(&key).unwrap(); + assert_eq!(entry.nodes.len(), 4); + assert!(entry.nodes.contains_key("node-1")); + assert!(entry.nodes.contains_key("node-2")); + assert!(entry.nodes.contains_key("node-3")); + assert!(entry.nodes.contains_key("node-4")); + } - table_entry.local_shards.clear(); - table_entry.insert_open_shards( - &local_node_id, - &remote_node_id, - &index_uid_0, - &[ShardId::from(2)], + #[test] + fn test_classify_az_locality() { + let table = RoutingTable::new(Some("az-1".to_string())); + let pool = IngesterPool::default(); + pool.insert("node-local".into(), mocked_ingester(Some("az-1"))); + pool.insert("node-remote".into(), mocked_ingester(Some("az-2"))); + pool.insert("node-no-az".into(), mocked_ingester(None)); + + assert_eq!( + table.classify_az_locality(&"node-local".into(), &pool), + "same_az" ); - - assert_eq!(table_entry.local_shards.len(), 0); - assert_eq!(table_entry.remote_shards.len(), 1); - - assert_eq!(table_entry.remote_shards[0].index_uid, index_uid_0); - assert_eq!(table_entry.remote_shards[0].source_id, source_id); - assert_eq!(table_entry.remote_shards[0].shard_id, ShardId::from(2)); - assert_eq!(table_entry.remote_shards[0].shard_state, ShardState::Open); - assert_eq!(table_entry.remote_shards[0].leader_id, remote_node_id); - - table_entry.remote_shards[0].shard_state = ShardState::Closed; - table_entry.insert_open_shards( - &local_node_id, - &remote_node_id, - &index_uid_0, - &[ShardId::from(1), ShardId::from(2)], + assert_eq!( + table.classify_az_locality(&"node-remote".into(), &pool), + "cross_az" ); - - assert_eq!(table_entry.local_shards.len(), 0); - assert_eq!(table_entry.remote_shards.len(), 2); - - assert_eq!(table_entry.remote_shards[0].shard_id, ShardId::from(1)); - assert_eq!(table_entry.remote_shards[0].shard_state, ShardState::Open); - assert_eq!(table_entry.remote_shards[1].shard_id, ShardId::from(2)); - assert_eq!(table_entry.remote_shards[1].shard_state, ShardState::Closed); - - // Update index incarnation. - let index_uid_1 = IndexUid::for_test("test-index", 1); - table_entry.insert_open_shards( - &local_node_id, - &local_node_id, - &index_uid_1, - &[ShardId::from(1)], + assert_eq!( + table.classify_az_locality(&"node-no-az".into(), &pool), + "az_unaware" ); - assert_eq!(table_entry.index_uid, index_uid_1); - assert_eq!(table_entry.local_shards.len(), 1); - assert_eq!(table_entry.remote_shards.len(), 0); - - assert_eq!(table_entry.local_shards[0].index_uid, index_uid_1); - assert_eq!(table_entry.local_shards[0].source_id, source_id); - assert_eq!(table_entry.local_shards[0].shard_id, ShardId::from(1)); - assert_eq!(table_entry.local_shards[0].shard_state, ShardState::Open); - assert_eq!(table_entry.local_shards[0].leader_id, local_node_id); - - // Ignore previous index incarnation. - table_entry.insert_open_shards( - &local_node_id, - &local_node_id, - &index_uid_0, - &[ShardId::from(12), ShardId::from(42), ShardId::from(1337)], + let table_no_az = RoutingTable::default(); + assert_eq!( + table_no_az.classify_az_locality(&"node-local".into(), &pool), + "az_unaware" ); - assert_eq!(table_entry.index_uid, index_uid_1); - assert_eq!(table_entry.local_shards.len(), 1); - assert_eq!(table_entry.remote_shards.len(), 0); } #[test] - fn test_routing_table_entry_close_shards() { - let index_uid = IndexUid::for_test("test-index", 0); - let source_id: SourceId = "test-source".into(); - - let mut table_entry = RoutingTableEntry::empty(index_uid.clone(), source_id.clone()); - table_entry.close_shards(&index_uid, &[]); - table_entry.close_shards(&index_uid, &[ShardId::from(1)]); - assert!(table_entry.local_shards.is_empty()); - assert!(table_entry.remote_shards.is_empty()); - - let mut table_entry = RoutingTableEntry { - index_uid: index_uid.clone(), - source_id: source_id.clone(), - local_shards: vec![ - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(1), - shard_state: ShardState::Open, - leader_id: "test-ingester-0".into(), - }, - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(2), - shard_state: ShardState::Open, - leader_id: "test-ingester-0".into(), - }, - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(3), - shard_state: ShardState::Open, - leader_id: "test-ingester-0".into(), - }, - ], - local_round_robin_idx: AtomicUsize::default(), - remote_shards: vec![ - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(5), - shard_state: ShardState::Open, - leader_id: "test-ingester-1".into(), - }, - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(6), - shard_state: ShardState::Open, - leader_id: "test-ingester-1".into(), - }, - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(7), - shard_state: ShardState::Open, - leader_id: "test-ingester-1".into(), - }, - ], - remote_round_robin_idx: AtomicUsize::default(), - }; - table_entry.close_shards( - &index_uid, - &[ - ShardId::from(1), - ShardId::from(3), - ShardId::from(4), - ShardId::from(6), - ShardId::from(8), - ], + fn test_incarnation_check_clears_stale_nodes() { + let mut table = RoutingTable::default(); + let key = ("test-index".to_string(), "test-source".to_string()); + + // Populate with incarnation 0: two nodes. + table.apply_capacity_update( + "node-1".into(), + IndexUid::for_test("test-index", 0), + "test-source".into(), + 8, + 3, ); - assert!(table_entry.local_shards[0].shard_state.is_closed()); - assert!(table_entry.local_shards[1].shard_state.is_open()); - assert!(table_entry.local_shards[2].shard_state.is_closed()); - assert!(table_entry.remote_shards[0].shard_state.is_open()); - assert!(table_entry.remote_shards[1].shard_state.is_closed()); - assert!(table_entry.remote_shards[2].shard_state.is_open()); - } - - #[test] - fn test_routing_table_entry_delete_shards() { - let index_uid = IndexUid::for_test("test-index", 0); - let source_id: SourceId = "test-source".into(); - - let mut table_entry = RoutingTableEntry::empty(index_uid.clone(), source_id.clone()); - table_entry.delete_shards(&index_uid, &[]); - table_entry.delete_shards(&index_uid, &[ShardId::from(1)]); - assert!(table_entry.local_shards.is_empty()); - assert!(table_entry.remote_shards.is_empty()); - - let mut table_entry = RoutingTableEntry { - index_uid: index_uid.clone(), - source_id: source_id.clone(), - local_shards: vec![ - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(1), - shard_state: ShardState::Open, - leader_id: "test-ingester-0".into(), - }, - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(2), - shard_state: ShardState::Open, - leader_id: "test-ingester-0".into(), - }, - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(3), - shard_state: ShardState::Open, - leader_id: "test-ingester-0".into(), - }, - ], - local_round_robin_idx: AtomicUsize::default(), - remote_shards: vec![ - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(5), - shard_state: ShardState::Open, - leader_id: "test-ingester-1".into(), - }, - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(6), - shard_state: ShardState::Open, - leader_id: "test-ingester-1".into(), - }, - RoutingEntry { - index_uid: index_uid.clone(), - source_id: "test-source".to_string(), - shard_id: ShardId::from(7), - shard_state: ShardState::Open, - leader_id: "test-ingester-1".into(), - }, - ], - remote_round_robin_idx: AtomicUsize::default(), - }; - table_entry.delete_shards( - &index_uid, - &[ - ShardId::from(1), - ShardId::from(3), - ShardId::from(4), - ShardId::from(6), - ShardId::from(8), - ], + table.apply_capacity_update( + "node-2".into(), + IndexUid::for_test("test-index", 0), + "test-source".into(), + 6, + 2, + ); + let entry = table.table.get(&key).unwrap(); + assert_eq!(entry.nodes.len(), 2); + assert_eq!(entry.index_uid, IndexUid::for_test("test-index", 0)); + + // Capacity update with incarnation 1 clears stale nodes. + table.apply_capacity_update( + "node-3".into(), + IndexUid::for_test("test-index", 1), + "test-source".into(), + 5, + 1, + ); + let entry = table.table.get(&key).unwrap(); + assert_eq!(entry.nodes.len(), 1); + assert!(entry.nodes.contains_key("node-3")); + assert!(!entry.nodes.contains_key("node-1")); + assert!(!entry.nodes.contains_key("node-2")); + assert_eq!(entry.index_uid, IndexUid::for_test("test-index", 1)); + + // merge_from_shards with incarnation 2 clears stale nodes. + let shards = vec![Shard { + index_uid: Some(IndexUid::for_test("test-index", 2)), + source_id: "test-source".to_string(), + shard_id: Some(ShardId::from(1u64)), + shard_state: ShardState::Open as i32, + leader_id: "node-4".to_string(), + ..Default::default() + }]; + table.merge_from_shards( + IndexUid::for_test("test-index", 2), + "test-source".into(), + shards, ); - assert_eq!(table_entry.local_shards.len(), 1); - assert_eq!(table_entry.local_shards[0].shard_id, ShardId::from(2)); - assert_eq!(table_entry.remote_shards.len(), 2); - assert_eq!(table_entry.remote_shards[0].shard_id, ShardId::from(5)); - assert_eq!(table_entry.remote_shards[1].shard_id, ShardId::from(7)); + let entry = table.table.get(&key).unwrap(); + assert_eq!(entry.nodes.len(), 1); + assert!(entry.nodes.contains_key("node-4")); + assert!(!entry.nodes.contains_key("node-3")); + assert_eq!(entry.index_uid, IndexUid::for_test("test-index", 2)); } } diff --git a/quickwit/quickwit-ingest/src/ingest_v2/state.rs b/quickwit/quickwit-ingest/src/ingest_v2/state.rs index 401e31e258e..bcf31b2f100 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/state.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/state.rs @@ -19,23 +19,28 @@ use std::path::Path; use std::sync::{Arc, Weak}; use std::time::{Duration, Instant}; +use bytesize::ByteSize; +use itertools::Itertools; use mrecordlog::error::{DeleteQueueError, TruncateError}; +use quickwit_cluster::Cluster; use quickwit_common::pretty::PrettyDisplay; use quickwit_common::rate_limiter::{RateLimiter, RateLimiterSettings}; +use quickwit_common::shared_consts::INGESTER_STATUS_KEY; use quickwit_doc_mapper::DocMapper; use quickwit_proto::control_plane::AdviseResetShardsResponse; use quickwit_proto::ingest::ingester::IngesterStatus; -use quickwit_proto::ingest::{IngestV2Error, IngestV2Result, ShardState}; -use quickwit_proto::types::{DocMappingUid, Position, QueueId}; +use quickwit_proto::ingest::{IngestV2Error, IngestV2Result, ShardIds, ShardState}; +use quickwit_proto::types::{DocMappingUid, IndexUid, Position, QueueId, SourceId, split_queue_id}; use tokio::sync::{Mutex, MutexGuard, RwLock, RwLockMappedWriteGuard, RwLockWriteGuard, watch}; use tracing::{error, info}; use super::models::IngesterShard; use super::rate_meter::RateMeter; use super::replication::{ReplicationStreamTaskHandle, ReplicationTaskHandle}; +use super::wal_capacity_tracker::WalCapacityTracker; use crate::ingest_v2::mrecordlog_utils::{force_delete_queue, queue_position_range}; use crate::mrecordlog_async::MultiRecordLogAsync; -use crate::{FollowerId, LeaderId}; +use crate::{FollowerId, LeaderId, OpenShardCounts}; /// Stores the state of the ingester and attempts to prevent deadlocks by exposing an API that /// guarantees that the internal data structures are always locked in the same order. @@ -54,11 +59,12 @@ pub(super) struct IngesterState { pub(super) struct InnerIngesterState { pub shards: HashMap, pub doc_mappers: HashMap>, - pub rate_trackers: HashMap, // Replication stream opened with followers. pub replication_streams: HashMap, // Replication tasks running for each replication stream opened with leaders. pub replication_tasks: HashMap, + cluster: Cluster, + pub wal_capacity_tracker: WalCapacityTracker, status: IngesterStatus, status_tx: watch::Sender, } @@ -68,25 +74,85 @@ impl InnerIngesterState { self.status } - pub fn set_status(&mut self, status: IngesterStatus) { + pub async fn set_status(&mut self, status: IngesterStatus) { self.status = status; self.status_tx.send(status).expect("channel should be open"); + self.cluster + .set_self_key_value(INGESTER_STATUS_KEY, status.as_json_str_name()) + .await; + } + + /// Returns the shard with the most available permits for this index and source. + pub fn find_most_capacity_shard_mut( + &mut self, + index_uid: &IndexUid, + source_id: &SourceId, + ) -> Option<&mut IngesterShard> { + self.shards + .values_mut() + .filter(|shard| { + shard.is_open() && shard.index_uid == *index_uid && shard.source_id == *source_id + }) + .map(|shard| (shard.rate_limiter.available_permits(), shard)) + .max_by_key(|(available_permits, _)| *available_permits) + .map(|(_, shard)| shard) + } + + /// Returns per-source open shard counts and closed shard IDs for all advertisable, + /// non-replica shards. + pub fn get_shard_snapshot(&self) -> (OpenShardCounts, Vec) { + let grouped = self + .shards + .values() + .filter(|shard| shard.is_advertisable && !shard.is_replica()) + .map(|shard| ((shard.index_uid.clone(), shard.source_id.clone()), shard)) + .into_group_map(); + + let mut open_counts = Vec::new(); + let mut closed_shards = Vec::new(); + + for ((index_uid, source_id), shards) in grouped { + let mut open_count = 0; + let mut closed_ids = Vec::new(); + + for shard in shards { + if shard.is_open() { + open_count += 1; + } else if shard.is_closed() { + closed_ids.push(shard.shard_id.clone()); + } + } + open_counts.push((index_uid.clone(), source_id.clone(), open_count)); + if !closed_ids.is_empty() { + closed_shards.push(ShardIds { + index_uid: Some(index_uid), + source_id, + shard_ids: closed_ids, + }); + } + } + (open_counts, closed_shards) } } impl IngesterState { - fn new() -> Self { + async fn create(cluster: Cluster, disk_capacity: ByteSize, memory_capacity: ByteSize) -> Self { let status = IngesterStatus::Initializing; let (status_tx, status_rx) = watch::channel(status); - let inner = InnerIngesterState { + let mut inner = InnerIngesterState { shards: Default::default(), doc_mappers: Default::default(), - rate_trackers: Default::default(), replication_streams: Default::default(), replication_tasks: Default::default(), + cluster, + wal_capacity_tracker: WalCapacityTracker::new(disk_capacity, memory_capacity), status, status_tx, }; + // We call `set_status` here instead of setting it directly because it also updates the + // ingester status in chitchat. + inner.set_status(IngesterStatus::Initializing).await; + let inner = Arc::new(Mutex::new(inner)); let mrecordlog = Arc::new(RwLock::new(None)); @@ -97,8 +163,14 @@ impl IngesterState { } } - pub fn load(wal_dir_path: &Path, rate_limiter_settings: RateLimiterSettings) -> Self { - let state = Self::new(); + pub async fn load( + cluster: Cluster, + wal_dir_path: &Path, + disk_capacity: ByteSize, + memory_capacity: ByteSize, + rate_limiter_settings: RateLimiterSettings, + ) -> Self { + let state = Self::create(cluster, disk_capacity, memory_capacity).await; let state_clone = state.clone(); let wal_dir_path = wal_dir_path.to_path_buf(); @@ -111,15 +183,26 @@ impl IngesterState { } #[cfg(test)] - pub async fn for_test() -> (tempfile::TempDir, Self) { + pub async fn for_test(cluster: Cluster) -> (tempfile::TempDir, Self) { + Self::for_test_with_disk_capacity(cluster, ByteSize::mb(256)).await + } + + #[cfg(test)] + pub async fn for_test_with_disk_capacity( + cluster: Cluster, + disk_capacity: ByteSize, + ) -> (tempfile::TempDir, Self) { let temp_dir = tempfile::tempdir().unwrap(); - let mut state = IngesterState::load(temp_dir.path(), RateLimiterSettings::default()); + let mut state = IngesterState::load( + cluster, + temp_dir.path(), + disk_capacity, + ByteSize::mb(256), + RateLimiterSettings::default(), + ) + .await; - state - .status_rx - .wait_for(|status| *status == IngesterStatus::Ready) - .await - .unwrap(); + state.wait_for_ready().await; (temp_dir, state) } @@ -128,8 +211,10 @@ impl IngesterState { /// queues. Empty queues are deleted, while non-empty queues are recovered. However, the /// corresponding shards are closed and become read-only. pub async fn init(&self, wal_dir_path: &Path, rate_limiter_settings: RateLimiterSettings) { - let mut inner_guard = self.inner.lock().await; + // Acquire locks in the same order as `lock_fully` (mrecordlog first, then inner) to + // prevent ABBA deadlocks with the broadcast capacity task. let mut mrecordlog_guard = self.mrecordlog.write().await; + let mut inner_guard = self.inner.lock().await; let now = Instant::now(); @@ -154,7 +239,7 @@ impl IngesterState { } Err(error) => { error!("failed to open WAL: {error}"); - inner_guard.set_status(IngesterStatus::Failed); + inner_guard.set_status(IngesterStatus::Failed).await; return; } }; @@ -172,6 +257,10 @@ impl IngesterState { for queue_id in queue_ids { if let Some(position_range) = queue_position_range(&mrecordlog, &queue_id) { + let Some((index_uid, source_id, shard_id)) = split_queue_id(&queue_id) else { + // `split_queue_id` already logs an error. + continue; + }; // The queue is not empty: recover it. let replication_position_inclusive = Position::offset(*position_range.end()); let truncation_position_inclusive = if *position_range.start() == 0 { @@ -179,23 +268,20 @@ impl IngesterState { } else { Position::offset(*position_range.start() - 1) }; - let mut solo_shard = IngesterShard::new_solo( - ShardState::Closed, - replication_position_inclusive, - truncation_position_inclusive, - None, - now, - false, - ); - // We want to advertise the shard as read-only right away. - solo_shard.is_advertisable = true; - inner_guard.shards.insert(queue_id.clone(), solo_shard); - let rate_limiter = RateLimiter::from_settings(rate_limiter_settings); let rate_meter = RateMeter::default(); - inner_guard - .rate_trackers - .insert(queue_id, (rate_limiter, rate_meter)); + // We want to advertise the shard as read-only right away. + let solo_shard = + IngesterShard::new_solo(index_uid.clone(), source_id.clone(), shard_id.clone()) + .with_state(ShardState::Closed) + .with_replication_position_inclusive(replication_position_inclusive) + .with_truncation_position_inclusive(truncation_position_inclusive) + .with_rate_limiter(rate_limiter) + .with_rate_meter(rate_meter) + .with_last_write(now) + .advertisable() // We want to advertise the shard as read-only right away. + .build(); + inner_guard.shards.insert(queue_id.clone(), solo_shard); num_closed_shards += 1; } else { @@ -214,7 +300,7 @@ impl IngesterState { info!("deleted {num_deleted_shards} empty shard(s)"); } mrecordlog_guard.replace(mrecordlog); - inner_guard.set_status(IngesterStatus::Ready); + inner_guard.set_status(IngesterStatus::Ready).await; } pub async fn wait_for_ready(&mut self) { @@ -336,14 +422,12 @@ impl DerefMut for FullyLockedIngesterState<'_> { impl FullyLockedIngesterState<'_> { /// Deletes the shard identified by `queue_id` from the ingester state. It removes the /// mrecordlog queue first and then removes the associated in-memory shard and rate trackers. - pub async fn delete_shard(&mut self, queue_id: &QueueId) { + pub async fn delete_shard(&mut self, queue_id: &QueueId, initiator: &'static str) { match self.mrecordlog.delete_queue(queue_id).await { Ok(_) | Err(DeleteQueueError::MissingQueue(_)) => { - self.rate_trackers.remove(queue_id); - // Log only if the shard was actually removed. if let Some(shard) = self.shards.remove(queue_id) { - info!("deleted shard `{queue_id}`"); + info!("deleted shard `{queue_id}` initiated via `{initiator}`"); if let Some(doc_mapper) = shard.doc_mapper_opt { // At this point, we hold the lock so we can safely check the strong count. @@ -371,37 +455,34 @@ impl FullyLockedIngesterState<'_> { &mut self, queue_id: &QueueId, truncate_up_to_position_inclusive: Position, + initiator: &'static str, ) { - // TODO: Replace with if-let-chains when stabilized. - let Some(truncate_up_to_offset_inclusive) = truncate_up_to_position_inclusive.as_u64() - else { - return; - }; - let Some(shard) = self.inner.shards.get_mut(queue_id) else { - return; - }; - if shard.truncation_position_inclusive >= truncate_up_to_position_inclusive { - return; - } - match self - .mrecordlog - .truncate(queue_id, truncate_up_to_offset_inclusive) - .await + if let Some(truncate_up_to_offset_inclusive) = truncate_up_to_position_inclusive.as_u64() + && let Some(shard) = self.inner.shards.get_mut(queue_id) + && shard.truncation_position_inclusive < truncate_up_to_position_inclusive { - Ok(_) => { - info!("truncated shard `{queue_id}` at {truncate_up_to_position_inclusive}"); - shard.truncation_position_inclusive = truncate_up_to_position_inclusive; - } - Err(TruncateError::MissingQueue(_)) => { - error!("failed to truncate shard `{queue_id}`: WAL queue not found"); - self.shards.remove(queue_id); - self.rate_trackers.remove(queue_id); - info!("deleted dangling shard `{queue_id}`"); - } - Err(TruncateError::IoError(io_error)) => { - error!("failed to truncate shard `{queue_id}`: {io_error}"); - } - }; + match self + .mrecordlog + .truncate(queue_id, truncate_up_to_offset_inclusive) + .await + { + Ok(_) => { + info!( + "truncated shard `{queue_id}` at {truncate_up_to_position_inclusive} \ + initiated via `{initiator}`" + ); + shard.truncation_position_inclusive = truncate_up_to_position_inclusive; + } + Err(TruncateError::MissingQueue(_)) => { + error!("failed to truncate shard `{queue_id}`: WAL queue not found"); + self.shards.remove(queue_id); + info!("deleted dangling shard `{queue_id}`"); + } + Err(TruncateError::IoError(io_error)) => { + error!("failed to truncate shard `{queue_id}`: {io_error}"); + } + }; + } } /// Deletes and truncates the shards as directed by the `advise_reset_shards_response` returned @@ -410,12 +491,18 @@ impl FullyLockedIngesterState<'_> { info!("resetting shards"); for shard_ids in &advise_reset_shards_response.shards_to_delete { for queue_id in shard_ids.queue_ids() { - self.delete_shard(&queue_id).await; + self.delete_shard(&queue_id, "control-plane-reset-shards-rpc") + .await; } } for shard_id_positions in &advise_reset_shards_response.shards_to_truncate { for (queue_id, publish_position) in shard_id_positions.queue_id_positions() { - self.truncate_shard(&queue_id, publish_position).await; + self.truncate_shard( + &queue_id, + publish_position, + "control-plane-reset-shards-rpc", + ) + .await; } } } @@ -444,13 +531,29 @@ impl WeakIngesterState { #[cfg(test)] mod tests { + use bytesize::ByteSize; + use quickwit_cluster::{ChannelTransport, create_cluster_for_test}; + use quickwit_config::service::QuickwitService; + use quickwit_proto::types::{NodeId, ShardId, SourceId}; use tokio::time::timeout; use super::*; + async fn test_cluster() -> Cluster { + create_cluster_for_test( + Vec::new(), + &[QuickwitService::Indexer.as_str()], + &ChannelTransport::default(), + true, + ) + .await + .unwrap() + } + #[tokio::test] async fn test_ingester_state_does_not_lock_while_initializing() { - let state = IngesterState::new(); + let cluster = test_cluster().await; + let state = IngesterState::create(cluster, ByteSize::mb(256), ByteSize::mb(256)).await; let inner_guard = state.inner.lock().await; assert_eq!(inner_guard.status(), IngesterStatus::Initializing); @@ -465,9 +568,15 @@ mod tests { #[tokio::test] async fn test_ingester_state_failed() { - let state = IngesterState::new(); + let cluster = test_cluster().await; + let state = IngesterState::create(cluster, ByteSize::mb(256), ByteSize::mb(256)).await; - state.inner.lock().await.set_status(IngesterStatus::Failed); + state + .inner + .lock() + .await + .set_status(IngesterStatus::Failed) + .await; let error = state.lock_partially().await.unwrap_err().to_string(); assert!(error.to_string().ends_with("failed to initialize ingester")); @@ -478,7 +587,8 @@ mod tests { #[tokio::test] async fn test_ingester_state_init() { - let mut state = IngesterState::new(); + let cluster = test_cluster().await; + let mut state = IngesterState::create(cluster, ByteSize::mb(256), ByteSize::mb(256)).await; let temp_dir = tempfile::tempdir().unwrap(); state @@ -495,4 +605,259 @@ mod tests { assert_eq!(locked_state.status(), IngesterStatus::Ready); assert_eq!(*locked_state.status_tx.borrow(), IngesterStatus::Ready); } + + fn insert_shard_with_used_capacity( + state: &mut InnerIngesterState, + index_uid: IndexUid, + source_id: SourceId, + shard_id: ShardId, + shard_state: ShardState, + used_capacity: ByteSize, + ) { + let mut shard = IngesterShard::new_solo(index_uid, source_id, shard_id) + .with_state(shard_state) + .build(); + shard.rate_limiter.acquire_bytes(used_capacity); + + let queue_id = shard.queue_id(); + state.shards.insert(queue_id, shard); + } + + #[tokio::test] + async fn test_find_most_capacity_shard_returns_shard_with_least_used_capacity() { + let cluster = create_cluster_for_test( + Vec::new(), + &[QuickwitService::Indexer.as_str()], + &ChannelTransport::default(), + true, + ) + .await + .unwrap(); + let (_temp_dir, state) = IngesterState::for_test(cluster).await; + let mut state_guard = state.lock_partially().await.unwrap(); + + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); + + // Shard 1: 1KB used (most available capacity) + // Shard 2: 2KB used + // ... + // Shard 5: 5KB used (least available capacity) + for i in 1..=5u64 { + insert_shard_with_used_capacity( + &mut state_guard, + index_uid.clone(), + source_id.clone(), + ShardId::from(i), + ShardState::Open, + ByteSize::kb(i), + ); + } + + let shard = state_guard + .find_most_capacity_shard_mut(&index_uid, &source_id) + .unwrap(); + + assert_eq!(shard.shard_id, ShardId::from(1)); + assert_eq!(shard.shard_state, ShardState::Open); + + let expected_available_permits = + RateLimiterSettings::default().burst_limit - ByteSize::kb(1).as_u64(); + assert_eq!( + shard.rate_limiter.available_permits(), + expected_available_permits + ); + } + + #[tokio::test] + async fn test_find_most_capacity_shard_skips_closed_shards() { + let cluster = create_cluster_for_test( + Vec::new(), + &[QuickwitService::Indexer.as_str()], + &ChannelTransport::default(), + true, + ) + .await + .unwrap(); + let (_temp_dir, state) = IngesterState::for_test(cluster).await; + let mut locked_state = state.lock_partially().await.unwrap(); + + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); + + insert_shard_with_used_capacity( + &mut locked_state, + index_uid.clone(), + source_id.clone(), + ShardId::from(1), + ShardState::Open, + ByteSize::kb(1), + ); + insert_shard_with_used_capacity( + &mut locked_state, + index_uid.clone(), + source_id.clone(), + ShardId::from(2), + ShardState::Open, + ByteSize::kb(2), + ); + + insert_shard_with_used_capacity( + &mut locked_state, + index_uid.clone(), + source_id.clone(), + ShardId::from(3), + ShardState::Closed, + ByteSize::kb(0), + ); + + let shard = locked_state + .find_most_capacity_shard_mut(&index_uid, &source_id) + .unwrap(); + + // Should pick shard 1 (most capacity among open shards), not shard 3 (closed) + assert_eq!(shard.shard_id, ShardId::from(1)); + } + + #[tokio::test] + async fn test_find_most_capacity_shard_returns_none_for_unknown_index_or_source() { + let cluster = create_cluster_for_test( + Vec::new(), + &[QuickwitService::Indexer.as_str()], + &ChannelTransport::default(), + true, + ) + .await + .unwrap(); + let (_temp_dir, state) = IngesterState::for_test(cluster).await; + let mut locked_state = state.lock_partially().await.unwrap(); + + let index_uid = IndexUid::for_test("test-index", 0); + let source_id = SourceId::from("test-source"); + + insert_shard_with_used_capacity( + &mut locked_state, + index_uid.clone(), + source_id.clone(), + ShardId::from(1), + ShardState::Open, + ByteSize::kb(0), + ); + + let shard_opt = locked_state + .find_most_capacity_shard_mut(&IndexUid::for_test("other-index", 0), &source_id); + assert!(shard_opt.is_none()); + + let shard_opt = + locked_state.find_most_capacity_shard_mut(&index_uid, &SourceId::from("other-source")); + assert!(shard_opt.is_none()); + } + + #[tokio::test] + async fn test_ingester_state_set_status() { + let cluster = test_cluster().await; + let state = + IngesterState::create(cluster.clone(), ByteSize::mb(256), ByteSize::mb(256)).await; + let temp_dir = tempfile::tempdir().unwrap(); + + state + .init(temp_dir.path(), RateLimiterSettings::default()) + .await; + + let mut state_guard = state.lock_fully().await.unwrap(); + state_guard.set_status(IngesterStatus::Failed).await; + assert_eq!(state_guard.status(), IngesterStatus::Failed); + assert_eq!(*state.status_rx.borrow(), IngesterStatus::Failed); + + let status_json_str = cluster + .get_self_key_value(INGESTER_STATUS_KEY) + .await + .unwrap(); + let status = IngesterStatus::from_json_str_name(&status_json_str).unwrap(); + assert_eq!(status, IngesterStatus::Failed); + } + + fn open_shard( + index_uid: IndexUid, + source_id: SourceId, + shard_id: ShardId, + is_replica: bool, + ) -> IngesterShard { + let builder = if is_replica { + IngesterShard::new_replica(index_uid, source_id, shard_id, NodeId::from("test-leader")) + } else { + IngesterShard::new_solo(index_uid, source_id, shard_id) + }; + builder.advertisable().build() + } + + #[tokio::test] + async fn test_get_shard_snapshot() { + let cluster = test_cluster().await; + let (_temp_dir, state) = IngesterState::for_test(cluster).await; + let mut state_guard = state.lock_partially().await.unwrap(); + + let index_uid = IndexUid::for_test("test-index", 0); + + // source-a: 2 open shards + 1 closed shard + 1 replica (ignored). + let s = open_shard( + index_uid.clone(), + "source-a".into(), + ShardId::from(1), + false, + ); + state_guard.shards.insert(s.queue_id(), s); + let s = open_shard( + index_uid.clone(), + "source-a".into(), + ShardId::from(2), + false, + ); + state_guard.shards.insert(s.queue_id(), s); + let s = IngesterShard::new_solo(index_uid.clone(), "source-a".into(), ShardId::from(3)) + .with_state(ShardState::Closed) + .advertisable() + .build(); + state_guard.shards.insert(s.queue_id(), s); + let s = open_shard(index_uid.clone(), "source-a".into(), ShardId::from(4), true); + state_guard.shards.insert(s.queue_id(), s); + + // source-b: 2 closed shards, no open shards. + let s = IngesterShard::new_solo(index_uid.clone(), "source-b".into(), ShardId::from(5)) + .with_state(ShardState::Closed) + .advertisable() + .build(); + state_guard.shards.insert(s.queue_id(), s); + let s = IngesterShard::new_solo(index_uid.clone(), "source-b".into(), ShardId::from(6)) + .with_state(ShardState::Closed) + .advertisable() + .build(); + state_guard.shards.insert(s.queue_id(), s); + + let (mut open_counts, mut closed_shards) = state_guard.get_shard_snapshot(); + + // Open counts: source-a has 2, source-b has 0. + open_counts.sort_by(|a, b| a.1.cmp(&b.1)); + assert_eq!(open_counts.len(), 2); + assert_eq!( + open_counts[0], + (index_uid.clone(), SourceId::from("source-a"), 2) + ); + assert_eq!( + open_counts[1], + (index_uid.clone(), SourceId::from("source-b"), 0) + ); + + // Closed shards: source-a has shard 3, source-b has shards 5 and 6. + closed_shards.sort_by(|a, b| a.source_id.cmp(&b.source_id)); + assert_eq!(closed_shards.len(), 2); + + assert_eq!(closed_shards[0].source_id, "source-a"); + assert_eq!(closed_shards[0].shard_ids, vec![ShardId::from(3)]); + + assert_eq!(closed_shards[1].source_id, "source-b"); + let mut source_b_ids = closed_shards[1].shard_ids.clone(); + source_b_ids.sort(); + assert_eq!(source_b_ids, vec![ShardId::from(5), ShardId::from(6)]); + } } diff --git a/quickwit/quickwit-ingest/src/ingest_v2/wal_capacity_tracker.rs b/quickwit/quickwit-ingest/src/ingest_v2/wal_capacity_tracker.rs new file mode 100644 index 00000000000..f24e8254053 --- /dev/null +++ b/quickwit/quickwit-ingest/src/ingest_v2/wal_capacity_tracker.rs @@ -0,0 +1,249 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use bytesize::ByteSize; +use quickwit_common::ring_buffer::RingBuffer; + +/// The lookback window length is meant to capture readings far enough back in time to give +/// a rough rate of change estimate. At size 6, with broadcast interval of 5 seconds, this would be +/// 30 seconds of readings. +const WAL_CAPACITY_LOOKBACK_WINDOW_LEN: usize = 6; + +/// The ring buffer stores one extra element so that `delta()` can compare the newest reading +/// with the one that is exactly `WAL_CAPACITY_LOOKBACK_WINDOW_LEN` steps ago. Otherwise, that +/// reading would be discarded when the next reading is inserted. +const WAL_CAPACITY_READINGS_LEN: usize = WAL_CAPACITY_LOOKBACK_WINDOW_LEN + 1; + +struct WalCapacityTimeSeries { + capacity: ByteSize, + readings: RingBuffer, +} + +impl WalCapacityTimeSeries { + fn new(capacity: ByteSize) -> Self { + #[cfg(not(test))] + assert!(capacity.as_u64() > 0); + Self { + capacity, + readings: RingBuffer::default(), + } + } + + fn record_and_score(&mut self, used: ByteSize) -> usize { + self.record(used); + let remaining = self.current().unwrap_or(1.0); + let delta = self.delta().unwrap_or(0.0); + compute_capacity_score(remaining, delta) + } + + fn score(&self, used: ByteSize) -> usize { + let remaining = 1.0 - (used.as_u64() as f64 / self.capacity.as_u64() as f64); + let delta = self.delta().unwrap_or(0.0); + compute_capacity_score(remaining, delta) + } + + fn record(&mut self, used: ByteSize) { + let remaining = 1.0 - (used.as_u64() as f64 / self.capacity.as_u64() as f64); + self.readings.push_back(remaining.clamp(0.0, 1.0)); + } + + fn current(&self) -> Option { + self.readings.last() + } + + fn delta(&self) -> Option { + let current = self.readings.last()?; + let oldest = self.readings.front()?; + Some(current - oldest) + } +} + +pub struct WalCapacityTracker { + disk: WalCapacityTimeSeries, + memory: WalCapacityTimeSeries, +} + +impl WalCapacityTracker { + pub fn new(disk_capacity: ByteSize, memory_capacity: ByteSize) -> Self { + Self { + disk: WalCapacityTimeSeries::new(disk_capacity), + memory: WalCapacityTimeSeries::new(memory_capacity), + } + } + + /// Records disk and memory usage readings and returns the resulting capacity score. + /// The score is the minimum of the individual disk and memory scores. + pub fn record_and_score(&mut self, disk_used: ByteSize, memory_used: ByteSize) -> usize { + let disk_score = self.disk.record_and_score(disk_used); + let memory_score = self.memory.record_and_score(memory_used); + disk_score.min(memory_score) + } + + /// Computes a capacity score for the given usage without recording it. + pub fn score(&self, disk_used: ByteSize, memory_used: ByteSize) -> usize { + let disk_score = self.disk.score(disk_used); + let memory_score = self.memory.score(memory_used); + disk_score.min(memory_score) + } +} + +/// Computes a capacity score from 0 to 10 using a PD controller. +/// +/// The score has two components: +/// +/// - **P (proportional):** How much WAL capacity remains right now. An ingester with 100% free +/// capacity gets `PROPORTIONAL_WEIGHT` points; 50% gets half; and so on. If remaining capacity +/// drops to `MIN_PERMISSIBLE_CAPACITY` or below, the score is immediately 0. +/// +/// - **D (derivative):** Up to `DERIVATIVE_WEIGHT` bonus points based on how fast remaining +/// capacity is changing over the lookback window. A higher drain rate is worse, so we invert it: +/// `drain / MAX_DRAIN_RATE` normalizes the drain to a 0–1 penalty, and subtracting from 1 +/// converts it into a 0–1 bonus. Multiplied by `DERIVATIVE_WEIGHT`, a stable node gets the full +/// bonus and a node draining at `MAX_DRAIN_RATE` or faster gets nothing. +/// +/// Putting it together: a completely idle ingester scores 10 (8 + 2). +/// One that is full but stable scores ~2. One that is draining rapidly scores less. +/// A score of 0 means the ingester is at or below minimum permissible capacity. +/// +/// Below this remaining capacity fraction, the score is immediately 0. +const MIN_PERMISSIBLE_CAPACITY: f64 = 0.05; +/// Weight of the proportional term (max points from P). +const PROPORTIONAL_WEIGHT: f64 = 8.0; +/// Weight of the derivative term (max points from D). +const DERIVATIVE_WEIGHT: f64 = 2.0; +/// The drain rate (as a fraction of total capacity over the lookback window) at which the +/// derivative penalty is fully applied. Drain rates beyond this are clamped. +const MAX_DRAIN_RATE: f64 = 0.10; + +fn compute_capacity_score(remaining_capacity: f64, capacity_delta: f64) -> usize { + if remaining_capacity <= MIN_PERMISSIBLE_CAPACITY { + return 0; + } + let p = PROPORTIONAL_WEIGHT * remaining_capacity; + let drain = (-capacity_delta).clamp(0.0, MAX_DRAIN_RATE); + let d = DERIVATIVE_WEIGHT * (1.0 - drain / MAX_DRAIN_RATE); + (p + d).clamp(0.0, 10.0) as usize +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ts() -> WalCapacityTimeSeries { + WalCapacityTimeSeries::new(ByteSize::b(100)) + } + + /// Helper: record a reading with `used` bytes against the series' fixed capacity. + fn record(series: &mut WalCapacityTimeSeries, used: u64) { + series.record(ByteSize::b(used)); + } + + #[test] + fn test_wal_disk_capacity_current_after_record() { + let mut series = WalCapacityTimeSeries::new(ByteSize::b(256)); + // 192 of 256 used => 25% remaining + series.record(ByteSize::b(192)); + assert_eq!(series.current(), Some(0.25)); + + // 16 of 256 used => 93.75% remaining + series.record(ByteSize::b(16)); + assert_eq!(series.current(), Some(0.9375)); + } + + #[test] + fn test_wal_disk_capacity_record_saturates_at_zero() { + let mut series = ts(); + // 200 used out of 100 capacity => clamped to 0.0 + record(&mut series, 200); + assert_eq!(series.current(), Some(0.0)); + } + + #[test] + fn test_wal_disk_capacity_delta_growing() { + let mut series = ts(); + // oldest: 60 of 100 used => 40% remaining + record(&mut series, 60); + // current: 20 of 100 used => 80% remaining + record(&mut series, 20); + // delta = 0.80 - 0.40 = 0.40 + assert_eq!(series.delta(), Some(0.40)); + } + + #[test] + fn test_wal_disk_capacity_delta_shrinking() { + let mut series = ts(); + // oldest: 20 of 100 used => 80% remaining + record(&mut series, 20); + // current: 60 of 100 used => 40% remaining + record(&mut series, 60); + // delta = 0.40 - 0.80 = -0.40 + assert_eq!(series.delta(), Some(-0.40)); + } + + #[test] + fn test_capacity_score_draining_vs_stable() { + // Node A: capacity draining — usage increases 10, 20, ..., 70 over 7 ticks. + let mut node_a = ts(); + for used in (10..=70).step_by(10) { + record(&mut node_a, used); + } + let a_remaining = node_a.current().unwrap(); + let a_delta = node_a.delta().unwrap(); + let a_score = compute_capacity_score(a_remaining, a_delta); + + // Node B: steady at 50% usage over 7 ticks. + let mut node_b = ts(); + for _ in 0..7 { + record(&mut node_b, 50); + } + let b_remaining = node_b.current().unwrap(); + let b_delta = node_b.delta().unwrap(); + let b_score = compute_capacity_score(b_remaining, b_delta); + + // p=2.4, d=0 (max drain) => 2 + assert_eq!(a_score, 2); + // p=4, d=2 (stable) => 6 + assert_eq!(b_score, 6); + assert!(b_score > a_score); + } + + #[test] + fn test_wal_disk_capacity_delta_spans_lookback_window() { + let mut series = ts(); + + // Fill to exactly the lookback window length (6 readings), all same value. + for _ in 0..WAL_CAPACITY_LOOKBACK_WINDOW_LEN { + record(&mut series, 50); + } + assert_eq!(series.delta(), Some(0.0)); + + // 7th reading fills the ring buffer. Delta spans 6 intervals. + record(&mut series, 0); + assert_eq!(series.delta(), Some(0.50)); + + // 8th reading evicts the oldest 50-remaining. Delta still spans 6 intervals. + record(&mut series, 0); + assert_eq!(series.delta(), Some(0.50)); + } + + #[test] + fn test_wal_capacity_tracker_returns_min() { + let mut tracker = WalCapacityTracker::new(ByteSize::b(100), ByteSize::b(100)); + // Disk 10% used (score 9), memory 90% used (score 2) → returns 2. + assert_eq!( + tracker.record_and_score(ByteSize::b(10), ByteSize::b(90)), + 2 + ); + } +} diff --git a/quickwit/quickwit-ingest/src/ingest_v2/workbench.rs b/quickwit/quickwit-ingest/src/ingest_v2/workbench.rs index 686c2fd9c0a..50f92654efb 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/workbench.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/workbench.rs @@ -19,14 +19,12 @@ use quickwit_common::rate_limited_error; use quickwit_proto::control_plane::{ GetOrCreateOpenShardsFailure, GetOrCreateOpenShardsFailureReason, }; -use quickwit_proto::ingest::ingester::{ - PersistFailure, PersistFailureReason, PersistRequest, PersistSuccess, -}; +use quickwit_proto::ingest::ingester::{PersistFailure, PersistFailureReason, PersistSuccess}; use quickwit_proto::ingest::router::{ IngestFailure, IngestFailureReason, IngestResponseV2, IngestSubrequest, IngestSuccess, }; -use quickwit_proto::ingest::{IngestV2Error, RateLimitingCause}; -use quickwit_proto::types::{NodeId, ShardId, SubrequestId}; +use quickwit_proto::ingest::{IngestV2Error, RateLimitingCause, ShardIds}; +use quickwit_proto::types::{NodeId, SubrequestId}; use tracing::warn; use super::publish_tracker::PublishTracker; @@ -37,7 +35,6 @@ use super::router::PersistRequestSummary; #[derive(Default)] pub(super) struct IngestWorkbench { pub subworkbenches: BTreeMap, - pub rate_limited_shards: HashSet, pub num_successes: usize, /// The number of batch persist attempts. This is not sum of the number of attempts for each /// subrequest. @@ -50,6 +47,7 @@ pub(super) struct IngestWorkbench { /// (The point here is to make sure we do not wait for the failure detection to kick the node /// out of the ingest node.) pub unavailable_leaders: HashSet, + pub closed_shards: Vec, publish_tracker: Option, } @@ -128,16 +126,6 @@ impl IngestWorkbench { .all(|subworbench| !subworbench.is_pending()) } - pub fn record_persist_request(&self, persist_request: &PersistRequest) { - if let Some(publish_tracker) = &self.publish_tracker { - let shards = persist_request - .subrequests - .iter() - .map(|subrequest| subrequest.shard_id()); - publish_tracker.register_requested_shards(shards); - } - } - pub fn record_get_or_create_open_shards_failure( &mut self, open_shards_failure: GetOrCreateOpenShardsFailure, @@ -169,13 +157,13 @@ impl IngestWorkbench { ); return; }; - if let Some(publish_tracker) = &mut self.publish_tracker { - if let Some(position) = &persist_success.replication_position_inclusive { - publish_tracker.track_persisted_shard_position( - persist_success.shard_id().clone(), - position.clone(), - ); - } + if let Some(publish_tracker) = &mut self.publish_tracker + && let Some(position) = &persist_success.replication_position_inclusive + { + publish_tracker.track_persisted_shard_position( + persist_success.shard_id().clone(), + position.clone(), + ); } self.num_successes += 1; subworkbench.num_attempts += 1; @@ -240,13 +228,6 @@ impl IngestWorkbench { self.record_failure(subrequest_id, SubworkbenchFailure::NoShardsAvailable); } - pub fn record_rate_limited(&mut self, subrequest_id: SubrequestId) { - self.record_failure( - subrequest_id, - SubworkbenchFailure::RateLimited(RateLimitingCause::ShardRateLimiting), - ); - } - /// Marks a node as unavailable for the span of the workbench. /// /// Remaining attempts will treat the node as if it was not in the ingester pool. @@ -411,7 +392,7 @@ mod tests { use std::time::Duration; use quickwit_proto::indexing::ShardPositionsUpdate; - use quickwit_proto::ingest::ingester::{PersistFailureReason, PersistSubrequest}; + use quickwit_proto::ingest::ingester::PersistFailureReason; use quickwit_proto::types::{IndexUid, Position, ShardId, SourceUid}; use super::*; @@ -445,7 +426,7 @@ mod tests { assert!(!subworkbench.last_failure_is_transient()); subworkbench.last_failure_opt = Some(SubworkbenchFailure::Persist( - PersistFailureReason::ShardRateLimited, + PersistFailureReason::NoShardsAvailable, )); assert!(subworkbench.is_pending()); assert!(subworkbench.last_failure_is_transient()); @@ -569,23 +550,6 @@ mod tests { assert_eq!(pending_subrequests(&workbench.subworkbenches).count(), 2); assert!(!workbench.is_complete()); - let persist_request = PersistRequest { - subrequests: vec![ - PersistSubrequest { - subrequest_id: 0, - shard_id: Some(shard_id_1.clone()), - ..Default::default() - }, - PersistSubrequest { - subrequest_id: 1, - shard_id: Some(shard_id_2.clone()), - ..Default::default() - }, - ], - ..Default::default() - }; - workbench.record_persist_request(&persist_request); - let persist_success = PersistSuccess { subrequest_id: 0, shard_id: Some(shard_id_1.clone()), @@ -596,7 +560,6 @@ mod tests { let persist_failure = PersistFailure { subrequest_id: 1, - shard_id: Some(shard_id_2.clone()), ..Default::default() }; workbench.record_persist_failure(&persist_failure); @@ -608,18 +571,6 @@ mod tests { ..Default::default() }; - // retry to persist shard 2 - - let persist_request = PersistRequest { - subrequests: vec![PersistSubrequest { - subrequest_id: 1, - shard_id: Some(shard_id_2.clone()), - ..Default::default() - }], - ..Default::default() - }; - workbench.record_persist_request(&persist_request); - workbench.record_persist_success(persist_success); assert!(workbench.is_complete()); @@ -662,23 +613,6 @@ mod tests { let mut workbench = IngestWorkbench::new_with_publish_tracking(ingest_subrequests, 1, event_broker.clone()); - let persist_request = PersistRequest { - subrequests: vec![ - PersistSubrequest { - subrequest_id: 0, - shard_id: Some(shard_id_1.clone()), - ..Default::default() - }, - PersistSubrequest { - subrequest_id: 1, - shard_id: Some(shard_id_2.clone()), - ..Default::default() - }, - ], - ..Default::default() - }; - workbench.record_persist_request(&persist_request); - let persist_success = PersistSuccess { subrequest_id: 0, shard_id: Some(shard_id_1.clone()), @@ -865,14 +799,13 @@ mod tests { let persist_failure = PersistFailure { subrequest_id: 42, - reason: PersistFailureReason::ShardRateLimited as i32, + reason: PersistFailureReason::NoShardsAvailable as i32, ..Default::default() }; workbench.record_persist_failure(&persist_failure); let persist_failure = PersistFailure { subrequest_id: 0, - shard_id: Some(ShardId::from(1)), reason: PersistFailureReason::WalFull as i32, ..Default::default() }; diff --git a/quickwit/quickwit-ingest/src/queue.rs b/quickwit/quickwit-ingest/src/queue.rs index 91b87520a7e..8b6ceebbd32 100644 --- a/quickwit/quickwit-ingest/src/queue.rs +++ b/quickwit/quickwit-ingest/src/queue.rs @@ -445,7 +445,8 @@ mod tests { use rand::rngs::StdRng; use rand::{Rng, SeedableRng}; - use rand_distr::{Distribution, LogNormal, WeightedIndex}; + use rand_distr::weighted::WeightedIndex; + use rand_distr::{Distribution, LogNormal}; const NUM_QUEUES: usize = 100; const NUM_RECORDS: usize = 1_000_000; @@ -467,7 +468,7 @@ mod tests { let records: Vec = record_queue_ids .into_iter() .map(|queue_id| { - let num_bytes: usize = rng.gen_range(80..800); + let num_bytes: usize = rng.random_range(80..800); let payload: Vec = repeat_with(rand::random::).take(num_bytes).collect(); Record { queue_id: queue_id.to_string(), diff --git a/quickwit/quickwit-integration-tests/Cargo.toml b/quickwit/quickwit-integration-tests/Cargo.toml index 23f2d884718..e7f1dab23db 100644 --- a/quickwit/quickwit-integration-tests/Cargo.toml +++ b/quickwit/quickwit-integration-tests/Cargo.toml @@ -27,6 +27,7 @@ hyper-util = { workspace = true } itertools = { workspace = true } rand = { workspace = true } reqwest = { workspace = true } +rustls = { workspace = true } serde_json = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true } @@ -39,6 +40,7 @@ quickwit-cli = { workspace = true } quickwit-common = { workspace = true, features = ["testsuite"] } quickwit-config = { workspace = true, features = ["testsuite"] } quickwit-indexing = { workspace = true, features = ["testsuite"] } +quickwit-ingest = { workspace = true, features = ["testsuite"] } quickwit-metastore = { workspace = true, features = ["testsuite"] } quickwit-opentelemetry = { workspace = true, features = ["testsuite"] } quickwit-proto = { workspace = true, features = ["testsuite"] } diff --git a/quickwit/quickwit-integration-tests/src/test_utils/cluster_sandbox.rs b/quickwit/quickwit-integration-tests/src/test_utils/cluster_sandbox.rs index 81889a9311a..a7385ca0946 100644 --- a/quickwit/quickwit-integration-tests/src/test_utils/cluster_sandbox.rs +++ b/quickwit/quickwit-integration-tests/src/test_utils/cluster_sandbox.rs @@ -43,7 +43,6 @@ use quickwit_serve::{ ListSplitsQueryParams, RestIngestResponse, SearchRequestQueryString, serve_quickwit, }; use quickwit_storage::StorageResolver; -use rand::Rng; use reqwest::Url; use serde_json::Value; use tempfile::TempDir; @@ -57,6 +56,39 @@ pub struct TestNodeConfig { pub enable_otlp: bool, } +impl TestNodeConfig { + async fn build_node_config( + &self, + node_idx: usize, + cluster_id: String, + temp_dir: &TempDir, + unique_dir_name: String, + tcp_listener_resolver: &TestTcpListenerResolver, + ) -> NodeConfig { + let socket: SocketAddr = ([127, 0, 0, 1], 0u16).into(); + let rest_tcp_listener = TcpListener::bind(socket).await.unwrap(); + let grpc_tcp_listener = TcpListener::bind(socket).await.unwrap(); + let mut config = NodeConfig::for_test_from_ports( + rest_tcp_listener.local_addr().unwrap().port(), + grpc_tcp_listener.local_addr().unwrap().port(), + ); + tcp_listener_resolver.add_listener(rest_tcp_listener).await; + tcp_listener_resolver.add_listener(grpc_tcp_listener).await; + config.indexer_config.enable_otlp_endpoint = self.enable_otlp; + config.enabled_services.clone_from(&self.services); + config.jaeger_config.enable_endpoint = true; + config.cluster_id.clone_from(&cluster_id); + config.node_id = NodeId::new(format!("test-node-{node_idx}")); + let root_data_dir = temp_dir.path().to_path_buf(); + config.data_dir_path = root_data_dir.join(config.node_id.as_str()); + config.metastore_uri = + QuickwitUri::from_str(&format!("ram:///{unique_dir_name}/metastore")).unwrap(); + config.default_index_root_uri = + QuickwitUri::from_str(&format!("ram:///{unique_dir_name}/indexes")).unwrap(); + config + } +} + pub struct ClusterSandboxBuilder { temp_dir: TempDir, node_configs: Vec, @@ -107,32 +139,21 @@ impl ClusterSandboxBuilder { /// - `default_index_root_uri` defined by `root_data_dir/indexes`. /// - `peers` defined by others nodes `gossip_advertise_addr`. pub async fn build_config(self) -> ResolvedClusterConfig { - let root_data_dir = self.temp_dir.path().to_path_buf(); let cluster_id = new_coolid("test-cluster"); let mut resolved_node_configs = Vec::new(); let mut peers: Vec = Vec::new(); let unique_dir_name = new_coolid("test-dir"); let tcp_listener_resolver = TestTcpListenerResolver::default(); for (node_idx, node_builder) in self.node_configs.iter().enumerate() { - let socket: SocketAddr = ([127, 0, 0, 1], 0u16).into(); - let rest_tcp_listener = TcpListener::bind(socket).await.unwrap(); - let grpc_tcp_listener = TcpListener::bind(socket).await.unwrap(); - let mut config = NodeConfig::for_test_from_ports( - rest_tcp_listener.local_addr().unwrap().port(), - grpc_tcp_listener.local_addr().unwrap().port(), - ); - tcp_listener_resolver.add_listener(rest_tcp_listener).await; - tcp_listener_resolver.add_listener(grpc_tcp_listener).await; - config.indexer_config.enable_otlp_endpoint = node_builder.enable_otlp; - config.enabled_services.clone_from(&node_builder.services); - config.jaeger_config.enable_endpoint = true; - config.cluster_id.clone_from(&cluster_id); - config.node_id = NodeId::new(format!("test-node-{node_idx}")); - config.data_dir_path = root_data_dir.join(config.node_id.as_str()); - config.metastore_uri = - QuickwitUri::from_str(&format!("ram:///{unique_dir_name}/metastore")).unwrap(); - config.default_index_root_uri = - QuickwitUri::from_str(&format!("ram:///{unique_dir_name}/indexes")).unwrap(); + let config = node_builder + .build_node_config( + node_idx, + cluster_id.clone(), + &self.temp_dir, + unique_dir_name.clone(), + &tcp_listener_resolver, + ) + .await; peers.push(config.gossip_advertise_addr.to_string()); resolved_node_configs.push((config, node_builder.services.clone())); } @@ -144,7 +165,9 @@ impl ClusterSandboxBuilder { .collect_vec(); } ResolvedClusterConfig { + cluster_id, temp_dir: self.temp_dir, + unique_dir_name, node_configs: resolved_node_configs, tcp_listener_resolver, } @@ -168,7 +191,9 @@ impl ClusterSandboxBuilder { /// Intermediate state where the ports of all the test cluster nodes have /// been reserved and the configurations have been generated. pub struct ResolvedClusterConfig { + cluster_id: String, temp_dir: TempDir, + unique_dir_name: String, pub node_configs: Vec<(NodeConfig, HashSet)>, tcp_listener_resolver: TestTcpListenerResolver, } @@ -176,49 +201,24 @@ pub struct ResolvedClusterConfig { impl ResolvedClusterConfig { /// Start a cluster using this config and waits for the nodes to be ready pub async fn start(self) -> ClusterSandbox { - let mut node_shutdown_handles = Vec::new(); - let runtimes_config = RuntimesConfig::light_for_tests(); - let storage_resolver = StorageResolver::unconfigured(); - let metastore_resolver = MetastoreResolver::unconfigured(); - let cluster_size = self.node_configs.len(); - for node_config in self.node_configs.iter() { - let mut shutdown_handler = - NodeShutdownHandle::new(node_config.0.node_id.clone(), node_config.1.clone()); - let shutdown_signal = shutdown_handler.shutdown_signal(); - let join_handle = tokio::spawn({ - let node_config = node_config.0.clone(); - let node_id = node_config.node_id.clone(); - let services = node_config.enabled_services.clone(); - let metastore_resolver = metastore_resolver.clone(); - let storage_resolver = storage_resolver.clone(); - let tcp_listener_resolver = self.tcp_listener_resolver.clone(); - - async move { - let result = serve_quickwit( - node_config, - runtimes_config, - metastore_resolver, - storage_resolver, - tcp_listener_resolver, - shutdown_signal, - quickwit_serve::do_nothing_env_filter_reload_fn(), - ) - .await?; - debug!("{node_id} stopped successfully ({:?})", services); - Result::<_, anyhow::Error>::Ok(result) - } - }); - shutdown_handler.set_node_join_handle(join_handle); - node_shutdown_handles.push(shutdown_handler); - } + quickwit_cli::install_default_crypto_ring_provider(); - let sandbox = ClusterSandbox { - node_configs: self.node_configs, - _temp_dir: self.temp_dir, - node_shutdown_handles, + let mut sandbox = ClusterSandbox { + cluster_id: self.cluster_id, + node_configs: Vec::new(), + temp_dir: self.temp_dir, + unique_dir_name: self.unique_dir_name, + node_shutdown_handles: Vec::new(), + tcp_listener_resolver: self.tcp_listener_resolver, + storage_resolver: StorageResolver::unconfigured(), + metastore_resolver: MetastoreResolver::unconfigured(), }; + for (config, services) in &self.node_configs { + sandbox.spawn_node(config.clone(), services.clone()); + } + sandbox.node_configs = self.node_configs; sandbox - .wait_for_cluster_num_ready_nodes(cluster_size) + .wait_for_cluster_num_ready_nodes(sandbox.node_configs.len()) .await .unwrap(); sandbox @@ -257,12 +257,74 @@ pub(crate) async fn ingest( /// A test environment where you can start a Quickwit cluster and use the gRPC /// or REST clients to test it. pub struct ClusterSandbox { + cluster_id: String, pub node_configs: Vec<(NodeConfig, HashSet)>, - _temp_dir: TempDir, + unique_dir_name: String, + temp_dir: TempDir, node_shutdown_handles: Vec, + tcp_listener_resolver: TestTcpListenerResolver, + storage_resolver: StorageResolver, + metastore_resolver: MetastoreResolver, } impl ClusterSandbox { + fn spawn_node(&mut self, config: NodeConfig, services: HashSet) { + let mut shutdown_handle = NodeShutdownHandle::new(config.node_id.clone(), services.clone()); + let shutdown_signal = shutdown_handle.shutdown_signal(); + let runtimes_config = RuntimesConfig::light_for_tests(); + let join_handle = tokio::spawn({ + let node_id = config.node_id.clone(); + let metastore_resolver = self.metastore_resolver.clone(); + let storage_resolver = self.storage_resolver.clone(); + let tcp_listener_resolver = self.tcp_listener_resolver.clone(); + async move { + let result = serve_quickwit( + config, + runtimes_config, + metastore_resolver, + storage_resolver, + tcp_listener_resolver, + shutdown_signal, + quickwit_serve::do_nothing_env_filter_reload_fn(), + ) + .await?; + debug!("{node_id} stopped successfully ({services:?})"); + Result::<_, anyhow::Error>::Ok(result) + } + }); + shutdown_handle.set_node_join_handle(join_handle); + self.node_shutdown_handles.push(shutdown_handle); + } + + /// Dynamically adds a node to the cluster. Does not wait for readiness. + pub async fn add_node(&mut self, services: impl IntoIterator) { + self.add_node_inner(TestNodeConfig { + services: HashSet::from_iter(services), + enable_otlp: false, + }) + .await; + } + + async fn add_node_inner(&mut self, config_builder: TestNodeConfig) { + let mut config = config_builder + .build_node_config( + self.node_configs.len() + 1, + self.cluster_id.clone(), + &self.temp_dir, + self.unique_dir_name.clone(), + &self.tcp_listener_resolver, + ) + .await; + config.peer_seeds = self + .node_configs + .iter() + .map(|config| config.0.gossip_advertise_addr.to_string()) + .collect_vec(); + self.spawn_node(config.clone(), config_builder.services.clone()); + self.node_configs + .push((config, config_builder.services.clone())); + } + fn find_node_for_service(&self, service: QuickwitService) -> NodeConfig { self.node_configs .iter() @@ -351,7 +413,7 @@ impl ClusterSandbox { TraceServiceClient::new(self.channel(QuickwitService::Indexer)) } - async fn wait_for_cluster_num_ready_nodes( + pub async fn wait_for_cluster_num_ready_nodes( &self, expected_num_ready_nodes: usize, ) -> anyhow::Result<()> { @@ -486,7 +548,7 @@ impl ClusterSandbox { let data_dir = test_conf .0 .data_dir_path - .join(rand::thread_rng().r#gen::().to_string()); + .join(rand::random::().to_string()); tokio::fs::create_dir(&data_dir).await?; let node_config = format!( r#" @@ -596,4 +658,48 @@ impl ClusterSandbox { self.shutdown_services(QuickwitService::supported_services()) .await } + + /// Remove a node from the sandbox and return its shutdown handle. + /// After this call, `rest_client` and other lookup methods skip the removed + /// node, so callers can trigger shutdown concurrently with other sandbox + /// operations. + pub fn remove_node_with_service(&mut self, service: QuickwitService) -> NodeShutdownHandle { + let idx = self + .node_shutdown_handles + .iter() + .position(|h| h.node_services.contains(&service)) + .unwrap_or_else(|| panic!("no node with service {service:?}")); + self.node_configs.remove(idx); + self.node_shutdown_handles.remove(idx) + } +} + +/// We don't usually test the tests, but the complexity of the sandbox setup code justifies it here. +#[tokio::test] +async fn test_sandbox_happy_path() { + let sandbox = ClusterSandboxBuilder::default() + .add_node([QuickwitService::ControlPlane, QuickwitService::Metastore]) + .add_node([QuickwitService::Searcher]) + .add_node([QuickwitService::Indexer]) + .build_and_start() + .await; + + sandbox.wait_for_cluster_num_ready_nodes(3).await.unwrap(); + sandbox.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_sandbox_add_node_dynamically() { + let mut sandbox = ClusterSandboxBuilder::default() + .add_node([QuickwitService::ControlPlane, QuickwitService::Metastore]) + .add_node([QuickwitService::Searcher]) + .build_and_start() + .await; + sandbox.wait_for_cluster_num_ready_nodes(2).await.unwrap(); + + // Later, add an indexer node to the running cluster + sandbox.add_node([QuickwitService::Indexer]).await; + + sandbox.wait_for_cluster_num_ready_nodes(3).await.unwrap(); + sandbox.shutdown().await.unwrap(); } diff --git a/quickwit/quickwit-integration-tests/src/tests/ingest_v2_tests.rs b/quickwit/quickwit-integration-tests/src/tests/ingest_v2_tests.rs index 4a5e29c9565..b63058b3d9c 100644 --- a/quickwit/quickwit-integration-tests/src/tests/ingest_v2_tests.rs +++ b/quickwit/quickwit-integration-tests/src/tests/ingest_v2_tests.rs @@ -856,3 +856,250 @@ async fn test_shutdown_indexer_first() { .unwrap() .unwrap(); } + +/// Tests that the graceful shutdown sequence works correctly in a multi-indexer +/// cluster: shutting down one indexer does NOT cause 500 errors or data loss, +/// and the cluster eventually rebalances. see #6158 +/// +/// We start with a single indexer so the shard for this index is guaranteed to +/// live on it. After ingesting, we dynamically add a second indexer, then shut +/// down the first one. This proves the decommission sequence correctly drains +/// in-flight data even when the shard owner is the node being removed. +#[tokio::test] +async fn test_graceful_shutdown_no_data_loss() { + let mut sandbox = ClusterSandboxBuilder::default() + .add_node([QuickwitService::Indexer]) + .add_node([ + QuickwitService::ControlPlane, + QuickwitService::Searcher, + QuickwitService::Metastore, + QuickwitService::Janitor, + ]) + .build_and_start() + .await; + let index_id = "test_graceful_shutdown_no_data_loss"; + + // Create index with a long commit timeout so documents stay uncommitted + // in the ingesters' WAL. The decommission sequence should commit + // them before the indexer quits. + sandbox + .rest_client(QuickwitService::Indexer) + .indexes() + .create( + format!( + r#" + version: 0.8 + index_id: {index_id} + doc_mapping: + field_mappings: + - name: body + type: text + indexing_settings: + commit_timeout_secs: 5 + "# + ), + ConfigFormat::Yaml, + false, + ) + .await + .unwrap(); + + // Ingest docs with auto-commit. With a 5s commit timeout, these documents + // sit uncommitted in the ingesters' WAL - exactly the in-flight state we + // want to exercise during draining. + ingest( + &sandbox.rest_client(QuickwitService::Indexer), + index_id, + ingest_json!({"body": "before-shutdown-1"}), + CommitType::Auto, + ) + .await + .unwrap(); + + ingest( + &sandbox.rest_client(QuickwitService::Indexer), + index_id, + ingest_json!({"body": "before-shutdown-2"}), + CommitType::Auto, + ) + .await + .unwrap(); + + // Add a second indexer after the shard has been created on the first one. + sandbox.add_node([QuickwitService::Indexer]).await; + sandbox.wait_for_cluster_num_ready_nodes(3).await.unwrap(); + + // Remove the first indexer (the shard owner) from the sandbox and get its + // shutdown handle. After this call, rest_client(Indexer) returns the + // second (surviving) indexer. + let shutdown_handle = sandbox.remove_node_with_service(QuickwitService::Indexer); + + // Concurrently: shut down the removed indexer AND ingest more data via the + // surviving indexer. This verifies the cluster stays operational and the + // router on the surviving node does not return 500 errors while one indexer + // is decommissioning. The control plane excludes the decommissioning + // ingester from shard allocation, so new shards go to the surviving one. + let ingest_client = sandbox.rest_client(QuickwitService::Indexer); + let (shutdown_result, ingest_result) = tokio::join!( + async { + tokio::time::timeout(Duration::from_secs(30), shutdown_handle.shutdown()) + .await + .expect("indexer shutdown timed out — decommission may be stuck") + }, + async { + // Small delay so the decommission sequence has started before we ingest. + tokio::time::sleep(Duration::from_millis(200)).await; + ingest( + &ingest_client, + index_id, + ingest_json!({"body": "during-shutdown"}), + CommitType::Auto, + ) + .await + }, + ); + shutdown_result.expect("indexer shutdown failed"); + ingest_result.expect("ingest during shutdown should succeed (no 500 errors)"); + + // All 3 documents should eventually be searchable. Documents 1 & 2 were + // in-flight on the decommissioning indexer and should have been committed during + // the decommission step. Document 3 was ingested to the surviving indexer. + wait_until_predicate( + || async { + match sandbox + .rest_client(QuickwitService::Searcher) + .search( + index_id, + quickwit_serve::SearchRequestQueryString { + query: "*".to_string(), + max_hits: 10, + ..Default::default() + }, + ) + .await + { + Ok(resp) => resp.num_hits == 3, + Err(_) => false, + } + }, + Duration::from_secs(30), + Duration::from_millis(500), + ) + .await + .expect("expected 3 documents after decommission shutdown, some data may have been lost"); + + // Verify the cluster sees 2 ready nodes (the surviving indexer + the + // control-plane/searcher/metastore/janitor node). + sandbox + .wait_for_cluster_num_ready_nodes(2) + .await + .expect("cluster should see 2 ready nodes after indexer shutdown"); + + // Clean shutdown of the remaining nodes. + tokio::time::timeout(Duration::from_secs(30), sandbox.shutdown()) + .await + .unwrap() + .unwrap(); +} + +/// Verifies that after deleting an index and recreating it with the same name, +/// ingest works correctly once the capacity broadcast has propagated (>2 broadcast +/// cycles). Uses 2 ingesters to exercise the Chitchat broadcast path. +#[tokio::test] +async fn test_ingest_after_index_recreate_multi_node() { + quickwit_common::setup_logging_for_tests(); + + let sandbox = ClusterSandboxBuilder::default() + .add_node([QuickwitService::Indexer, QuickwitService::Janitor]) + .add_node([QuickwitService::Indexer, QuickwitService::Janitor]) + .add_node([ + QuickwitService::ControlPlane, + QuickwitService::Metastore, + QuickwitService::Searcher, + ]) + .build_and_start() + .await; + + let index_id = "test-recreate-multi-node"; + let index_config = format!( + r#" + version: 0.8 + index_id: {index_id} + doc_mapping: + field_mappings: + - name: body + type: text + indexing_settings: + commit_timeout_secs: 1 + ingest_settings: + min_shards: 2 + "# + ); + + // Step 1: Create index and ingest to seed the routing table. + let original_metadata = sandbox + .rest_client(QuickwitService::Indexer) + .indexes() + .create(index_config.clone(), ConfigFormat::Yaml, false) + .await + .unwrap(); + + ingest( + &sandbox.rest_client(QuickwitService::Indexer), + index_id, + ingest_json!({"body": "first incarnation"}), + CommitType::Force, + ) + .await + .unwrap(); + + // Wait for the broadcast to propagate routing entries to all routers. + tokio::time::sleep(Duration::from_millis(60)).await; + + // Step 2: Delete the index. + sandbox + .rest_client(QuickwitService::Indexer) + .indexes() + .delete(index_id, false) + .await + .unwrap(); + + // Step 3: Wait for 2+ broadcast cycles (50ms each with testsuite feature) so that + // ingesters broadcast open_shard_count=0 and routers clear stale entries. + tokio::time::sleep(Duration::from_millis(120)).await; + + // Step 4: Recreate with the same name — new incarnation. + let new_metadata = sandbox + .rest_client(QuickwitService::Indexer) + .indexes() + .create(index_config, ConfigFormat::Yaml, false) + .await + .unwrap(); + + assert_ne!( + original_metadata.index_uid.incarnation_id, + new_metadata.index_uid.incarnation_id + ); + + // Step 5: Ingest into the recreated index. If stale routing entries weren't + // cleared, this would fail with NoShardsAvailable after exhausting retries. + let ingest_resp = ingest( + &sandbox.rest_client(QuickwitService::Indexer), + index_id, + ingest_json!({"body": "second incarnation"}), + CommitType::Force, + ) + .await + .unwrap(); + assert_eq!(ingest_resp.num_ingested_docs, Some(1)); + + // Cleanup. + sandbox + .rest_client(QuickwitService::Indexer) + .indexes() + .delete(index_id, false) + .await + .unwrap(); + + sandbox.shutdown().await.unwrap(); +} diff --git a/quickwit/quickwit-jaeger/src/lib.rs b/quickwit/quickwit-jaeger/src/lib.rs index 5749842655a..a2aedd4d9d8 100644 --- a/quickwit/quickwit-jaeger/src/lib.rs +++ b/quickwit/quickwit-jaeger/src/lib.rs @@ -18,7 +18,6 @@ use std::ops::{Bound, RangeInclusive}; use std::sync::Arc; use std::time::Instant; -use async_trait::async_trait; use itertools::{Either, Itertools}; use prost::Message; use prost_types::{Duration as WellKnownDuration, Timestamp as WellKnownTimestamp}; @@ -26,13 +25,11 @@ use quickwit_config::JaegerConfig; use quickwit_opentelemetry::otlp::{ Event as QwEvent, Link as QwLink, OTEL_TRACES_INDEX_ID, Span as QwSpan, SpanFingerprint, SpanId, SpanKind as QwSpanKind, SpanStatus as QwSpanStatus, TraceId, - extract_otel_traces_index_id_patterns_from_metadata, }; use quickwit_proto::jaeger::api_v2::{ KeyValue as JaegerKeyValue, Log as JaegerLog, Process as JaegerProcess, Span as JaegerSpan, SpanRef as JaegerSpanRef, SpanRefType as JaegerSpanRefType, ValueType, }; -use quickwit_proto::jaeger::storage::v1::span_reader_plugin_server::SpanReaderPlugin; use quickwit_proto::jaeger::storage::v1::{ FindTraceIDsRequest, FindTraceIDsResponse, FindTracesRequest, GetOperationsRequest, GetOperationsResponse, GetServicesRequest, GetServicesResponse, GetTraceRequest, Operation, @@ -50,22 +47,27 @@ use time::OffsetDateTime; use time::format_description::well_known::Rfc3339; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; -use tonic::{Request, Response, Status}; +use tonic::Status; use tracing::field::Empty; use tracing::{Span as RuntimeSpan, debug, error, instrument, warn}; -use crate::metrics::JAEGER_SERVICE_METRICS; +pub(crate) use crate::metrics::JAEGER_SERVICE_METRICS; mod metrics; +mod v1; +mod v2; // OpenTelemetry to Jaeger Transformation // type TimeIntervalSecs = RangeInclusive; -type JaegerResult = Result; +pub(crate) type JaegerResult = Result; -type SpanStream = ReceiverStream>; +pub(crate) type SpanStream = ReceiverStream>; + +pub(crate) type TracesDataStream = + ReceiverStream>; #[derive(Clone)] pub struct JaegerService { @@ -88,34 +90,16 @@ impl JaegerService { #[instrument("get_services", skip_all)] pub async fn get_services_for_indexes( &self, - request: GetServicesRequest, + _request: GetServicesRequest, index_id_patterns: Vec, ) -> JaegerResult { - debug!(request=?request, index_ids=?index_id_patterns, "`get_services` request"); - - let max_hits = Some(1_000); - let start_timestamp = - Some(OffsetDateTime::now_utc().unix_timestamp() - self.lookback_period_secs); - - let search_request = ListTermsRequest { + let services = get_services_impl( + self.search_service.clone(), + self.lookback_period_secs, index_id_patterns, - field: "service_name".to_string(), - max_hits, - start_timestamp, - end_timestamp: None, - start_key: None, - end_key: None, - }; - let search_response = self.search_service.root_list_terms(search_request).await?; - let services: Vec = search_response - .terms - .into_iter() - .map(|term_bytes| extract_term(&term_bytes)) - .sorted() - .collect(); - let response = GetServicesResponse { services }; - debug!(response=?response, "`get_services` response"); - Ok(response) + ) + .await?; + Ok(GetServicesResponse { services }) } #[instrument("get_operations", skip_all, fields(service=%request.service, span_kind=%request.span_kind))] @@ -124,38 +108,18 @@ impl JaegerService { request: GetOperationsRequest, index_id_patterns: Vec, ) -> JaegerResult { - debug!(request=?request, request=?request, index_ids=?index_id_patterns, "`get_operations` request"); - - let max_hits = Some(1_000); - let start_timestamp = - Some(OffsetDateTime::now_utc().unix_timestamp() - self.lookback_period_secs); - - let span_kind_opt = request.span_kind.parse().ok(); - let start_key = SpanFingerprint::start_key(&request.service, span_kind_opt.clone()); - let end_key = SpanFingerprint::end_key(&request.service, span_kind_opt); - - let search_request = ListTermsRequest { + let operations = get_operations_impl( + self.search_service.clone(), + self.lookback_period_secs, + request.service, + request.span_kind, index_id_patterns, - field: "span_fingerprint".to_string(), - max_hits, - start_timestamp, - end_timestamp: None, - start_key, - end_key, - }; - let search_response = self.search_service.root_list_terms(search_request).await?; - let operations: Vec = search_response - .terms - .into_iter() - .map(|term_json| extract_operation(&term_json)) - .sorted() - .collect(); - debug!(operations=?operations, "`get_operations` response"); - let response = GetOperationsResponse { + ) + .await?; + Ok(GetOperationsResponse { operations, operation_names: Vec::new(), // `operation_names` is deprecated. - }; - Ok(response) + }) } // Instrumentation happens in `find_trace_ids`. @@ -247,50 +211,81 @@ impl JaegerService { trace_query: TraceQueryParameters, index_id_patterns: Vec, ) -> Result<(Vec, TimeIntervalSecs), Status> { - let span_kind_opt = None; - let min_span_start_timestamp_secs_opt = trace_query.start_time_min.map(|ts| ts.seconds); - let max_span_start_timestamp_secs_opt = trace_query.start_time_max.map(|ts| ts.seconds); - let min_span_duration_millis_opt = trace_query + let min_start_secs = trace_query.start_time_min.map(|ts| ts.seconds); + let max_start_secs = trace_query.start_time_max.map(|ts| ts.seconds); + let min_duration_millis = trace_query .duration_min .and_then(|d| to_duration_millis(&d)); - let max_span_duration_millis_opt = trace_query + let max_duration_millis = trace_query .duration_max .and_then(|d| to_duration_millis(&d)); - let query = build_search_query( + + find_trace_ids_common( + self.search_service.clone(), &trace_query.service_name, - span_kind_opt, &trace_query.operation_name, trace_query.tags, - min_span_start_timestamp_secs_opt, - max_span_start_timestamp_secs_opt, - min_span_duration_millis_opt, - max_span_duration_millis_opt, - ); - let query_ast = - serde_json::to_string(&query).map_err(|err| Status::internal(err.to_string()))?; - let aggregation_query = build_aggregations_query(trace_query.num_traces as usize); - let max_hits = 0; - let search_request = SearchRequest { + min_start_secs, + max_start_secs, + min_duration_millis, + max_duration_millis, + trace_query.num_traces as usize, index_id_patterns, - query_ast, - aggregation_request: Some(aggregation_query), - max_hits, - start_timestamp: min_span_start_timestamp_secs_opt, - end_timestamp: max_span_start_timestamp_secs_opt, - count_hits: CountHits::Underestimate.into(), - ..Default::default() - }; - let search_response = self.search_service.root_search(search_request).await?; - - let Some(agg_result_postcard) = search_response.aggregation_postcard else { - debug!("the query matched no traces"); - return Ok((Vec::new(), 0..=0)); - }; - let trace_ids = collect_trace_ids(&agg_result_postcard)?; - debug!("the query matched {} traces.", trace_ids.0.len()); - Ok(trace_ids) + ) + .await } +} +#[instrument("find_trace_ids_common", skip_all)] +#[allow(clippy::too_many_arguments)] +pub(crate) async fn find_trace_ids_common( + search_service: Arc, + service_name: &str, + operation_name: &str, + tags: HashMap, + min_start_secs: Option, + max_start_secs: Option, + min_duration_millis: Option, + max_duration_millis: Option, + num_traces: usize, + index_id_patterns: Vec, +) -> Result<(Vec, TimeIntervalSecs), Status> { + let query_ast = build_search_query( + service_name, + None, + operation_name, + tags, + min_start_secs, + max_start_secs, + min_duration_millis, + max_duration_millis, + ); + + let search_request = SearchRequest { + index_id_patterns, + query_ast: serde_json::to_string(&query_ast) + .map_err(|err| Status::internal(err.to_string()))?, + aggregation_request: Some(build_aggregations_query(num_traces)), + max_hits: 0, + start_timestamp: min_start_secs, + end_timestamp: max_start_secs, + count_hits: CountHits::Underestimate.into(), + ..Default::default() + }; + + let search_response = search_service.root_search(search_request).await?; + + let Some(agg_result_postcard) = search_response.aggregation_postcard else { + debug!("the query matched no traces"); + return Ok((Vec::new(), 0..=0)); + }; + + let trace_ids = collect_trace_ids(&agg_result_postcard)?; + debug!("the query matched {} traces.", trace_ids.0.len()); + Ok(trace_ids) +} + +impl JaegerService { #[instrument("stream_spans", skip_all, fields(num_traces=%trace_ids.len(), num_spans=Empty, num_bytes=Empty))] async fn stream_spans( &self, @@ -435,29 +430,7 @@ impl JaegerService { } } -macro_rules! metrics { - ($expr:expr, [$operation:ident, $($label:expr),*]) => { - let start = std::time::Instant::now(); - let labels = [stringify!($operation), $($label,)*]; - JAEGER_SERVICE_METRICS.requests_total.with_label_values(labels).inc(); - let (res, is_error) = match $expr { - ok @ Ok(_) => { - (ok, "false") - }, - err @ Err(_) => { - JAEGER_SERVICE_METRICS.request_errors_total.with_label_values(labels).inc(); - (err, "true") - }, - }; - let elapsed = start.elapsed().as_secs_f64(); - let labels = [stringify!($operation), $($label,)* is_error]; - JAEGER_SERVICE_METRICS.request_duration_seconds.with_label_values(labels).observe(elapsed); - - return res.map(Response::new); - }; -} - -fn record_error(operation_name: &'static str, request_start: Instant) { +pub(crate) fn record_error(operation_name: &'static str, request_start: Instant) { JAEGER_SERVICE_METRICS .request_errors_total .with_label_values([operation_name, OTEL_TRACES_INDEX_ID]) @@ -470,7 +443,7 @@ fn record_error(operation_name: &'static str, request_start: Instant) { .observe(elapsed); } -fn record_send(operation_name: &'static str, num_spans: usize, num_bytes: usize) { +pub(crate) fn record_send(operation_name: &'static str, num_spans: usize, num_bytes: usize) { JAEGER_SERVICE_METRICS .fetched_spans_total .with_label_values([operation_name, OTEL_TRACES_INDEX_ID]) @@ -481,87 +454,7 @@ fn record_send(operation_name: &'static str, num_spans: usize, num_bytes: usize) .inc_by(num_bytes as u64); } -#[async_trait] -impl SpanReaderPlugin for JaegerService { - type GetTraceStream = SpanStream; - - type FindTracesStream = SpanStream; - - async fn get_services( - &self, - request: Request, - ) -> Result, Status> { - let index_id_patterns = - extract_otel_traces_index_id_patterns_from_metadata(request.metadata())?; - metrics!( - self.get_services_for_indexes(request.into_inner(), index_id_patterns) - .await, - [get_services, OTEL_TRACES_INDEX_ID] - ); - } - - async fn get_operations( - &self, - request: Request, - ) -> Result, Status> { - let index_id_patterns = - extract_otel_traces_index_id_patterns_from_metadata(request.metadata())?; - metrics!( - self.get_operations_for_indexes(request.into_inner(), index_id_patterns) - .await, - [get_operations, OTEL_TRACES_INDEX_ID] - ); - } - - async fn find_trace_i_ds( - &self, - request: Request, - ) -> Result, Status> { - let index_id_patterns = - extract_otel_traces_index_id_patterns_from_metadata(request.metadata())?; - metrics!( - self.find_trace_ids_for_indexes(request.into_inner(), index_id_patterns) - .await, - [find_trace_ids, OTEL_TRACES_INDEX_ID] - ); - } - - async fn find_traces( - &self, - request: Request, - ) -> Result, Status> { - let index_id_patterns = - extract_otel_traces_index_id_patterns_from_metadata(request.metadata())?; - self.find_traces_for_indexes( - request.into_inner(), - "find_traces", - Instant::now(), - index_id_patterns, - false, /* if we use true, Jaeger will display "1 Span", and display an empty trace - * when clicking on the ui (but display the full trace after reloading the - * page) */ - ) - .await - .map(Response::new) - } - - async fn get_trace( - &self, - request: Request, - ) -> Result, Status> { - let index_id_patterns = - extract_otel_traces_index_id_patterns_from_metadata(request.metadata())?; - self.get_trace_for_indexes( - request.into_inner(), - "get_trace", - Instant::now(), - index_id_patterns, - ) - .await - .map(Response::new) - } -} - +#[allow(deprecated)] fn extract_term(term_bytes: &[u8]) -> String { tantivy::Term::wrap(term_bytes) .value() @@ -588,9 +481,77 @@ fn extract_operation(term_bytes: &[u8]) -> Operation { } } +#[instrument("get_services", skip_all)] +pub(crate) async fn get_services_impl( + search_service: Arc, + lookback_period_secs: i64, + index_id_patterns: Vec, +) -> Result, Status> { + debug!(index_ids=?index_id_patterns, "`get_services` request"); + + let max_hits = Some(1_000); + let start_timestamp = Some(OffsetDateTime::now_utc().unix_timestamp() - lookback_period_secs); + + let search_request = ListTermsRequest { + index_id_patterns, + field: "service_name".to_string(), + max_hits, + start_timestamp, + end_timestamp: None, + start_key: None, + end_key: None, + }; + let search_response = search_service.root_list_terms(search_request).await?; + let services: Vec = search_response + .terms + .into_iter() + .map(|term_bytes| extract_term(&term_bytes)) + .sorted() + .collect(); + debug!(services=?services, "`get_services` response"); + Ok(services) +} + +#[instrument("get_operations", skip_all, fields(service=%service, span_kind=%span_kind))] +pub(crate) async fn get_operations_impl( + search_service: Arc, + lookback_period_secs: i64, + service: String, + span_kind: String, + index_id_patterns: Vec, +) -> Result, Status> { + debug!(service=%service, span_kind=%span_kind, index_ids=?index_id_patterns, "`get_operations` request"); + + let max_hits = Some(1_000); + let start_timestamp = Some(OffsetDateTime::now_utc().unix_timestamp() - lookback_period_secs); + + let span_kind_opt = span_kind.parse().ok(); + let start_key = SpanFingerprint::start_key(&service, span_kind_opt.clone()); + let end_key = SpanFingerprint::end_key(&service, span_kind_opt); + + let search_request = ListTermsRequest { + index_id_patterns, + field: "span_fingerprint".to_string(), + max_hits, + start_timestamp, + end_timestamp: None, + start_key, + end_key, + }; + let search_response = search_service.root_list_terms(search_request).await?; + let operations: Vec = search_response + .terms + .into_iter() + .map(|term_json| extract_operation(&term_json)) + .sorted() + .collect(); + debug!(operations=?operations, "`get_operations` response"); + Ok(operations) +} + // TODO: builder pattern #[allow(clippy::too_many_arguments)] -fn build_search_query( +pub(crate) fn build_search_query( service_name: &str, span_kind_opt: Option, span_name: &str, @@ -745,7 +706,7 @@ fn build_search_query( } } -fn build_aggregations_query(num_traces: usize) -> String { +pub(crate) fn build_aggregations_query(num_traces: usize) -> String { let query = serde_json::to_string(&FindTraceIdsCollector { num_traces, trace_id_field_name: "trace_id".to_string(), @@ -806,7 +767,7 @@ fn qw_span_to_jaeger_span(qw_span_json: &str) -> Result { Ok(span) } -fn to_duration_millis(duration: &WellKnownDuration) -> Option { +pub(crate) fn to_duration_millis(duration: &WellKnownDuration) -> Option { let duration_millis = duration.seconds * 1_000 + (duration.nanos as i64) / 1_000_000; if duration_millis == 0 { None @@ -1113,7 +1074,9 @@ fn collect_trace_ids( #[allow(clippy::result_large_err)] fn json_deserialize<'a, T>(json: &'a str, label: &'static str) -> Result -where T: Deserialize<'a> { +where + T: Deserialize<'a>, +{ match serde_json::from_str(json) { Ok(deserialized) => Ok(deserialized), Err(error) => { @@ -1127,7 +1090,9 @@ where T: Deserialize<'a> { #[allow(clippy::result_large_err)] fn postcard_deserialize<'a, T>(json: &'a [u8], label: &'static str) -> Result -where T: Deserialize<'a> { +where + T: Deserialize<'a>, +{ match postcard::from_bytes(json) { Ok(deserialized) => Ok(deserialized), Err(error) => { @@ -1143,6 +1108,7 @@ where T: Deserialize<'a> { mod tests { use quickwit_opentelemetry::otlp::{OTEL_TRACES_INDEX_ID_PATTERN, OtelSignal}; use quickwit_proto::jaeger::api_v2::ValueType; + use quickwit_proto::jaeger::storage::v1::span_reader_plugin_server::SpanReaderPlugin; use quickwit_search::{MockSearchService, QuickwitAggregations, encode_term_for_test}; use serde_json::json; @@ -2631,4 +2597,210 @@ mod tests { let response = jaeger.get_services(request).await.unwrap().into_inner(); assert_eq!(response.services, &["service1", "service2", "service3"]); } + + #[tokio::test] + async fn test_v2_get_services() { + let mut service = MockSearchService::new(); + service + .expect_root_list_terms() + .withf(|req| { + req.index_id_patterns == vec![OTEL_TRACES_INDEX_ID_PATTERN] + && req.field == "service_name" + && req.start_timestamp.is_some() + }) + .return_once(|_| { + Ok(quickwit_proto::search::ListTermsResponse { + num_hits: 3, + terms: vec![ + encode_term_for_test!("service1"), + encode_term_for_test!("service2"), + encode_term_for_test!("service3"), + ], + elapsed_time_micros: 0, + errors: Vec::new(), + }) + }); + + let service = Arc::new(service); + let jaeger = JaegerService::new(JaegerConfig::default(), service); + + let request = + tonic::Request::new(quickwit_proto::jaeger::storage::v2::GetServicesRequest {}); + let response = + quickwit_proto::jaeger::storage::v2::trace_reader_server::TraceReader::get_services( + &jaeger, request, + ) + .await + .unwrap() + .into_inner(); + assert_eq!(response.services, &["service1", "service2", "service3"]); + } + + #[tokio::test] + async fn test_v2_get_operations() { + let mut service = MockSearchService::new(); + service + .expect_root_list_terms() + .withf(|req| { + req.index_id_patterns == vec![OTEL_TRACES_INDEX_ID_PATTERN] + && req.field == "span_fingerprint" + && req.start_timestamp.is_some() + }) + .return_once(|_| { + let fingerprint1 = + SpanFingerprint::new("test-service", QwSpanKind::from(2), "GET /api"); + let fingerprint2 = + SpanFingerprint::new("test-service", QwSpanKind::from(3), "POST /data"); + + Ok(quickwit_proto::search::ListTermsResponse { + num_hits: 2, + terms: vec![ + encode_term_for_test!(fingerprint1.as_str()), + encode_term_for_test!(fingerprint2.as_str()), + ], + elapsed_time_micros: 0, + errors: Vec::new(), + }) + }); + + let service = Arc::new(service); + let jaeger = JaegerService::new(JaegerConfig::default(), service); + + let request = + tonic::Request::new(quickwit_proto::jaeger::storage::v2::GetOperationsRequest { + service: "test-service".to_string(), + span_kind: String::new(), + }); + let response = + quickwit_proto::jaeger::storage::v2::trace_reader_server::TraceReader::get_operations( + &jaeger, request, + ) + .await + .unwrap() + .into_inner(); + assert_eq!(response.operations.len(), 2); + assert_eq!(response.operations[0].name, "GET /api"); + assert_eq!(response.operations[0].span_kind, "server"); + assert_eq!(response.operations[1].name, "POST /data"); + assert_eq!(response.operations[1].span_kind, "client"); + } + + #[tokio::test] + async fn test_v2_find_trace_ids() { + let mut service = MockSearchService::new(); + service + .expect_root_search() + .withf(|req| { + req.index_id_patterns == vec![OTEL_TRACES_INDEX_ID_PATTERN] + && req.start_timestamp.is_some() + && req.end_timestamp.is_some() + }) + .return_once(|_| { + use quickwit_search::Span as TraceSpan; + use tantivy::DateTime; + + let trace_id_1 = + TraceId::new([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]); + let trace_id_2 = TraceId::new([ + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]); + + let spans = vec![ + TraceSpan { + trace_id: trace_id_1, + span_timestamp: DateTime::from_timestamp_secs(1500), + }, + TraceSpan { + trace_id: trace_id_2, + span_timestamp: DateTime::from_timestamp_secs(1600), + }, + ]; + + let aggregation_postcard = postcard::to_allocvec(&spans).unwrap(); + + Ok(quickwit_proto::search::SearchResponse { + num_hits: 2, + hits: vec![], + elapsed_time_micros: 100, + errors: Vec::new(), + aggregation_postcard: Some(aggregation_postcard), + scroll_id: None, + failed_splits: Vec::new(), + num_successful_splits: 1, + }) + }); + + let service = Arc::new(service); + let jaeger = JaegerService::new(JaegerConfig::default(), service); + + let request = tonic::Request::new(quickwit_proto::jaeger::storage::v2::FindTracesRequest { + query: Some(quickwit_proto::jaeger::storage::v2::TraceQueryParameters { + service_name: "test-service".to_string(), + operation_name: String::new(), + attributes: vec![], + start_time_min: Some(prost_types::Timestamp { + seconds: 1000, + nanos: 0, + }), + start_time_max: Some(prost_types::Timestamp { + seconds: 2000, + nanos: 0, + }), + duration_min: None, + duration_max: None, + search_depth: 10, + }), + }); + let response = + quickwit_proto::jaeger::storage::v2::trace_reader_server::TraceReader::find_trace_i_ds( + &jaeger, request, + ) + .await + .unwrap() + .into_inner(); + assert_eq!(response.trace_ids.len(), 2); + assert_eq!(response.trace_ids[0].trace_id.len(), 16); + assert_eq!(response.trace_ids[1].trace_id.len(), 16); + } + + #[test] + fn test_convert_v2_attributes_to_v1_tags() { + let attributes = vec![ + quickwit_proto::jaeger::storage::v2::KeyValue { + key: "http.method".to_string(), + value: Some(quickwit_proto::jaeger::storage::v2::AnyValue { + value: Some( + quickwit_proto::jaeger::storage::v2::any_value::Value::StringValue( + "GET".to_string(), + ), + ), + }), + }, + quickwit_proto::jaeger::storage::v2::KeyValue { + key: "http.status_code".to_string(), + value: Some(quickwit_proto::jaeger::storage::v2::AnyValue { + value: Some( + quickwit_proto::jaeger::storage::v2::any_value::Value::IntValue(200), + ), + }), + }, + quickwit_proto::jaeger::storage::v2::KeyValue { + key: "error".to_string(), + value: Some(quickwit_proto::jaeger::storage::v2::AnyValue { + value: Some( + quickwit_proto::jaeger::storage::v2::any_value::Value::BoolValue(false), + ), + }), + }, + ]; + + let tags = crate::v2::convert_v2_attributes_to_v1_tags(attributes); + assert_eq!(tags.len(), 3); + assert_eq!(tags.get("http.method"), Some(&"GET".to_string())); + assert_eq!(tags.get("http.status_code"), Some(&"200".to_string())); + assert_eq!(tags.get("error"), Some(&"false".to_string())); + } + + // Note: test_spans_to_otel_traces_data was removed as v2 now works directly with + // native OpenTelemetry format (QwSpan) instead of converting from Jaeger v1 format } diff --git a/quickwit/quickwit-jaeger/src/v1.rs b/quickwit/quickwit-jaeger/src/v1.rs new file mode 100644 index 00000000000..11d6935db4e --- /dev/null +++ b/quickwit/quickwit-jaeger/src/v1.rs @@ -0,0 +1,134 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Jaeger v1 API implementation (SpanReaderPlugin) + +use std::time::Instant; + +use async_trait::async_trait; +use quickwit_opentelemetry::otlp::{ + OTEL_TRACES_INDEX_ID, extract_otel_traces_index_id_patterns_from_metadata, +}; +use quickwit_proto::jaeger::storage::v1::span_reader_plugin_server::SpanReaderPlugin; +use quickwit_proto::jaeger::storage::v1::{ + FindTraceIDsRequest, FindTraceIDsResponse, FindTracesRequest, GetOperationsRequest, + GetOperationsResponse, GetServicesRequest, GetServicesResponse, GetTraceRequest, +}; +use tonic::{Request, Response, Status}; + +use crate::metrics::JAEGER_SERVICE_METRICS; +use crate::{JaegerService, SpanStream}; + +macro_rules! metrics { + ($expr:expr, [$operation:ident, $($label:expr),*]) => { + let start = std::time::Instant::now(); + let labels = [stringify!($operation), $($label,)*]; + JAEGER_SERVICE_METRICS.requests_total.with_label_values(labels).inc(); + let (res, is_error) = match $expr { + ok @ Ok(_) => { + (ok, "false") + }, + err @ Err(_) => { + JAEGER_SERVICE_METRICS.request_errors_total.with_label_values(labels).inc(); + (err, "true") + }, + }; + let elapsed = start.elapsed().as_secs_f64(); + let labels = [stringify!($operation), $($label,)* is_error]; + JAEGER_SERVICE_METRICS.request_duration_seconds.with_label_values(labels).observe(elapsed); + + return res.map(Response::new); + }; +} + +#[async_trait] +impl SpanReaderPlugin for JaegerService { + type GetTraceStream = SpanStream; + + type FindTracesStream = SpanStream; + + async fn get_services( + &self, + request: Request, + ) -> Result, Status> { + let index_id_patterns = + extract_otel_traces_index_id_patterns_from_metadata(request.metadata())?; + metrics!( + self.get_services_for_indexes(request.into_inner(), index_id_patterns) + .await, + [get_services, OTEL_TRACES_INDEX_ID] + ); + } + + async fn get_operations( + &self, + request: Request, + ) -> Result, Status> { + let index_id_patterns = + extract_otel_traces_index_id_patterns_from_metadata(request.metadata())?; + metrics!( + self.get_operations_for_indexes(request.into_inner(), index_id_patterns) + .await, + [get_operations, OTEL_TRACES_INDEX_ID] + ); + } + + async fn find_trace_i_ds( + &self, + request: Request, + ) -> Result, Status> { + let index_id_patterns = + extract_otel_traces_index_id_patterns_from_metadata(request.metadata())?; + metrics!( + self.find_trace_ids_for_indexes(request.into_inner(), index_id_patterns) + .await, + [find_trace_ids, OTEL_TRACES_INDEX_ID] + ); + } + + async fn find_traces( + &self, + request: Request, + ) -> Result, Status> { + let index_id_patterns = + extract_otel_traces_index_id_patterns_from_metadata(request.metadata())?; + self.find_traces_for_indexes( + request.into_inner(), + "find_traces", + Instant::now(), + index_id_patterns, + false, /* if we use true, Jaeger will display "1 Span", and display an empty trace + * when clicking on the ui (but display the full trace after reloading the + * page) */ + ) + .await + .map(Response::new) + } + + async fn get_trace( + &self, + request: Request, + ) -> Result, Status> { + let index_id_patterns = + extract_otel_traces_index_id_patterns_from_metadata(request.metadata())?; + self.get_trace_for_indexes( + request.into_inner(), + "get_trace", + Instant::now(), + index_id_patterns, + ) + .await + .map(Response::new) + } +} diff --git a/quickwit/quickwit-jaeger/src/v2.rs b/quickwit/quickwit-jaeger/src/v2.rs new file mode 100644 index 00000000000..e355c18a8c3 --- /dev/null +++ b/quickwit/quickwit-jaeger/src/v2.rs @@ -0,0 +1,639 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Jaeger v2 API implementation (TraceReader) +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Instant; + +use async_trait::async_trait; +use prost_types::Timestamp as WellKnownTimestamp; +use quickwit_opentelemetry::otlp::{ + OTEL_TRACES_INDEX_ID, Span as QwSpan, TraceId, + extract_otel_traces_index_id_patterns_from_metadata, +}; +use quickwit_proto::jaeger::storage::v2::trace_reader_server::TraceReader; +use quickwit_proto::jaeger::storage::v2::{ + FindTracesRequest, FoundTraceId, GetOperationsRequest, GetOperationsResponse, + GetServicesRequest, GetServicesResponse, GetTracesRequest, Operation, +}; +use quickwit_proto::opentelemetry::proto::common::v1::any_value::Value as OtelValue; +use quickwit_proto::opentelemetry::proto::common::v1::{ + AnyValue as OtelAnyValue, InstrumentationScope, KeyValue as OtelKeyValue, +}; +use quickwit_proto::opentelemetry::proto::resource::v1::Resource as OtelResource; +use quickwit_proto::opentelemetry::proto::trace::v1 as otel_trace; +use quickwit_proto::opentelemetry::proto::trace::v1::status::StatusCode as OtelStatusCode; +use quickwit_proto::opentelemetry::proto::trace::v1::{ + ResourceSpans, ScopeSpans, Span as OtelSpan, Status as OtelStatus, +}; +use quickwit_proto::search::{CountHits, SearchRequest}; +use quickwit_query::BooleanOperand; +use quickwit_query::query_ast::{BoolQuery, QueryAst, TermQuery, UserInputQuery}; +use quickwit_search::SearchService; +use serde_json::Value as JsonValue; +use time::OffsetDateTime; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{Request, Response, Status}; +use tracing::field::Empty; +use tracing::{Span as RuntimeSpan, debug, error, instrument}; + +use crate::metrics::JAEGER_SERVICE_METRICS; +use crate::{ + JaegerService, TimeIntervalSecs, TracesDataStream, get_operations_impl, get_services_impl, + json_deserialize, record_error, record_send, to_duration_millis, +}; + +macro_rules! metrics { + ($expr:expr, [$operation:ident, $($label:expr),*]) => { + let start = std::time::Instant::now(); + let labels = [stringify!($operation), $($label,)*]; + JAEGER_SERVICE_METRICS.requests_total.with_label_values(labels).inc(); + let (res, is_error) = match $expr { + ok @ Ok(_) => { + (ok, "false") + }, + err @ Err(_) => { + JAEGER_SERVICE_METRICS.request_errors_total.with_label_values(labels).inc(); + (err, "true") + }, + }; + let elapsed = start.elapsed().as_secs_f64(); + let labels = [stringify!($operation), $($label,)* is_error]; + JAEGER_SERVICE_METRICS.request_duration_seconds.with_label_values(labels).observe(elapsed); + + return res.map(Response::new); + }; +} + +#[async_trait] +impl TraceReader for JaegerService { + async fn get_services( + &self, + request: Request, + ) -> Result, Status> { + let index_id_patterns = + extract_otel_traces_index_id_patterns_from_metadata(request.metadata())?; + + let services = get_services_impl( + self.search_service.clone(), + self.lookback_period_secs, + index_id_patterns, + ) + .await?; + + let response = GetServicesResponse { services }; + metrics!(Ok(response), [get_services_v2, OTEL_TRACES_INDEX_ID]); + } + + async fn get_operations( + &self, + request: Request, + ) -> Result, Status> { + let index_id_patterns = + extract_otel_traces_index_id_patterns_from_metadata(request.metadata())?; + + let req = request.into_inner(); + + let operations = get_operations_impl( + self.search_service.clone(), + self.lookback_period_secs, + req.service, + req.span_kind, + index_id_patterns, + ) + .await? + .into_iter() + .map(|op| Operation { + name: op.name, + span_kind: op.span_kind, + }) + .collect(); + + let response = GetOperationsResponse { operations }; + metrics!(Ok(response), [get_operations_v2, OTEL_TRACES_INDEX_ID]); + } + + type GetTracesStream = TracesDataStream; + + async fn get_traces( + &self, + request: Request, + ) -> Result, Status> { + let request_start = Instant::now(); + let index_id_patterns = + extract_otel_traces_index_id_patterns_from_metadata(request.metadata())?; + + let (tx, rx) = mpsc::channel(2); + let search_service = self.search_service.clone(); + let max_fetch_spans = self.max_fetch_spans; + let lookback_period_secs = self.lookback_period_secs; + let query_list = request.into_inner().query; + + tokio::task::spawn(async move { + for query_params in query_list { + let trace_id = match TraceId::try_from(query_params.trace_id) { + Ok(id) => id, + Err(error) => { + let _ = tx + .send(Err(Status::invalid_argument(error.to_string()))) + .await; + return; + } + }; + + let end = OffsetDateTime::now_utc().unix_timestamp(); + let search_window = (end - lookback_period_secs)..=end; + + let otel_spans = match stream_otel_spans_impl( + search_service.clone(), + max_fetch_spans, + &[trace_id], + search_window, + "get_traces_v2", + request_start, + index_id_patterns.clone(), + false, + ) + .await + { + Ok(spans) => spans, + Err(e) => { + let _ = tx.send(Err(e)).await; + return; + } + }; + + if tx + .send(Ok(qw_spans_to_otel_traces_data(otel_spans))) + .await + .is_err() + { + return; + } + } + }); + + Ok(Response::new(ReceiverStream::new(rx))) + } + + type FindTracesStream = TracesDataStream; + + async fn find_traces( + &self, + request: Request, + ) -> Result, Status> { + let request_start = Instant::now(); + + let index_id_patterns = + extract_otel_traces_index_id_patterns_from_metadata(request.metadata())?; + + let query = request + .into_inner() + .query + .ok_or_else(|| Status::invalid_argument("Query is empty."))?; + + let (trace_ids, span_timestamps_range) = find_trace_ids_impl( + self.search_service.clone(), + self.max_trace_duration_secs, + query, + index_id_patterns.clone(), + ) + .await?; + + let search_window = (span_timestamps_range.start() - self.max_trace_duration_secs) + ..=(span_timestamps_range.end() + self.max_trace_duration_secs); + + let (tx, rx) = mpsc::channel(2); + let search_service = self.search_service.clone(); + let max_fetch_spans = self.max_fetch_spans; + + tokio::task::spawn(async move { + let all_spans = match stream_otel_spans_impl( + search_service, + max_fetch_spans, + &trace_ids, + search_window, + "find_traces_v2", + request_start, + index_id_patterns, + false, + ) + .await + { + Ok(spans) => spans, + Err(e) => { + let _ = tx.send(Err(e)).await; + return; + } + }; + + // Group by trace_id and send each trace + let mut spans_by_trace: HashMap, Vec> = HashMap::new(); + for span in all_spans { + spans_by_trace + .entry(span.trace_id.to_vec()) + .or_default() + .push(span); + } + + for spans in spans_by_trace.into_values() { + if tx + .send(Ok(qw_spans_to_otel_traces_data(spans))) + .await + .is_err() + { + return; + } + } + }); + + Ok(Response::new(ReceiverStream::new(rx))) + } + + async fn find_trace_i_ds( + &self, + request: Request, + ) -> Result, Status> { + let index_id_patterns = + extract_otel_traces_index_id_patterns_from_metadata(request.metadata())?; + + let query = request + .into_inner() + .query + .ok_or_else(|| Status::invalid_argument("Query is empty."))?; + + let (trace_ids, time_range) = find_trace_ids_impl( + self.search_service.clone(), + self.max_trace_duration_secs, + query, + index_id_patterns, + ) + .await?; + + let trace_ids = trace_ids + .into_iter() + .map(|trace_id| FoundTraceId { + trace_id: trace_id.to_vec(), + start: Some(WellKnownTimestamp { + seconds: *time_range.start(), + nanos: 0, + }), + end: Some(WellKnownTimestamp { + seconds: *time_range.end(), + nanos: 0, + }), + }) + .collect(); + + let response = quickwit_proto::jaeger::storage::v2::FindTraceIDsResponse { trace_ids }; + metrics!(Ok(response), [find_trace_ids_v2, OTEL_TRACES_INDEX_ID]); + } +} + +// === Helper functions === +#[instrument("find_trace_ids", skip_all)] +async fn find_trace_ids_impl( + search_service: Arc, + _max_trace_duration_secs: i64, + query: quickwit_proto::jaeger::storage::v2::TraceQueryParameters, + index_id_patterns: Vec, +) -> Result<(Vec, TimeIntervalSecs), Status> { + debug!(service_name=%query.service_name, operation_name=%query.operation_name, "`find_trace_ids` request"); + + let min_start_secs = query.start_time_min.as_ref().map(|ts| ts.seconds); + let max_start_secs = query.start_time_max.as_ref().map(|ts| ts.seconds); + let min_duration_millis = query.duration_min.as_ref().and_then(to_duration_millis); + let max_duration_millis = query.duration_max.as_ref().and_then(to_duration_millis); + let tags = convert_v2_attributes_to_v1_tags(query.attributes); + + crate::find_trace_ids_common( + search_service, + &query.service_name, + &query.operation_name, + tags, + min_start_secs, + max_start_secs, + min_duration_millis, + max_duration_millis, + query.search_depth as usize, + index_id_patterns, + ) + .await +} + +#[instrument("stream_otel_spans", skip_all, fields(num_traces=%trace_ids.len(), num_spans=Empty, num_bytes=Empty))] +#[allow(clippy::too_many_arguments)] +async fn stream_otel_spans_impl( + search_service: Arc, + max_fetch_spans: u64, + trace_ids: &[TraceId], + search_window: TimeIntervalSecs, + operation_name: &'static str, + request_start: Instant, + index_id_patterns: Vec, + root_only: bool, +) -> Result, Status> { + if trace_ids.is_empty() { + return Ok(Vec::new()); + } + + let mut query = BoolQuery::default(); + + for trace_id in trace_ids { + let value = trace_id.hex_display(); + let term_query = TermQuery { + field: "trace_id".to_string(), + value, + }; + query.should.push(term_query.into()); + } + + if root_only { + let is_root = UserInputQuery { + user_text: "NOT is_root:false".to_string(), + default_fields: None, + default_operator: BooleanOperand::And, + lenient: true, + }; + let mut new_query = BoolQuery::default(); + new_query.must.push(query.into()); + new_query.must.push(is_root.into()); + query = new_query; + } + + let query_ast: QueryAst = query.into(); + let query_ast = + serde_json::to_string(&query_ast).map_err(|err| Status::internal(err.to_string()))?; + + let search_request = SearchRequest { + index_id_patterns, + query_ast, + start_timestamp: Some(*search_window.start()), + end_timestamp: Some(*search_window.end()), + max_hits: max_fetch_spans, + count_hits: CountHits::Underestimate.into(), + ..Default::default() + }; + + let search_response = match search_service.root_search(search_request).await { + Ok(search_response) => search_response, + Err(search_error) => { + error!(search_error=?search_error, "failed to fetch spans"); + record_error(operation_name, request_start); + return Err(Status::internal("Failed to fetch spans.")); + } + }; + + let mut qw_spans: Vec = Vec::with_capacity(search_response.hits.len()); + + for hit in search_response.hits { + match qw_span_from_json(&hit.json) { + Ok(span) => { + qw_spans.push(span); + } + Err(status) => { + record_error(operation_name, request_start); + return Err(status); + } + }; + } + + if trace_ids.len() > 1 { + qw_spans.sort_unstable_by(|left, right| left.trace_id.cmp(&right.trace_id)); + } + + let num_spans = qw_spans.len(); + let num_bytes = qw_spans + .iter() + .map(|span| serde_json::to_string(span).unwrap_or_default().len()) + .sum::(); + + RuntimeSpan::current().record("num_spans", num_spans); + RuntimeSpan::current().record("num_bytes", num_bytes); + + record_send(operation_name, num_spans, num_bytes); + + JAEGER_SERVICE_METRICS + .fetched_traces_total + .with_label_values([operation_name, OTEL_TRACES_INDEX_ID]) + .inc_by(trace_ids.len() as u64); + + let elapsed = request_start.elapsed().as_secs_f64(); + JAEGER_SERVICE_METRICS + .request_duration_seconds + .with_label_values([operation_name, OTEL_TRACES_INDEX_ID, "false"]) + .observe(elapsed); + + Ok(qw_spans) +} + +// === Conversion functions === +// Note: record_error and record_send are now shared in lib.rs + +/// Direct conversion from Quickwit's native OpenTelemetry span to Jaeger v2's OpenTelemetry format +fn qw_spans_to_otel_traces_data( + qw_spans: Vec, +) -> quickwit_proto::opentelemetry::proto::trace::v1::TracesData { + // Group spans by service + let mut spans_by_service: HashMap> = HashMap::new(); + for span in qw_spans { + spans_by_service + .entry(span.service_name.clone()) + .or_default() + .push(span); + } + + let resource_spans = spans_by_service + .into_iter() + .map(|(service_name, spans)| { + // Get resource attributes from first span before grouping + let first_span_attrs = spans + .first() + .map(|span| span.resource_attributes.clone()) + .unwrap_or_default(); + + // Group by scope + let mut spans_by_scope: HashMap<(Option, Option), Vec> = + HashMap::new(); + for span in spans { + let key = (span.scope_name.clone(), span.scope_version.clone()); + spans_by_scope.entry(key).or_default().push(span); + } + + let scope_spans = spans_by_scope + .into_iter() + .map(|((scope_name, scope_version), spans)| { + let otel_spans = spans.into_iter().map(qw_span_to_otel_span).collect(); + + ScopeSpans { + scope: Some(InstrumentationScope { + name: scope_name.unwrap_or_default(), + version: scope_version.unwrap_or_default(), + attributes: vec![], + dropped_attributes_count: 0, + }), + spans: otel_spans, + schema_url: String::new(), + } + }) + .collect(); + + let mut resource_attrs = vec![OtelKeyValue { + key: "service.name".to_string(), + value: Some(OtelAnyValue { + value: Some(OtelValue::StringValue(service_name)), + }), + }]; + + // Add other resource attributes + for (key, value) in first_span_attrs { + resource_attrs.push(json_value_to_otel_kv(key, value)); + } + + ResourceSpans { + resource: Some(OtelResource { + attributes: resource_attrs, + dropped_attributes_count: 0, + }), + scope_spans, + schema_url: String::new(), + } + }) + .collect(); + + quickwit_proto::opentelemetry::proto::trace::v1::TracesData { resource_spans } +} + +/// Convert a Quickwit span (native OTEL format) to Jaeger v2 OTEL span +fn qw_span_to_otel_span(qw_span: QwSpan) -> OtelSpan { + OtelSpan { + trace_id: qw_span.trace_id.to_vec(), + span_id: qw_span.span_id.to_vec(), + trace_state: qw_span.trace_state.unwrap_or_default(), + parent_span_id: qw_span + .parent_span_id + .map(|id| id.to_vec()) + .unwrap_or_default(), + name: qw_span.span_name, + kind: qw_span.span_kind as i32, + start_time_unix_nano: qw_span.span_start_timestamp_nanos, + end_time_unix_nano: qw_span.span_end_timestamp_nanos, + attributes: qw_span + .span_attributes + .into_iter() + .map(|(k, v)| json_value_to_otel_kv(k, v)) + .collect(), + dropped_attributes_count: qw_span.span_dropped_attributes_count, + events: qw_span + .events + .into_iter() + .map(|event| otel_trace::span::Event { + time_unix_nano: event.event_timestamp_nanos, + name: event.event_name, + attributes: event + .event_attributes + .into_iter() + .map(|(k, v)| json_value_to_otel_kv(k, v)) + .collect(), + dropped_attributes_count: event.event_dropped_attributes_count, + }) + .collect(), + dropped_events_count: qw_span.span_dropped_events_count, + links: qw_span + .links + .into_iter() + .map(|link| otel_trace::span::Link { + trace_id: link.link_trace_id.to_vec(), + span_id: link.link_span_id.to_vec(), + trace_state: link.link_trace_state.unwrap_or_default(), + attributes: link + .link_attributes + .into_iter() + .map(|(k, v)| json_value_to_otel_kv(k, v)) + .collect(), + dropped_attributes_count: link.link_dropped_attributes_count, + }) + .collect(), + dropped_links_count: qw_span.span_dropped_links_count, + status: Some(OtelStatus { + message: qw_span.span_status.message.unwrap_or_default(), + code: match qw_span.span_status.code { + quickwit_proto::opentelemetry::proto::trace::v1::status::StatusCode::Unset => { + OtelStatusCode::Unset as i32 + } + quickwit_proto::opentelemetry::proto::trace::v1::status::StatusCode::Ok => { + OtelStatusCode::Ok as i32 + } + quickwit_proto::opentelemetry::proto::trace::v1::status::StatusCode::Error => { + OtelStatusCode::Error as i32 + } + }, + }), + } +} + +fn json_value_to_otel_kv(key: String, value: JsonValue) -> OtelKeyValue { + let otel_value = match value { + JsonValue::String(s) => OtelValue::StringValue(s), + JsonValue::Number(n) => { + if let Some(i) = n.as_i64() { + OtelValue::IntValue(i) + } else if let Some(f) = n.as_f64() { + OtelValue::DoubleValue(f) + } else { + OtelValue::StringValue(n.to_string()) + } + } + JsonValue::Bool(b) => OtelValue::BoolValue(b), + JsonValue::Array(_) | JsonValue::Object(_) => OtelValue::StringValue(value.to_string()), + JsonValue::Null => OtelValue::StringValue(String::new()), + }; + + OtelKeyValue { + key, + value: Some(OtelAnyValue { + value: Some(otel_value), + }), + } +} + +#[allow(clippy::result_large_err)] +fn qw_span_from_json(qw_span_json: &str) -> Result { + json_deserialize(qw_span_json, "span") +} + +pub(crate) fn convert_v2_attributes_to_v1_tags( + attributes: Vec, +) -> HashMap { + attributes + .into_iter() + .filter_map(|kv| { + let value = kv.value?.value?; + let string_value = match value { + quickwit_proto::jaeger::storage::v2::any_value::Value::StringValue(s) => s, + quickwit_proto::jaeger::storage::v2::any_value::Value::IntValue(i) => i.to_string(), + quickwit_proto::jaeger::storage::v2::any_value::Value::DoubleValue(d) => { + d.to_string() + } + quickwit_proto::jaeger::storage::v2::any_value::Value::BoolValue(b) => { + b.to_string() + } + _ => return None, + }; + Some((kv.key, string_value)) + }) + .collect() +} diff --git a/quickwit/quickwit-lambda-client/Cargo.toml b/quickwit/quickwit-lambda-client/Cargo.toml new file mode 100644 index 00000000000..257172068b3 --- /dev/null +++ b/quickwit/quickwit-lambda-client/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "quickwit-lambda-client" +description = "AWS Lambda client for Quickwit leaf search invocation and deployment" + +version.workspace = true +edition.workspace = true +homepage.workspace = true +documentation.workspace = true +repository.workspace = true +authors.workspace = true +license.workspace = true + +[dependencies] +anyhow = { workspace = true } +async-trait = { workspace = true } +aws-config = { workspace = true } +aws-sdk-lambda = { workspace = true } +base64 = { workspace = true } +md5 = { workspace = true } +prost = { workspace = true } +serde_json = { workspace = true } +once_cell = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } + +quickwit-common = { workspace = true } +quickwit-config = { workspace = true } +quickwit-lambda-server = { workspace = true } +quickwit-proto = { workspace = true } +quickwit-search = { workspace = true } + +[dev-dependencies] +aws-smithy-mocks = { workspace = true } +aws-sdk-lambda = { workspace = true, features = ["test-util"] } +bytesize = { workspace = true } +tokio = { workspace = true, features = ["test-util", "macros"] } + +# Required for complicated reasons. quickwit-storage checks that we +# do use preserve order with serde. aws forces that feature. We disable +# the check by switching on its testsuite feature. +quickwit-storage = { workspace = true, features = ["testsuite"] } + +[build-dependencies] +sha2 = { workspace = true } +ureq = { workspace = true } diff --git a/quickwit/quickwit-lambda-client/README.md b/quickwit/quickwit-lambda-client/README.md new file mode 100644 index 00000000000..94d4fbf965b --- /dev/null +++ b/quickwit/quickwit-lambda-client/README.md @@ -0,0 +1,51 @@ +# Quickwit Lambda + +Quickwit supports offloading leaf search to AWS Lambda for horizontal scaling. +The Lambda function is built separately and embedded into Quickwit's binary, +allowing Quickwit to auto-deploy the function at startup. + +## Architecture + +- **quickwit-lambda-server**: The Lambda function binary that executes leaf searches +- **quickwit-lambda-client**: The client that invokes Lambda and embeds the Lambda zip for auto-deployment + +## Release Process + +### 1. Tag the release + +Push a tag with the `lambda-` prefix: + +```bash +git tag lambda-v0.8.0 +git push origin lambda-v0.8.0 +``` + +This triggers the `publish_lambda.yaml` GitHub Action which: +- Cross-compiles the Lambda binary for ARM64 +- Creates a zip file named `quickwit-aws-lambda-v0.8.0-aarch64.zip` +- Uploads it as a **draft** GitHub release + +### 2. Publish the release + +Go to GitHub releases and manually publish the draft release to make the +artifact URL publicly accessible. + +### 3. Update the embedded Lambda URL + +Update `LAMBDA_ZIP_URL` in `quickwit-lambda-client/build.rs` to point to the +new release: + +```rust +const LAMBDA_ZIP_URL: &str = "https://github.com/quickwit-oss/quickwit/releases/download/lambda-v0.8.0/quickwit-aws-lambda-v0.8.0-aarch64.zip"; +``` + +### 4. Versioning + +The Lambda client uses content-based versioning: +- An MD5 hash of the Lambda zip is computed at build time +- This hash is embedded in the Lambda function description as `quickwit:{version}-{hash_short}` +- When Quickwit starts, it checks if a matching version exists before deploying + +This ensures that: +- Different Quickwit builds with the same Lambda binary share the same Lambda version +- Updating the Lambda binary automatically triggers a new deployment diff --git a/quickwit/quickwit-lambda-client/build.rs b/quickwit/quickwit-lambda-client/build.rs new file mode 100644 index 00000000000..6223ed7668c --- /dev/null +++ b/quickwit/quickwit-lambda-client/build.rs @@ -0,0 +1,122 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Build script for quickwit-lambda-client. +//! +//! This script downloads the pre-built Lambda zip from a GitHub release +//! and places it in OUT_DIR for embedding via include_bytes! +//! +//! The Lambda binary is built separately in CI and published as a GitHub release. + +use std::env; +use std::path::{Path, PathBuf}; + +use sha2::{Digest, Sha256}; + +/// URL to download the pre-built Lambda zip from GitHub releases. +/// This should be updated when a new Lambda binary is released. +const LAMBDA_ZIP_URL: &str = "https://github.com/quickwit-oss/quickwit/releases/download/lambda-ff6fdfa5/quickwit-aws-lambda--aarch64.zip"; + +/// Expected SHA256 hash of the Lambda zip artifact. +/// Must be updated alongside LAMBDA_ZIP_URL when a new Lambda binary is released. +const LAMBDA_ZIP_SHA256: &str = "fa940f44178e28460c21e44bb2610b776542b9b97db66a53bc65b10cad653b90"; + +/// AWS Lambda direct upload limit is 50MB. +/// Larger artifacts must be uploaded via S3. +const MAX_LAMBDA_ZIP_SIZE: usize = 50 * 1024 * 1024; + +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + + let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR not set")); + let zip_path = out_dir.join("lambda_bootstrap.zip"); + + fetch_lambda_zip(&zip_path); + + // Export first 8 hex chars of the SHA256 as environment variable. + // This is used to create a unique qualifier for Lambda versioning. + let hash_short = &LAMBDA_ZIP_SHA256[..8]; + println!("cargo:rustc-env=LAMBDA_BINARY_HASH={}", hash_short); + println!("lambda binary hash (short): {}", hash_short); +} + +/// Fetch the Lambda zip and save it to `local_cache_path`. +/// +/// If a cached file already exists with the correct SHA256, this is a no-op. +/// If the hash doesn't match (stale artifact), the file is deleted and re-downloaded. +/// If no cached file exists, the artifact is downloaded fresh. +/// +/// This function panics if a problem happens. +fn fetch_lambda_zip(local_cache_path: &Path) { + // Try the cache first. + if let Ok(data) = std::fs::read(local_cache_path) { + let actual_hash = sha256_hex(&data); + if actual_hash == LAMBDA_ZIP_SHA256 { + println!("using cached Lambda zip from {:?}", local_cache_path); + return; + } + println!("cargo:warning=cached Lambda zip has wrong SHA256, re-downloading"); + std::fs::remove_file(local_cache_path).expect("failed to delete stale cached zip"); + } + + // Download from the remote URL. + println!( + "cargo:warning=downloading Lambda zip from: {}", + LAMBDA_ZIP_URL + ); + let data = download_lambda_zip(LAMBDA_ZIP_URL).expect("failed to download Lambda zip"); + + // Verify SHA256 BEFORE writing to disk. + let actual_hash = sha256_hex(&data); + if actual_hash != LAMBDA_ZIP_SHA256 { + panic!( + "SHA256 mismatch for Lambda zip!\n expected: {LAMBDA_ZIP_SHA256}\n actual: \ + {actual_hash}\nThe artifact at {LAMBDA_ZIP_URL} may have been tampered with." + ); + } + + std::fs::write(local_cache_path, &data).expect("failed to write zip file"); + println!( + "cargo:warning=downloaded Lambda zip to {:?} ({} bytes)", + local_cache_path, + data.len() + ); +} + +fn sha256_hex(data: &[u8]) -> String { + format!("{:x}", Sha256::digest(data)) +} + +fn download_lambda_zip(url: &str) -> Result, String> { + let response = ureq::get(url) + .call() + .map_err(|err| format!("HTTP request failed: {err}"))?; + // Set limit higher than MAX_LAMBDA_ZIP_SIZE so we can detect oversized artifacts. + let data = response + .into_body() + .with_config() + .limit(MAX_LAMBDA_ZIP_SIZE as u64 + 1) + .read_to_vec() + .map_err(|err| format!("failed to read response body: {err}"))?; + if data.len() > MAX_LAMBDA_ZIP_SIZE { + return Err(format!( + "Lambda zip is too large ({} bytes, max {} bytes).\nAWS Lambda does not support \ + direct upload of binaries larger than 50MB.\nWorkaround: upload the Lambda zip to S3 \ + and deploy from there instead.", + data.len(), + MAX_LAMBDA_ZIP_SIZE + )); + } + Ok(data) +} diff --git a/quickwit/quickwit-lambda-client/src/deploy.rs b/quickwit/quickwit-lambda-client/src/deploy.rs new file mode 100644 index 00000000000..aacef0929ea --- /dev/null +++ b/quickwit/quickwit-lambda-client/src/deploy.rs @@ -0,0 +1,1010 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Lambda function deployment for auto-deploy feature. +//! +//! This module provides functionality to automatically deploy or update +//! the Lambda function used for leaf search operations. +//! +//! # Versioning Strategy +//! +//! We use AWS Lambda published versions with description-based identification: +//! - Each published version has a description like `quickwit:0_8_0-fa752891` +//! - We list versions to find one matching our qualifier +//! - We invoke the specific version number (not $LATEST) +//! - Old versions are garbage collected (keep current + top 5 most recent) + +use std::collections::HashMap; +use std::sync::OnceLock; + +use anyhow::{Context, anyhow}; +use aws_sdk_lambda::Client as LambdaClient; +use aws_sdk_lambda::error::SdkError; +use aws_sdk_lambda::primitives::Blob; +use aws_sdk_lambda::types::{ + Architecture, Environment, FunctionCode, LastUpdateStatus, Runtime, State, +}; +use quickwit_config::{LambdaConfig, LambdaDeployConfig}; +use quickwit_search::LambdaLeafSearchInvoker; +use tracing::{debug, info}; + +use crate::invoker::create_lambda_invoker_for_version; + +/// Embedded Lambda binary (arm64, compressed). +/// This is included at compile time. +const LAMBDA_BINARY: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/lambda_bootstrap.zip")); + +/// Prefix for version descriptions to identify Quickwit-managed versions. +const VERSION_DESCRIPTION_PREFIX: &str = "quickwit"; + +/// Number of recent versions to keep during garbage collection (in addition to current). +const GC_KEEP_RECENT_VERSIONS: usize = 5; + +/// Returns the Lambda qualifier combining version and binary hash. +/// Format: "{quickwit_version}-{hash_short}" with dots replaced by underscores. +/// Example: "0_8_0-fa752891" +fn lambda_qualifier() -> &'static str { + static LAMBDA_QUALIFIER: OnceLock = OnceLock::new(); + LAMBDA_QUALIFIER + .get_or_init(|| { + format!( + "{}_{}", + env!("CARGO_PKG_VERSION").replace('.', "_"), + env!("LAMBDA_BINARY_HASH") + ) + }) + .as_str() +} + +/// Returns the version description for our qualifier. +/// +/// We also pass the deploy config, as we want the function to be redeployed +/// if the deploy config is changed. +fn version_description(deploy_config_opt: Option<&LambdaDeployConfig>) -> String { + if let Some(deploy_config) = deploy_config_opt { + let memory_size_mib = deploy_config.memory_size.as_mib() as u64; + let execution_role_arn_digest: String = format!( + "{:x}", + md5::compute(deploy_config.execution_role_arn.as_bytes()) + ); + format!( + "{}_{}_{}_{}s_{}", + VERSION_DESCRIPTION_PREFIX, + lambda_qualifier(), + memory_size_mib, + deploy_config.invocation_timeout_secs, + &execution_role_arn_digest[..5] + ) + } else { + format!( + "{}_{}_nodeploy", + VERSION_DESCRIPTION_PREFIX, + lambda_qualifier() + ) + } +} + +/// Get or deploy the Lambda function and return an invoker. +/// +/// This function: +/// 1. Lists existing Lambda versions to find one matching our description +/// 2. If not found, (and if a deploy config is provided) attempt to deploy the embedded Lambda +/// binary +/// 3. Garbage collects old versions (keeps current + 5 most recent) +/// 4. Returns an invoker configured to call the specific version +/// +/// The qualifier is computed from the Quickwit version and Lambda binary hash, +/// ensuring the deployed Lambda matches the embedded binary. +pub async fn try_get_or_deploy_invoker( + lambda_config: &LambdaConfig, +) -> anyhow::Result { + let aws_config = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await; + let client = LambdaClient::new(&aws_config); + let function_name = &lambda_config.function_name; + let target_description = version_description(lambda_config.auto_deploy.as_ref()); + + info!( + function_name = %function_name, + qualifier = %lambda_qualifier(), + "looking for Lambda function version" + ); + + let version = find_or_deploy_version( + &client, + function_name, + &target_description, + lambda_config.auto_deploy.as_ref(), + ) + .await?; + + // Spawn background garbage collection (best effort, non-blocking) + let gc_client = client.clone(); + let gc_function_name = function_name.clone(); + let gc_version = version.clone(); + tokio::spawn(async move { + if let Err(e) = + garbage_collect_old_versions(&gc_client, &gc_function_name, &gc_version).await + { + info!(error = %e, "failed to garbage collect old Lambda versions"); + } + }); + + // Create and return the invoker + let invoker = create_lambda_invoker_for_version(function_name.clone(), version) + .await + .context("failed to create Lambda invoker")?; + + info!("created the lambda invoker"); + + Ok(invoker) +} + +/// Find a Lambda version with a description matching our qualifier. +/// +/// If none is found and a deploy config is provided, attempt to deploy a new version. +/// +/// Returns the version number as a string (because it is a string on AWS side, e.g.: "7") if found. +async fn find_or_deploy_version( + client: &LambdaClient, + function_name: &str, + target_description: &str, + deploy_config: Option<&LambdaDeployConfig>, +) -> anyhow::Result { + if let Some(version) = find_matching_version(client, function_name, target_description).await? { + info!( + function_name = %function_name, + version = %version, + "found existing Lambda version" + ); + return Ok(version); + } + + let deploy_config = deploy_config.with_context(|| { + format!( + "no Lambda version found with description '{}' and auto_deploy is not configured. \ + Either deploy the Lambda function manually or enable auto_deploy.", + target_description + ) + })?; + + info!( + function_name = %function_name, + "no matching version found, deploying Lambda function" + ); + + deploy_lambda_function(client, function_name, deploy_config).await +} + +async fn find_matching_version( + client: &LambdaClient, + function_name: &str, + target_description: &str, +) -> anyhow::Result> { + let mut marker: Option = None; + + loop { + let mut request = client + .list_versions_by_function() + .function_name(function_name); + + if let Some(m) = marker { + request = request.marker(m); + } + + let response = match request.send().await { + Ok(resp) => resp, + Err(SdkError::ServiceError(err)) if err.err().is_resource_not_found_exception() => { + info!( + function_name = %function_name, + "lambda function does not exist yet" + ); + return Ok(None); + } + Err(e) => { + return Err(anyhow!( + "failed to list Lambda versions for '{}': {}", + function_name, + e + )); + } + }; + + for version in response.versions() { + if let Some(description) = version.description() + && description == target_description + && let Some(ver) = version.version() + && ver != "$LATEST" + { + return Ok(Some(ver.to_string())); + } + } + + marker = response.next_marker().map(|s| s.to_string()); + if marker.is_none() { + break; + } + } + + Ok(None) +} + +/// Deploy the Lambda function and publish a new version. +/// AWS's API is pretty terrible. +/// +/// Lambda's version are integer generated by AWS (we don't have control over them). +/// To publish a new version, we need to implement two paths: +/// - If the function doesn't exist yet, `create_function(publish=true)` atomically creates it and +/// publishes a version in one call. +/// - If the function already exists, we first update the code. We do not publish because strangely +/// the API call does not make it possible to change the description. Updating the code has the +/// effect of create a version $LATEST. +/// - We publish the version $LATEST. That's the moment AWS attributes a version number. That call +/// allows us to change the description. We pass the sha256 hash of the code to ensure that +/// $LATEST has not been overwritten by another concurrent update. +async fn deploy_lambda_function( + client: &LambdaClient, + function_name: &str, + deploy_config: &LambdaDeployConfig, +) -> anyhow::Result { + // This looks overly complicated but this is not AI slop. + // The AWS API forces us to go through a bunch of hoops to update our function + // in a safe manner. + let description = version_description(Some(deploy_config)); + + // Fast path: if the function does not exist, we can create and publish the function atomically. + if let Some(version) = + try_create_function(client, function_name, deploy_config, &description).await? + { + return Ok(version); + } + + // Function already exists — we need to update the code. + // This will create or update a version called "$LATEST" (that's the actual string) + // + // We cannot directly publish here, because updating the function code does not allow + // us to pass a different description. + let code_sha256 = update_function_code(client, function_name).await?; + + // We can now publish that new uploaded version. + // We pass the code_sha256 guard to make sure a race condition does not cause + // us to publish a different version. + // + // Publishing will create an actual version (a number as a string) and return it. + publish_version(client, function_name, &code_sha256, &description).await +} + +/// Try to create the Lambda function with `publish=true`. +/// +/// Returns `Some(version)` if the function was created and published. +/// Returns `None` if the function already exists (`ResourceConflictException`). +async fn try_create_function( + client: &LambdaClient, + function_name: &str, + deploy_config: &LambdaDeployConfig, + description: &str, +) -> anyhow::Result> { + let memory_size_mb = deploy_config + .memory_size + .as_u64() + .div_ceil(1024u64 * 1024u64) as i32; + let timeout_secs = deploy_config.invocation_timeout_secs as i32; + + info!( + function_name = %function_name, + memory_size_mb = memory_size_mb, + timeout_secs = timeout_secs, + "attempting to create Lambda function" + ); + + let function_code = FunctionCode::builder() + .zip_file(Blob::new(LAMBDA_BINARY)) + .build(); + + let create_result = client + .create_function() + .function_name(function_name) + .runtime(Runtime::Providedal2023) + .role(&deploy_config.execution_role_arn) + .handler("bootstrap") + .description(description) + .code(function_code) + .architectures(Architecture::Arm64) + .memory_size(memory_size_mb) + .timeout(timeout_secs) + .environment(build_environment()) + .set_tags(Some(build_tags())) + .publish(true) + .send() + .await; + + match create_result { + Ok(output) => { + let version = output + .version() + .ok_or_else(|| anyhow!("created function has no version number"))? + .to_string(); + info!( + function_name = %function_name, + version = %version, + "lambda function created and published" + ); + Ok(Some(version)) + } + Err(SdkError::ServiceError(err)) if err.err().is_resource_conflict_exception() => { + debug!( + function_name = %function_name, + "lambda function already exists" + ); + Ok(None) + } + Err(e) => Err(anyhow!( + "failed to create Lambda function '{}': {}", + function_name, + e + )), + } +} + +/// Update `$LATEST` to our embedded binary. +/// +/// Returns the `code_sha256` of the uploaded code, to be used as a guard +/// when publishing the version (detects if another process overwrote `$LATEST` +/// between our update and publish). +async fn update_function_code( + client: &LambdaClient, + function_name: &str, +) -> anyhow::Result { + info!( + function_name = %function_name, + "updating Lambda function code to current binary" + ); + + let response = client + .update_function_code() + .function_name(function_name) + .zip_file(Blob::new(LAMBDA_BINARY)) + .architectures(Architecture::Arm64) + .send() + .await + .context("failed to update Lambda function code")?; + + let code_sha256 = response + .code_sha256() + .ok_or_else(|| anyhow!("update_function_code response missing code_sha256"))? + .to_string(); + + wait_for_function_ready(client, function_name).await?; + + Ok(code_sha256) +} + +/// Publish a new immutable version from `$LATEST` with our description. +/// +/// The `code_sha256` parameter guards against races: if another process +/// overwrote `$LATEST` since our `update_function_code` call, AWS will +/// reject the publish. +/// +/// Returns the version number (e.g., "8"). +async fn publish_version( + client: &LambdaClient, + function_name: &str, + code_sha256: &str, + description: &str, +) -> anyhow::Result { + info!( + function_name = %function_name, + description = %description, + "publishing new Lambda version" + ); + + let publish_response = client + .publish_version() + .function_name(function_name) + .description(description) + .code_sha256(code_sha256) + .send() + .await + .context( + "failed to publish Lambda version (code_sha256 mismatch means a concurrent deploy \ + race)", + )?; + + let version = publish_response + .version() + .context("published version has no version number")? + .to_string(); + + info!( + function_name = %function_name, + version = %version, + "lambda version published successfully" + ); + + Ok(version) +} + +/// Wait for the Lambda function to be ready. +/// +/// "Ready" means `State == Active` and no update is in progress +/// (`LastUpdateStatus` is absent or `Successful`). +/// +/// This matters because: +/// - After `create_function`: `State` transitions `Pending → Active` +/// - After `update_function_code`: `State` stays `Active` but `LastUpdateStatus` transitions +/// `InProgress → Successful` +async fn wait_for_function_ready(client: &LambdaClient, function_name: &str) -> anyhow::Result<()> { + const MAX_WAIT_ATTEMPTS: u32 = 30; + const WAIT_INTERVAL: tokio::time::Duration = tokio::time::Duration::from_secs(1); + + let mut interval = tokio::time::interval(WAIT_INTERVAL); + + for attempt in 0..MAX_WAIT_ATTEMPTS { + interval.tick().await; + + let response = client + .get_function() + .function_name(function_name) + .send() + .await + .context("failed to get function status")?; + + let Some(config) = response.configuration() else { + continue; + }; + + // Check for terminal failure states. + if config.state() == Some(&State::Failed) { + let reason = config.state_reason().unwrap_or("unknown reason"); + anyhow::bail!( + "lambda function '{}' is in Failed state: {}", + function_name, + reason + ); + } + + let last_update_status: &LastUpdateStatus = config + .last_update_status() + .unwrap_or(&LastUpdateStatus::Successful); + + if last_update_status == &LastUpdateStatus::Failed { + let reason = config + .last_update_status_reason() + .unwrap_or("unknown reason"); + anyhow::bail!( + "lambda function '{}' last update failed: {}", + function_name, + reason + ); + } + + // Ready = Active state with no update in progress. + let is_active = config.state() == Some(&State::Active); + if is_active && last_update_status == &LastUpdateStatus::Successful { + info!( + function_name = %function_name, + attempts = attempt + 1, + "lambda function is ready" + ); + return Ok(()); + } + + info!( + function_name = %function_name, + state = ?config.state(), + last_update_status = ?config.last_update_status(), + attempt = attempt + 1, + "waiting for Lambda function to be ready" + ); + } + + anyhow::bail!( + "lambda function '{}' did not become ready within {} seconds", + function_name, + MAX_WAIT_ATTEMPTS as u64 * WAIT_INTERVAL.as_secs() + ) +} + +/// Garbage collect old Lambda versions, keeping the current + 5 most recent. +async fn garbage_collect_old_versions( + client: &LambdaClient, + function_name: &str, + current_version: &str, +) -> anyhow::Result<()> { + let mut quickwit_lambda_versions: Vec<(u64, String)> = Vec::new(); + let mut marker: Option = None; + + // Collect all Quickwit-managed versions + loop { + let mut request = client + .list_versions_by_function() + .function_name(function_name); + + if let Some(m) = marker { + request = request.marker(m); + } + + let response = request + .send() + .await + .context("failed to list Lambda versions for garbage collection")?; + + for version in response.versions() { + let Some(version_str) = version.version() else { + continue; + }; + if version_str == "$LATEST" { + continue; + } + // Only consider Quickwit-managed versions + let Some(description) = version.description() else { + continue; + }; + if description.starts_with(VERSION_DESCRIPTION_PREFIX) + && let Ok(version_num) = version_str.parse::() + { + quickwit_lambda_versions.push((version_num, version_str.to_string())); + } + } + + marker = response.next_marker().map(ToString::to_string); + if marker.is_none() { + break; + } + } + + // Sort by version number ascending (oldest first) + quickwit_lambda_versions.sort(); + + // We keep the last 5 versions. + quickwit_lambda_versions.truncate( + quickwit_lambda_versions + .len() + .saturating_sub(GC_KEEP_RECENT_VERSIONS), + ); + + if let Some(pos) = quickwit_lambda_versions + .iter() + .position(|(_version, version_str)| version_str == current_version) + { + quickwit_lambda_versions.swap_remove(pos); + } + + // Delete old versions + for (version, version_str) in quickwit_lambda_versions { + info!( + function_name = %function_name, + version = %version_str, + "deleting old Lambda version" + ); + + if let Err(e) = client + .delete_function() + .function_name(function_name) + .qualifier(&version_str) + .send() + .await + { + info!( + function_name = %function_name, + version = %version, + error = %e, + "failed to delete old Lambda version" + ); + } + } + + Ok(()) +} + +/// Build environment variables for the Lambda function. +fn build_environment() -> Environment { + let mut env_vars = HashMap::new(); + env_vars.insert("RUST_LOG".to_string(), "info".to_string()); + env_vars.insert("RUST_BACKTRACE".to_string(), "1".to_string()); + Environment::builder().set_variables(Some(env_vars)).build() +} + +/// Build tags for the Lambda function. +fn build_tags() -> HashMap { + let mut tags = HashMap::new(); + tags.insert("managed_by".to_string(), "quickwit".to_string()); + tags +} + +#[cfg(test)] +mod tests { + use aws_sdk_lambda::operation::create_function::{CreateFunctionError, CreateFunctionOutput}; + use aws_sdk_lambda::operation::delete_function::DeleteFunctionOutput; + use aws_sdk_lambda::operation::get_function::GetFunctionOutput; + use aws_sdk_lambda::operation::list_versions_by_function::{ + ListVersionsByFunctionError, ListVersionsByFunctionOutput, + }; + use aws_sdk_lambda::operation::publish_version::PublishVersionOutput; + use aws_sdk_lambda::operation::update_function_code::UpdateFunctionCodeOutput; + use aws_sdk_lambda::types::FunctionConfiguration; + use aws_sdk_lambda::types::error::{ResourceConflictException, ResourceNotFoundException}; + use aws_smithy_mocks::{RuleMode, mock, mock_client}; + use bytesize::ByteSize; + + use super::*; + + fn make_version(version: &str, description: &str) -> FunctionConfiguration { + FunctionConfiguration::builder() + .version(version) + .description(description) + .build() + } + + fn test_deploy_config() -> LambdaDeployConfig { + LambdaDeployConfig { + execution_role_arn: "arn:aws:iam::123456789:role/test-role".to_string(), + memory_size: ByteSize::gib(5), + invocation_timeout_secs: 60, + } + } + + fn test_description() -> String { + version_description(None) + } + + #[test] + fn test_version_description() { + let lambda_deploy_config = test_deploy_config(); + let description = version_description(Some(&lambda_deploy_config)); + assert!(description.ends_with("_60s_6c3b2")); + let description = version_description(None); + assert!(description.ends_with("_nodeploy")); + } + + // --- find_matching_version tests --- + + #[tokio::test] + async fn test_find_matching_version_found() { + let target = "quickwit:test_version"; + let rule = mock!(aws_sdk_lambda::Client::list_versions_by_function).then_output(|| { + ListVersionsByFunctionOutput::builder() + .versions(make_version("$LATEST", "")) + .versions(make_version("1", "quickwit:old_version")) + .versions(make_version("7", "quickwit:test_version")) + .build() + }); + let client = mock_client!(aws_sdk_lambda, [&rule]); + + let matching_version_opt = find_matching_version(&client, "my-fn", target) + .await + .unwrap(); + assert_eq!(matching_version_opt, Some("7".to_string())); + } + + #[tokio::test] + async fn test_find_matching_version_not_found() { + let rule = mock!(aws_sdk_lambda::Client::list_versions_by_function).then_output(|| { + ListVersionsByFunctionOutput::builder() + .versions(make_version("$LATEST", "")) + .versions(make_version("1", "quickwit:other")) + .build() + }); + let client = mock_client!(aws_sdk_lambda, [&rule]); + + let result = find_matching_version(&client, "my-fn", "quickwit:no_match") + .await + .unwrap(); + assert_eq!(result, None); + } + + #[tokio::test] + async fn test_find_matching_version_function_does_not_exist() { + let rule = mock!(aws_sdk_lambda::Client::list_versions_by_function).then_error(|| { + ListVersionsByFunctionError::ResourceNotFoundException( + ResourceNotFoundException::builder().build(), + ) + }); + let client = mock_client!(aws_sdk_lambda, [&rule]); + + let result = find_matching_version(&client, "no-such-fn", "quickwit:x") + .await + .unwrap(); + assert_eq!(result, None); + } + + #[tokio::test] + async fn test_find_matching_version_skips_latest_even_if_description_matches() { + let rule = mock!(aws_sdk_lambda::Client::list_versions_by_function).then_output(|| { + ListVersionsByFunctionOutput::builder() + .versions(make_version("$LATEST", "quickwit:match")) + .build() + }); + let client = mock_client!(aws_sdk_lambda, [&rule]); + + let result = find_matching_version(&client, "my-fn", "quickwit:match") + .await + .unwrap(); + assert_eq!(result, None); + } + + // --- try_create_function tests --- + + #[tokio::test] + async fn test_try_create_function_success() { + let rule = mock!(aws_sdk_lambda::Client::create_function).then_output(|| { + CreateFunctionOutput::builder() + .version("1") + .function_name("my-fn") + .build() + }); + let client = mock_client!(aws_sdk_lambda, [&rule]); + let config = test_deploy_config(); + + let result = try_create_function(&client, "my-fn", &config, &test_description()) + .await + .unwrap(); + assert_eq!(result, Some("1".to_string())); + } + + #[tokio::test] + async fn test_try_create_function_already_exists() { + let rule = mock!(aws_sdk_lambda::Client::create_function).then_error(|| { + CreateFunctionError::ResourceConflictException( + ResourceConflictException::builder().build(), + ) + }); + let client = mock_client!(aws_sdk_lambda, [&rule]); + let config = test_deploy_config(); + + let result = try_create_function(&client, "my-fn", &config, &test_description()) + .await + .unwrap(); + assert_eq!(result, None); + } + + // --- deploy (update path) tests --- + + #[tokio::test] + async fn test_deploy_update_path() { + // create_function → conflict (function exists) + let create_rule = mock!(aws_sdk_lambda::Client::create_function).then_error(|| { + CreateFunctionError::ResourceConflictException( + ResourceConflictException::builder().build(), + ) + }); + // update_function_code → success with code_sha256 + let update_rule = mock!(aws_sdk_lambda::Client::update_function_code).then_output(|| { + UpdateFunctionCodeOutput::builder() + .code_sha256("abc123hash") + .build() + }); + // get_function → active and ready (for wait_for_function_ready) + let get_rule = mock!(aws_sdk_lambda::Client::get_function).then_output(|| { + GetFunctionOutput::builder() + .configuration( + FunctionConfiguration::builder() + .state(State::Active) + .last_update_status(LastUpdateStatus::Successful) + .build(), + ) + .build() + }); + // publish_version → success + let publish_rule = mock!(aws_sdk_lambda::Client::publish_version) + .then_output(|| PublishVersionOutput::builder().version("8").build()); + + let client = mock_client!( + aws_sdk_lambda, + RuleMode::MatchAny, + [&create_rule, &update_rule, &get_rule, &publish_rule] + ); + let config = test_deploy_config(); + + tokio::time::pause(); + let version = deploy_lambda_function(&client, "my-fn", &config) + .await + .unwrap(); + assert_eq!(version, "8"); + } + + // --- wait_for_function_ready tests --- + + #[tokio::test] + async fn test_wait_for_function_ready_immediate() { + let rule = mock!(aws_sdk_lambda::Client::get_function).then_output(|| { + GetFunctionOutput::builder() + .configuration( + FunctionConfiguration::builder() + .state(State::Active) + .last_update_status(LastUpdateStatus::Successful) + .build(), + ) + .build() + }); + let client = mock_client!(aws_sdk_lambda, [&rule]); + + tokio::time::pause(); + wait_for_function_ready(&client, "my-fn").await.unwrap(); + } + + #[tokio::test] + async fn test_wait_for_function_ready_after_update_in_progress() { + let rule = mock!(aws_sdk_lambda::Client::get_function) + .sequence() + .output(|| { + GetFunctionOutput::builder() + .configuration( + FunctionConfiguration::builder() + .state(State::Active) + .last_update_status(LastUpdateStatus::InProgress) + .build(), + ) + .build() + }) + .output(|| { + GetFunctionOutput::builder() + .configuration( + FunctionConfiguration::builder() + .state(State::Active) + .last_update_status(LastUpdateStatus::Successful) + .build(), + ) + .build() + }) + .build(); + let client = mock_client!(aws_sdk_lambda, RuleMode::Sequential, [&rule]); + + tokio::time::pause(); + wait_for_function_ready(&client, "my-fn").await.unwrap(); + assert_eq!(rule.num_calls(), 2); + } + + #[tokio::test] + async fn test_wait_for_function_ready_fails_on_failed_state() { + let rule = mock!(aws_sdk_lambda::Client::get_function).then_output(|| { + GetFunctionOutput::builder() + .configuration( + FunctionConfiguration::builder() + .state(State::Failed) + .state_reason("Something broke") + .build(), + ) + .build() + }); + let client = mock_client!(aws_sdk_lambda, [&rule]); + + tokio::time::pause(); + let err = wait_for_function_ready(&client, "my-fn").await.unwrap_err(); + assert!( + err.to_string().contains("Failed state"), + "unexpected error: {}", + err + ); + } + + #[tokio::test] + async fn test_wait_for_function_ready_fails_on_last_update_failed() { + let rule = mock!(aws_sdk_lambda::Client::get_function).then_output(|| { + GetFunctionOutput::builder() + .configuration( + FunctionConfiguration::builder() + .state(State::Active) + .last_update_status(LastUpdateStatus::Failed) + .last_update_status_reason("Update broke") + .build(), + ) + .build() + }); + let client = mock_client!(aws_sdk_lambda, [&rule]); + + tokio::time::pause(); + let err = wait_for_function_ready(&client, "my-fn").await.unwrap_err(); + assert!( + err.to_string().contains("last update failed"), + "unexpected error: {}", + err + ); + } + + // --- garbage_collect_old_versions tests --- + + #[tokio::test] + async fn test_gc_deletes_old_versions_keeps_recent() { + // 8 quickwit versions (1..=8) + $LATEST + one non-quickwit version + let list_rule = + mock!(aws_sdk_lambda::Client::list_versions_by_function).then_output(|| { + let mut builder = ListVersionsByFunctionOutput::builder() + .versions(make_version("$LATEST", "")) + .versions(make_version("99", "not-quickwit")); + for i in 1..=8 { + builder = builder + .versions(make_version(&i.to_string(), &format!("quickwit:ver_{}", i))); + } + builder.build() + }); + + let delete_rule = mock!(aws_sdk_lambda::Client::delete_function) + .then_output(|| DeleteFunctionOutput::builder().build()); + + let client = mock_client!( + aws_sdk_lambda, + RuleMode::MatchAny, + [&list_rule, &delete_rule] + ); + + // Current version is "7", so keep 7 + the 5 most recent (4,5,6,7,8). + // Should delete versions 1, 2, 3. + garbage_collect_old_versions(&client, "my-fn", "7") + .await + .unwrap(); + + assert_eq!(delete_rule.num_calls(), 3); + } + + #[tokio::test] + async fn test_gc_nothing_to_delete() { + // Only 3 quickwit versions — below the GC_KEEP_RECENT_VERSIONS threshold. + let list_rule = + mock!(aws_sdk_lambda::Client::list_versions_by_function).then_output(|| { + ListVersionsByFunctionOutput::builder() + .versions(make_version("$LATEST", "")) + .versions(make_version("1", "quickwit:v1")) + .versions(make_version("2", "quickwit:v2")) + .versions(make_version("3", "quickwit:v3")) + .build() + }); + + let delete_rule = mock!(aws_sdk_lambda::Client::delete_function) + .then_output(|| DeleteFunctionOutput::builder().build()); + + let client = mock_client!( + aws_sdk_lambda, + RuleMode::MatchAny, + [&list_rule, &delete_rule] + ); + + garbage_collect_old_versions(&client, "my-fn", "3") + .await + .unwrap(); + + assert_eq!(delete_rule.num_calls(), 0); + } + + #[tokio::test] + async fn test_gc_does_not_delete_current_version() { + // 7 quickwit versions, current is "1" (the oldest). + // Without the current-version guard, version 1 would be deleted. + let list_rule = + mock!(aws_sdk_lambda::Client::list_versions_by_function).then_output(|| { + let mut builder = + ListVersionsByFunctionOutput::builder().versions(make_version("$LATEST", "")); + for i in 1..=7 { + builder = builder + .versions(make_version(&i.to_string(), &format!("quickwit:ver_{}", i))); + } + builder.build() + }); + + let delete_rule = mock!(aws_sdk_lambda::Client::delete_function) + .then_output(|| DeleteFunctionOutput::builder().build()); + + let client = mock_client!( + aws_sdk_lambda, + RuleMode::MatchAny, + [&list_rule, &delete_rule] + ); + + // Current version is "1". Without guard: would delete 1,2. With guard: only deletes 2. + garbage_collect_old_versions(&client, "my-fn", "1") + .await + .unwrap(); + + assert_eq!(delete_rule.num_calls(), 1); + } +} diff --git a/quickwit/quickwit-lambda-client/src/invoker.rs b/quickwit/quickwit-lambda-client/src/invoker.rs new file mode 100644 index 00000000000..c8ffa0716a0 --- /dev/null +++ b/quickwit/quickwit-lambda-client/src/invoker.rs @@ -0,0 +1,299 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::time::Duration; + +use anyhow::Context as _; +use async_trait::async_trait; +use aws_sdk_lambda::Client as LambdaClient; +use aws_sdk_lambda::error::{DisplayErrorContext, SdkError}; +use aws_sdk_lambda::operation::invoke::InvokeError; +use aws_sdk_lambda::primitives::Blob; +use aws_sdk_lambda::types::InvocationType; +use base64::prelude::*; +use prost::Message; +use quickwit_common::retry::RetryParams; +use quickwit_lambda_server::{LambdaSearchRequestPayload, LambdaSearchResponsePayload}; +use quickwit_proto::search::{LambdaSearchResponses, LambdaSingleSplitResult, LeafSearchRequest}; +use quickwit_search::{LambdaLeafSearchInvoker, SearchError}; +use tracing::{debug, info, instrument, warn}; + +use crate::metrics::LAMBDA_METRICS; + +/// Upper bound on the retry-after hint we will honor from Lambda rate-limit responses. +const MAX_RETRY_AFTER: Duration = Duration::from_secs(10); + +/// Richer error type used internally by the invoker so that rate-limit retry-after hints +/// are not lost before the retry loop can consume them. +enum LambdaInvokeError { + /// Lambda returned a throttling error. The optional duration is the `Retry-After` hint + /// provided by Lambda; `None` means no hint was present. + RateLimited(Option), + /// The invocation timed out. + Timeout(String), + /// A non-retryable error. + Permanent(SearchError), +} + +impl LambdaInvokeError { + fn into_search_error(self) -> SearchError { + match self { + Self::RateLimited(_) => SearchError::TooManyRequests, + Self::Timeout(msg) => SearchError::Timeout(msg), + Self::Permanent(err) => err, + } + } +} + +impl From for LambdaInvokeError { + fn from(err: SearchError) -> Self { + LambdaInvokeError::Permanent(err) + } +} + +fn invoke_error_to_lambda_error(error: SdkError) -> LambdaInvokeError { + if let SdkError::ServiceError(ref service_error) = error { + match service_error.err() { + InvokeError::TooManyRequestsException(exc) => { + let retry_after = exc + .retry_after_seconds() + .and_then(|raw| raw.parse::().ok()) + .filter(|secs| secs.is_finite() && *secs > 0.0) + .map(|secs| Duration::from_secs_f64(secs).min(MAX_RETRY_AFTER)); + return LambdaInvokeError::RateLimited(retry_after); + } + InvokeError::EniLimitReachedException(_) + | InvokeError::SubnetIpAddressLimitReachedException(_) + | InvokeError::Ec2ThrottledException(_) + | InvokeError::ResourceConflictException(_) => { + return LambdaInvokeError::RateLimited(None); + } + _ => {} + } + } + + let is_timeout = match &error { + SdkError::TimeoutError(_) => true, + SdkError::DispatchFailure(failure) => failure.is_timeout(), + SdkError::ServiceError(service_error) => matches!( + service_error.err(), + InvokeError::EfsMountTimeoutException(_) | InvokeError::SnapStartTimeoutException(_) + ), + _ => false, + }; + + let error_msg = format!("lambda invocation failed: {}", DisplayErrorContext(&error)); + + if is_timeout { + LambdaInvokeError::Timeout(error_msg) + } else { + LambdaInvokeError::Permanent(SearchError::Internal(error_msg)) + } +} + +/// Create a Lambda invoker for a specific version. +/// +/// The version number is used as the qualifier when invoking, ensuring we call +/// the exact published version (not $LATEST). +pub(crate) async fn create_lambda_invoker_for_version( + function_name: String, + version: String, +) -> anyhow::Result { + let aws_config = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await; + let client = LambdaClient::new(&aws_config); + let invoker = AwsLambdaInvoker { + client, + function_name, + version, + }; + invoker.validate().await?; + Ok(invoker) +} + +/// AWS Lambda implementation of RemoteFunctionInvoker. +pub(crate) struct AwsLambdaInvoker { + client: LambdaClient, + function_name: String, + /// The version number to invoke (e.g., "7", "12"). + version: String, +} + +impl AwsLambdaInvoker { + /// Validate that the Lambda function version exists and is invocable. + /// Uses DryRun invocation type - validates without executing. + async fn validate(&self) -> anyhow::Result<()> { + info!("lambda invoker dry run"); + let request = self + .client + .invoke() + .function_name(&self.function_name) + .qualifier(&self.version) + .invocation_type(InvocationType::DryRun); + + request.send().await.with_context(|| { + format!( + "failed to validate Lambda function '{}:{}'", + self.function_name, self.version + ) + })?; + + info!("the lambda invoker dry run was successful"); + Ok(()) + } +} + +/// Retry parameters used for exponential backoff when no `Retry-After` hint is available. +const LAMBDA_RETRY_PARAMS: RetryParams = RetryParams { + base_delay: Duration::from_secs(1), + max_delay: Duration::from_secs(10), + max_attempts: 3, +}; + +#[async_trait] +impl LambdaLeafSearchInvoker for AwsLambdaInvoker { + #[instrument(skip(self, request), fields(function_name = %self.function_name, version = %self.version))] + async fn invoke_leaf_search( + &self, + request: LeafSearchRequest, + ) -> Result, SearchError> { + let start = std::time::Instant::now(); + let result = self.invoke_leaf_search_with_retry(request).await; + let elapsed = start.elapsed().as_secs_f64(); + let status = if result.is_ok() { "success" } else { "error" }; + LAMBDA_METRICS + .leaf_search_requests_total + .with_label_values([status]) + .inc(); + LAMBDA_METRICS + .leaf_search_duration_seconds + .with_label_values([status]) + .observe(elapsed); + result + } +} + +impl AwsLambdaInvoker { + async fn invoke_leaf_search_with_retry( + &self, + request: LeafSearchRequest, + ) -> Result, SearchError> { + let mut error = match self.invoke_leaf_search_once(request.clone()).await { + Ok(results) => return Ok(results), + Err(error) => error, + }; + + for num_attempts in 1..LAMBDA_RETRY_PARAMS.max_attempts { + // Determine whether to retry and how long to wait. + let delay = match &error { + LambdaInvokeError::RateLimited(retry_after) => { + retry_after.unwrap_or_else(|| LAMBDA_RETRY_PARAMS.compute_delay(num_attempts)) + } + LambdaInvokeError::Timeout(_) => LAMBDA_RETRY_PARAMS.compute_delay(num_attempts), + LambdaInvokeError::Permanent(_) => return Err(error.into_search_error()), + }; + + warn!( + num_attempts = num_attempts, + delay_ms = delay.as_millis(), + "lambda invocation failed, retrying" + ); + tokio::time::sleep(delay).await; + + match self.invoke_leaf_search_once(request.clone()).await { + Ok(results) => return Ok(results), + Err(e) => error = e, + }; + } + + Err(error.into_search_error()) + } + + async fn invoke_leaf_search_once( + &self, + request: LeafSearchRequest, + ) -> Result, LambdaInvokeError> { + // Serialize request to protobuf bytes, then base64 encode + let request_bytes = request.encode_to_vec(); + let payload = LambdaSearchRequestPayload { + payload: BASE64_STANDARD.encode(&request_bytes), + }; + + let payload_json = serde_json::to_vec(&payload) + .map_err(|e| SearchError::Internal(format!("JSON serialization error: {}", e)))?; + + LAMBDA_METRICS + .leaf_search_request_payload_size_bytes + .observe(payload_json.len() as f64); + + debug!( + payload_size = payload_json.len(), + version = %self.version, + "invoking Lambda function" + ); + + // Invoke the specific version + let invoke_builder = self + .client + .invoke() + .function_name(&self.function_name) + .qualifier(&self.version) + .invocation_type(InvocationType::RequestResponse) + .payload(Blob::new(payload_json)); + + let response = invoke_builder + .send() + .await + .map_err(invoke_error_to_lambda_error)?; + + // Check for function error + if let Some(error) = response.function_error() { + let error_payload = response + .payload() + .map(|b| String::from_utf8_lossy(b.as_ref()).to_string()) + .unwrap_or_default(); + return Err(SearchError::Internal(format!( + "lambda function error: {}: {}", + error, error_payload + )) + .into()); + } + + // Deserialize response + let response_payload = response + .payload() + .ok_or_else(|| SearchError::Internal("no response payload from Lambda".into()))?; + + LAMBDA_METRICS + .leaf_search_response_payload_size_bytes + .observe(response_payload.as_ref().len() as f64); + + let lambda_response: LambdaSearchResponsePayload = + serde_json::from_slice(response_payload.as_ref()) + .map_err(|e| SearchError::Internal(format!("json deserialization error: {}", e)))?; + + let response_bytes = BASE64_STANDARD + .decode(&lambda_response.payload) + .map_err(|e| SearchError::Internal(format!("base64 decode error: {}", e)))?; + + let leaf_responses = LambdaSearchResponses::decode(&response_bytes[..]) + .map_err(|e| SearchError::Internal(format!("protobuf decode error: {}", e)))?; + + debug!( + num_results = leaf_responses.split_results.len(), + "lambda invocation completed" + ); + + Ok(leaf_responses.split_results) + } +} diff --git a/quickwit/quickwit-lambda-client/src/lib.rs b/quickwit/quickwit-lambda-client/src/lib.rs new file mode 100644 index 00000000000..aebf264df8c --- /dev/null +++ b/quickwit/quickwit-lambda-client/src/lib.rs @@ -0,0 +1,37 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! AWS Lambda client for Quickwit leaf search operations. +//! +//! This crate provides: +//! - An AWS Lambda implementation of the `LambdaLeafSearchInvoker` trait used by `quickwit-search` +//! - Auto-deployment functionality for Lambda functions +//! +//! # Usage +//! +//! Use `try_get_or_deploy_invoker` to get an invoker that will automatically deploy +//! the Lambda function if needed: +//! +//! ```ignore +//! let invoker = try_get_or_deploy_invoker(&function_name, &deploy_config).await?; +//! ``` + +mod deploy; +mod invoker; +mod metrics; + +pub use deploy::try_get_or_deploy_invoker; +pub use metrics::LAMBDA_METRICS; +// Re-export payload types from server crate for convenience +pub use quickwit_lambda_server::{LambdaSearchRequestPayload, LambdaSearchResponsePayload}; diff --git a/quickwit/quickwit-lambda-client/src/metrics.rs b/quickwit/quickwit-lambda-client/src/metrics.rs new file mode 100644 index 00000000000..4325c9b3a95 --- /dev/null +++ b/quickwit/quickwit-lambda-client/src/metrics.rs @@ -0,0 +1,74 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// See https://prometheus.io/docs/practices/naming/ + +use once_cell::sync::Lazy; +use quickwit_common::metrics::{ + Histogram, HistogramVec, IntCounterVec, exponential_buckets, new_counter_vec, new_histogram, + new_histogram_vec, +}; + +/// From 100ms to 73s seconds +fn duration_buckets() -> Vec { + exponential_buckets(0.100, 3f64.sqrt(), 13).unwrap() +} + +/// From 1KB to 16MB +fn payload_size_buckets() -> Vec { + exponential_buckets(1024.0, 4.0, 8).unwrap() +} + +pub struct LambdaMetrics { + pub leaf_search_requests_total: IntCounterVec<1>, + pub leaf_search_duration_seconds: HistogramVec<1>, + pub leaf_search_request_payload_size_bytes: Histogram, + pub leaf_search_response_payload_size_bytes: Histogram, +} + +impl Default for LambdaMetrics { + fn default() -> Self { + LambdaMetrics { + leaf_search_requests_total: new_counter_vec( + "leaf_search_requests_total", + "Total number of Lambda leaf search invocations.", + "lambda", + &[], + ["status"], + ), + leaf_search_duration_seconds: new_histogram_vec( + "leaf_search_duration_seconds", + "Duration of Lambda leaf search invocations in seconds.", + "lambda", + &[], + ["status"], + duration_buckets(), + ), + leaf_search_request_payload_size_bytes: new_histogram( + "leaf_search_request_payload_size_bytes", + "Size of the request payload sent to Lambda in bytes.", + "lambda", + payload_size_buckets(), + ), + leaf_search_response_payload_size_bytes: new_histogram( + "leaf_search_response_payload_size_bytes", + "Size of the response payload received from Lambda in bytes.", + "lambda", + payload_size_buckets(), + ), + } + } +} + +pub static LAMBDA_METRICS: Lazy = Lazy::new(LambdaMetrics::default); diff --git a/quickwit/quickwit-lambda-server/Cargo.toml b/quickwit/quickwit-lambda-server/Cargo.toml new file mode 100644 index 00000000000..e362c329e1f --- /dev/null +++ b/quickwit/quickwit-lambda-server/Cargo.toml @@ -0,0 +1,54 @@ +[package] +name = "quickwit-lambda-server" +description = "AWS Lambda handler for Quickwit leaf search" + +version.workspace = true +edition.workspace = true +homepage.workspace = true +documentation.workspace = true +repository.workspace = true +authors.workspace = true +license.workspace = true + +[package.metadata.cargo-machete] +# Its here even though it is not useful, in order to enable its "vendor" feature, +# allowing the cross-build. +ignored = ["openssl"] + +[dependencies] +anyhow = { workspace = true } +base64 = { workspace = true } +bytesize = { workspace = true } +lambda_runtime = { workspace = true } +prost = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true, features = ["env-filter", "json"] } + +openssl = { workspace = true, optional = true } + +quickwit-common = { workspace = true } +quickwit-config = { workspace = true } +quickwit-doc-mapper = { workspace = true } +quickwit-proto = { workspace = true } +quickwit-search = { workspace = true } +quickwit-storage = { workspace = true } + +[[bin]] +name = "quickwit-aws-lambda-leaf-search" +path = "src/bin/leaf_search.rs" + +[features] +default = [] +testsuite = [] + +# Keep this in sync with quickwit-cli! +lambda-release = [ + # The vendored OpenSSL will be compiled from source during the + # build, avoiding the pkg-config dependency issue during + # cross-compilation. + "openssl/vendored", +] diff --git a/quickwit/quickwit-lambda-server/src/bin/leaf_search.rs b/quickwit/quickwit-lambda-server/src/bin/leaf_search.rs new file mode 100644 index 00000000000..0ccdbc741e0 --- /dev/null +++ b/quickwit/quickwit-lambda-server/src/bin/leaf_search.rs @@ -0,0 +1,52 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! AWS Lambda binary entry point for Quickwit leaf search. + +use std::sync::Arc; + +use lambda_runtime::{Error, LambdaEvent, service_fn}; +use quickwit_lambda_server::{ + LambdaSearchRequestPayload, LambdaSearcherContext, handle_leaf_search, +}; +use tracing::info; +use tracing_subscriber::EnvFilter; + +#[tokio::main] +async fn main() -> Result<(), Error> { + // Initialize tracing with JSON output for CloudWatch + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::from_default_env()) + .json() + .init(); + + // Initialize context on cold start (wrapped in Arc for sharing across invocations) + let context = Arc::new(LambdaSearcherContext::try_from_env()?); + + info!("lambda context initialized, starting handler loop"); + + // Run the Lambda handler + lambda_runtime::run(service_fn( + |event: LambdaEvent| { + let ctx = Arc::clone(&context); + async move { + let (payload, _event_ctx) = event.into_parts(); + handle_leaf_search(payload, &ctx) + .await + .map_err(|e| lambda_runtime::Error::from(e.to_string())) + } + }, + )) + .await +} diff --git a/quickwit/quickwit-lambda-server/src/context.rs b/quickwit/quickwit-lambda-server/src/context.rs new file mode 100644 index 00000000000..d3b9167414f --- /dev/null +++ b/quickwit/quickwit-lambda-server/src/context.rs @@ -0,0 +1,70 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use anyhow::Context as _; +use bytesize::ByteSize; +use quickwit_config::{CacheConfig, SearcherConfig}; +use quickwit_search::SearcherContext; +use quickwit_storage::StorageResolver; +use tracing::info; + +/// Lambda-specific searcher context that holds resources for search execution. +pub struct LambdaSearcherContext { + pub searcher_context: Arc, + pub storage_resolver: StorageResolver, +} + +impl LambdaSearcherContext { + /// Create a new Lambda searcher context from environment variables. + pub fn try_from_env() -> anyhow::Result { + info!("initializing lambda searcher context"); + + let searcher_config = try_searcher_config_from_env()?; + let searcher_context = + Arc::new(SearcherContext::new_without_invoker(searcher_config, None)); + let storage_resolver = StorageResolver::configured(&Default::default()); + + Ok(Self { + searcher_context, + storage_resolver, + }) + } +} + +/// Create a Lambda-optimized searcher config based on the `AWS_LAMBDA_FUNCTION_MEMORY_SIZE` +/// environment variable. +fn try_searcher_config_from_env() -> anyhow::Result { + let lambda_memory_mib: u64 = quickwit_common::get_from_env_opt( + "AWS_LAMBDA_FUNCTION_MEMORY_SIZE", + /* sensitive */ false, + ) + .context("could not get aws lambda function memory size from ENV")?; + let lambda_memory = ByteSize::mib(lambda_memory_mib); + anyhow::ensure!( + lambda_memory >= ByteSize::gib(1u64), + "lambda memory must be at least 1GB" + ); + let warmup_memory_budget = ByteSize::b(lambda_memory.as_u64() - ByteSize::mib(500).as_u64()); + + let mut searcher_config = SearcherConfig::default(); + searcher_config.max_num_concurrent_split_searches = 20; + searcher_config.warmup_memory_budget = warmup_memory_budget; + searcher_config.fast_field_cache = CacheConfig::no_cache(); + searcher_config.split_footer_cache = CacheConfig::no_cache(); + searcher_config.predicate_cache = CacheConfig::no_cache(); + searcher_config.partial_request_cache = CacheConfig::no_cache(); + Ok(searcher_config) +} diff --git a/quickwit/quickwit-lambda-server/src/error.rs b/quickwit/quickwit-lambda-server/src/error.rs new file mode 100644 index 00000000000..7f60f33b9b8 --- /dev/null +++ b/quickwit/quickwit-lambda-server/src/error.rs @@ -0,0 +1,69 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use quickwit_search::SearchError; +use thiserror::Error; + +/// Result type for Lambda operations. +pub type LambdaResult = Result; + +/// Errors that can occur during Lambda handler operations. +#[derive(Debug, Error)] +pub enum LambdaError { + /// Error serializing/deserializing protobuf. + #[error("serialization error: {0}")] + Serialization(String), + /// Error from the search operation. + #[error("search error: {0}")] + Search(#[from] SearchError), + /// Internal error. + #[error("internal error: {0}")] + Internal(String), + /// Task was cancelled. + #[error("cancelled")] + Cancelled, +} + +impl From for LambdaError { + fn from(err: prost::DecodeError) -> Self { + LambdaError::Serialization(format!("protobuf decode error: {}", err)) + } +} + +impl From for LambdaError { + fn from(err: prost::EncodeError) -> Self { + LambdaError::Serialization(format!("protobuf encode error: {}", err)) + } +} + +impl From for LambdaError { + fn from(err: base64::DecodeError) -> Self { + LambdaError::Serialization(format!("base64 decode error: {}", err)) + } +} + +impl From for LambdaError { + fn from(err: serde_json::Error) -> Self { + LambdaError::Serialization(format!("json error: {}", err)) + } +} + +impl From for SearchError { + fn from(err: LambdaError) -> Self { + match err { + LambdaError::Search(search_err) => search_err, + other => SearchError::Internal(other.to_string()), + } + } +} diff --git a/quickwit/quickwit-lambda-server/src/handler.rs b/quickwit/quickwit-lambda-server/src/handler.rs new file mode 100644 index 00000000000..e0467af21ef --- /dev/null +++ b/quickwit/quickwit-lambda-server/src/handler.rs @@ -0,0 +1,200 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::str::FromStr; +use std::sync::Arc; + +use base64::prelude::*; +use prost::Message; +use quickwit_common::uri::Uri; +use quickwit_doc_mapper::DocMapper; +use quickwit_proto::search::lambda_single_split_result::Outcome; +use quickwit_proto::search::{ + LambdaSearchResponses, LambdaSingleSplitResult, LeafRequestRef, LeafSearchRequest, + SearchRequest, +}; +use quickwit_search::leaf::single_doc_mapping_leaf_search; +use quickwit_storage::Storage; +use serde::{Deserialize, Serialize}; +use tracing::{error, info, instrument, warn}; + +use crate::context::LambdaSearcherContext; +use crate::error::{LambdaError, LambdaResult}; + +/// Payload for leaf search Lambda invocation. +#[derive(Debug, Serialize, Deserialize)] +pub struct LambdaSearchRequestPayload { + /// Base64-encoded serialized LeafSearchRequest protobuf. + pub payload: String, +} + +/// Response from leaf search Lambda invocation. +#[derive(Debug, Serialize, Deserialize)] +pub struct LambdaSearchResponsePayload { + /// Base64-encoded serialized `LambdaSearchResponses` protobuf (one per split). + pub payload: String, +} + +/// Handle a leaf search request in Lambda. +/// +/// Returns one `LambdaSingleSplitResult` per split, each tagged with its +/// split_id. Individual split failures are reported per-split rather than +/// failing the entire invocation, so the caller can retry only failed splits. +#[instrument(skip(ctx), fields(request_id))] +pub async fn handle_leaf_search( + event: LambdaSearchRequestPayload, + ctx: &LambdaSearcherContext, +) -> LambdaResult { + // Decode base64 payload + let request_bytes: Vec = BASE64_STANDARD + .decode(&event.payload) + .map_err(|err| LambdaError::Serialization(format!("base64 decode error: {}", err)))?; + + // Deserialize LeafSearchRequest + let leaf_search_request = LeafSearchRequest::decode(&request_bytes[..])?; + + // Unpack the shared fields once instead of cloning per split. + let search_request: Arc = leaf_search_request + .search_request + .ok_or_else(|| LambdaError::Internal("no search request".to_string()))? + .into(); + + let doc_mappers: Vec> = leaf_search_request + .doc_mappers + .iter() + .map(String::as_str) + .map(serde_json::from_str::>) + .collect::, _>>() + .map_err(|err| { + LambdaError::Internal(format!("failed to deserialize doc mapper: `{err}`")) + })?; + + // Resolve storage for every index URI upfront. + let mut storages: Vec> = + Vec::with_capacity(leaf_search_request.index_uris.len()); + for uri_str in &leaf_search_request.index_uris { + let uri = Uri::from_str(uri_str) + .map_err(|err| LambdaError::Internal(format!("invalid index uri: {err}")))?; + let storage = + ctx.storage_resolver.resolve(&uri).await.map_err(|err| { + LambdaError::Internal(format!("failed to resolve storage: {err}")) + })?; + storages.push(storage); + } + + let split_results: Vec = lambda_leaf_search( + search_request, + leaf_search_request.leaf_requests, + &doc_mappers[..], + &storages[..], + ctx, + ) + .await?; + let wrapper = LambdaSearchResponses { split_results }; + let response_bytes = wrapper.encode_to_vec(); + let payload = BASE64_STANDARD.encode(&response_bytes); + + Ok(LambdaSearchResponsePayload { payload }) +} + +/// Lambda leaf search returns individual split results. +async fn lambda_leaf_search( + search_request: Arc, + leaf_req_ref: Vec, + doc_mappers: &[Arc], + storages: &[Arc], + ctx: &LambdaSearcherContext, +) -> LambdaResult> { + // Flatten leaf_requests into per-split tasks using pre-resolved Arc references. + let mut split_search_joinset: tokio::task::JoinSet<(String, Result<_, String>)> = + tokio::task::JoinSet::new(); + + for leaf_req in leaf_req_ref { + let doc_mapper = doc_mappers + .get(leaf_req.doc_mapper_ord as usize) + .ok_or_else(|| { + LambdaError::Internal(format!( + "doc_mapper_ord out of bounds: {}", + leaf_req.doc_mapper_ord + )) + })? + .clone(); + let storage = storages[leaf_req.index_uri_ord as usize].clone(); + + for split_id_and_footer_offsets in leaf_req.split_offsets { + let split_id = split_id_and_footer_offsets.split_id.clone(); + let searcher_context = ctx.searcher_context.clone(); + let search_request = search_request.clone(); + let doc_mapper = doc_mapper.clone(); + let storage = storage.clone(); + let split = split_id_and_footer_offsets.clone(); + split_search_joinset.spawn(async move { + let result = single_doc_mapping_leaf_search( + searcher_context, + search_request, + storage, + vec![split], + doc_mapper, + ) + .await + .map_err(|err| format!("{err}")); + (split_id, result) + }); + } + } + + let num_splits = split_search_joinset.len(); + info!(num_splits, "processing leaf search request (per-split)"); + + // Collect results. Order is irrelevant: each result is tagged with its split_id. + let mut split_results: Vec = Vec::with_capacity(num_splits); + let mut num_successes: usize = 0; + let mut num_failures: usize = 0; + while let Some(join_result) = split_search_joinset.join_next().await { + match join_result { + Ok((split_id, Ok(response))) => { + num_successes += 1; + split_results.push(LambdaSingleSplitResult { + split_id, + outcome: Some(Outcome::Response(response)), + }); + } + Ok((split_id, Err(error_msg))) => { + num_failures += 1; + warn!(split_id = %split_id, error = %error_msg, "split search failed"); + split_results.push(LambdaSingleSplitResult { + split_id, + outcome: Some(Outcome::Error(error_msg)), + }); + } + Err(join_error) if join_error.is_cancelled() => { + warn!("search task was cancelled"); + return Err(LambdaError::Cancelled); + } + Err(join_error) => { + // Panics lose the captured split_id, so we fail the entire invocation. + error!(error = %join_error, "search task panicked"); + return Err(LambdaError::Internal(format!( + "search task panicked: {join_error}" + ))); + } + } + } + info!( + num_successes, + num_failures, "leaf search completed (per-split)" + ); + + Ok(split_results) +} diff --git a/quickwit/quickwit-lambda-server/src/lib.rs b/quickwit/quickwit-lambda-server/src/lib.rs new file mode 100644 index 00000000000..da2519b3b33 --- /dev/null +++ b/quickwit/quickwit-lambda-server/src/lib.rs @@ -0,0 +1,26 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! AWS Lambda handler for Quickwit leaf search operations. +//! +//! This crate provides the Lambda handler that executes leaf search requests. +//! It is designed to be deployed as an AWS Lambda function. + +mod context; +mod error; +mod handler; + +pub use context::LambdaSearcherContext; +pub use error::{LambdaError, LambdaResult}; +pub use handler::{LambdaSearchRequestPayload, LambdaSearchResponsePayload, handle_leaf_search}; diff --git a/quickwit/quickwit-lambda/Cargo.toml b/quickwit/quickwit-lambda/Cargo.toml deleted file mode 100644 index a43cce32913..00000000000 --- a/quickwit/quickwit-lambda/Cargo.toml +++ /dev/null @@ -1,72 +0,0 @@ -[package] -name = "quickwit-lambda" -description = "Serverless Quickwit on AWS Lambda" - -version.workspace = true -edition.workspace = true -homepage.workspace = true -documentation.workspace = true -repository.workspace = true -authors.workspace = true -license.workspace = true - -[[bin]] -name = "indexer" -path = "src/bin/indexer.rs" - -[[bin]] -name = "searcher" -path = "src/bin/searcher.rs" - -[features] -s3-localstack-tests = [] - -[dependencies] -anyhow = { workspace = true } -aws_lambda_events = "0.16" -bytesize = { workspace = true } -chitchat = { workspace = true } -chrono = { workspace = true } -flate2 = { workspace = true } -http = { workspace = true } -lambda_http = "0.14" -lambda_runtime = "0.13" -mime_guess = { workspace = true } -once_cell = { workspace = true } -opentelemetry = { workspace = true } -opentelemetry_sdk = { workspace = true } -opentelemetry-otlp = { workspace = true, features = [ - "reqwest-client", - "reqwest-rustls", - "http-proto", -] } -rand = { workspace = true } -reqwest = { workspace = true } -serde = { workspace = true } -serde_json = { workspace = true } -time = { workspace = true } -tokio = { workspace = true } -tracing = { workspace = true } -tracing-opentelemetry = { workspace = true } -tracing-subscriber = { workspace = true, features = ["json"] } -warp = { workspace = true, features = ["compression-gzip"] } - - -quickwit-actors = { workspace = true } -quickwit-cli = { workspace = true } -quickwit-cluster = { workspace = true } -quickwit-common = { workspace = true } -quickwit-config = { workspace = true } -quickwit-index-management = { workspace = true } -quickwit-indexing = { workspace = true } -quickwit-ingest = { workspace = true } -quickwit-janitor = { workspace = true } -quickwit-metastore = { workspace = true } -quickwit-proto = { workspace = true } -quickwit-search = { workspace = true } -quickwit-serve = { workspace = true } -quickwit-storage = { workspace = true } -quickwit-telemetry = { workspace = true } - -[dev-dependencies] -serial_test = { workspace = true } diff --git a/quickwit/quickwit-lambda/src/bin/searcher.rs b/quickwit/quickwit-lambda/src/bin/searcher.rs deleted file mode 100644 index b961271275b..00000000000 --- a/quickwit/quickwit-lambda/src/bin/searcher.rs +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use quickwit_lambda::logger; -use quickwit_lambda::searcher::{setup_searcher_api, warp_lambda}; - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - logger::setup_lambda_tracer(tracing::Level::INFO)?; - let routes = setup_searcher_api().await?; - let warp_service = warp::service(routes); - warp_lambda::run(warp_service) - .await - .map_err(|e| anyhow::anyhow!(e)) -} diff --git a/quickwit/quickwit-lambda/src/environment.rs b/quickwit/quickwit-lambda/src/environment.rs deleted file mode 100644 index 7b26129cf54..00000000000 --- a/quickwit/quickwit-lambda/src/environment.rs +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::env::var; - -use once_cell::sync::Lazy; -use quickwit_common::get_bool_from_env; - -pub static INDEX_ID: Lazy = Lazy::new(|| { - var("QW_LAMBDA_INDEX_ID").expect("environment variable `QW_LAMBDA_INDEX_ID` should be set") -}); - -/// Configures the fmt tracing subscriber to log as json and include span -/// boundaries. This is very verbose and is only used to generate advanced KPIs -/// from Lambda runs (e.g. for blog post benchmarks) -pub static ENABLE_VERBOSE_JSON_LOGS: Lazy = - Lazy::new(|| get_bool_from_env("QW_LAMBDA_ENABLE_VERBOSE_JSON_LOGS", false)); - -pub static OPENTELEMETRY_URL: Lazy> = - Lazy::new(|| var("QW_LAMBDA_OPENTELEMETRY_URL").ok()); - -pub static OPENTELEMETRY_AUTHORIZATION: Lazy> = - Lazy::new(|| var("QW_LAMBDA_OPENTELEMETRY_AUTHORIZATION").ok()); diff --git a/quickwit/quickwit-lambda/src/indexer/environment.rs b/quickwit/quickwit-lambda/src/indexer/environment.rs deleted file mode 100644 index f2f960f224b..00000000000 --- a/quickwit/quickwit-lambda/src/indexer/environment.rs +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::env::var; - -use once_cell::sync::Lazy; -use quickwit_common::get_bool_from_env; - -pub const CONFIGURATION_TEMPLATE: &str = r#" -version: 0.8 -node_id: lambda-indexer -cluster_id: lambda-ephemeral -metastore_uri: s3://${QW_LAMBDA_METASTORE_BUCKET}/${QW_LAMBDA_METASTORE_PREFIX:-index} -default_index_root_uri: s3://${QW_LAMBDA_INDEX_BUCKET}/${QW_LAMBDA_INDEX_PREFIX:-index} -data_dir: /tmp -"#; - -pub static INDEX_CONFIG_URI: Lazy = Lazy::new(|| { - var("QW_LAMBDA_INDEX_CONFIG_URI") - .expect("environment variable `QW_LAMBDA_INDEX_CONFIG_URI` should be set") -}); - -pub static DISABLE_MERGE: Lazy = - Lazy::new(|| get_bool_from_env("QW_LAMBDA_DISABLE_MERGE", false)); - -pub static DISABLE_JANITOR: Lazy = - Lazy::new(|| get_bool_from_env("QW_LAMBDA_DISABLE_JANITOR", false)); - -pub static MAX_CHECKPOINTS: Lazy = Lazy::new(|| { - var("QW_LAMBDA_MAX_CHECKPOINTS").map_or(100, |v| { - v.parse() - .expect("QW_LAMBDA_MAX_CHECKPOINTS must be a positive integer") - }) -}); - -#[cfg(test)] -mod tests { - - use quickwit_config::{ConfigFormat, NodeConfig}; - - use super::*; - - #[tokio::test] - #[serial_test::file_serial(with_env)] - async fn test_load_config() { - // SAFETY: this test may not be entirely sound if not run with nextest or --test-threads=1 - // as this is only a test, and it would be extremly inconvenient to run it in a different - // way, we are keeping it that way - // file_serial may not be enough, given other tests not ran serially could read env - - let bucket = "mock-test-bucket"; - unsafe { - std::env::set_var("QW_LAMBDA_METASTORE_BUCKET", bucket); - std::env::set_var("QW_LAMBDA_INDEX_BUCKET", bucket); - std::env::set_var( - "QW_LAMBDA_INDEX_CONFIG_URI", - "s3://mock-index-config-bucket", - ); - std::env::set_var("QW_LAMBDA_INDEX_ID", "lambda-test"); - }; - let node_config = NodeConfig::load(ConfigFormat::Yaml, CONFIGURATION_TEMPLATE.as_bytes()) - .await - .unwrap(); - // - assert_eq!( - node_config.data_dir_path.to_string_lossy(), - "/tmp", - "only `/tmp` is writeable in AWS Lambda" - ); - assert_eq!( - node_config.default_index_root_uri, - "s3://mock-test-bucket/index" - ); - assert_eq!( - node_config.metastore_uri.to_string(), - "s3://mock-test-bucket/index" - ); - - unsafe { - std::env::remove_var("QW_LAMBDA_METASTORE_BUCKET"); - std::env::remove_var("QW_LAMBDA_INDEX_BUCKET"); - std::env::remove_var("QW_LAMBDA_INDEX_CONFIG_URI"); - std::env::remove_var("QW_LAMBDA_INDEX_ID"); - } - } -} diff --git a/quickwit/quickwit-lambda/src/indexer/handler.rs b/quickwit/quickwit-lambda/src/indexer/handler.rs deleted file mode 100644 index 87468c476f1..00000000000 --- a/quickwit/quickwit-lambda/src/indexer/handler.rs +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use lambda_runtime::{Error, LambdaEvent}; -use serde_json::Value; -use tracing::{Instrument, debug_span, error, info, info_span}; - -use super::environment::{DISABLE_JANITOR, DISABLE_MERGE, INDEX_CONFIG_URI}; -use super::ingest::{IngestArgs, ingest}; -use super::model::IndexerEvent; -use crate::environment::INDEX_ID; -use crate::logger; -use crate::utils::LambdaContainerContext; - -async fn indexer_handler(event: LambdaEvent) -> Result { - let container_ctx = LambdaContainerContext::load(); - let memory = event.context.env_config.memory; - let payload = serde_json::from_value::(event.payload)?; - - let ingest_res = ingest(IngestArgs { - input_path: payload.uri()?, - input_format: quickwit_config::SourceInputFormat::Json, - vrl_script: None, - // TODO: instead of clearing the cache, we use a cache and set its max - // size with indexer_config.split_store_max_num_bytes - clear_cache: true, - }) - .instrument(debug_span!( - "ingest", - memory, - env.INDEX_CONFIG_URI = *INDEX_CONFIG_URI, - env.INDEX_ID = *INDEX_ID, - env.DISABLE_MERGE = *DISABLE_MERGE, - env.DISABLE_JANITOR = *DISABLE_JANITOR, - cold = container_ctx.cold, - container_id = container_ctx.container_id, - )) - .await; - - match ingest_res { - Ok(stats) => { - info!(stats=?stats, "Indexing succeeded"); - Ok(serde_json::to_value(stats)?) - } - Err(e) => { - error!(err=?e, "Indexing failed"); - Err(anyhow::anyhow!("Indexing failed").into()) - } - } -} - -pub async fn handler(event: LambdaEvent) -> Result { - let request_id = event.context.request_id.clone(); - let mut response = indexer_handler(event) - .instrument(info_span!("indexer_handler", request_id)) - .await; - if let Err(e) = &response { - error!(err=?e, "Handler failed"); - } - if let Ok(Value::Object(ref mut map)) = response { - map.insert("request_id".to_string(), Value::String(request_id)); - } - logger::flush_tracer(); - response -} diff --git a/quickwit/quickwit-lambda/src/indexer/ingest/helpers.rs b/quickwit/quickwit-lambda/src/indexer/ingest/helpers.rs deleted file mode 100644 index a44ea9e1f44..00000000000 --- a/quickwit/quickwit-lambda/src/indexer/ingest/helpers.rs +++ /dev/null @@ -1,347 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::HashSet; -use std::num::NonZeroUsize; -use std::path::Path; - -use anyhow::{Context, bail}; -use chitchat::FailureDetectorConfig; -use chitchat::transport::ChannelTransport; -use quickwit_actors::{ActorHandle, Mailbox, Universe}; -use quickwit_cluster::{Cluster, ClusterMember}; -use quickwit_common::pubsub::EventBroker; -use quickwit_common::runtimes::RuntimesConfig; -use quickwit_common::uri::Uri; -use quickwit_config::merge_policy_config::MergePolicyConfig; -use quickwit_config::service::QuickwitService; -use quickwit_config::{ - ConfigFormat, IndexConfig, NodeConfig, SourceConfig, SourceInputFormat, SourceParams, - TransformConfig, load_index_config_from_user_config, -}; -use quickwit_indexing::IndexingPipeline; -use quickwit_indexing::actors::{IndexingService, MergePipeline, MergeSchedulerService}; -use quickwit_indexing::models::{DetachIndexingPipeline, DetachMergePipeline, SpawnPipeline}; -use quickwit_ingest::IngesterPool; -use quickwit_janitor::{JanitorService, start_janitor_service}; -use quickwit_metastore::{ - AddSourceRequestExt, CreateIndexRequestExt, CreateIndexResponseExt, IndexMetadata, - IndexMetadataResponseExt, -}; -use quickwit_proto::indexing::CpuCapacity; -use quickwit_proto::metastore::{ - AddSourceRequest, CreateIndexRequest, IndexMetadataRequest, MetastoreError, MetastoreService, - MetastoreServiceClient, ResetSourceCheckpointRequest, -}; -use quickwit_proto::types::PipelineUid; -use quickwit_search::SearchJobPlacer; -use quickwit_storage::StorageResolver; -use quickwit_telemetry::payload::{QuickwitFeature, QuickwitTelemetryInfo, TelemetryEvent}; -use tracing::{debug, info, instrument}; - -use crate::environment::INDEX_ID; -use crate::indexer::environment::{ - DISABLE_JANITOR, DISABLE_MERGE, INDEX_CONFIG_URI, MAX_CHECKPOINTS, -}; - -const LAMBDA_SOURCE_ID: &str = "ingest-lambda-source"; - -/// The indexing service needs to update its cluster chitchat state so that the control plane is -/// aware of the running tasks. We thus create a fake cluster to instantiate the indexing service -/// and avoid impacting potential control plane running on the cluster. -pub(super) async fn create_empty_cluster( - config: &NodeConfig, - services: &[QuickwitService], -) -> anyhow::Result { - let self_node = ClusterMember { - node_id: config.node_id.clone(), - generation_id: quickwit_cluster::GenerationId::now(), - is_ready: false, - enabled_services: HashSet::from_iter(services.to_owned()), - gossip_advertise_addr: config.gossip_advertise_addr, - grpc_advertise_addr: config.grpc_advertise_addr, - indexing_tasks: Vec::new(), - indexing_cpu_capacity: CpuCapacity::zero(), - }; - let cluster = Cluster::join( - config.cluster_id.clone(), - self_node, - config.gossip_advertise_addr, - Vec::new(), - config.gossip_interval, - FailureDetectorConfig::default(), - &ChannelTransport::default(), - Default::default(), - ) - .await?; - Ok(cluster) -} - -/// TODO refactor with `dir_and_filename` in file source -fn dir_and_filename(filepath: &Path) -> anyhow::Result<(Uri, &Path)> { - let dir_uri: Uri = filepath - .parent() - .context("Parent directory could not be resolved")? - .to_str() - .context("Path cannot be turned to string")? - .parse()?; - let file_name = filepath - .file_name() - .context("Path does not appear to be a file")?; - Ok((dir_uri, file_name.as_ref())) -} - -#[instrument(level = "debug", skip(resolver))] -pub(super) async fn load_index_config( - resolver: &StorageResolver, - default_index_root_uri: &Uri, -) -> anyhow::Result { - let (dir, file) = dir_and_filename(Path::new(&*INDEX_CONFIG_URI))?; - let index_config_storage = resolver.resolve(&dir).await?; - let bytes = index_config_storage.get_all(file).await?; - let mut index_config = load_index_config_from_user_config( - ConfigFormat::Yaml, - bytes.as_slice(), - default_index_root_uri, - )?; - if *DISABLE_MERGE { - debug!("force disable merges"); - index_config.indexing_settings.merge_policy = MergePolicyConfig::Nop; - } - Ok(index_config) -} - -pub(super) async fn send_telemetry() { - let services: HashSet = - HashSet::from_iter([QuickwitService::Indexer.as_str().to_string()]); - let telemetry_info = - QuickwitTelemetryInfo::new(services, HashSet::from_iter([QuickwitFeature::AwsLambda])); - let _telemetry_handle_opt = quickwit_telemetry::start_telemetry_loop(telemetry_info); - quickwit_telemetry::send_telemetry_event(TelemetryEvent::RunCommand).await; -} - -/// Convert the incoming file path to a source config -pub(super) async fn configure_source( - input_uri: Uri, - input_format: SourceInputFormat, - vrl_script: Option, -) -> anyhow::Result { - let transform_config = vrl_script.map(|vrl_script| TransformConfig::new(vrl_script, None)); - let source_params = SourceParams::file_from_uri(input_uri); - Ok(SourceConfig { - source_id: LAMBDA_SOURCE_ID.to_owned(), - num_pipelines: NonZeroUsize::MIN, - enabled: true, - source_params, - transform_config, - input_format, - }) -} - -/// Check if the index exists, creating it if necessary -/// -/// If the index exists but without the Lambda source ([`LAMBDA_SOURCE_ID`]), -/// the source is added. -pub(super) async fn init_index_if_necessary( - metastore: &mut MetastoreServiceClient, - storage_resolver: &StorageResolver, - default_index_root_uri: &Uri, - source_config: &SourceConfig, -) -> anyhow::Result { - let metadata_result = metastore - .index_metadata(IndexMetadataRequest::for_index_id(INDEX_ID.clone())) - .await; - let metadata = match metadata_result { - Ok(metadata_resp) => { - let current_metadata = metadata_resp.deserialize_index_metadata()?; - if !current_metadata.sources.contains_key(LAMBDA_SOURCE_ID) { - let add_source_request = AddSourceRequest::try_from_source_config( - current_metadata.index_uid.clone(), - source_config, - )?; - metastore.add_source(add_source_request).await?; - metastore - .index_metadata(IndexMetadataRequest::for_index_id(INDEX_ID.clone())) - .await? - .deserialize_index_metadata()? - } else { - current_metadata - } - } - Err(MetastoreError::NotFound(_)) => { - info!( - index_id = *INDEX_ID, - index_config_uri = *INDEX_CONFIG_URI, - "Index not found, creating it" - ); - let index_config = load_index_config(storage_resolver, default_index_root_uri).await?; - if index_config.index_id != *INDEX_ID { - bail!( - "Expected index ID was {} but config file had {}", - *INDEX_ID, - index_config.index_id, - ); - } - let create_index_request = CreateIndexRequest::try_from_index_and_source_configs( - &index_config, - std::slice::from_ref(source_config), - )?; - let create_resp = metastore.create_index(create_index_request).await?; - - info!(index_uid = %create_resp.index_uid(), "index created"); - create_resp.deserialize_index_metadata()? - } - Err(e) => bail!(e), - }; - Ok(metadata) -} - -pub(super) async fn spawn_services( - universe: &Universe, - cluster: Cluster, - metastore: MetastoreServiceClient, - storage_resolver: StorageResolver, - node_config: &NodeConfig, - runtime_config: RuntimesConfig, -) -> anyhow::Result<( - ActorHandle, - Option>, -)> { - let event_broker = EventBroker::default(); - - // spawn merge scheduler service - let merge_scheduler_service = - MergeSchedulerService::new(node_config.indexer_config.merge_concurrency.get()); - let (merge_scheduler_service_mailbox, _) = - universe.spawn_builder().spawn(merge_scheduler_service); - - // spawn indexer service - let indexing_service = IndexingService::new( - node_config.node_id.clone(), - node_config.data_dir_path.clone(), - node_config.indexer_config.clone(), - runtime_config.num_threads_blocking, - cluster, - metastore.clone(), - None, - merge_scheduler_service_mailbox.clone(), - IngesterPool::default(), - storage_resolver.clone(), - event_broker.clone(), - ) - .await?; - let (_, indexing_service_handle) = universe.spawn_builder().spawn(indexing_service); - - // spawn janitor service - let janitor_service_opt = if *DISABLE_JANITOR { - None - } else { - Some( - start_janitor_service( - universe, - node_config, - metastore, - SearchJobPlacer::default(), - storage_resolver, - event_broker, - false, - ) - .await?, - ) - }; - Ok((indexing_service_handle, janitor_service_opt)) -} - -/// Spawn and split an indexing pipeline -pub(super) async fn spawn_pipelines( - indexing_server_mailbox: &Mailbox, - source_config: SourceConfig, -) -> anyhow::Result<(ActorHandle, ActorHandle)> { - let pipeline_id = indexing_server_mailbox - .ask_for_res(SpawnPipeline { - index_id: INDEX_ID.clone(), - source_config, - pipeline_uid: PipelineUid::default(), - }) - .await?; - let merge_pipeline_handle = indexing_server_mailbox - .ask_for_res(DetachMergePipeline { - pipeline_id: pipeline_id.merge_pipeline_id(), - }) - .await?; - let indexing_pipeline_handle = indexing_server_mailbox - .ask_for_res(DetachIndexingPipeline { pipeline_id }) - .await?; - Ok((indexing_pipeline_handle, merge_pipeline_handle)) -} - -/// Prune old Lambda file checkpoints if there are too many -/// -/// Without pruning checkpoints accumulate indefinitely. This is particularly -/// problematic when indexing a lot of small files, as the metastore will grow -/// large even for a small index. -/// -/// The current implementation just deletes all checkpoints if there are more -/// than QW_LAMBDA_MAX_CHECKPOINTS. When this purging is performed, the Lambda -/// indexer might ingest the same file again if it receives a duplicate -/// notification. -pub(super) async fn prune_lambda_source( - metastore: &mut MetastoreServiceClient, - index_metadata: IndexMetadata, -) -> anyhow::Result<()> { - let lambda_checkpoint_opt = index_metadata - .checkpoint - .source_checkpoint(LAMBDA_SOURCE_ID); - - if let Some(lambda_checkpoint) = lambda_checkpoint_opt { - if lambda_checkpoint.num_partitions() > *MAX_CHECKPOINTS { - info!( - partitions = lambda_checkpoint.num_partitions(), - "prune Lambda checkpoints" - ); - metastore - .reset_source_checkpoint(ResetSourceCheckpointRequest { - index_uid: Some(index_metadata.index_uid.clone()), - source_id: LAMBDA_SOURCE_ID.to_owned(), - }) - .await?; - } - } - - Ok(()) -} - -/// Observe the merge pipeline until there are no more ongoing merges -pub(super) async fn wait_for_merges( - merge_pipeline_handle: ActorHandle, -) -> anyhow::Result<()> { - // TODO: find a way to stop the MergePlanner actor in the MergePipeline, - // otherwise a new merge might be scheduled after this loop. That shouldn't - // have any concrete impact as the merge will be immediately cancelled, but - // it might generate errors during the universe shutdown (i.e "Failed to - // acquire permit") - loop { - let state = merge_pipeline_handle.state(); - let obs = merge_pipeline_handle.observe().await; - debug!(state=?state, ongoing=obs.num_ongoing_merges, "merge pipeline state"); - if obs.num_ongoing_merges == 0 { - break; - } - // We tolerate a relatively low refresh rate because the indexer - // typically runs for longer periods of times and merges happen only - // occasionally. - tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; - } - Ok(()) -} diff --git a/quickwit/quickwit-lambda/src/indexer/ingest/mod.rs b/quickwit/quickwit-lambda/src/indexer/ingest/mod.rs deleted file mode 100644 index 4f7295bc89a..00000000000 --- a/quickwit/quickwit-lambda/src/indexer/ingest/mod.rs +++ /dev/null @@ -1,263 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod helpers; - -use std::collections::HashSet; - -use anyhow::bail; -use helpers::{ - configure_source, create_empty_cluster, init_index_if_necessary, send_telemetry, - spawn_pipelines, spawn_services, -}; -use quickwit_actors::Universe; -use quickwit_cli::start_actor_runtimes; -use quickwit_cli::tool::start_statistics_reporting_loop; -use quickwit_common::runtimes::RuntimesConfig; -use quickwit_common::uri::Uri; -use quickwit_config::SourceInputFormat; -use quickwit_config::service::QuickwitService; -use quickwit_index_management::clear_cache_directory; -use quickwit_indexing::models::IndexingStatistics; -use tracing::{debug, info}; - -use crate::indexer::environment::{CONFIGURATION_TEMPLATE, DISABLE_JANITOR}; -use crate::indexer::ingest::helpers::{prune_lambda_source, wait_for_merges}; -use crate::utils::load_node_config; - -#[derive(Debug, Eq, PartialEq)] -pub struct IngestArgs { - pub input_path: Uri, - pub input_format: SourceInputFormat, - pub vrl_script: Option, - pub clear_cache: bool, -} - -pub async fn ingest(args: IngestArgs) -> anyhow::Result { - debug!(args=?args, "lambda-ingest"); - - send_telemetry().await; - - let (config, storage_resolver, mut metastore) = - load_node_config(CONFIGURATION_TEMPLATE).await?; - - let source_config = - configure_source(args.input_path, args.input_format, args.vrl_script).await?; - - let index_metadata = init_index_if_necessary( - &mut metastore, - &storage_resolver, - &config.default_index_root_uri, - &source_config, - ) - .await?; - - let mut services = vec![QuickwitService::Indexer]; - if !*DISABLE_JANITOR { - services.push(QuickwitService::Janitor); - } - let cluster = create_empty_cluster(&config, &services[..]).await?; - let universe = Universe::new(); - let runtimes_config = RuntimesConfig::default(); - - start_actor_runtimes(runtimes_config, &HashSet::from_iter(services))?; - - let (indexing_service_handle, _janitor_service_guard) = spawn_services( - &universe, - cluster, - metastore.clone(), - storage_resolver.clone(), - &config, - runtimes_config, - ) - .await?; - - let (indexing_pipeline_handle, merge_pipeline_handle) = - spawn_pipelines(indexing_service_handle.mailbox(), source_config).await?; - - prune_lambda_source(&mut metastore, index_metadata).await?; - - debug!("wait for indexing to complete"); - let statistics = start_statistics_reporting_loop(indexing_pipeline_handle, false).await?; - - debug!("wait for merges to complete"); - wait_for_merges(merge_pipeline_handle).await?; - - debug!("indexing completed, tearing down actors"); - // TODO: is it really necessary to terminate the indexing service? - // Quitting the universe should be enough. - universe - .send_exit_with_success(indexing_service_handle.mailbox()) - .await?; - indexing_service_handle.join().await; - debug!("quitting universe"); - universe.quit().await; - debug!("universe.quit() awaited"); - - if args.clear_cache { - info!("clearing local cache directory"); - clear_cache_directory(&config.data_dir_path).await?; - info!("local cache directory cleared"); - } - - if statistics.num_invalid_docs > 0 { - bail!("Failed to ingest {} documents", statistics.num_invalid_docs) - } - Ok(statistics) -} - -#[cfg(all(test, feature = "s3-localstack-tests"))] -mod tests { - use std::path::PathBuf; - use std::str::FromStr; - - use quickwit_common::new_coolid; - use quickwit_storage::StorageResolver; - - use super::*; - - async fn put_object( - storage_resolver: StorageResolver, - bucket: &str, - prefix: &str, - filename: &str, - data: Vec, - ) -> Uri { - let src_location = format!("s3://{bucket}/{prefix}"); - let storage_uri = Uri::from_str(&src_location).unwrap(); - let storage = storage_resolver.resolve(&storage_uri).await.unwrap(); - storage - .put(&PathBuf::from(filename), Box::new(data)) - .await - .unwrap(); - storage_uri.join(filename).unwrap() - } - - #[tokio::test] - #[serial_test::file_serial(with_env)] - async fn test_ingest() -> anyhow::Result<()> { - // SAFETY: this test may not be entirely sound if not run with nextest or --test-threads=1 - // as this is only a test, and it would be extremly inconvenient to run it in a different - // way, we are keeping it that way - // file_serial may not be enough, given other tests not ran serially could read env - - quickwit_common::setup_logging_for_tests(); - let bucket = "quickwit-integration-tests"; - let prefix = new_coolid("lambda-ingest-test"); - let storage_resolver = StorageResolver::unconfigured(); - - let index_config = br#" - version: 0.8 - index_id: lambda-test - doc_mapping: - field_mappings: - - name: timestamp - type: datetime - input_formats: - - unix_timestamp - fast: true - timestamp_field: timestamp - "#; - let config_uri = put_object( - storage_resolver.clone(), - bucket, - &prefix, - "index-config.yaml", - index_config.to_vec(), - ) - .await; - - // TODO use dependency injection instead of lazy static for env configs - unsafe { - std::env::set_var("QW_LAMBDA_METASTORE_BUCKET", bucket); - std::env::set_var("QW_LAMBDA_INDEX_BUCKET", bucket); - std::env::set_var("QW_LAMBDA_METASTORE_PREFIX", &prefix); - std::env::set_var("QW_LAMBDA_INDEX_PREFIX", &prefix); - std::env::set_var("QW_LAMBDA_INDEX_CONFIG_URI", config_uri.as_str()); - std::env::set_var("QW_LAMBDA_INDEX_ID", "lambda-test"); - } - - // first ingestion creates the index metadata - let test_data_1 = br#"{"timestamp": 1724140899, "field1": "value1"}"#; - let test_data_1_uri = put_object( - storage_resolver.clone(), - bucket, - &prefix, - "data.json", - test_data_1.to_vec(), - ) - .await; - - { - let args = IngestArgs { - input_path: test_data_1_uri.clone(), - input_format: SourceInputFormat::Json, - vrl_script: None, - clear_cache: true, - }; - let stats = ingest(args).await?; - assert_eq!(stats.num_invalid_docs, 0); - assert_eq!(stats.num_docs, 1); - } - - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - - { - // ingesting the same data again is a no-op - let args = IngestArgs { - input_path: test_data_1_uri, - input_format: SourceInputFormat::Json, - vrl_script: None, - clear_cache: true, - }; - let stats = ingest(args).await?; - assert_eq!(stats.num_invalid_docs, 0); - assert_eq!(stats.num_docs, 0); - } - - { - // second ingestion should not fail when metadata already exists - let test_data = br#"{"timestamp": 1724149900, "field1": "value2"}"#; - let test_data_uri = put_object( - storage_resolver.clone(), - bucket, - &prefix, - "data2.json", - test_data.to_vec(), - ) - .await; - - let args = IngestArgs { - input_path: test_data_uri, - input_format: SourceInputFormat::Json, - vrl_script: None, - clear_cache: true, - }; - let stats = ingest(args).await?; - assert_eq!(stats.num_invalid_docs, 0); - assert_eq!(stats.num_docs, 1); - } - - unsafe { - std::env::remove_var("QW_LAMBDA_METASTORE_BUCKET"); - std::env::remove_var("QW_LAMBDA_INDEX_BUCKET"); - std::env::remove_var("QW_LAMBDA_METASTORE_PREFIX"); - std::env::remove_var("QW_LAMBDA_INDEX_PREFIX"); - std::env::remove_var("QW_LAMBDA_INDEX_CONFIG_URI"); - std::env::remove_var("QW_LAMBDA_INDEX_ID"); - } - - Ok(()) - } -} diff --git a/quickwit/quickwit-lambda/src/indexer/model.rs b/quickwit/quickwit-lambda/src/indexer/model.rs deleted file mode 100644 index c87958c9ea8..00000000000 --- a/quickwit/quickwit-lambda/src/indexer/model.rs +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::str::FromStr; - -use aws_lambda_events::event::s3::S3Event; -use quickwit_common::uri::Uri; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Debug, Deserialize, Serialize)] -#[serde(untagged)] -/// Event types that can be used to invoke the indexer Lambda. -pub enum IndexerEvent { - Custom { source_uri: String }, - S3(S3Event), -} - -impl IndexerEvent { - pub fn uri(&self) -> anyhow::Result { - let path: String = match self { - IndexerEvent::Custom { source_uri } => source_uri.clone(), - IndexerEvent::S3(event) => [ - "s3://", - event.records[0].s3.bucket.name.as_ref().unwrap(), - "/", - event.records[0].s3.object.key.as_ref().unwrap(), - ] - .join(""), - }; - Uri::from_str(&path) - } -} - -#[cfg(test)] -mod tests { - use serde_json::json; - - use super::*; - - #[test] - fn test_custom_event_uri() { - let cust_event = json!({ - "source_uri": "s3://quickwit-test/test.json" - }); - let parsed_cust_event: IndexerEvent = serde_json::from_value(cust_event).unwrap(); - assert_eq!( - parsed_cust_event.uri().unwrap(), - Uri::from_str("s3://quickwit-test/test.json").unwrap(), - ); - } - - #[test] - fn test_s3_event_uri() { - let s3_event = json!({ - "Records": [ - { - "eventVersion": "2.0", - "eventSource": "aws:s3", - "awsRegion": "us-east-1", - "eventTime": "1970-01-01T00:00:00.000Z", - "eventName": "ObjectCreated:Put", - "userIdentity": { - "principalId": "EXAMPLE" - }, - "requestParameters": { - "sourceIPAddress": "127.0.0.1" - }, - "responseElements": { - "x-amz-request-id": "EXAMPLE123456789", - "x-amz-id-2": "EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH" - }, - "s3": { - "s3SchemaVersion": "1.0", - "configurationId": "testConfigRule", - "bucket": { - "name": "quickwit-test", - "ownerIdentity": { - "principalId": "EXAMPLE" - }, - "arn": "arn:aws:s3:::quickwit-test" - }, - "object": { - "key": "test.json", - "size": 1024, - "eTag": "0123456789abcdef0123456789abcdef", - "sequencer": "0A1B2C3D4E5F678901" - } - } - } - ] - }); - let s3_event: IndexerEvent = serde_json::from_value(s3_event).unwrap(); - assert_eq!( - s3_event.uri().unwrap(), - Uri::from_str("s3://quickwit-test/test.json").unwrap(), - ); - } -} diff --git a/quickwit/quickwit-lambda/src/logger.rs b/quickwit/quickwit-lambda/src/logger.rs deleted file mode 100644 index c722f19f9a4..00000000000 --- a/quickwit/quickwit-lambda/src/logger.rs +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use anyhow::Context; -use once_cell::sync::OnceCell; -use opentelemetry::trace::TracerProvider as _; -use opentelemetry::{KeyValue, global}; -use opentelemetry_otlp::{WithExportConfig, WithHttpConfig}; -use opentelemetry_sdk::propagation::TraceContextPropagator; -use opentelemetry_sdk::trace::{BatchConfigBuilder, TracerProvider}; -use opentelemetry_sdk::{Resource, trace}; -use quickwit_serve::BuildInfo; -use tracing::{Level, debug}; -use tracing_subscriber::fmt::format::{FmtSpan, JsonFields}; -use tracing_subscriber::fmt::time::UtcTime; -use tracing_subscriber::layer::SubscriberExt; -use tracing_subscriber::prelude::*; -use tracing_subscriber::registry::LookupSpan; -use tracing_subscriber::{EnvFilter, Layer}; - -use crate::environment::{ - ENABLE_VERBOSE_JSON_LOGS, OPENTELEMETRY_AUTHORIZATION, OPENTELEMETRY_URL, -}; - -static TRACER_PROVIDER: OnceCell = OnceCell::new(); -pub(crate) const RUNTIME_CONTEXT_SPAN: &str = "runtime_context"; - -fn fmt_env_filter(level: Level) -> EnvFilter { - let default_directive = format!("quickwit={level}") - .parse() - .expect("default directive should be valid"); - EnvFilter::builder() - .with_default_directive(default_directive) - .from_env_lossy() -} - -fn fmt_time_format() -> UtcTime>> { - // We do not rely on the Rfc3339 implementation, because it has a nanosecond precision. - // See discussion here: https://github.com/time-rs/time/discussions/418 - UtcTime::new( - time::format_description::parse( - "[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond digits:3]Z", - ) - .expect("Time format invalid."), - ) -} - -fn compact_fmt_layer(level: Level) -> impl Layer -where - S: for<'a> LookupSpan<'a>, - S: tracing::Subscriber, -{ - let event_format = tracing_subscriber::fmt::format() - .with_target(true) - .with_timer(fmt_time_format()) - .compact(); - - tracing_subscriber::fmt::layer::() - .event_format(event_format) - .with_ansi(false) - .with_filter(fmt_env_filter(level)) -} - -fn json_fmt_layer(level: Level) -> impl Layer -where - S: for<'a> LookupSpan<'a>, - S: tracing::Subscriber, -{ - let event_format = tracing_subscriber::fmt::format() - .with_target(true) - .with_timer(fmt_time_format()) - .json(); - tracing_subscriber::fmt::layer::() - .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE) - .event_format(event_format) - .fmt_fields(JsonFields::default()) - .with_ansi(false) - .with_filter(fmt_env_filter(level)) -} - -fn fmt_layer(level: Level) -> Box + Send + Sync + 'static> -where - S: for<'a> LookupSpan<'a>, - S: tracing::Subscriber, -{ - if *ENABLE_VERBOSE_JSON_LOGS { - json_fmt_layer(level).boxed() - } else { - compact_fmt_layer(level).boxed() - } -} - -fn otlp_layer( - ot_url: String, - ot_auth: String, - level: Level, - build_info: &BuildInfo, -) -> anyhow::Result> -where - S: for<'a> LookupSpan<'a>, - S: tracing::Subscriber, -{ - let headers = std::collections::HashMap::from([("Authorization".to_string(), ot_auth)]); - let otlp_exporter = opentelemetry_otlp::SpanExporter::builder() - .with_http() - .with_endpoint(ot_url) - .with_headers(headers) - .build() - .context("failed to initialize OpenTelemetry OTLP exporter")?; - let batch_processor = - trace::BatchSpanProcessor::builder(otlp_exporter, opentelemetry_sdk::runtime::Tokio) - .with_batch_config( - BatchConfigBuilder::default() - // Quickwit can generate a lot of spans, especially in debug mode, and the - // default queue size of 2048 is too small. - .with_max_queue_size(32_768) - .build(), - ) - .build(); - let provider = opentelemetry_sdk::trace::TracerProvider::builder() - .with_span_processor(batch_processor) - .with_resource(Resource::new([ - KeyValue::new("service.name", "quickwit"), - KeyValue::new("service.version", build_info.version.clone()), - ])) - .build(); - TRACER_PROVIDER - .set(provider.clone()) - .expect("cell should be empty"); - let tracer = provider.tracer("quickwit"); - let env_filter = std::env::var(EnvFilter::DEFAULT_ENV) - .map(|_| EnvFilter::from_default_env()) - .or_else(|_| { - // record the runtime context span for trace querying - EnvFilter::try_new(format!( - "quickwit={level},quickwit[{RUNTIME_CONTEXT_SPAN}]=trace" - )) - }) - .expect("Failed to set up OTLP tracing filter."); - Ok(tracing_opentelemetry::layer() - .with_tracer(tracer) - .with_filter(env_filter)) -} - -pub fn setup_lambda_tracer(level: Level) -> anyhow::Result<()> { - global::set_text_map_propagator(TraceContextPropagator::new()); - let registry = tracing_subscriber::registry(); - let build_info = BuildInfo::get(); - if let (Some(ot_url), Some(ot_auth)) = ( - OPENTELEMETRY_URL.clone(), - OPENTELEMETRY_AUTHORIZATION.clone(), - ) { - registry - .with(fmt_layer(level)) - .with(otlp_layer(ot_url, ot_auth, level, build_info)?) - .try_init() - .context("failed to register tracing subscriber")?; - } else { - registry - .with(fmt_layer(level)) - .try_init() - .context("failed to register tracing subscriber")?; - } - Ok(()) -} - -pub fn flush_tracer() { - if let Some(tracer_provider) = TRACER_PROVIDER.get() { - debug!("flush tracers"); - for res in tracer_provider.force_flush() { - if let Err(err) = res { - debug!(?err, "failed to flush tracer"); - } - } - } -} diff --git a/quickwit/quickwit-lambda/src/searcher/api.rs b/quickwit/quickwit-lambda/src/searcher/api.rs deleted file mode 100644 index 7ba034de5a4..00000000000 --- a/quickwit/quickwit-lambda/src/searcher/api.rs +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::HashSet; -use std::net::{Ipv4Addr, SocketAddr}; -use std::sync::Arc; - -use quickwit_config::SearcherConfig; -use quickwit_config::service::QuickwitService; -use quickwit_proto::metastore::MetastoreServiceClient; -use quickwit_search::{ - ClusterClient, SearchJobPlacer, SearchService, SearchServiceClient, SearchServiceImpl, - SearcherContext, SearcherPool, -}; -use quickwit_serve::lambda_search_api::*; -use quickwit_storage::StorageResolver; -use quickwit_telemetry::payload::{QuickwitFeature, QuickwitTelemetryInfo, TelemetryEvent}; -use tracing::{error, info}; -use warp::Filter; -use warp::filters::path::FullPath; -use warp::reject::Rejection; - -use crate::searcher::environment::CONFIGURATION_TEMPLATE; -use crate::utils::load_node_config; - -async fn create_local_search_service( - searcher_config: SearcherConfig, - metastore: MetastoreServiceClient, - storage_resolver: StorageResolver, -) -> Arc { - let searcher_pool = SearcherPool::default(); - let search_job_placer = SearchJobPlacer::new(searcher_pool.clone()); - let cluster_client = ClusterClient::new(search_job_placer); - // TODO configure split cache - let searcher_context = Arc::new(SearcherContext::new(searcher_config, None)); - let search_service = Arc::new(SearchServiceImpl::new( - metastore, - storage_resolver, - cluster_client.clone(), - searcher_context.clone(), - )); - // Add search service to pool to avoid "no available searcher nodes in the pool" error - let socket_addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 7280u16); - let search_service_client = - SearchServiceClient::from_service(search_service.clone(), socket_addr); - searcher_pool.insert(socket_addr, search_service_client); - search_service -} - -fn native_api( - search_service: Arc, -) -> impl Filter + Clone { - search_get_handler(search_service.clone()).or(search_post_handler(search_service)) -} - -fn es_compat_api( - search_service: Arc, - metastore: MetastoreServiceClient, -) -> impl Filter + Clone { - es_compat_search_handler(search_service.clone()) - .or(es_compat_index_search_handler(search_service.clone())) - .or(es_compat_index_count_handler(search_service.clone())) - .or(es_compat_scroll_handler(search_service.clone())) - .or(es_compat_index_multi_search_handler(search_service.clone())) - .or(es_compat_index_field_capabilities_handler( - search_service.clone(), - )) - .or(es_compat_index_stats_handler(metastore.clone())) - .or(es_compat_stats_handler(metastore.clone())) - .or(es_compat_index_cat_indices_handler(metastore.clone())) - .or(es_compat_cat_indices_handler(metastore.clone())) - .or(es_compat_resolve_index_handler(metastore.clone())) -} - -fn index_api( - metastore: MetastoreServiceClient, -) -> impl Filter + Clone { - get_index_metadata_handler(metastore) -} - -fn v1_searcher_api( - search_service: Arc, - metastore: MetastoreServiceClient, -) -> impl Filter + Clone { - warp::path!("api" / "v1" / ..) - .and( - native_api(search_service.clone()) - .or(es_compat_api(search_service, metastore.clone())) - .or(index_api(metastore)), - ) - .with(warp::filters::compression::gzip()) - .recover(|rejection| { - error!(?rejection, "request rejected"); - recover_fn(rejection) - }) -} - -pub async fn setup_searcher_api() --> anyhow::Result + Clone> { - let (node_config, storage_resolver, metastore) = - load_node_config(CONFIGURATION_TEMPLATE).await?; - - let telemetry_info = QuickwitTelemetryInfo::new( - HashSet::from_iter([QuickwitService::Searcher.as_str().to_string()]), - HashSet::from_iter([QuickwitFeature::AwsLambda]), - ); - let _telemetry_handle_opt = quickwit_telemetry::start_telemetry_loop(telemetry_info); - - let search_service = create_local_search_service( - node_config.searcher_config, - metastore.clone(), - storage_resolver, - ) - .await; - - let before_hook = warp::path::full() - .and(warp::method()) - .and_then(|route: FullPath, method: warp::http::Method| async move { - info!( - method = method.as_str(), - route = route.as_str(), - "new request" - ); - quickwit_telemetry::send_telemetry_event(TelemetryEvent::RunCommand).await; - Ok::<_, std::convert::Infallible>(()) - }) - .untuple_one(); - - let after_hook = warp::log::custom(|info| { - info!(status = info.status().as_str(), "request completed"); - }); - - let api = warp::any() - .and(before_hook) - .and(v1_searcher_api(search_service, metastore)) - .with(after_hook); - - Ok(api) -} diff --git a/quickwit/quickwit-lambda/src/searcher/environment.rs b/quickwit/quickwit-lambda/src/searcher/environment.rs deleted file mode 100644 index 25c4793e771..00000000000 --- a/quickwit/quickwit-lambda/src/searcher/environment.rs +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pub(crate) const CONFIGURATION_TEMPLATE: &str = r#" -version: 0.8 -node_id: lambda-searcher -metastore_uri: s3://${QW_LAMBDA_METASTORE_BUCKET}/${QW_LAMBDA_METASTORE_PREFIX:-index}#polling_interval=${QW_LAMBDA_SEARCHER_METASTORE_POLLING_INTERVAL_SECONDS:-60}s -default_index_root_uri: s3://${QW_LAMBDA_INDEX_BUCKET}/${QW_LAMBDA_INDEX_PREFIX:-index} -data_dir: /tmp -searcher: - partial_request_cache_capacity: ${QW_LAMBDA_PARTIAL_REQUEST_CACHE_CAPACITY:-64M} -"#; - -#[cfg(test)] -mod tests { - - use bytesize::ByteSize; - use quickwit_config::{ConfigFormat, NodeConfig}; - - use super::*; - - #[tokio::test] - #[serial_test::file_serial(with_env)] - async fn test_load_config() { - // SAFETY: this test may not be entirely sound if not run with nextest or --test-threads=1 - // as this is only a test, and it would be extremly inconvenient to run it in a different - // way, we are keeping it that way - // file_serial may not be enough, given other tests not ran serially could read env - - let bucket = "mock-test-bucket"; - unsafe { - std::env::set_var("QW_LAMBDA_METASTORE_BUCKET", bucket); - std::env::set_var("QW_LAMBDA_INDEX_BUCKET", bucket); - std::env::set_var( - "QW_LAMBDA_INDEX_CONFIG_URI", - "s3://mock-index-config-bucket", - ); - std::env::set_var("QW_LAMBDA_INDEX_ID", "lambda-test"); - } - - let node_config = NodeConfig::load(ConfigFormat::Yaml, CONFIGURATION_TEMPLATE.as_bytes()) - .await - .unwrap(); - assert_eq!( - node_config.data_dir_path.to_string_lossy(), - "/tmp", - "only `/tmp` is writeable in AWS Lambda" - ); - assert_eq!( - node_config.default_index_root_uri, - "s3://mock-test-bucket/index" - ); - assert_eq!( - node_config.metastore_uri.to_string(), - "s3://mock-test-bucket/index#polling_interval=60s" - ); - assert_eq!( - node_config.searcher_config.partial_request_cache_capacity, - ByteSize::mb(64) - ); - - unsafe { - std::env::remove_var("QW_LAMBDA_METASTORE_BUCKET"); - std::env::remove_var("QW_LAMBDA_INDEX_BUCKET"); - std::env::remove_var("QW_LAMBDA_INDEX_CONFIG_URI"); - std::env::remove_var("QW_LAMBDA_INDEX_ID"); - } - } -} diff --git a/quickwit/quickwit-lambda/src/searcher/warp_lambda.rs b/quickwit/quickwit-lambda/src/searcher/warp_lambda.rs deleted file mode 100644 index 03937c2ebba..00000000000 --- a/quickwit/quickwit-lambda/src/searcher/warp_lambda.rs +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Based on https://github.com/aslamplr/warp_lambda under MIT license - -use core::future::Future; -use std::collections::HashSet; -use std::convert::Infallible; -use std::marker::PhantomData; -use std::pin::Pin; -use std::str::FromStr; -use std::task::{Context, Poll}; - -use anyhow::anyhow; -use lambda_http::http::HeaderValue; -use lambda_http::{ - Adapter, Body as LambdaBody, Error as LambdaError, Request, RequestExt, Response, Service, - lambda_runtime, -}; -use mime_guess::{Mime, mime}; -use once_cell::sync::Lazy; -use tracing::{Instrument, info_span}; -use warp::hyper::Body as WarpBody; -pub use {lambda_http, warp}; - -pub type WarpRequest = warp::http::Request; -pub type WarpResponse = warp::http::Response; - -static PLAINTEXT_MIMES: Lazy> = Lazy::new(|| { - HashSet::from_iter([ - mime::APPLICATION_JAVASCRIPT, - mime::APPLICATION_JAVASCRIPT_UTF_8, - mime::APPLICATION_JSON, - ]) -}); - -pub async fn run<'a, S>(service: S) -> Result<(), LambdaError> -where - S: Service + Send + 'a, - S::Future: Send + 'a, -{ - lambda_runtime::run(Adapter::from(WarpAdapter::new(service))).await -} - -#[derive(Clone)] -pub struct WarpAdapter<'a, S> -where - S: Service, - S::Future: Send + 'a, -{ - warp_service: S, - _phantom_data: PhantomData<&'a WarpResponse>, -} - -impl<'a, S> WarpAdapter<'a, S> -where - S: Service, - S::Future: Send + 'a, -{ - pub fn new(warp_service: S) -> Self { - Self { - warp_service, - _phantom_data: PhantomData, - } - } -} - -impl<'a, S> Service for WarpAdapter<'a, S> -where - S: Service + 'a, - S::Future: Send + 'a, -{ - type Response = Response; - type Error = LambdaError; - type Future = Pin> + Send + 'a>>; - - fn poll_ready(&mut self, ctx: &mut Context<'_>) -> Poll> { - self.warp_service - .poll_ready(ctx) - .map_err(|error| match error {}) - } - - fn call(&mut self, request: Request) -> Self::Future { - let query_params = request.query_string_parameters(); - let request_id = request.lambda_context().request_id.clone(); - let (parts, body) = request.into_parts(); - let mut warp_parts = lambda_parts_to_warp_parts(&parts); - let (content_len, warp_body) = match body { - LambdaBody::Empty => (0, WarpBody::empty()), - LambdaBody::Text(text) => (text.len(), WarpBody::from(text.into_bytes())), - LambdaBody::Binary(bytes) => (bytes.len(), WarpBody::from(bytes)), - }; - let mut uri = format!("http://{}{}", "127.0.0.1", parts.uri.path()); - if !query_params.is_empty() { - let url_res = reqwest::Url::parse_with_params(&uri, query_params.iter()); - if let Ok(url) = url_res { - uri = url.into(); - } else { - return Box::pin(async move { Err(anyhow!("invalid url: {uri}").into()) }); - } - } - warp_parts.uri = warp::hyper::Uri::from_str(uri.as_str()).unwrap(); - // REST API Gateways swallow the content-length header which is required - // by many Quickwit routes (`warp::body::content_length_limit(xxx)`) - if let warp::http::header::Entry::Vacant(entry) = warp_parts.headers.entry("Content-Length") - { - entry.insert(content_len.into()); - } - let warp_request = WarpRequest::from_parts(warp_parts, warp_body); - - // Call warp service with warp request, save future - let warp_fut = self.warp_service.call(warp_request); - - // Create lambda future - let fut = async move { - let warp_response = warp_fut.await?; - let (warp_parts, warp_body): (_, _) = warp_response.into_parts(); - let parts = warp_parts_to_lambda_parts(&warp_parts); - let body = warp_body_to_lambda_body(&parts, warp_body).await?; - let lambda_response = Response::from_parts(parts, body); - Ok::(lambda_response) - } - .instrument(info_span!("searcher request", request_id)); - Box::pin(fut) - } -} - -fn lambda_parts_to_warp_parts( - parts: &lambda_http::http::request::Parts, -) -> warp::http::request::Parts { - let mut builder = warp::http::Request::builder() - .method(lambda_method_to_warp_method(&parts.method)) - .uri(lambda_uri_to_warp_uri(&parts.uri)) - .version(lambda_version_to_warp_version(parts.version)); - - for (name, value) in parts.headers.iter() { - builder = builder.header(name.as_str(), value.as_bytes()); - } - let request = builder.body(()).unwrap(); - let (parts, _) = request.into_parts(); - parts -} - -fn warp_parts_to_lambda_parts( - parts: &warp::http::response::Parts, -) -> lambda_http::http::response::Parts { - let mut builder = lambda_http::http::Response::builder() - .status(parts.status.as_u16()) - .version(warp_version_to_lambda_version(parts.version)); - - for (name, value) in parts.headers.iter() { - builder = builder.header(name.as_str(), value.as_bytes()); - } - let response = builder.body(()).unwrap(); - let (parts, _) = response.into_parts(); - parts -} - -async fn warp_body_to_lambda_body( - parts: &lambda_http::http::response::Parts, - warp_body: WarpBody, -) -> Result { - // Concatenate all bytes into a single buffer - let body_bytes = warp::hyper::body::to_bytes(warp_body).await?.to_vec(); - - // Attempt to determine the Content-Type - let content_type_opt: Option<&HeaderValue> = parts.headers.get("Content-Type"); - let content_encoding_opt: Option<&HeaderValue> = parts.headers.get("Content-Encoding"); - - // If Content-Encoding is present, assume compression - // If Content-Type is not present, don't assume is a string - if let (Some(content_type), None) = (content_type_opt, content_encoding_opt) { - let content_type_str = content_type.to_str()?; - let mime = content_type_str.parse::()?; - - if PLAINTEXT_MIMES.contains(&mime) || mime.type_() == mime::TEXT { - return Ok(LambdaBody::Text(String::from_utf8(body_bytes)?)); - } - } - // Not a text response, make binary - Ok(LambdaBody::Binary(body_bytes)) -} - -fn lambda_method_to_warp_method(method: &lambda_http::http::Method) -> warp::http::Method { - method.as_str().parse::().unwrap() -} - -fn lambda_uri_to_warp_uri(uri: &lambda_http::http::Uri) -> warp::http::Uri { - uri.to_string().parse::().unwrap() -} - -fn lambda_version_to_warp_version(version: lambda_http::http::Version) -> warp::http::Version { - if version == lambda_http::http::Version::HTTP_09 { - warp::http::Version::HTTP_09 - } else if version == lambda_http::http::Version::HTTP_10 { - warp::http::Version::HTTP_10 - } else if version == lambda_http::http::Version::HTTP_11 { - warp::http::Version::HTTP_11 - } else if version == lambda_http::http::Version::HTTP_2 { - warp::http::Version::HTTP_2 - } else if version == lambda_http::http::Version::HTTP_3 { - warp::http::Version::HTTP_3 - } else { - panic!("invalid HTTP version: {version:?}"); - } -} - -fn warp_version_to_lambda_version(version: warp::http::Version) -> lambda_http::http::Version { - if version == warp::http::Version::HTTP_09 { - lambda_http::http::Version::HTTP_09 - } else if version == warp::http::Version::HTTP_10 { - lambda_http::http::Version::HTTP_10 - } else if version == warp::http::Version::HTTP_11 { - lambda_http::http::Version::HTTP_11 - } else if version == warp::http::Version::HTTP_2 { - lambda_http::http::Version::HTTP_2 - } else if version == warp::http::Version::HTTP_3 { - lambda_http::http::Version::HTTP_3 - } else { - panic!("invalid HTTP version: {version:?}"); - } -} diff --git a/quickwit/quickwit-lambda/src/utils.rs b/quickwit/quickwit-lambda/src/utils.rs deleted file mode 100644 index 84ce69e3c27..00000000000 --- a/quickwit/quickwit-lambda/src/utils.rs +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::atomic::AtomicU32; -use std::sync::atomic::Ordering::SeqCst; - -use anyhow::Context; -use quickwit_config::{ConfigFormat, NodeConfig}; -use quickwit_metastore::MetastoreResolver; -use quickwit_proto::metastore::MetastoreServiceClient; -use quickwit_storage::StorageResolver; -use tracing::info; - -pub(crate) async fn load_node_config( - config_template: &str, -) -> anyhow::Result<(NodeConfig, StorageResolver, MetastoreServiceClient)> { - let config = NodeConfig::load(ConfigFormat::Yaml, config_template.as_bytes()) - .await - .with_context(|| format!("Failed to parse node config `{config_template}`."))?; - info!(config=?config, "loaded node config"); - let storage_resolver = StorageResolver::configured(&config.storage_configs); - let metastore_resolver = - MetastoreResolver::configured(storage_resolver.clone(), &config.metastore_configs); - let metastore: MetastoreServiceClient = - metastore_resolver.resolve(&config.metastore_uri).await?; - Ok((config, storage_resolver, metastore)) -} - -static CONTAINER_ID: AtomicU32 = AtomicU32::new(0); - -pub struct LambdaContainerContext { - pub container_id: u32, - pub cold: bool, -} - -impl LambdaContainerContext { - /// Configure and return the Lambda container context. - /// - /// The `cold` field returned will be `true` only the first time this - /// function is called. - pub fn load() -> Self { - let mut container_id = CONTAINER_ID.load(SeqCst); - let mut cold = false; - if container_id == 0 { - container_id = rand::random(); - CONTAINER_ID.store(container_id, SeqCst); - cold = true; - } - Self { container_id, cold } - } -} diff --git a/quickwit/quickwit-metastore/Cargo.toml b/quickwit/quickwit-metastore/Cargo.toml index 3fd9fbea63e..0f362f49ceb 100644 --- a/quickwit/quickwit-metastore/Cargo.toml +++ b/quickwit/quickwit-metastore/Cargo.toml @@ -37,11 +37,13 @@ tokio-stream = { workspace = true } tower = { workspace = true } tracing = { workspace = true } ulid = { workspace = true, features = ["serde"] } +uuid = { workspace = true } utoipa = { workspace = true } quickwit-common = { workspace = true } quickwit-config = { workspace = true } quickwit-doc-mapper = { workspace = true } +quickwit-parquet-engine = { workspace = true } quickwit-proto = { workspace = true } quickwit-query = { workspace = true } quickwit-storage = { workspace = true } @@ -65,5 +67,5 @@ quickwit-storage = { workspace = true, features = ["testsuite"] } [features] ci-test = [] -postgres = ["quickwit-proto/postgres", "sea-query", "sea-query-binder", "sqlx"] +postgres = ["quickwit-proto/postgres", "quickwit-parquet-engine/postgres", "sea-query", "sea-query-binder", "sqlx"] testsuite = ["mockall", "tempfile", "quickwit-config/testsuite"] diff --git a/quickwit/quickwit-metastore/migrations/postgresql/24_add-arbitrary-kv.down.sql b/quickwit/quickwit-metastore/migrations/postgresql/24_add-arbitrary-kv.down.sql new file mode 100644 index 00000000000..7ec25a79a65 --- /dev/null +++ b/quickwit/quickwit-metastore/migrations/postgresql/24_add-arbitrary-kv.down.sql @@ -0,0 +1 @@ +DROP TABLE kv; diff --git a/quickwit/quickwit-metastore/migrations/postgresql/24_add-arbitrary-kv.up.sql b/quickwit/quickwit-metastore/migrations/postgresql/24_add-arbitrary-kv.up.sql new file mode 100644 index 00000000000..d303988db80 --- /dev/null +++ b/quickwit/quickwit-metastore/migrations/postgresql/24_add-arbitrary-kv.up.sql @@ -0,0 +1,4 @@ +CREATE TABLE IF NOT EXISTS kv ( + key VARCHAR(50) PRIMARY KEY, + value TEXT NOT NULL +); diff --git a/quickwit/quickwit-metastore/migrations/postgresql/25_add-split-size.down.sql b/quickwit/quickwit-metastore/migrations/postgresql/25_add-split-size.down.sql new file mode 100644 index 00000000000..3e1c1746b04 --- /dev/null +++ b/quickwit/quickwit-metastore/migrations/postgresql/25_add-split-size.down.sql @@ -0,0 +1,3 @@ +DROP INDEX IF EXISTS idx_splits_stats; + +ALTER TABLE splits DROP COLUMN IF EXISTS split_size_bytes; diff --git a/quickwit/quickwit-metastore/migrations/postgresql/25_add-split-size.up.sql b/quickwit/quickwit-metastore/migrations/postgresql/25_add-split-size.up.sql new file mode 100644 index 00000000000..407809cc6b2 --- /dev/null +++ b/quickwit/quickwit-metastore/migrations/postgresql/25_add-split-size.up.sql @@ -0,0 +1,3 @@ +ALTER TABLE splits ADD COLUMN IF NOT EXISTS split_size_bytes BIGINT NOT NULL GENERATED ALWAYS AS ((split_metadata_json::json->'footer_offsets'->>'end')::bigint) STORED; + +CREATE INDEX IF NOT EXISTS idx_splits_stats ON splits (index_uid, split_state) INCLUDE (split_size_bytes); diff --git a/quickwit/quickwit-metastore/migrations/postgresql/26_create-metrics-splits.down.sql b/quickwit/quickwit-metastore/migrations/postgresql/26_create-metrics-splits.down.sql new file mode 100644 index 00000000000..8e09bdd2ac8 --- /dev/null +++ b/quickwit/quickwit-metastore/migrations/postgresql/26_create-metrics-splits.down.sql @@ -0,0 +1,13 @@ +-- Drop indexes first (implicitly dropped with table, but explicit for clarity) +DROP INDEX IF EXISTS idx_metrics_splits_published_time; +DROP INDEX IF EXISTS idx_metrics_splits_state; +DROP INDEX IF EXISTS idx_metrics_splits_tag_host; +DROP INDEX IF EXISTS idx_metrics_splits_tag_region; +DROP INDEX IF EXISTS idx_metrics_splits_tag_datacenter; +DROP INDEX IF EXISTS idx_metrics_splits_tag_env; +DROP INDEX IF EXISTS idx_metrics_splits_tag_service; +DROP INDEX IF EXISTS idx_metrics_splits_metric_names; +DROP INDEX IF EXISTS idx_metrics_splits_time; + +-- Drop the table +DROP TABLE IF EXISTS metrics_splits; diff --git a/quickwit/quickwit-metastore/migrations/postgresql/26_create-metrics-splits.up.sql b/quickwit/quickwit-metastore/migrations/postgresql/26_create-metrics-splits.up.sql new file mode 100644 index 00000000000..d5a9e349f03 --- /dev/null +++ b/quickwit/quickwit-metastore/migrations/postgresql/26_create-metrics-splits.up.sql @@ -0,0 +1,60 @@ +-- Metrics splits table for two-tier pruning strategy. +-- Tier 1: Postgres with GIN indexes for low-cardinality tag filtering. +-- Tier 2: Parquet bloom filters for high-cardinality tags (not stored here). +CREATE TABLE IF NOT EXISTS metrics_splits ( + -- Identity + split_id VARCHAR(50) PRIMARY KEY, + split_state VARCHAR(30) NOT NULL, + index_uid VARCHAR(282) NOT NULL, + + -- Temporal pruning (always used in metrics queries) + time_range_start BIGINT NOT NULL, + time_range_end BIGINT NOT NULL, + + -- Metric name pruning (100% of queries filter by metric name) + metric_names TEXT[] NOT NULL, + + -- Low-cardinality tag pruning (<1000 unique values per key) + -- Stored as TEXT[] for GIN indexing. NULL means tag not present in split. + tag_service TEXT[], + tag_env TEXT[], + tag_datacenter TEXT[], + tag_region TEXT[], + tag_host TEXT[], + + -- High-cardinality tag keys (values stored in Parquet bloom filters) + high_cardinality_tag_keys TEXT[] NOT NULL DEFAULT '{}', + + -- Planning metadata + num_rows BIGINT NOT NULL, + size_bytes BIGINT NOT NULL, + + -- Full metadata for flexibility and future-proofing + split_metadata_json TEXT NOT NULL, + + -- Timestamps + create_timestamp TIMESTAMP NOT NULL DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'), + update_timestamp TIMESTAMP NOT NULL DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'), + publish_timestamp TIMESTAMP, + + -- Foreign key to indexes table (metrics indices are still registered there) + FOREIGN KEY(index_uid) REFERENCES indexes(index_uid) +); + +-- Indexes optimized for metrics query patterns + +-- Time range: Composite index for range overlap queries +-- Query pattern: time_range_start <= :query_end AND time_range_end >= :query_start +CREATE INDEX idx_metrics_splits_time ON metrics_splits (time_range_start, time_range_end); + +-- Metric names: GIN index for array containment +-- Query pattern: $$cpu.usage$$ = ANY(metric_names) +CREATE INDEX idx_metrics_splits_metric_names ON metrics_splits USING GIN (metric_names); + +-- Per-tag GIN indexes for efficient filtering +-- Query pattern: $$web$$ = ANY(tag_service) +CREATE INDEX idx_metrics_splits_tag_service ON metrics_splits USING GIN (tag_service); +CREATE INDEX idx_metrics_splits_tag_env ON metrics_splits USING GIN (tag_env); +CREATE INDEX idx_metrics_splits_tag_datacenter ON metrics_splits USING GIN (tag_datacenter); +CREATE INDEX idx_metrics_splits_tag_region ON metrics_splits USING GIN (tag_region); +CREATE INDEX idx_metrics_splits_tag_host ON metrics_splits USING GIN (tag_host); diff --git a/quickwit/quickwit-metastore/src/backward_compatibility_tests/mod.rs b/quickwit/quickwit-metastore/src/backward_compatibility_tests/mod.rs index baec93647e3..6cd153bd06a 100644 --- a/quickwit/quickwit-metastore/src/backward_compatibility_tests/mod.rs +++ b/quickwit/quickwit-metastore/src/backward_compatibility_tests/mod.rs @@ -58,14 +58,18 @@ fn test_global_version(serializable: &T) -> anyhow::Result<()> { } fn deserialize_json_file(path: &Path) -> anyhow::Result -where for<'a> T: Deserialize<'a> { +where + for<'a> T: Deserialize<'a>, +{ let payload = std::fs::read(path)?; let deserialized: T = serde_json::from_slice(&payload)?; Ok(deserialized) } fn test_backward_compatibility_single_case(path: &Path) -> anyhow::Result<()> -where T: TestableForRegression + std::fmt::Debug { +where + T: TestableForRegression + std::fmt::Debug, +{ println!("---\nTest deserialization of {}", path.display()); let deserialized: T = deserialize_json_file(path)?; let expected_path = path.to_string_lossy().replace(".json", ".expected.json"); @@ -79,7 +83,9 @@ where T: TestableForRegression + std::fmt::Debug { /// For each pair of `x.json` and `x.expected.json` in `test_dir`, assert that the deserialized /// versions are equal according to `T::assert_equality`. fn test_backward_compatibility(test_dir: &Path) -> anyhow::Result<()> -where T: TestableForRegression + std::fmt::Debug { +where + T: TestableForRegression + std::fmt::Debug, +{ for entry in fs::read_dir(test_dir).with_context(|| format!("failed to read {}", test_dir.display()))? { @@ -97,7 +103,9 @@ where T: TestableForRegression + std::fmt::Debug { } fn test_and_update_expected_files_single_case(expected_path: &Path) -> anyhow::Result -where for<'a> T: std::fmt::Debug + Serialize + Deserialize<'a> { +where + for<'a> T: std::fmt::Debug + Serialize + Deserialize<'a>, +{ let expected: T = deserialize_json_file(Path::new(&expected_path))?; let expected_old_json_value: JsonValue = deserialize_json_file(Path::new(&expected_path))?; let expected_new_json_value: JsonValue = serde_json::to_value(&expected)?; @@ -123,7 +131,9 @@ where for<'a> T: std::fmt::Debug + Serialize + Deserialize<'a> { /// /// Returns the proposed updated files (xxx.expected.modified.json). fn test_and_update_old_expected_files(test_dir: &Path) -> anyhow::Result> -where for<'a> T: std::fmt::Debug + Deserialize<'a> + Serialize { +where + for<'a> T: std::fmt::Debug + Deserialize<'a> + Serialize, +{ let mut updated_expected_files = Vec::new(); for entry in fs::read_dir(test_dir)? { let entry = entry?; @@ -153,7 +163,9 @@ where for<'a> T: std::fmt::Debug + Deserialize<'a> + Serialize { /// /// Both generated files have identical contents. fn test_and_create_new_test(test_dir: &Path, sample: T) -> anyhow::Result> -where for<'a> T: Serialize { +where + for<'a> T: Serialize, +{ let sample_json_value = serde_json::to_value(&sample)?; let mut sample_json = serde_json::to_string_pretty(&sample_json_value)?; sample_json.push('\n'); @@ -198,7 +210,9 @@ where for<'a> T: Serialize { /// /// - `test_name` is just the subdirectory name, for the type being test. pub(crate) fn test_json_backward_compatibility_helper(test_name: &str) -> anyhow::Result<()> -where T: TestableForRegression + std::fmt::Debug { +where + T: TestableForRegression + std::fmt::Debug, +{ let sample_instance: T = T::sample_for_regression(); test_global_version(&sample_instance).unwrap(); diff --git a/quickwit/quickwit-metastore/src/checkpoint.rs b/quickwit/quickwit-metastore/src/checkpoint.rs index 7cc25e80c51..634fd1e504f 100644 --- a/quickwit/quickwit-metastore/src/checkpoint.rs +++ b/quickwit/quickwit-metastore/src/checkpoint.rs @@ -201,7 +201,9 @@ impl SourceCheckpoint { /// ``` impl FromIterator<(PartitionId, Position)> for SourceCheckpoint { fn from_iter(iter: I) -> SourceCheckpoint - where I: IntoIterator { + where + I: IntoIterator, + { SourceCheckpoint { per_partition: iter.into_iter().collect(), } @@ -210,7 +212,9 @@ impl FromIterator<(PartitionId, Position)> for SourceCheckpoint { impl Serialize for SourceCheckpoint { fn serialize(&self, serializer: S) -> Result - where S: serde::Serializer { + where + S: serde::Serializer, + { let mut map = serializer.serialize_map(Some(self.per_partition.len()))?; for (partition, position) in &self.per_partition { map.serialize_entry(&*partition.0, position)?; @@ -221,7 +225,9 @@ impl Serialize for SourceCheckpoint { impl<'de> Deserialize<'de> for SourceCheckpoint { fn deserialize(deserializer: D) -> Result - where D: serde::Deserializer<'de> { + where + D: serde::Deserializer<'de>, + { let string_to_string_map: BTreeMap = BTreeMap::deserialize(deserializer)?; let per_partition: BTreeMap = string_to_string_map .into_iter() diff --git a/quickwit/quickwit-metastore/src/lib.rs b/quickwit/quickwit-metastore/src/lib.rs index 9a29391e3db..e8fffeb7962 100644 --- a/quickwit/quickwit-metastore/src/lib.rs +++ b/quickwit/quickwit-metastore/src/lib.rs @@ -45,9 +45,11 @@ pub use metastore::postgres::PostgresqlMetastore; pub use metastore::{ AddSourceRequestExt, CreateIndexRequestExt, CreateIndexResponseExt, IndexMetadata, IndexMetadataResponseExt, IndexesMetadataResponseExt, ListIndexesMetadataResponseExt, + ListMetricsSplitsQuery, ListMetricsSplitsRequestExt, ListMetricsSplitsResponseExt, ListSplitsQuery, ListSplitsRequestExt, ListSplitsResponseExt, MetastoreServiceExt, - MetastoreServiceStreamSplitsExt, PublishSplitsRequestExt, StageSplitsRequestExt, - UpdateIndexRequestExt, UpdateSourceRequestExt, file_backed, + MetastoreServiceStreamSplitsExt, PublishMetricsSplitsRequestExt, PublishSplitsRequestExt, + StageMetricsSplitsRequestExt, StageSplitsRequestExt, UpdateIndexRequestExt, + UpdateSourceRequestExt, file_backed, }; pub use metastore_factory::{MetastoreFactory, UnsupportedMetastore}; pub use metastore_resolver::MetastoreResolver; diff --git a/quickwit/quickwit-metastore/src/metastore/control_plane_metastore.rs b/quickwit/quickwit-metastore/src/metastore/control_plane_metastore.rs index 1ed30cabe08..44dc4f58282 100644 --- a/quickwit/quickwit-metastore/src/metastore/control_plane_metastore.rs +++ b/quickwit/quickwit-metastore/src/metastore/control_plane_metastore.rs @@ -20,19 +20,22 @@ use quickwit_proto::control_plane::{ControlPlaneService, ControlPlaneServiceClie use quickwit_proto::metastore::{ AcquireShardsRequest, AcquireShardsResponse, AddSourceRequest, CreateIndexRequest, CreateIndexResponse, CreateIndexTemplateRequest, DeleteIndexRequest, - DeleteIndexTemplatesRequest, DeleteQuery, DeleteShardsRequest, DeleteShardsResponse, - DeleteSourceRequest, DeleteSplitsRequest, DeleteTask, EmptyResponse, - FindIndexTemplateMatchesRequest, FindIndexTemplateMatchesResponse, GetIndexTemplateRequest, - GetIndexTemplateResponse, IndexMetadataRequest, IndexMetadataResponse, IndexesMetadataRequest, - IndexesMetadataResponse, LastDeleteOpstampRequest, LastDeleteOpstampResponse, - ListDeleteTasksRequest, ListDeleteTasksResponse, ListIndexTemplatesRequest, - ListIndexTemplatesResponse, ListIndexesMetadataRequest, ListIndexesMetadataResponse, + DeleteIndexTemplatesRequest, DeleteMetricsSplitsRequest, DeleteQuery, DeleteShardsRequest, + DeleteShardsResponse, DeleteSourceRequest, DeleteSplitsRequest, DeleteTask, EmptyResponse, + FindIndexTemplateMatchesRequest, FindIndexTemplateMatchesResponse, GetClusterIdentityRequest, + GetClusterIdentityResponse, GetIndexTemplateRequest, GetIndexTemplateResponse, + IndexMetadataRequest, IndexMetadataResponse, IndexesMetadataRequest, IndexesMetadataResponse, + LastDeleteOpstampRequest, LastDeleteOpstampResponse, ListDeleteTasksRequest, + ListDeleteTasksResponse, ListIndexStatsRequest, ListIndexStatsResponse, + ListIndexTemplatesRequest, ListIndexTemplatesResponse, ListIndexesMetadataRequest, + ListIndexesMetadataResponse, ListMetricsSplitsRequest, ListMetricsSplitsResponse, ListShardsRequest, ListShardsResponse, ListSplitsRequest, ListSplitsResponse, - ListStaleSplitsRequest, MarkSplitsForDeletionRequest, MetastoreResult, MetastoreService, - MetastoreServiceClient, MetastoreServiceStream, OpenShardsRequest, OpenShardsResponse, - PruneShardsRequest, PublishSplitsRequest, ResetSourceCheckpointRequest, StageSplitsRequest, - ToggleSourceRequest, UpdateIndexRequest, UpdateSourceRequest, UpdateSplitsDeleteOpstampRequest, - UpdateSplitsDeleteOpstampResponse, + ListStaleSplitsRequest, MarkMetricsSplitsForDeletionRequest, MarkSplitsForDeletionRequest, + MetastoreResult, MetastoreService, MetastoreServiceClient, MetastoreServiceStream, + OpenShardsRequest, OpenShardsResponse, PruneShardsRequest, PublishMetricsSplitsRequest, + PublishSplitsRequest, ResetSourceCheckpointRequest, StageMetricsSplitsRequest, + StageSplitsRequest, ToggleSourceRequest, UpdateIndexRequest, UpdateSourceRequest, + UpdateSplitsDeleteOpstampRequest, UpdateSplitsDeleteOpstampResponse, }; /// A [`MetastoreService`] implementation that proxies some requests to the control plane so it can @@ -162,6 +165,13 @@ impl MetastoreService for ControlPlaneMetastore { self.metastore.list_splits(request).await } + async fn list_index_stats( + &self, + request: ListIndexStatsRequest, + ) -> MetastoreResult { + self.metastore.list_index_stats(request).await + } + async fn list_stale_splits( &self, request: ListStaleSplitsRequest, @@ -274,4 +284,50 @@ impl MetastoreService for ControlPlaneMetastore { ) -> MetastoreResult { self.metastore.delete_index_templates(request).await } + + async fn get_cluster_identity( + &self, + request: GetClusterIdentityRequest, + ) -> MetastoreResult { + self.metastore.get_cluster_identity(request).await + } + + // Metrics Splits API - Proxy to underlying metastore + + async fn stage_metrics_splits( + &self, + request: StageMetricsSplitsRequest, + ) -> MetastoreResult { + self.metastore.stage_metrics_splits(request).await + } + + async fn publish_metrics_splits( + &self, + request: PublishMetricsSplitsRequest, + ) -> MetastoreResult { + self.metastore.publish_metrics_splits(request).await + } + + async fn list_metrics_splits( + &self, + request: ListMetricsSplitsRequest, + ) -> MetastoreResult { + self.metastore.list_metrics_splits(request).await + } + + async fn mark_metrics_splits_for_deletion( + &self, + request: MarkMetricsSplitsForDeletionRequest, + ) -> MetastoreResult { + self.metastore + .mark_metrics_splits_for_deletion(request) + .await + } + + async fn delete_metrics_splits( + &self, + request: DeleteMetricsSplitsRequest, + ) -> MetastoreResult { + self.metastore.delete_metrics_splits(request).await + } } diff --git a/quickwit/quickwit-metastore/src/metastore/file_backed/file_backed_index/mod.rs b/quickwit/quickwit-metastore/src/metastore/file_backed/file_backed_index/mod.rs index f2d332faa51..6fd5ce244be 100644 --- a/quickwit/quickwit-metastore/src/metastore/file_backed/file_backed_index/mod.rs +++ b/quickwit/quickwit-metastore/src/metastore/file_backed/file_backed_index/mod.rs @@ -28,10 +28,12 @@ use quickwit_common::pretty::PrettySample; use quickwit_config::{ DocMapping, IndexingSettings, IngestSettings, RetentionPolicy, SearchSettings, SourceConfig, }; +use quickwit_parquet_engine::split::{MetricsSplitMetadata, MetricsSplitState}; use quickwit_proto::metastore::{ AcquireShardsRequest, AcquireShardsResponse, DeleteQuery, DeleteShardsRequest, - DeleteShardsResponse, DeleteTask, EntityKind, ListShardsSubrequest, ListShardsSubresponse, - MetastoreError, MetastoreResult, OpenShardSubrequest, OpenShardSubresponse, PruneShardsRequest, + DeleteShardsResponse, DeleteTask, EntityKind, IndexStats, ListShardsSubrequest, + ListShardsSubresponse, MetastoreError, MetastoreResult, OpenShardSubrequest, + OpenShardSubresponse, PruneShardsRequest, SplitStats, }; use quickwit_proto::types::{IndexUid, PublishToken, SourceId, SplitId}; use serde::{Deserialize, Serialize}; @@ -42,9 +44,20 @@ use tracing::{info, warn}; use super::MutationOccurred; use crate::checkpoint::IndexCheckpointDelta; -use crate::metastore::{SortBy, use_shard_api}; +use crate::metastore::{ListMetricsSplitsQuery, SortBy, use_shard_api}; use crate::{IndexMetadata, ListSplitsQuery, Split, SplitMetadata, SplitState, split_tag_filter}; +/// A stored metrics split with its state. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct StoredMetricsSplit { + /// The split metadata. + pub metadata: MetricsSplitMetadata, + /// The split state. + pub state: MetricsSplitState, + /// Update timestamp (Unix epoch seconds). + pub update_timestamp: i64, +} + /// A `FileBackedIndex` object carries an index metadata and its split metadata. // This struct is meant to be used only within the [`FileBackedMetastore`]. The public visibility is #[derive(Clone, Debug, Serialize, Deserialize)] @@ -59,6 +72,8 @@ pub(crate) struct FileBackedIndex { per_source_shards: HashMap, /// Delete tasks. delete_tasks: Vec, + /// Metrics splits (for metrics pipeline). + metrics_splits: HashMap, /// Stamper. stamper: Stamper, /// Flag used to avoid polling the metastore if @@ -147,6 +162,7 @@ impl From for FileBackedIndex { splits: Default::default(), per_source_shards, delete_tasks: Default::default(), + metrics_splits: Default::default(), stamper: Default::default(), recently_modified: false, discarded: false, @@ -163,11 +179,23 @@ enum DeleteSplitOutcome { impl FileBackedIndex { /// Constructor. + #[allow(dead_code)] pub fn new( metadata: IndexMetadata, splits: Vec, per_source_shards: HashMap, delete_tasks: Vec, + ) -> Self { + Self::new_with_metrics_splits(metadata, splits, per_source_shards, delete_tasks, vec![]) + } + + /// Constructor with metrics splits. + pub fn new_with_metrics_splits( + metadata: IndexMetadata, + splits: Vec, + per_source_shards: HashMap, + delete_tasks: Vec, + metrics_splits: Vec, ) -> Self { let last_opstamp = delete_tasks .iter() @@ -178,11 +206,16 @@ impl FileBackedIndex { .into_iter() .map(|split| (split.split_id().to_string(), split)) .collect(); + let metrics_splits = metrics_splits + .into_iter() + .map(|split| (split.metadata.split_id.to_string(), split)) + .collect(); Self { metadata, splits, per_source_shards, delete_tasks, + metrics_splits, stamper: Stamper::new(last_opstamp), recently_modified: false, discarded: false, @@ -221,7 +254,7 @@ impl FileBackedIndex { ingest_settings: IngestSettings, search_settings: SearchSettings, retention_policy_opt: Option, - ) -> bool { + ) -> MetastoreResult { self.metadata.update_index_config( doc_mapping, indexing_settings, @@ -246,14 +279,14 @@ impl FileBackedIndex { // Check whether the split exists. // If the split exists, we check what state it is in. If it's anything other than `Staged` // something has gone very wrong and we should abort the operation. - if let Some(split) = self.splits.get(split_metadata.split_id()) { - if split.split_state != SplitState::Staged { - let entity = EntityKind::Split { - split_id: split.split_id().to_string(), - }; - let message = "split is not staged".to_string(); - return Err(MetastoreError::FailedPrecondition { entity, message }); - } + if let Some(split) = self.splits.get(split_metadata.split_id()) + && split.split_state != SplitState::Staged + { + let entity = EntityKind::Split { + split_id: split.split_id().to_string(), + }; + let message = "split is not staged".to_string(); + return Err(MetastoreError::FailedPrecondition { entity, message }); } let now_timestamp = OffsetDateTime::now_utc().unix_timestamp(); let split = Split { @@ -497,6 +530,34 @@ impl FileBackedIndex { Ok(()) } + /// Gets IndexStats for this index + pub(crate) fn get_stats(&self) -> MetastoreResult { + let mut staged_stats = SplitStats::default(); + let mut published_stats = SplitStats::default(); + let mut marked_for_deletion_stats = SplitStats::default(); + + for split in self.splits.values() { + match split.split_state { + SplitState::Staged => { + staged_stats.add_split(split.split_metadata.footer_offsets.end) + } + SplitState::Published => { + published_stats.add_split(split.split_metadata.footer_offsets.end) + } + SplitState::MarkedForDeletion => { + marked_for_deletion_stats.add_split(split.split_metadata.footer_offsets.end) + } + } + } + + Ok(IndexStats { + index_uid: Some(self.index_uid().clone()), + staged: Some(staged_stats), + published: Some(published_stats), + marked_for_deletion: Some(marked_for_deletion_stats), + }) + } + /// Adds a source. pub(crate) fn add_source(&mut self, source_config: SourceConfig) -> MetastoreResult<()> { let index_uid = self.index_uid().clone(); @@ -668,6 +729,257 @@ impl FileBackedIndex { self.get_shards_for_source_mut(&checkpoint_delta.source_id)? .try_apply_delta(checkpoint_delta.source_delta, publish_token) } + + // Metrics Splits API + + /// Stages metrics splits. + pub(crate) fn stage_metrics_splits( + &mut self, + splits_metadata: Vec, + ) -> MetastoreResult { + if splits_metadata.is_empty() { + return Ok(false); + } + let now = OffsetDateTime::now_utc().unix_timestamp(); + for metadata in splits_metadata { + let split_id = metadata.split_id.to_string(); + + // Reject if the split already exists in a non-Staged state. + if let Some(existing) = self.metrics_splits.get(&split_id) + && existing.state != MetricsSplitState::Staged + { + let entity = EntityKind::Split { + split_id: split_id.clone(), + }; + let message = "split is not in Staged state".to_string(); + return Err(MetastoreError::FailedPrecondition { entity, message }); + } + + let stored = StoredMetricsSplit { + metadata, + state: MetricsSplitState::Staged, + update_timestamp: now, + }; + self.metrics_splits.insert(split_id, stored); + } + Ok(true) + } + + /// Publishes metrics splits (transitions from Staged to Published). + pub(crate) fn publish_metrics_splits( + &mut self, + staged_split_ids: &[String], + replaced_split_ids: &[String], + checkpoint_delta_opt: Option, + publish_token_opt: Option, + ) -> MetastoreResult { + // Apply checkpoint delta to index metadata, matching publish_splits behavior. + if let Some(checkpoint_delta) = checkpoint_delta_opt { + let source_id = checkpoint_delta.source_id.clone(); + let source = self.metadata.sources.get(&source_id).ok_or_else(|| { + MetastoreError::NotFound(EntityKind::Source { + index_id: self.index_id().to_string(), + source_id: source_id.clone(), + }) + })?; + + if use_shard_api(&source.source_params) { + let publish_token = publish_token_opt.ok_or_else(|| { + let message = format!( + "publish token is required for publishing splits for source `{source_id}`" + ); + MetastoreError::InvalidArgument { message } + })?; + self.try_apply_delta_v2(checkpoint_delta, publish_token)?; + } else { + self.metadata + .checkpoint + .try_apply_delta(checkpoint_delta) + .map_err(|error| { + quickwit_common::rate_limited_error!( + limit_per_min = 6, + index = self.index_id(), + "failed to apply checkpoint delta" + ); + let entity = EntityKind::CheckpointDelta { + index_id: self.index_id().to_string(), + source_id, + }; + let message = error.to_string(); + MetastoreError::FailedPrecondition { entity, message } + })?; + } + } + + let now = OffsetDateTime::now_utc().unix_timestamp(); + + // Verify all staged splits exist and are in Staged state + for split_id in staged_split_ids { + let split = self.metrics_splits.get(split_id).ok_or_else(|| { + MetastoreError::NotFound(EntityKind::Splits { + split_ids: vec![split_id.clone()], + }) + })?; + if split.state != MetricsSplitState::Staged { + return Err(MetastoreError::FailedPrecondition { + entity: EntityKind::Splits { + split_ids: vec![split_id.clone()], + }, + message: format!("split {} is not in Staged state", split_id), + }); + } + } + + // Transition staged splits to Published + for split_id in staged_split_ids { + if let Some(split) = self.metrics_splits.get_mut(split_id) { + split.state = MetricsSplitState::Published; + split.update_timestamp = now; + } + } + + // Mark replaced splits for deletion + for split_id in replaced_split_ids { + if let Some(split) = self.metrics_splits.get_mut(split_id) { + split.state = MetricsSplitState::MarkedForDeletion; + split.update_timestamp = now; + } + } + + Ok(true) + } + + /// Lists metrics splits matching the query. + pub(crate) fn list_metrics_splits( + &self, + query: &ListMetricsSplitsQuery, + ) -> Vec { + self.metrics_splits + .values() + .filter(|split| metrics_split_matches_query(split, query)) + .cloned() + .collect() + } + + /// Marks metrics splits for deletion. + pub(crate) fn mark_metrics_splits_for_deletion( + &mut self, + split_ids: &[String], + ) -> MetastoreResult { + let now = OffsetDateTime::now_utc().unix_timestamp(); + let mut mutated = false; + for split_id in split_ids { + if let Some(split) = self.metrics_splits.get_mut(split_id) { + split.state = MetricsSplitState::MarkedForDeletion; + split.update_timestamp = now; + mutated = true; + } + } + Ok(mutated) + } + + /// Deletes metrics splits (must be MarkedForDeletion). + pub(crate) fn delete_metrics_splits(&mut self, split_ids: &[String]) -> MetastoreResult { + for split_id in split_ids { + if let Some(split) = self.metrics_splits.get(split_id) + && split.state != MetricsSplitState::MarkedForDeletion + { + return Err(MetastoreError::FailedPrecondition { + entity: EntityKind::Splits { + split_ids: vec![split_id.clone()], + }, + message: format!("split {} is not marked for deletion", split_id), + }); + } + } + let mut mutated = false; + for split_id in split_ids { + if self.metrics_splits.remove(split_id).is_some() { + mutated = true; + } + } + Ok(mutated) + } +} + +/// Checks if a metrics split matches the query criteria. +fn metrics_split_matches_query(split: &StoredMetricsSplit, query: &ListMetricsSplitsQuery) -> bool { + // Filter by state + if !query.split_states.is_empty() { + let state_str = split.state.as_str(); + if !query.split_states.iter().any(|s| s == state_str) { + return false; + } + } + + // Filter by time range + if let Some(start) = query.time_range_start + && (split.metadata.time_range.end_secs as i64) < start + { + return false; + } + if let Some(end) = query.time_range_end + && (split.metadata.time_range.start_secs as i64) > end + { + return false; + } + + // Filter by metric names + if !query.metric_names.is_empty() { + let has_match = query + .metric_names + .iter() + .any(|name| split.metadata.metric_names.contains(name)); + if !has_match { + return false; + } + } + + // Filter by tags (service, env, datacenter, region, host) + if let Some(service) = &query.tag_service { + match split.metadata.service_names() { + Some(services) if services.contains(service) => {} + _ => return false, + } + } + if let Some(env) = &query.tag_env { + if let Some(envs) = split.metadata.low_cardinality_tags.get("env") { + if !envs.contains(env) { + return false; + } + } else { + return false; + } + } + if let Some(datacenter) = &query.tag_datacenter { + if let Some(dcs) = split.metadata.low_cardinality_tags.get("datacenter") { + if !dcs.contains(datacenter) { + return false; + } + } else { + return false; + } + } + if let Some(region) = &query.tag_region { + if let Some(regions) = split.metadata.low_cardinality_tags.get("region") { + if !regions.contains(region) { + return false; + } + } else { + return false; + } + } + if let Some(host) = &query.tag_host { + if let Some(hosts) = split.metadata.low_cardinality_tags.get("host") { + if !hosts.contains(host) { + return false; + } + } else { + return false; + } + } + + true } /// Stamper provides Opstamps, which is just an auto-increment id to label @@ -735,17 +1047,17 @@ fn split_query_predicate(split: &&Split, query: &ListSplitsQuery) -> bool { if !query.time_range.overlaps_with(range.clone()) { return false; } - if let Some(v) = query.max_time_range_end { - if range.end() > &v { - return false; - } + if let Some(v) = query.max_time_range_end + && range.end() > &v + { + return false; } } - if let Some(node_id) = &query.node_id { - if split.split_metadata.node_id != *node_id { - return false; - } + if let Some(node_id) = &query.node_id + && split.split_metadata.node_id != *node_id + { + return false; } if let Some((index_uid, split_id)) = &query.after_split { @@ -764,16 +1076,16 @@ fn split_query_predicate(split: &&Split, query: &ListSplitsQuery) -> bool { #[cfg(test)] mod tests { - use std::collections::BTreeSet; + use std::collections::{BTreeSet, HashMap}; use quickwit_doc_mapper::tag_pruning::TagFilterAst; use quickwit_proto::ingest::Shard; - use quickwit_proto::metastore::ListShardsSubrequest; + use quickwit_proto::metastore::{ListShardsSubrequest, SplitStats}; use quickwit_proto::types::{IndexUid, SourceId}; use super::FileBackedIndex; use crate::file_backed::file_backed_index::split_query_predicate; - use crate::{ListSplitsQuery, Split, SplitMetadata, SplitState}; + use crate::{IndexMetadata, ListSplitsQuery, Split, SplitMetadata, SplitState}; impl FileBackedIndex { pub(crate) fn insert_shards(&mut self, source_id: &SourceId, shards: Vec) { @@ -804,6 +1116,7 @@ mod tests { time_range: Some(32..=40), tags: BTreeSet::from(["tag-1".to_string()]), create_timestamp: 12, + footer_offsets: 0..2048, ..Default::default() }, split_state: SplitState::Staged, @@ -817,6 +1130,7 @@ mod tests { time_range: None, tags: BTreeSet::from(["tag-2".to_string(), "tag-3".to_string()]), create_timestamp: 5, + footer_offsets: 0..1024, ..Default::default() }, split_state: SplitState::MarkedForDeletion, @@ -830,6 +1144,7 @@ mod tests { time_range: Some(0..=90), tags: BTreeSet::from(["tag-2".to_string(), "tag-4".to_string()]), create_timestamp: 64, + footer_offsets: 0..512, ..Default::default() }, split_state: SplitState::Published, @@ -944,4 +1259,30 @@ mod tests { assert!(!split_query_predicate(&&split_2, &query)); assert!(!split_query_predicate(&&split_3, &query)); } + + #[test] + fn test_get_stats() { + let index_id = "test-index"; + let index_metadata = IndexMetadata::for_test(index_id, "file:///qwdata/indexes/test-index"); + let index = + FileBackedIndex::new(index_metadata, make_splits().into(), HashMap::new(), vec![]); + + let expected_staged = Some(SplitStats { + num_splits: 1, + total_size_bytes: 2048, + }); + let expected_published = Some(SplitStats { + num_splits: 1, + total_size_bytes: 512, + }); + let expected_marked_for_deletion = Some(SplitStats { + num_splits: 1, + total_size_bytes: 1024, + }); + let stats = index.get_stats().unwrap(); + + assert_eq!(stats.staged, expected_staged); + assert_eq!(stats.published, expected_published); + assert_eq!(stats.marked_for_deletion, expected_marked_for_deletion); + } } diff --git a/quickwit/quickwit-metastore/src/metastore/file_backed/file_backed_index/serialize.rs b/quickwit/quickwit-metastore/src/metastore/file_backed/file_backed_index/serialize.rs index 8d4cf13e22d..dc7293f7b72 100644 --- a/quickwit/quickwit-metastore/src/metastore/file_backed/file_backed_index/serialize.rs +++ b/quickwit/quickwit-metastore/src/metastore/file_backed/file_backed_index/serialize.rs @@ -20,6 +20,7 @@ use quickwit_proto::metastore::SourceType; use quickwit_proto::types::{DocMappingUid, SourceId}; use serde::{Deserialize, Serialize}; +use super::StoredMetricsSplit; use super::shards::Shards; use crate::file_backed::file_backed_index::FileBackedIndex; use crate::metastore::DeleteTask; @@ -68,6 +69,9 @@ pub(crate) struct FileBackedIndexV0_8 { shards: HashMap>, #[serde(default)] delete_tasks: Vec, + /// Metrics splits (for metrics pipeline). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + metrics_splits: Vec, } impl From for FileBackedIndexV0_8 { @@ -97,11 +101,17 @@ impl From for FileBackedIndexV0_8 { .into_iter() .sorted_by_key(|delete_task| delete_task.opstamp) .collect(); + let metrics_splits = index + .metrics_splits + .into_values() + .sorted_by_key(|split| split.update_timestamp) + .collect(); Self { metadata: index.metadata, splits, shards, delete_tasks, + metrics_splits, } } } @@ -129,11 +139,12 @@ impl From for FileBackedIndex { per_source_shards.insert(source_id.clone(), Shards::empty(index_uid, source_id)); } } - Self::new( + Self::new_with_metrics_splits( index.metadata, index.splits, per_source_shards, index.delete_tasks, + index.metrics_splits, ) } } diff --git a/quickwit/quickwit-metastore/src/metastore/file_backed/lazy_file_backed_index.rs b/quickwit/quickwit-metastore/src/metastore/file_backed/lazy_file_backed_index.rs index 34ac2641a1d..c13711de0b2 100644 --- a/quickwit/quickwit-metastore/src/metastore/file_backed/lazy_file_backed_index.rs +++ b/quickwit/quickwit-metastore/src/metastore/file_backed/lazy_file_backed_index.rs @@ -44,15 +44,15 @@ impl LazyFileBackedIndex { let index_mutex_opt = file_backed_index.map(|index| Arc::new(Mutex::new(index))); // If the polling interval is configured and the index is already loaded, // spawn immediately the polling task - if let Some(index_mutex) = &index_mutex_opt { - if let Some(polling_interval) = polling_interval_opt { - spawn_index_metadata_polling_task( - storage.clone(), - index_id.clone(), - Arc::downgrade(index_mutex), - polling_interval, - ); - } + if let Some(index_mutex) = &index_mutex_opt + && let Some(polling_interval) = polling_interval_opt + { + spawn_index_metadata_polling_task( + storage.clone(), + index_id.clone(), + Arc::downgrade(index_mutex), + polling_interval, + ); } Self { index_id, diff --git a/quickwit/quickwit-metastore/src/metastore/file_backed/manifest.rs b/quickwit/quickwit-metastore/src/metastore/file_backed/manifest.rs index 450ede00f8e..cec811bd3e4 100644 --- a/quickwit/quickwit-metastore/src/metastore/file_backed/manifest.rs +++ b/quickwit/quickwit-metastore/src/metastore/file_backed/manifest.rs @@ -23,6 +23,7 @@ use quickwit_proto::types::{DocMappingUid, IndexId}; use quickwit_storage::{OwnedBytes, Storage, StorageError, StorageErrorKind, StorageResult}; use serde::{Deserialize, Serialize}; use tracing::error; +use uuid::Uuid; pub(super) const MANIFEST_FILE_NAME: &str = "manifest.json"; @@ -40,6 +41,7 @@ impl LegacyManifest { Manifest { indexes: self.indexes, templates: HashMap::new(), + identity: Uuid::nil(), } } } @@ -64,6 +66,7 @@ pub(crate) struct Manifest { // The templates are serialized as a sorted `Vec` so the btree map is // unnecessary here and we can pass the hash map as is to the `MetastoreState` pub templates: HashMap, + pub identity: Uuid, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -103,6 +106,8 @@ impl From for Manifest { struct ManifestV0_8 { indexes: BTreeMap, templates: Vec, + #[serde(default, skip_serializing_if = "Uuid::is_nil")] + identity: Uuid, } impl From for ManifestV0_8 { @@ -115,6 +120,7 @@ impl From for ManifestV0_8 { ManifestV0_8 { indexes: manifest.indexes, templates, + identity: manifest.identity, } } } @@ -127,7 +133,11 @@ impl From for Manifest { .into_iter() .map(|template| (template.template_id.clone(), template)) .collect(); - Manifest { indexes, templates } + Manifest { + indexes, + templates, + identity: manifest.identity, + } } } @@ -144,7 +154,11 @@ impl quickwit_config::TestableForRegression for Manifest { "test-template-1".to_string(), IndexTemplate::sample_for_regression(), ); - Manifest { indexes, templates } + Manifest { + indexes, + templates, + identity: Uuid::nil(), + } } fn assert_equality(&self, other: &Self) { @@ -320,7 +334,11 @@ mod tests { IndexTemplate::for_test("test-template-2", &["test-index-bar*"], 200), ), ]); - let manifest = Manifest { indexes, templates }; + let manifest = Manifest { + indexes, + templates, + identity: Uuid::nil(), + }; let manifest_json = serde_json::to_string_pretty(&manifest).unwrap(); let manifest_deserialized: Manifest = serde_json::from_str(&manifest_json).unwrap(); assert_eq!(manifest, manifest_deserialized); diff --git a/quickwit/quickwit-metastore/src/metastore/file_backed/mod.rs b/quickwit/quickwit-metastore/src/metastore/file_backed/mod.rs index 7c6e9de065f..e8540acc0a4 100644 --- a/quickwit/quickwit-metastore/src/metastore/file_backed/mod.rs +++ b/quickwit/quickwit-metastore/src/metastore/file_backed/mod.rs @@ -42,18 +42,21 @@ use quickwit_config::IndexTemplate; use quickwit_proto::metastore::{ AcquireShardsRequest, AcquireShardsResponse, AddSourceRequest, CreateIndexRequest, CreateIndexResponse, CreateIndexTemplateRequest, DeleteIndexRequest, - DeleteIndexTemplatesRequest, DeleteQuery, DeleteShardsRequest, DeleteShardsResponse, - DeleteSourceRequest, DeleteSplitsRequest, DeleteTask, EmptyResponse, EntityKind, - FindIndexTemplateMatchesRequest, FindIndexTemplateMatchesResponse, GetIndexTemplateRequest, + DeleteIndexTemplatesRequest, DeleteMetricsSplitsRequest, DeleteQuery, DeleteShardsRequest, + DeleteShardsResponse, DeleteSourceRequest, DeleteSplitsRequest, DeleteTask, EmptyResponse, + EntityKind, FindIndexTemplateMatchesRequest, FindIndexTemplateMatchesResponse, + GetClusterIdentityRequest, GetClusterIdentityResponse, GetIndexTemplateRequest, GetIndexTemplateResponse, IndexMetadataFailure, IndexMetadataFailureReason, IndexMetadataRequest, IndexMetadataResponse, IndexTemplateMatch, IndexesMetadataRequest, IndexesMetadataResponse, LastDeleteOpstampRequest, LastDeleteOpstampResponse, - ListDeleteTasksRequest, ListDeleteTasksResponse, ListIndexTemplatesRequest, - ListIndexTemplatesResponse, ListIndexesMetadataRequest, ListIndexesMetadataResponse, + ListDeleteTasksRequest, ListDeleteTasksResponse, ListIndexStatsRequest, ListIndexStatsResponse, + ListIndexTemplatesRequest, ListIndexTemplatesResponse, ListIndexesMetadataRequest, + ListIndexesMetadataResponse, ListMetricsSplitsRequest, ListMetricsSplitsResponse, ListShardsRequest, ListShardsResponse, ListSplitsRequest, ListSplitsResponse, - ListStaleSplitsRequest, MarkSplitsForDeletionRequest, MetastoreError, MetastoreResult, - MetastoreService, MetastoreServiceStream, OpenShardSubrequest, OpenShardsRequest, - OpenShardsResponse, PruneShardsRequest, PublishSplitsRequest, ResetSourceCheckpointRequest, + ListStaleSplitsRequest, MarkMetricsSplitsForDeletionRequest, MarkSplitsForDeletionRequest, + MetastoreError, MetastoreResult, MetastoreService, MetastoreServiceStream, OpenShardSubrequest, + OpenShardsRequest, OpenShardsResponse, PruneShardsRequest, PublishMetricsSplitsRequest, + PublishSplitsRequest, ResetSourceCheckpointRequest, StageMetricsSplitsRequest, StageSplitsRequest, ToggleSourceRequest, UpdateIndexRequest, UpdateSourceRequest, UpdateSplitsDeleteOpstampRequest, UpdateSplitsDeleteOpstampResponse, serde_utils, }; @@ -62,6 +65,7 @@ use quickwit_storage::Storage; use time::OffsetDateTime; use tokio::sync::{Mutex, OwnedMutexGuard, RwLock}; use ulid::Ulid; +use uuid::Uuid; use self::file_backed_index::FileBackedIndex; pub use self::file_backed_metastore_factory::FileBackedMetastoreFactory; @@ -72,9 +76,11 @@ use self::state::MetastoreState; use self::store_operations::{delete_index, index_exists, load_index, put_index}; use super::{ AddSourceRequestExt, CreateIndexRequestExt, IndexMetadataResponseExt, - IndexesMetadataResponseExt, ListIndexesMetadataResponseExt, ListSplitsRequestExt, - ListSplitsResponseExt, PublishSplitsRequestExt, STREAM_SPLITS_CHUNK_SIZE, - StageSplitsRequestExt, UpdateIndexRequestExt, UpdateSourceRequestExt, + IndexesMetadataResponseExt, ListIndexesMetadataResponseExt, ListMetricsSplitsRequestExt, + ListMetricsSplitsResponseExt, ListSplitsRequestExt, ListSplitsResponseExt, + PublishMetricsSplitsRequestExt, PublishSplitsRequestExt, STREAM_SPLITS_CHUNK_SIZE, + StageMetricsSplitsRequestExt, StageSplitsRequestExt, UpdateIndexRequestExt, + UpdateSourceRequestExt, }; use crate::checkpoint::IndexCheckpointDelta; use crate::{IndexMetadata, ListSplitsQuery, MetastoreServiceExt, Split, SplitState}; @@ -252,7 +258,9 @@ impl FileBackedMetastore { } async fn read(&self, index_uid: &IndexUid, view: F) -> MetastoreResult - where F: FnOnce(&FileBackedIndex) -> MetastoreResult { + where + F: FnOnce(&FileBackedIndex) -> MetastoreResult, + { self.read_any( index_uid.index_id.as_str(), Some(index_uid.incarnation_id), @@ -270,12 +278,12 @@ impl FileBackedMetastore { view: impl FnOnce(&FileBackedIndex) -> MetastoreResult, ) -> MetastoreResult { let locked_index = self.get_locked_index(index_id).await?; - if let Some(incarnation_id) = incarnation_id_opt { - if locked_index.index_uid().incarnation_id != incarnation_id { - return Err(MetastoreError::NotFound(EntityKind::Index { - index_id: index_id.to_string(), - })); - } + if let Some(incarnation_id) = incarnation_id_opt + && locked_index.index_uid().incarnation_id != incarnation_id + { + return Err(MetastoreError::NotFound(EntityKind::Index { + index_id: index_id.to_string(), + })); } view(&locked_index) } @@ -375,13 +383,13 @@ impl FileBackedMetastore { return Err((metastore_error, index_id_opt, index_uid_opt)); } }; - if let Some(index_uid) = &index_uid_opt { - if index_metadata.index_uid != *index_uid { - let metastore_error = MetastoreError::NotFound(EntityKind::Index { - index_id: index_id.to_string(), - }); - return Err((metastore_error, index_id_opt, index_uid_opt)); - } + if let Some(index_uid) = &index_uid_opt + && index_metadata.index_uid != *index_uid + { + let metastore_error = MetastoreError::NotFound(EntityKind::Index { + index_id: index_id.to_string(), + }); + return Err((metastore_error, index_id_opt, index_uid_opt)); } Ok(index_metadata) } @@ -573,7 +581,7 @@ impl MetastoreService for FileBackedMetastore { ingest_settings, search_settings, retention_policy_opt, - ); + )?; let index_metadata = index.metadata().clone(); if mutation_occurred { @@ -807,6 +815,51 @@ impl MetastoreService for FileBackedMetastore { Ok(ServiceStream::new(splits_responses_stream)) } + async fn list_index_stats( + &self, + request: ListIndexStatsRequest, + ) -> MetastoreResult { + let index_id_matcher = + IndexIdMatcher::try_from_index_id_patterns(&request.index_id_patterns)?; + let index_ids: Vec = { + let inner_rlock_guard = self.state.read().await; + inner_rlock_guard + .indexes + .iter() + .filter_map(|(index_id, index_state)| match index_state { + LazyIndexStatus::Active(_) if index_id_matcher.is_match(index_id) => { + Some(index_id) + } + _ => None, + }) + .cloned() + .collect() + }; + + let mut index_read_futures = FuturesUnordered::new(); + for index_id in index_ids { + let index_read_future = async move { + self.read_any(&index_id, None, |index| index.get_stats()) + .await + }; + index_read_futures.push(index_read_future); + } + + let mut index_stats = Vec::new(); + while let Some(index_read_result) = index_read_futures.next().await { + match index_read_result { + Ok(stats) => index_stats.push(stats), + Err(MetastoreError::NotFound(_)) => { + // If the index does not exist, we just skip it. + continue; + } + Err(error) => return Err(error), + } + } + + Ok(ListIndexStatsResponse { index_stats }) + } + async fn list_stale_splits( &self, request: ListStaleSplitsRequest, @@ -1202,6 +1255,167 @@ impl MetastoreService for FileBackedMetastore { } Ok(EmptyResponse {}) } + + // Get cluster identity api + + // this returns a constant uuid. on first call, it generate said uuid if it doesn't already + // exists + async fn get_cluster_identity( + &self, + _: GetClusterIdentityRequest, + ) -> MetastoreResult { + let mut state_wlock_guard = self.state.write().await; + + if state_wlock_guard.identity.is_nil() { + state_wlock_guard.identity = Uuid::new_v4(); + + let manifest = state_wlock_guard.as_manifest(); + + if let Err(error) = save_manifest(&*self.storage, &manifest).await { + state_wlock_guard.identity = Uuid::nil(); + return Err(error); + } + } + + Ok(GetClusterIdentityResponse { + uuid: state_wlock_guard.identity.hyphenated().to_string(), + }) + } + + // Metrics Splits API + + async fn stage_metrics_splits( + &self, + request: StageMetricsSplitsRequest, + ) -> MetastoreResult { + use quickwit_parquet_engine::split::MetricsSplitMetadata; + + let index_uid = request.index_uid().clone(); + let splits_metadata: Vec = request.deserialize_splits_metadata()?; + + if splits_metadata.is_empty() { + return Ok(EmptyResponse {}); + } + + self.mutate(&index_uid, |index| { + let mutated = index.stage_metrics_splits(splits_metadata)?; + if mutated { + Ok(MutationOccurred::Yes(())) + } else { + Ok(MutationOccurred::No(())) + } + }) + .await?; + + Ok(EmptyResponse {}) + } + + async fn publish_metrics_splits( + &self, + request: PublishMetricsSplitsRequest, + ) -> MetastoreResult { + let index_checkpoint_delta: Option = + request.deserialize_index_checkpoint()?; + let index_uid = request.index_uid().clone(); + let staged_split_ids = request.staged_split_ids; + let replaced_split_ids = request.replaced_split_ids; + let publish_token_opt = request.publish_token_opt; + + if staged_split_ids.is_empty() && replaced_split_ids.is_empty() { + return Ok(EmptyResponse {}); + } + + self.mutate(&index_uid, |index| { + let mutated = index.publish_metrics_splits( + &staged_split_ids, + &replaced_split_ids, + index_checkpoint_delta, + publish_token_opt, + )?; + if mutated { + Ok(MutationOccurred::Yes(())) + } else { + Ok(MutationOccurred::No(())) + } + }) + .await?; + + Ok(EmptyResponse {}) + } + + async fn list_metrics_splits( + &self, + request: ListMetricsSplitsRequest, + ) -> MetastoreResult { + use quickwit_parquet_engine::split::MetricsSplitRecord; + + let index_uid = request.index_uid().clone(); + let query = request.deserialize_query()?; + + let stored_splits = self + .read(&index_uid, |index| Ok(index.list_metrics_splits(&query))) + .await?; + + // Convert StoredMetricsSplit to MetricsSplitRecord for the response + let split_records: Vec = stored_splits + .into_iter() + .map(|s| MetricsSplitRecord { + state: s.state, + update_timestamp: s.update_timestamp, + metadata: s.metadata, + }) + .collect(); + + ListMetricsSplitsResponse::try_from_splits(&split_records) + } + + async fn mark_metrics_splits_for_deletion( + &self, + request: MarkMetricsSplitsForDeletionRequest, + ) -> MetastoreResult { + let index_uid = request.index_uid().clone(); + let split_ids = request.split_ids; + + if split_ids.is_empty() { + return Ok(EmptyResponse {}); + } + + self.mutate(&index_uid, |index| { + let mutated = index.mark_metrics_splits_for_deletion(&split_ids)?; + if mutated { + Ok(MutationOccurred::Yes(())) + } else { + Ok(MutationOccurred::No(())) + } + }) + .await?; + + Ok(EmptyResponse {}) + } + + async fn delete_metrics_splits( + &self, + request: DeleteMetricsSplitsRequest, + ) -> MetastoreResult { + let index_uid = request.index_uid().clone(); + let split_ids = request.split_ids; + + if split_ids.is_empty() { + return Ok(EmptyResponse {}); + } + + self.mutate(&index_uid, |index| { + let mutated = index.delete_metrics_splits(&split_ids)?; + if mutated { + Ok(MutationOccurred::Yes(())) + } else { + Ok(MutationOccurred::No(())) + } + }) + .await?; + + Ok(EmptyResponse {}) + } } impl MetastoreServiceExt for FileBackedMetastore {} @@ -1670,9 +1884,9 @@ mod tests { // Stage splits in multiple threads let mut handles = Vec::new(); - let mut random_generator = rand::thread_rng(); + let mut random_generator = rand::rng(); for i in 1..=20 { - let sleep_duration = Duration::from_millis(random_generator.gen_range(0..=200)); + let sleep_duration = Duration::from_millis(random_generator.random_range(0..=200)); let metastore = metastore.clone(); let current_timestamp = OffsetDateTime::now_utc().unix_timestamp(); let handle = tokio::spawn({ diff --git a/quickwit/quickwit-metastore/src/metastore/file_backed/state.rs b/quickwit/quickwit-metastore/src/metastore/file_backed/state.rs index 49f0029cf33..0d42408f430 100644 --- a/quickwit/quickwit-metastore/src/metastore/file_backed/state.rs +++ b/quickwit/quickwit-metastore/src/metastore/file_backed/state.rs @@ -20,6 +20,7 @@ use quickwit_config::{IndexTemplate, IndexTemplateId}; use quickwit_proto::metastore::MetastoreResult; use quickwit_proto::types::IndexId; use quickwit_storage::Storage; +use uuid::Uuid; use super::LazyIndexStatus; use super::index_template_matcher::IndexTemplateMatcher; @@ -31,6 +32,7 @@ pub(super) struct MetastoreState { pub indexes: HashMap, pub templates: HashMap, pub template_matcher: IndexTemplateMatcher, + pub identity: Uuid, } impl MetastoreState { @@ -64,6 +66,7 @@ impl MetastoreState { indexes, templates: manifest.templates, template_matcher, + identity: manifest.identity, }; Ok(state) } @@ -82,6 +85,10 @@ impl MetastoreState { }) .collect(); let templates = self.templates.clone(); - Manifest { indexes, templates } + Manifest { + indexes, + templates, + identity: self.identity, + } } } diff --git a/quickwit/quickwit-metastore/src/metastore/index_metadata/mod.rs b/quickwit/quickwit-metastore/src/metastore/index_metadata/mod.rs index 4e40b0dfcda..e7a5677099e 100644 --- a/quickwit/quickwit-metastore/src/metastore/index_metadata/mod.rs +++ b/quickwit/quickwit-metastore/src/metastore/index_metadata/mod.rs @@ -20,7 +20,7 @@ use std::collections::hash_map::Entry; use quickwit_common::uri::Uri; use quickwit_config::{ DocMapping, IndexConfig, IndexingSettings, IngestSettings, RetentionPolicy, SearchSettings, - SourceConfig, + SourceConfig, prepare_doc_mapping_update, }; use quickwit_proto::metastore::{EntityKind, MetastoreError, MetastoreResult}; use quickwit_proto::types::{IndexUid, SourceId}; @@ -106,13 +106,16 @@ impl IndexMetadata { ingest_settings: IngestSettings, search_settings: SearchSettings, retention_policy_opt: Option, - ) -> bool { - let mut mutation_occurred = false; - - if doc_mapping != self.index_config.doc_mapping { - self.index_config.doc_mapping = doc_mapping; - mutation_occurred = true; - } + ) -> MetastoreResult { + let (updated_doc_mapping, mut mutation_occurred) = prepare_doc_mapping_update( + doc_mapping, + &self.index_config.doc_mapping, + &search_settings, + ) + .map_err(|error| MetastoreError::InvalidArgument { + message: error.to_string(), + })?; + self.index_config.doc_mapping = updated_doc_mapping; if indexing_settings != self.index_config.indexing_settings { self.index_config.indexing_settings = indexing_settings; mutation_occurred = true; @@ -129,7 +132,7 @@ impl IndexMetadata { self.index_config.retention_policy_opt = retention_policy_opt; mutation_occurred = true; } - mutation_occurred + Ok(mutation_occurred) } /// Adds a source to the index. Returns an error if the source already exists. @@ -234,3 +237,96 @@ impl quickwit_config::TestableForRegression for IndexMetadata { assert_eq!(self.sources, other.sources); } } + +#[cfg(test)] +mod tests { + use quickwit_doc_mapper::Mode; + use quickwit_proto::types::DocMappingUid; + + use super::*; + + #[test] + fn test_update_index_config() { + let current_index_config = IndexConfig::for_test("test-index", "s3://test-index"); + let mut current_index_metadata = IndexMetadata::new(current_index_config.clone()); + + let mutation_occurred = current_index_metadata + .update_index_config( + current_index_config.doc_mapping.clone(), + current_index_config.indexing_settings.clone(), + current_index_config.ingest_settings.clone(), + current_index_config.search_settings.clone(), + current_index_config.retention_policy_opt.clone(), + ) + .unwrap(); + assert!(!mutation_occurred); + + let new_search_settings = SearchSettings { + default_search_fields: vec!["message".to_string(), "status".to_string()], + }; + let mutation_occurred = current_index_metadata + .update_index_config( + current_index_config.doc_mapping.clone(), + current_index_config.indexing_settings.clone(), + current_index_config.ingest_settings.clone(), + new_search_settings, + current_index_config.retention_policy_opt.clone(), + ) + .unwrap(); + assert!(mutation_occurred); + assert_eq!( + current_index_metadata + .index_config() + .search_settings + .default_search_fields, + ["message", "status"] + ); + } + + #[test] + fn test_update_doc_mapping() { + let current_index_config = IndexConfig::for_test("test-index", "s3://test-index"); + let mut current_index_metadata = IndexMetadata::new(current_index_config.clone()); + + let mut new_doc_mapping = current_index_config.doc_mapping.clone(); + new_doc_mapping.doc_mapping_uid = DocMappingUid::random(); + new_doc_mapping.timestamp_field = Some("ts".to_string()); // This is set to `timestamp` for the current doc mapping. + + current_index_metadata + .update_index_config( + new_doc_mapping, + current_index_config.indexing_settings.clone(), + current_index_config.ingest_settings.clone(), + current_index_config.search_settings.clone(), + current_index_config.retention_policy_opt.clone(), + ) + .unwrap_err(); + + let mut new_doc_mapping = current_index_config.doc_mapping.clone(); + let new_doc_mapping_uid = DocMappingUid::random(); + new_doc_mapping.doc_mapping_uid = new_doc_mapping_uid; + new_doc_mapping.mode = Mode::Strict; + + let mutation_occurred = current_index_metadata + .update_index_config( + new_doc_mapping, + current_index_config.indexing_settings, + current_index_config.ingest_settings, + current_index_config.search_settings, + current_index_config.retention_policy_opt, + ) + .unwrap(); + assert!(mutation_occurred); + assert_eq!( + current_index_metadata + .index_config() + .doc_mapping + .doc_mapping_uid, + new_doc_mapping_uid + ); + assert_eq!( + current_index_metadata.index_config().doc_mapping.mode, + Mode::Strict + ); + } +} diff --git a/quickwit/quickwit-metastore/src/metastore/mod.rs b/quickwit/quickwit-metastore/src/metastore/mod.rs index db50d097ce7..ddfee25afab 100644 --- a/quickwit/quickwit-metastore/src/metastore/mod.rs +++ b/quickwit/quickwit-metastore/src/metastore/mod.rs @@ -33,19 +33,82 @@ use quickwit_config::{ SearchSettings, SourceConfig, SourceParams, }; use quickwit_doc_mapper::tag_pruning::TagFilterAst; +use quickwit_parquet_engine::split::{MetricsSplitMetadata, MetricsSplitRecord}; use quickwit_proto::metastore::{ AddSourceRequest, CreateIndexRequest, CreateIndexResponse, DeleteTask, IndexMetadataFailure, IndexMetadataRequest, IndexMetadataResponse, IndexesMetadataResponse, - ListIndexesMetadataResponse, ListSplitsRequest, ListSplitsResponse, MetastoreError, - MetastoreResult, MetastoreService, MetastoreServiceClient, MetastoreServiceStream, - PublishSplitsRequest, StageSplitsRequest, UpdateIndexRequest, UpdateSourceRequest, serde_utils, + ListIndexesMetadataResponse, ListMetricsSplitsRequest, ListMetricsSplitsResponse, + ListSplitsRequest, ListSplitsResponse, MetastoreError, MetastoreResult, MetastoreService, + MetastoreServiceClient, MetastoreServiceStream, PublishMetricsSplitsRequest, + PublishSplitsRequest, StageMetricsSplitsRequest, StageSplitsRequest, UpdateIndexRequest, + UpdateSourceRequest, serde_utils, }; use quickwit_proto::types::{IndexUid, NodeId, SplitId}; +use serde::{Deserialize, Serialize}; use time::OffsetDateTime; use crate::checkpoint::IndexCheckpointDelta; use crate::{Split, SplitMetadata, SplitState}; +/// Query parameters for listing metrics splits. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ListMetricsSplitsQuery { + /// Index UID to filter by (required). + pub index_uid: IndexUid, + /// Split states to include. + #[serde(default)] + pub split_states: Vec, + /// Time range start (inclusive). + pub time_range_start: Option, + /// Time range end (inclusive). + pub time_range_end: Option, + /// Metric names to filter by (any match). + #[serde(default)] + pub metric_names: Vec, + /// Service tag filter. + pub tag_service: Option, + /// Environment tag filter. + pub tag_env: Option, + /// Datacenter tag filter. + pub tag_datacenter: Option, + /// Region tag filter. + pub tag_region: Option, + /// Host tag filter. + pub tag_host: Option, + /// Limit number of results. + pub limit: Option, +} + +impl ListMetricsSplitsQuery { + /// Creates a query for all splits in an index. + pub fn for_index(index_uid: impl Into) -> Self { + Self { + index_uid: index_uid.into(), + split_states: vec!["Published".to_string()], + ..Default::default() + } + } + + /// Filter by split states. + pub fn with_split_states(mut self, states: Vec) -> Self { + self.split_states = states; + self + } + + /// Filter by time range. + pub fn with_time_range(mut self, start: i64, end: i64) -> Self { + self.time_range_start = Some(start); + self.time_range_end = Some(end); + self + } + + /// Filter by metric names. + pub fn with_metric_names(mut self, names: Vec) -> Self { + self.metric_names = names; + self + } +} + /// Splits batch size returned by the stream splits API pub(crate) const STREAM_SPLITS_CHUNK_SIZE: usize = 100; @@ -586,6 +649,22 @@ impl PublishSplitsRequestExt for PublishSplitsRequest { } } +/// Helper trait for [`PublishMetricsSplitsRequest`] to deserialize its payload. +pub trait PublishMetricsSplitsRequestExt { + /// Deserializes the `index_checkpoint_delta_json_opt` field of a + /// [`PublishMetricsSplitsRequest`] into an [`Option`]. + fn deserialize_index_checkpoint(&self) -> MetastoreResult>; +} + +impl PublishMetricsSplitsRequestExt for PublishMetricsSplitsRequest { + fn deserialize_index_checkpoint(&self) -> MetastoreResult> { + self.index_checkpoint_delta_json_opt + .as_ref() + .map(|value| serde_utils::from_json_str(value)) + .transpose() + } +} + #[async_trait] impl ListSplitsResponseExt for ListSplitsResponse { fn empty() -> Self { @@ -630,6 +709,123 @@ impl ListSplitsResponseExt for ListSplitsResponse { } } +// ===================================================== +// Metrics Splits Extension Traits +// ===================================================== + +/// Helper trait to build a [`StageMetricsSplitsRequest`]. +pub trait StageMetricsSplitsRequestExt { + /// Creates a new [`StageMetricsSplitsRequest`] from an index UID and split metadata. + fn try_from_splits_metadata( + index_uid: impl Into, + splits_metadata: &[MetricsSplitMetadata], + ) -> MetastoreResult; + + /// Deserializes the splits metadata. + fn deserialize_splits_metadata(&self) -> MetastoreResult>; +} + +impl StageMetricsSplitsRequestExt for StageMetricsSplitsRequest { + fn try_from_splits_metadata( + index_uid: impl Into, + splits_metadata: &[MetricsSplitMetadata], + ) -> MetastoreResult { + let splits_metadata_json = splits_metadata + .iter() + .map(serde_utils::to_json_str) + .collect::>>()?; + Ok(StageMetricsSplitsRequest { + index_uid: Some(index_uid.into()), + splits_metadata_json, + }) + } + + fn deserialize_splits_metadata(&self) -> MetastoreResult> { + self.splits_metadata_json + .iter() + .map(|s| serde_utils::from_json_str(s)) + .collect() + } +} + +/// Helper trait to build a [`ListMetricsSplitsRequest`]. +pub trait ListMetricsSplitsRequestExt { + /// Creates a new [`ListMetricsSplitsRequest`] from an index UID. + fn for_index(index_uid: impl Into) -> MetastoreResult; + + /// Creates a new [`ListMetricsSplitsRequest`] from an index UID and query. + fn try_from_query( + index_uid: impl Into, + query: &ListMetricsSplitsQuery, + ) -> MetastoreResult; + + /// Deserializes the `query_json` field. + fn deserialize_query(&self) -> MetastoreResult; +} + +impl ListMetricsSplitsRequestExt for ListMetricsSplitsRequest { + fn for_index(index_uid: impl Into) -> MetastoreResult { + let index_uid = index_uid.into(); + let query = ListMetricsSplitsQuery::for_index(index_uid.clone()); + Self::try_from_query(index_uid, &query) + } + + fn try_from_query( + index_uid: impl Into, + query: &ListMetricsSplitsQuery, + ) -> MetastoreResult { + let query_json = serde_utils::to_json_str(query)?; + Ok(ListMetricsSplitsRequest { + index_uid: Some(index_uid.into()), + query_json, + }) + } + + fn deserialize_query(&self) -> MetastoreResult { + serde_utils::from_json_str(&self.query_json) + } +} + +/// Helper trait to build and deserialize [`ListMetricsSplitsResponse`]. +pub trait ListMetricsSplitsResponseExt { + /// Creates an empty response. + fn empty() -> Self; + + /// Creates a response from a list of splits. + fn try_from_splits(splits: &[MetricsSplitRecord]) + -> MetastoreResult; + + /// Deserializes the splits in the response. + fn deserialize_splits(&self) -> MetastoreResult>; +} + +impl ListMetricsSplitsResponseExt for ListMetricsSplitsResponse { + fn empty() -> Self { + Self { + splits_serialized_json: Vec::new(), + } + } + + fn try_from_splits( + splits: &[MetricsSplitRecord], + ) -> MetastoreResult { + let splits_serialized_json = splits + .iter() + .map(serde_utils::to_json_str) + .collect::>>()?; + Ok(ListMetricsSplitsResponse { + splits_serialized_json, + }) + } + + fn deserialize_splits(&self) -> MetastoreResult> { + self.splits_serialized_json + .iter() + .map(|s| serde_utils::from_json_str(s)) + .collect() + } +} + #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] /// A query builder for listing splits within the metastore. pub struct ListSplitsQuery { diff --git a/quickwit/quickwit-metastore/src/metastore/postgres/metastore.rs b/quickwit/quickwit-metastore/src/metastore/postgres/metastore.rs index b85b5a8b533..d459ceb243b 100644 --- a/quickwit/quickwit-metastore/src/metastore/postgres/metastore.rs +++ b/quickwit/quickwit-metastore/src/metastore/postgres/metastore.rs @@ -14,6 +14,7 @@ use std::collections::HashMap; use std::fmt::{self, Write}; +use std::str::FromStr; use std::time::Duration; use async_trait::async_trait; @@ -29,21 +30,25 @@ use quickwit_proto::ingest::{Shard, ShardState}; use quickwit_proto::metastore::{ AcquireShardsRequest, AcquireShardsResponse, AddSourceRequest, CreateIndexRequest, CreateIndexResponse, CreateIndexTemplateRequest, DeleteIndexRequest, - DeleteIndexTemplatesRequest, DeleteQuery, DeleteShardsRequest, DeleteShardsResponse, - DeleteSourceRequest, DeleteSplitsRequest, DeleteTask, EmptyResponse, EntityKind, - FindIndexTemplateMatchesRequest, FindIndexTemplateMatchesResponse, GetIndexTemplateRequest, + DeleteIndexTemplatesRequest, DeleteMetricsSplitsRequest, DeleteQuery, DeleteShardsRequest, + DeleteShardsResponse, DeleteSourceRequest, DeleteSplitsRequest, DeleteTask, EmptyResponse, + EntityKind, FindIndexTemplateMatchesRequest, FindIndexTemplateMatchesResponse, + GetClusterIdentityRequest, GetClusterIdentityResponse, GetIndexTemplateRequest, GetIndexTemplateResponse, IndexMetadataFailure, IndexMetadataFailureReason, - IndexMetadataRequest, IndexMetadataResponse, IndexTemplateMatch, IndexesMetadataRequest, - IndexesMetadataResponse, LastDeleteOpstampRequest, LastDeleteOpstampResponse, - ListDeleteTasksRequest, ListDeleteTasksResponse, ListIndexTemplatesRequest, + IndexMetadataRequest, IndexMetadataResponse, IndexStats, IndexTemplateMatch, + IndexesMetadataRequest, IndexesMetadataResponse, LastDeleteOpstampRequest, + LastDeleteOpstampResponse, ListDeleteTasksRequest, ListDeleteTasksResponse, + ListIndexStatsRequest, ListIndexStatsResponse, ListIndexTemplatesRequest, ListIndexTemplatesResponse, ListIndexesMetadataRequest, ListIndexesMetadataResponse, - ListShardsRequest, ListShardsResponse, ListShardsSubresponse, ListSplitsRequest, - ListSplitsResponse, ListStaleSplitsRequest, MarkSplitsForDeletionRequest, MetastoreError, + ListMetricsSplitsRequest, ListMetricsSplitsResponse, ListShardsRequest, ListShardsResponse, + ListShardsSubresponse, ListSplitsRequest, ListSplitsResponse, ListStaleSplitsRequest, + MarkMetricsSplitsForDeletionRequest, MarkSplitsForDeletionRequest, MetastoreError, MetastoreResult, MetastoreService, MetastoreServiceStream, OpenShardSubrequest, OpenShardSubresponse, OpenShardsRequest, OpenShardsResponse, PruneShardsRequest, - PublishSplitsRequest, ResetSourceCheckpointRequest, StageSplitsRequest, ToggleSourceRequest, - UpdateIndexRequest, UpdateSourceRequest, UpdateSplitsDeleteOpstampRequest, - UpdateSplitsDeleteOpstampResponse, serde_utils, + PublishMetricsSplitsRequest, PublishSplitsRequest, ResetSourceCheckpointRequest, SplitStats, + StageMetricsSplitsRequest, StageSplitsRequest, ToggleSourceRequest, UpdateIndexRequest, + UpdateSourceRequest, UpdateSplitsDeleteOpstampRequest, UpdateSplitsDeleteOpstampResponse, + serde_utils, }; use quickwit_proto::types::{IndexId, IndexUid, Position, PublishToken, ShardId, SourceId}; use sea_query::{Alias, Asterisk, Expr, Func, PostgresQueryBuilder, Query, UnionType}; @@ -51,6 +56,7 @@ use sea_query_binder::SqlxBinder; use sqlx::{Acquire, Executor, Postgres, Transaction}; use time::OffsetDateTime; use tracing::{debug, info, instrument, warn}; +use uuid::Uuid; use super::error::convert_sqlx_err; use super::migrator::run_migrations; @@ -69,7 +75,8 @@ use crate::file_backed::MutationOccurred; use crate::metastore::postgres::model::Shards; use crate::metastore::postgres::utils::split_maturity_timestamp; use crate::metastore::{ - IndexesMetadataResponseExt, PublishSplitsRequestExt, STREAM_SPLITS_CHUNK_SIZE, + IndexesMetadataResponseExt, ListMetricsSplitsResponseExt, PublishMetricsSplitsRequestExt, + PublishSplitsRequestExt, STREAM_SPLITS_CHUNK_SIZE, StageMetricsSplitsRequestExt, UpdateSourceRequestExt, use_shard_api, }; use crate::{ @@ -430,7 +437,7 @@ impl MetastoreService for PostgresqlMetastore { ingest_settings, search_settings, retention_policy_opt, - ); + )?; Ok(MutationOccurred::from(mutation_occurred)) }) .await @@ -903,6 +910,82 @@ impl MetastoreService for PostgresqlMetastore { Ok(service_stream) } + async fn list_index_stats( + &self, + request: ListIndexStatsRequest, + ) -> MetastoreResult { + let index_pattern_sql = build_index_id_patterns_sql_query(&request.index_id_patterns) + .map_err(|error| MetastoreError::Internal { + message: "failed to build `list_index_stats` SQL query".to_string(), + cause: error.to_string(), + })?; + let sql = format!( + "SELECT + i.index_uid, + s.split_state, + COUNT(s.split_state) AS num_splits, + COALESCE(SUM(s.split_size_bytes)::BIGINT, 0) AS total_size_bytes + FROM ({index_pattern_sql}) i + LEFT JOIN splits s ON s.index_uid = i.index_uid + GROUP BY i.index_uid, s.split_state" + ); + + let rows: Vec<(String, Option, i64, i64)> = sqlx::query_as(&sql) + .fetch_all(&self.connection_pool) + .await?; + + let mut index_stats = HashMap::new(); + for (index_uid_str, split_state, num_splits, total_size_bytes) in rows { + let Ok(index_uid) = IndexUid::from_str(&index_uid_str) else { + return Err(MetastoreError::Internal { + message: "failed to parse index_uid".to_string(), + cause: index_uid_str.to_string(), + }); + }; + let stats = index_stats + .entry(index_uid_str) + .or_insert_with(|| IndexStats { + index_uid: Some(index_uid), + staged: Some(SplitStats::default()), + published: Some(SplitStats::default()), + marked_for_deletion: Some(SplitStats::default()), + }); + let num_splits = num_splits as u64; + let total_size_bytes = total_size_bytes as u64; + match split_state.as_deref() { + Some("Staged") => { + stats.staged = Some(SplitStats { + num_splits, + total_size_bytes, + }); + } + Some("Published") => { + stats.published = Some(SplitStats { + num_splits, + total_size_bytes, + }); + } + Some("MarkedForDeletion") => { + stats.marked_for_deletion = Some(SplitStats { + num_splits, + total_size_bytes, + }); + } + None => {} // if an index has no splits, we can keep the defaults + Some(split_state) => { + return Err(MetastoreError::Internal { + message: "invalid split state".to_string(), + cause: split_state.to_string(), + }); + } + } + } + + Ok(ListIndexStatsResponse { + index_stats: index_stats.into_values().collect(), + }) + } + #[instrument(skip(self))] async fn mark_splits_for_deletion( &self, @@ -1664,6 +1747,691 @@ impl MetastoreService for PostgresqlMetastore { .await?; Ok(EmptyResponse {}) } + + async fn get_cluster_identity( + &self, + _: GetClusterIdentityRequest, + ) -> MetastoreResult { + // `ON CONFLICT DO NOTHING RETURNING` returns NULL if no insert happens. + // To always get the value, we use this pattern: + let (uuid,) = sqlx::query_as( + r" + INSERT INTO kv (key, value) + VALUES ('cluster_identity', $1) + ON CONFLICT (key) DO UPDATE SET key = EXCLUDED.key + RETURNING value + ", + ) + .bind(Uuid::new_v4().hyphenated().to_string()) + .fetch_one(&self.connection_pool) + .await?; + Ok(GetClusterIdentityResponse { uuid }) + } + + // Metrics Splits API + + async fn stage_metrics_splits( + &self, + request: StageMetricsSplitsRequest, + ) -> MetastoreResult { + use quickwit_parquet_engine::split::{ + InsertableMetricsSplit, MetricsSplitMetadata, MetricsSplitState, + }; + + let index_uid = request.index_uid().clone(); + + // Deserialize the splits metadata + let splits_metadata: Vec = request.deserialize_splits_metadata()?; + + if splits_metadata.is_empty() { + return Ok(EmptyResponse {}); + } + + // Convert to insertable rows - use JSON strings for array fields due to sqlx limitations + let mut split_ids = Vec::with_capacity(splits_metadata.len()); + let mut split_states = Vec::with_capacity(splits_metadata.len()); + let mut index_uids = Vec::with_capacity(splits_metadata.len()); + let mut time_range_starts = Vec::with_capacity(splits_metadata.len()); + let mut time_range_ends = Vec::with_capacity(splits_metadata.len()); + let mut metric_names_json = Vec::with_capacity(splits_metadata.len()); + let mut tag_service_json = Vec::with_capacity(splits_metadata.len()); + let mut tag_env_json = Vec::with_capacity(splits_metadata.len()); + let mut tag_datacenter_json = Vec::with_capacity(splits_metadata.len()); + let mut tag_region_json = Vec::with_capacity(splits_metadata.len()); + let mut tag_host_json = Vec::with_capacity(splits_metadata.len()); + let mut high_card_tag_keys_json = Vec::with_capacity(splits_metadata.len()); + let mut num_rows_list = Vec::with_capacity(splits_metadata.len()); + let mut size_bytes_list = Vec::with_capacity(splits_metadata.len()); + let mut split_metadata_jsons = Vec::with_capacity(splits_metadata.len()); + + for metadata in &splits_metadata { + let insertable = + InsertableMetricsSplit::from_metadata(metadata, MetricsSplitState::Staged) + .map_err(|e| MetastoreError::JsonSerializeError { + struct_name: "MetricsSplitMetadata".to_string(), + message: e.to_string(), + })?; + + split_ids.push(insertable.split_id); + split_states.push(insertable.split_state); + index_uids.push(insertable.index_uid); + time_range_starts.push(insertable.time_range_start); + time_range_ends.push(insertable.time_range_end); + // Convert array fields to JSON strings (owned) + let json_err = |e: serde_json::Error| MetastoreError::JsonSerializeError { + struct_name: "MetricsSplitMetadata".to_string(), + message: e.to_string(), + }; + metric_names_json + .push(serde_json::to_string(&insertable.metric_names).map_err(json_err)?); + tag_service_json + .push(serde_json::to_string(&insertable.tag_service).map_err(json_err)?); + tag_env_json.push(serde_json::to_string(&insertable.tag_env).map_err(json_err)?); + tag_datacenter_json + .push(serde_json::to_string(&insertable.tag_datacenter).map_err(json_err)?); + tag_region_json.push(serde_json::to_string(&insertable.tag_region).map_err(json_err)?); + tag_host_json.push(serde_json::to_string(&insertable.tag_host).map_err(json_err)?); + high_card_tag_keys_json.push( + serde_json::to_string(&insertable.high_cardinality_tag_keys).map_err(json_err)?, + ); + num_rows_list.push(insertable.num_rows); + size_bytes_list.push(insertable.size_bytes); + split_metadata_jsons.push(insertable.split_metadata_json); + } + + info!( + index_uid = %request.index_uid(), + num_splits = split_ids.len(), + "staging metrics splits" + ); + + // Insert into metrics_splits table using JSON for array fields + const STAGE_METRICS_SPLITS_QUERY: &str = r#" + INSERT INTO metrics_splits ( + split_id, + split_state, + index_uid, + time_range_start, + time_range_end, + metric_names, + tag_service, + tag_env, + tag_datacenter, + tag_region, + tag_host, + high_cardinality_tag_keys, + num_rows, + size_bytes, + split_metadata_json, + create_timestamp, + update_timestamp + ) + SELECT + split_id, + split_state, + index_uid, + time_range_start, + time_range_end, + ARRAY(SELECT json_array_elements_text(metric_names_json::json)), + CASE WHEN tag_service_json::text = 'null' THEN NULL + ELSE ARRAY(SELECT json_array_elements_text(tag_service_json::json)) END, + CASE WHEN tag_env_json::text = 'null' THEN NULL + ELSE ARRAY(SELECT json_array_elements_text(tag_env_json::json)) END, + CASE WHEN tag_datacenter_json::text = 'null' THEN NULL + ELSE ARRAY(SELECT json_array_elements_text(tag_datacenter_json::json)) END, + CASE WHEN tag_region_json::text = 'null' THEN NULL + ELSE ARRAY(SELECT json_array_elements_text(tag_region_json::json)) END, + CASE WHEN tag_host_json::text = 'null' THEN NULL + ELSE ARRAY(SELECT json_array_elements_text(tag_host_json::json)) END, + ARRAY(SELECT json_array_elements_text(high_cardinality_tag_keys_json::json)), + num_rows, + size_bytes, + split_metadata_json, + (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'), + (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') + FROM UNNEST( + $1::text[], + $2::text[], + $3::text[], + $4::bigint[], + $5::bigint[], + $6::text[], + $7::text[], + $8::text[], + $9::text[], + $10::text[], + $11::text[], + $12::text[], + $13::bigint[], + $14::bigint[], + $15::text[] + ) AS staged( + split_id, + split_state, + index_uid, + time_range_start, + time_range_end, + metric_names_json, + tag_service_json, + tag_env_json, + tag_datacenter_json, + tag_region_json, + tag_host_json, + high_cardinality_tag_keys_json, + num_rows, + size_bytes, + split_metadata_json + ) + ON CONFLICT (split_id) DO UPDATE + SET + split_state = EXCLUDED.split_state, + time_range_start = EXCLUDED.time_range_start, + time_range_end = EXCLUDED.time_range_end, + metric_names = EXCLUDED.metric_names, + tag_service = EXCLUDED.tag_service, + tag_env = EXCLUDED.tag_env, + tag_datacenter = EXCLUDED.tag_datacenter, + tag_region = EXCLUDED.tag_region, + tag_host = EXCLUDED.tag_host, + high_cardinality_tag_keys = EXCLUDED.high_cardinality_tag_keys, + num_rows = EXCLUDED.num_rows, + size_bytes = EXCLUDED.size_bytes, + split_metadata_json = EXCLUDED.split_metadata_json, + update_timestamp = (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') + WHERE metrics_splits.split_state = 'Staged' + RETURNING split_id + "#; + + let split_ids_snapshot = split_ids.clone(); + let index_id_for_err = index_uid.index_id.clone(); + let upserted_split_ids: Vec = + run_with_tx!(self.connection_pool, tx, "stage metrics splits", { + sqlx::query_scalar(STAGE_METRICS_SPLITS_QUERY) + .bind(&split_ids) + .bind(&split_states) + .bind(&index_uids) + .bind(&time_range_starts) + .bind(&time_range_ends) + .bind(&metric_names_json) + .bind(&tag_service_json) + .bind(&tag_env_json) + .bind(&tag_datacenter_json) + .bind(&tag_region_json) + .bind(&tag_host_json) + .bind(&high_card_tag_keys_json) + .bind(&num_rows_list) + .bind(&size_bytes_list) + .bind(&split_metadata_jsons) + .fetch_all(tx.as_mut()) + .await + .map_err(|sqlx_error| convert_sqlx_err(&index_id_for_err, sqlx_error)) + })?; + + if upserted_split_ids.len() != split_ids_snapshot.len() { + let failed_split_ids: Vec = split_ids_snapshot + .into_iter() + .filter(|split_id| !upserted_split_ids.contains(split_id)) + .collect(); + let entity = EntityKind::Splits { + split_ids: failed_split_ids.clone(), + }; + let message = "splits are not in the Staged state".to_string(); + warn!( + %index_uid, + failed_split_ids = ?failed_split_ids, + "failed to stage some metrics splits" + ); + return Err(MetastoreError::FailedPrecondition { entity, message }); + } + + info!( + %index_uid, + num_staged = upserted_split_ids.len(), + "staged metrics splits successfully" + ); + Ok(EmptyResponse {}) + } + + #[instrument(skip_all, fields(index_uid = %request.index_uid()))] + async fn publish_metrics_splits( + &self, + request: PublishMetricsSplitsRequest, + ) -> MetastoreResult { + let checkpoint_delta_opt: Option = + request.deserialize_index_checkpoint()?; + let index_uid = request.index_uid().clone(); + let staged_split_ids = request.staged_split_ids; + let replaced_split_ids = request.replaced_split_ids; + + info!( + %index_uid, + staged_splits = staged_split_ids.len(), + replaced_splits = replaced_split_ids.len(), + "publishing metrics splits" + ); + + run_with_tx!(self.connection_pool, tx, "publish metrics splits", { + let mut index_metadata = index_metadata(tx, &index_uid.index_id, true).await?; + let index_uid = index_metadata.index_uid.clone(); + + if let Some(checkpoint_delta) = checkpoint_delta_opt { + let source_id = checkpoint_delta.source_id.clone(); + let source = index_metadata.sources.get(&source_id).ok_or_else(|| { + MetastoreError::NotFound(EntityKind::Source { + index_id: index_uid.index_id.to_string(), + source_id: source_id.to_string(), + }) + })?; + + if use_shard_api(&source.source_params) { + let publish_token = request.publish_token_opt.ok_or_else(|| { + let message = format!( + "publish token is required for publishing splits for source \ + `{source_id}`" + ); + MetastoreError::InvalidArgument { message } + })?; + try_apply_delta_v2( + tx, + &index_uid, + &source_id, + checkpoint_delta.source_delta, + publish_token, + ) + .await?; + } else { + index_metadata + .checkpoint + .try_apply_delta(checkpoint_delta) + .map_err(|error| { + let entity = EntityKind::CheckpointDelta { + index_id: index_uid.index_id.to_string(), + source_id, + }; + let message = error.to_string(); + MetastoreError::FailedPrecondition { entity, message } + })?; + } + } + let index_metadata_json = serde_utils::to_json_str(&index_metadata)?; + + // Publish staged splits, mark replaced splits for deletion, and update + // index metadata (checkpoint) atomically. + const PUBLISH_METRICS_SPLITS_QUERY: &str = r#" + WITH publish AS ( + UPDATE metrics_splits + SET + split_state = 'Published', + update_timestamp = (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') + WHERE + index_uid = $1 + AND split_id = ANY($3) + AND split_state = 'Staged' + RETURNING split_id + ), + mark_for_deletion AS ( + UPDATE metrics_splits + SET + split_state = 'MarkedForDeletion', + update_timestamp = (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') + WHERE + index_uid = $1 + AND split_id = ANY($4) + AND split_state = 'Published' + RETURNING split_id + ), + updated_index_metadata AS ( + UPDATE indexes + SET + index_metadata_json = $2 + WHERE + index_uid = $1 + ) + SELECT + (SELECT COUNT(*) FROM publish) as published_count, + (SELECT COUNT(*) FROM mark_for_deletion) as marked_count + "#; + + let (published_count, marked_count): (i64, i64) = + sqlx::query_as(PUBLISH_METRICS_SPLITS_QUERY) + .bind(&index_uid) + .bind(index_metadata_json) + .bind(&staged_split_ids) + .bind(&replaced_split_ids) + .fetch_one(tx.as_mut()) + .await + .map_err(|sqlx_error| convert_sqlx_err(&index_uid.index_id, sqlx_error))?; + + // Verify all staged splits were published + if published_count as usize != staged_split_ids.len() { + let entity = EntityKind::Splits { + split_ids: staged_split_ids.clone(), + }; + let message = format!( + "expected to publish {} splits, but only {} were in Staged state", + staged_split_ids.len(), + published_count + ); + return Err(MetastoreError::FailedPrecondition { entity, message }); + } + + // Verify all replaced splits were marked for deletion + if marked_count as usize != replaced_split_ids.len() { + let entity = EntityKind::Splits { + split_ids: replaced_split_ids.clone(), + }; + let message = format!( + "expected to mark {} splits for deletion, but only {} were in Published state", + replaced_split_ids.len(), + marked_count + ); + return Err(MetastoreError::FailedPrecondition { entity, message }); + } + + info!( + %index_uid, + published_count, + marked_count, + "published metrics splits successfully" + ); + Ok(EmptyResponse {}) + }) + } + + #[instrument(skip_all, fields(index_uid = %request.index_uid()))] + async fn list_metrics_splits( + &self, + request: ListMetricsSplitsRequest, + ) -> MetastoreResult { + use quickwit_parquet_engine::split::MetricsSplitRecord; + + use crate::metastore::ListMetricsSplitsQuery; + + // Deserialize the query + let query: ListMetricsSplitsQuery = serde_utils::from_json_str(&request.query_json)?; + + // Build dynamic SQL query based on filters + let mut sql = String::from( + r#" + SELECT + split_id, + split_state, + index_uid, + time_range_start, + time_range_end, + metric_names, + tag_service, + tag_env, + tag_datacenter, + tag_region, + tag_host, + high_cardinality_tag_keys, + num_rows, + size_bytes, + split_metadata_json, + EXTRACT(EPOCH FROM update_timestamp)::bigint as update_timestamp + FROM metrics_splits + WHERE index_uid = $1 + "#, + ); + + let mut param_idx = 2; + + // Add state filter + if !query.split_states.is_empty() { + sql.push_str(&format!(" AND split_state = ANY(${}::text[])", param_idx)); + param_idx += 1; + } + + // Add time range filter + if query.time_range_start.is_some() { + sql.push_str(&format!(" AND time_range_end >= ${}", param_idx)); + param_idx += 1; + } + if query.time_range_end.is_some() { + sql.push_str(&format!(" AND time_range_start <= ${}", param_idx)); + param_idx += 1; + } + + // Add metric names filter (ANY overlap) + if !query.metric_names.is_empty() { + sql.push_str(&format!(" AND metric_names && ${}::text[]", param_idx)); + param_idx += 1; + } + + // Add tag filters + if query.tag_service.is_some() { + sql.push_str(&format!(" AND ${} = ANY(tag_service)", param_idx)); + param_idx += 1; + } + if query.tag_env.is_some() { + sql.push_str(&format!(" AND ${} = ANY(tag_env)", param_idx)); + param_idx += 1; + } + if query.tag_datacenter.is_some() { + sql.push_str(&format!(" AND ${} = ANY(tag_datacenter)", param_idx)); + param_idx += 1; + } + if query.tag_region.is_some() { + sql.push_str(&format!(" AND ${} = ANY(tag_region)", param_idx)); + param_idx += 1; + } + if query.tag_host.is_some() { + sql.push_str(&format!(" AND ${} = ANY(tag_host)", param_idx)); + param_idx += 1; + } + + sql.push_str(" ORDER BY time_range_start ASC"); + + // Add limit + if query.limit.is_some() { + sql.push_str(&format!(" LIMIT ${}", param_idx)); + } + + // Execute query with bindings + let mut query_builder = sqlx::query_as::< + _, + ( + String, // split_id + String, // split_state + String, // index_uid + i64, // time_range_start + i64, // time_range_end + Vec, // metric_names + Option>, // tag_service + Option>, // tag_env + Option>, // tag_datacenter + Option>, // tag_region + Option>, // tag_host + Vec, // high_cardinality_tag_keys + i64, // num_rows + i64, // size_bytes + String, // split_metadata_json + i64, // update_timestamp + ), + >(&sql); + + query_builder = query_builder.bind(query.index_uid.to_string()); + + if !query.split_states.is_empty() { + query_builder = query_builder.bind(&query.split_states); + } + if let Some(start) = query.time_range_start { + query_builder = query_builder.bind(start); + } + if let Some(end) = query.time_range_end { + query_builder = query_builder.bind(end); + } + if !query.metric_names.is_empty() { + query_builder = query_builder.bind(&query.metric_names); + } + if let Some(ref service) = query.tag_service { + query_builder = query_builder.bind(service); + } + if let Some(ref env) = query.tag_env { + query_builder = query_builder.bind(env); + } + if let Some(ref dc) = query.tag_datacenter { + query_builder = query_builder.bind(dc); + } + if let Some(ref region) = query.tag_region { + query_builder = query_builder.bind(region); + } + if let Some(ref host) = query.tag_host { + query_builder = query_builder.bind(host); + } + if let Some(limit) = query.limit { + query_builder = query_builder.bind(limit as i64); + } + + let rows = query_builder + .fetch_all(&self.connection_pool) + .await + .map_err(|sqlx_error| convert_sqlx_err(&query.index_uid.index_id, sqlx_error))?; + + // Convert rows to MetricsSplitRecord + let splits: Vec = rows + .into_iter() + .filter_map(|row| { + use quickwit_parquet_engine::split::{MetricsSplitState, PgMetricsSplit}; + + let pg_split = PgMetricsSplit { + split_id: row.0, + split_state: row.1, + index_uid: row.2, + time_range_start: row.3, + time_range_end: row.4, + metric_names: row.5, + tag_service: row.6, + tag_env: row.7, + tag_datacenter: row.8, + tag_region: row.9, + tag_host: row.10, + high_cardinality_tag_keys: row.11, + num_rows: row.12, + size_bytes: row.13, + split_metadata_json: row.14, + update_timestamp: row.15, + }; + + let state = pg_split.split_state().unwrap_or(MetricsSplitState::Staged); + let metadata = pg_split.to_metadata().ok()?; + + Some(MetricsSplitRecord { + state, + update_timestamp: row.15, + metadata, + }) + }) + .collect(); + + ListMetricsSplitsResponse::try_from_splits(&splits) + } + + #[instrument(skip_all, fields(index_uid = %request.index_uid()))] + async fn mark_metrics_splits_for_deletion( + &self, + request: MarkMetricsSplitsForDeletionRequest, + ) -> MetastoreResult { + if request.split_ids.is_empty() { + return Ok(EmptyResponse {}); + } + + info!( + index_uid = %request.index_uid(), + split_ids = ?request.split_ids, + "marking metrics splits for deletion" + ); + + const MARK_FOR_DELETION_QUERY: &str = r#" + UPDATE metrics_splits + SET + split_state = 'MarkedForDeletion', + update_timestamp = (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') + WHERE + index_uid = $1 + AND split_id = ANY($2) + AND split_state IN ('Staged', 'Published') + RETURNING split_id + "#; + + let marked_split_ids: Vec = sqlx::query_scalar(MARK_FOR_DELETION_QUERY) + .bind(request.index_uid()) + .bind(&request.split_ids) + .fetch_all(&self.connection_pool) + .await + .map_err(|sqlx_error| convert_sqlx_err(&request.index_uid().index_id, sqlx_error))?; + + info!( + index_uid = %request.index_uid(), + marked_count = marked_split_ids.len(), + "marked metrics splits for deletion" + ); + Ok(EmptyResponse {}) + } + + #[instrument(skip_all, fields(index_uid = %request.index_uid()))] + async fn delete_metrics_splits( + &self, + request: DeleteMetricsSplitsRequest, + ) -> MetastoreResult { + if request.split_ids.is_empty() { + return Ok(EmptyResponse {}); + } + + info!( + index_uid = %request.index_uid(), + split_ids = ?request.split_ids, + "deleting metrics splits" + ); + + // Only delete splits that are marked for deletion + const DELETE_SPLITS_QUERY: &str = r#" + DELETE FROM metrics_splits + WHERE + index_uid = $1 + AND split_id = ANY($2) + AND split_state = 'MarkedForDeletion' + RETURNING split_id + "#; + + let deleted_split_ids: Vec = sqlx::query_scalar(DELETE_SPLITS_QUERY) + .bind(request.index_uid()) + .bind(&request.split_ids) + .fetch_all(&self.connection_pool) + .await + .map_err(|sqlx_error| convert_sqlx_err(&request.index_uid().index_id, sqlx_error))?; + + // Check if any splits could not be deleted + if deleted_split_ids.len() != request.split_ids.len() { + let not_deleted: Vec = request + .split_ids + .iter() + .filter(|id| !deleted_split_ids.contains(id)) + .cloned() + .collect(); + + if !not_deleted.is_empty() { + warn!( + index_uid = %request.index_uid(), + not_deleted = ?not_deleted, + "some metrics splits were not in MarkedForDeletion state" + ); + let entity = EntityKind::Splits { + split_ids: not_deleted, + }; + let message = "splits are not marked for deletion".to_string(); + return Err(MetastoreError::FailedPrecondition { entity, message }); + } + } + + info!( + index_uid = %request.index_uid(), + deleted_count = deleted_split_ids.len(), + "deleted metrics splits successfully" + ); + Ok(EmptyResponse {}) + } } async fn open_or_fetch_shard<'e>( diff --git a/quickwit/quickwit-metastore/src/metastore/postgres/pool.rs b/quickwit/quickwit-metastore/src/metastore/postgres/pool.rs index f0437300095..fe6a4584092 100644 --- a/quickwit/quickwit-metastore/src/metastore/postgres/pool.rs +++ b/quickwit/quickwit-metastore/src/metastore/postgres/pool.rs @@ -15,7 +15,6 @@ use futures::future::BoxFuture; use futures::stream::BoxStream; use quickwit_common::metrics::GaugeGuard; -use sqlx::database::HasStatement; use sqlx::pool::PoolConnection; use sqlx::pool::maybe::MaybePoolConnection; use sqlx::{ @@ -71,13 +70,18 @@ impl<'a, DB: Database> Acquire<'a> for &TrackedPool { let acquire_conn_fut = self.acquire(); Box::pin(async move { - Transaction::begin(MaybePoolConnection::PoolConnection(acquire_conn_fut.await?)).await + Transaction::begin( + MaybePoolConnection::PoolConnection(acquire_conn_fut.await?), + None, + ) + .await }) } } impl Executor<'_> for &TrackedPool -where for<'c> &'c mut DB::Connection: Executor<'c, Database = DB> +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, { type Database = DB; @@ -105,7 +109,7 @@ where for<'c> &'c mut DB::Connection: Executor<'c, Database = DB> self, sql: &'q str, parameters: &'e [::TypeInfo], - ) -> BoxFuture<'e, Result<>::Statement, Error>> { + ) -> BoxFuture<'e, Result<::Statement<'q>, Error>> { self.inner_pool.prepare_with(sql, parameters) } diff --git a/quickwit/quickwit-metastore/src/tests/get_identity.rs b/quickwit/quickwit-metastore/src/tests/get_identity.rs new file mode 100644 index 00000000000..9cbda1bef13 --- /dev/null +++ b/quickwit/quickwit-metastore/src/tests/get_identity.rs @@ -0,0 +1,48 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Index API tests +// +// - create_index +// - index_exists +// - index_metadata +// - list_indexes +// - delete_index + +use quickwit_proto::metastore::{GetClusterIdentityRequest, MetastoreService}; +use uuid::Uuid; + +use super::DefaultForTest; +use crate::MetastoreServiceExt; + +pub async fn test_metastore_get_identity< + MetastoreToTest: MetastoreService + MetastoreServiceExt + DefaultForTest, +>() { + let metastore = MetastoreToTest::default_for_test().await; + + let identity_1 = metastore + .get_cluster_identity(GetClusterIdentityRequest {}) + .await + .unwrap() + .uuid; + + let identity_2 = metastore + .get_cluster_identity(GetClusterIdentityRequest {}) + .await + .unwrap() + .uuid; + + assert_eq!(identity_1, identity_2); + assert_ne!(identity_1, Uuid::nil().hyphenated().to_string()); +} diff --git a/quickwit/quickwit-metastore/src/tests/index.rs b/quickwit/quickwit-metastore/src/tests/index.rs index 6d7adf8ced7..c093c7b1a6c 100644 --- a/quickwit/quickwit-metastore/src/tests/index.rs +++ b/quickwit/quickwit-metastore/src/tests/index.rs @@ -31,9 +31,9 @@ use quickwit_config::{ use quickwit_doc_mapper::{Cardinality, FieldMappingEntry, FieldMappingType, QuickwitJsonOptions}; use quickwit_proto::metastore::{ CreateIndexRequest, DeleteIndexRequest, EntityKind, IndexMetadataFailure, - IndexMetadataFailureReason, IndexMetadataRequest, IndexMetadataSubrequest, - IndexesMetadataRequest, ListIndexesMetadataRequest, MetastoreError, MetastoreService, - StageSplitsRequest, UpdateIndexRequest, + IndexMetadataFailureReason, IndexMetadataRequest, IndexMetadataSubrequest, IndexStats, + IndexesMetadataRequest, ListIndexStatsRequest, ListIndexesMetadataRequest, MetastoreError, + MetastoreService, PublishSplitsRequest, SplitStats, StageSplitsRequest, UpdateIndexRequest, }; use quickwit_proto::types::{DocMappingUid, IndexUid}; @@ -160,6 +160,7 @@ pub async fn test_metastore_update_ingest_settings< let ingest_settings = IngestSettings { min_shards: NonZeroUsize::new(12).unwrap(), + ..Default::default() }; let index_update_request = UpdateIndexRequest::try_from_updates( index_uid.clone(), @@ -677,9 +678,14 @@ pub async fn test_metastore_list_indexes() { + let metastore = MetastoreToTest::default_for_test().await; + + let index_id_1 = append_random_suffix("test-list-index-stats"); + let index_uid_1 = IndexUid::new_with_random_ulid(&index_id_1); + let index_uri_1 = format!("ram:///indexes/{index_id_1}"); + let index_config_1 = IndexConfig::for_test(&index_id_1, &index_uri_1); + + let index_id_2 = append_random_suffix("test-list-index-stats"); + let index_uid_2 = IndexUid::new_with_random_ulid(&index_id_2); + let index_uri_2 = format!("ram:///indexes/{index_id_2}"); + let index_config_2 = IndexConfig::for_test(&index_id_2, &index_uri_2); + + let split_id_1 = format!("{index_id_1}--split-1"); + let split_metadata_1 = SplitMetadata { + split_id: split_id_1.clone(), + index_uid: index_uid_1.clone(), + footer_offsets: 0..2048, + ..Default::default() + }; + + let split_id_2 = format!("{index_id_1}--split-2"); + let split_metadata_2 = SplitMetadata { + split_id: split_id_2.clone(), + index_uid: index_uid_1.clone(), + footer_offsets: 0..2048, + ..Default::default() + }; + + let split_id_3 = format!("{index_id_1}--split-3"); + let split_metadata_3 = SplitMetadata { + split_id: split_id_3.clone(), + index_uid: index_uid_2.clone(), + footer_offsets: 0..1000, + ..Default::default() + }; + + // add split-1 and split-2 to index-1 + let create_index_request = CreateIndexRequest::try_from_index_config(&index_config_1).unwrap(); + let index_uid_1: IndexUid = metastore + .create_index(create_index_request) + .await + .unwrap() + .index_uid() + .clone(); + + let stage_splits_request = StageSplitsRequest::try_from_splits_metadata( + index_uid_1.clone(), + vec![split_metadata_1.clone(), split_metadata_2.clone()], + ) + .unwrap(); + metastore.stage_splits(stage_splits_request).await.unwrap(); + + let publish_splits_request = PublishSplitsRequest { + index_uid: Some(index_uid_1.clone()), + staged_split_ids: vec![split_id_1.clone(), split_id_2.clone()], + ..Default::default() + }; + metastore + .publish_splits(publish_splits_request) + .await + .unwrap(); + + // add split-3 to index-2 + let create_index_request = CreateIndexRequest::try_from_index_config(&index_config_2).unwrap(); + let index_uid_2: IndexUid = metastore + .create_index(create_index_request) + .await + .unwrap() + .index_uid() + .clone(); + + let stage_splits_request = StageSplitsRequest::try_from_splits_metadata( + index_uid_2.clone(), + vec![split_metadata_3.clone()], + ) + .unwrap(); + metastore.stage_splits(stage_splits_request).await.unwrap(); + + let expected_stats_1 = IndexStats { + index_uid: Some(index_uid_1.clone()), + staged: Some(SplitStats { + num_splits: 0, + total_size_bytes: 0, + }), + published: Some(SplitStats { + num_splits: 2, + total_size_bytes: 4096, + }), + marked_for_deletion: Some(SplitStats { + num_splits: 0, + total_size_bytes: 0, + }), + }; + let expected_stats_2 = IndexStats { + index_uid: Some(index_uid_2.clone()), + staged: Some(SplitStats { + num_splits: 1, + total_size_bytes: 1000, + }), + published: Some(SplitStats { + num_splits: 0, + total_size_bytes: 0, + }), + marked_for_deletion: Some(SplitStats { + num_splits: 0, + total_size_bytes: 0, + }), + }; + + let response = metastore + .list_index_stats(ListIndexStatsRequest { + index_id_patterns: vec!["test-list-index-stats*".to_string()], + }) + .await + .unwrap(); + + let index_stats_1 = response + .index_stats + .iter() + .find(|index| index.index_uid == Some(index_uid_1.clone())) + .expect("Should find index 1"); + + assert_eq!(index_stats_1, &expected_stats_1); + + let index_stats_2 = response + .index_stats + .iter() + .find(|index| index.index_uid == Some(index_uid_2.clone())) + .expect("Should find index 2"); + assert_eq!(index_stats_2, &expected_stats_2); +} + +pub async fn test_metastore_list_index_stats_no_splits< + MetastoreToTest: MetastoreServiceExt + DefaultForTest, +>() { + let metastore = MetastoreToTest::default_for_test().await; + + let index_id = append_random_suffix("test-list-index-stats-no-splits"); + let index_uri = format!("ram:///indexes/{index_id}"); + let index_config = IndexConfig::for_test(&index_id, &index_uri); + let create_index_request = CreateIndexRequest::try_from_index_config(&index_config).unwrap(); + + let index_uid: IndexUid = metastore + .create_index(create_index_request) + .await + .unwrap() + .index_uid() + .clone(); + + let expected_stats = IndexStats { + index_uid: Some(index_uid.clone()), + staged: Some(SplitStats { + num_splits: 0, + total_size_bytes: 0, + }), + published: Some(SplitStats { + num_splits: 0, + total_size_bytes: 0, + }), + marked_for_deletion: Some(SplitStats { + num_splits: 0, + total_size_bytes: 0, + }), + }; + + let response = metastore + .list_index_stats(ListIndexStatsRequest { + index_id_patterns: vec!["test-list-index-stats-no-splits*".to_string()], + }) + .await + .unwrap(); + + let index_stats = response + .index_stats + .iter() + .find(|index| index.index_uid == Some(index_uid.clone())) + .expect("Should find index"); + + assert_eq!(index_stats, &expected_stats); +} diff --git a/quickwit/quickwit-metastore/src/tests/mod.rs b/quickwit/quickwit-metastore/src/tests/mod.rs index 8fcd6af59ca..d6e549baf25 100644 --- a/quickwit/quickwit-metastore/src/tests/mod.rs +++ b/quickwit/quickwit-metastore/src/tests/mod.rs @@ -26,6 +26,7 @@ use quickwit_proto::tonic::transport::Channel; use quickwit_proto::types::IndexUid; pub(crate) mod delete_task; +pub(crate) mod get_identity; pub(crate) mod index; pub(crate) mod list_splits; pub(crate) mod shard; @@ -177,6 +178,7 @@ macro_rules! metastore_test_suite { // - indexes_metadata // - list_indexes // - delete_index + // - list_index_stats #[tokio::test] #[serial_test::file_serial] @@ -279,6 +281,20 @@ macro_rules! metastore_test_suite { $crate::tests::index::test_metastore_delete_index::<$metastore_type>().await; } + #[tokio::test] + #[serial_test::file_serial] + async fn test_metastore_list_index_stats() { + let _ = tracing_subscriber::fmt::try_init(); + $crate::tests::index::test_metastore_list_index_stats::<$metastore_type>().await; + } + + #[tokio::test] + #[serial_test::file_serial] + async fn test_metastore_list_index_stats_no_splits() { + let _ = tracing_subscriber::fmt::try_init(); + $crate::tests::index::test_metastore_list_index_stats_no_splits::<$metastore_type>().await; + } + // Split API tests // // - stage_splits @@ -552,6 +568,13 @@ macro_rules! metastore_test_suite { async fn test_metastore_delete_index_templates() { $crate::tests::template::test_metastore_delete_index_templates::<$metastore_type>().await; } + + #[tokio::test] + #[serial_test::file_serial] + async fn test_metastore_get_identity() { + let _ = tracing_subscriber::fmt::try_init(); + $crate::tests::get_identity::test_metastore_get_identity::<$metastore_type>().await; + } } }; } diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.7.expected.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.7.expected.json index 9f7f0e27f23..cb00de2fbd8 100644 --- a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.7.expected.json +++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.7.expected.json @@ -96,7 +96,7 @@ "maturation_period": "2days" }, "resources": { - "heap_size": "50.0 MB" + "heap_size": 50000000 } }, "ingest_settings": { diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.8.expected.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.8.expected.json index 9f7f0e27f23..cb00de2fbd8 100644 --- a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.8.expected.json +++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.8.expected.json @@ -96,7 +96,7 @@ "maturation_period": "2days" }, "resources": { - "heap_size": "50.0 MB" + "heap_size": 50000000 } }, "ingest_settings": { diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.9.expected.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.9.expected.json index f9ecb6a7bcb..cf23e2349e5 100644 --- a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.9.expected.json +++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.9.expected.json @@ -96,7 +96,7 @@ "maturation_period": "2days" }, "resources": { - "heap_size": "50.0 MB" + "heap_size": 50000000 } }, "ingest_settings": { diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.9.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.9.json index f9ecb6a7bcb..cf23e2349e5 100644 --- a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.9.json +++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.9.json @@ -96,7 +96,7 @@ "maturation_period": "2days" }, "resources": { - "heap_size": "50.0 MB" + "heap_size": 50000000 } }, "ingest_settings": { diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.7.expected.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.7.expected.json index 96710cd3f53..2287c4d5f42 100644 --- a/quickwit/quickwit-metastore/test-data/index-metadata/v0.7.expected.json +++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.7.expected.json @@ -96,7 +96,7 @@ "type": "stable_log" }, "resources": { - "heap_size": "50.0 MB" + "heap_size": 50000000 }, "split_num_docs_target": 10000001 }, diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.8.expected.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.8.expected.json index 96710cd3f53..2287c4d5f42 100644 --- a/quickwit/quickwit-metastore/test-data/index-metadata/v0.8.expected.json +++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.8.expected.json @@ -96,7 +96,7 @@ "type": "stable_log" }, "resources": { - "heap_size": "50.0 MB" + "heap_size": 50000000 }, "split_num_docs_target": 10000001 }, diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.9.expected.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.9.expected.json index f3c28fdbfbc..8fca7405352 100644 --- a/quickwit/quickwit-metastore/test-data/index-metadata/v0.9.expected.json +++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.9.expected.json @@ -96,7 +96,7 @@ "type": "stable_log" }, "resources": { - "heap_size": "50.0 MB" + "heap_size": 50000000 }, "split_num_docs_target": 10000001 }, diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.9.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.9.json index f3c28fdbfbc..8fca7405352 100644 --- a/quickwit/quickwit-metastore/test-data/index-metadata/v0.9.json +++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.9.json @@ -96,7 +96,7 @@ "type": "stable_log" }, "resources": { - "heap_size": "50.0 MB" + "heap_size": 50000000 }, "split_num_docs_target": 10000001 }, diff --git a/quickwit/quickwit-metastore/test-data/manifest/v0.7.expected.json b/quickwit/quickwit-metastore/test-data/manifest/v0.7.expected.json index 01c6595e1c3..674e583d56f 100644 --- a/quickwit/quickwit-metastore/test-data/manifest/v0.7.expected.json +++ b/quickwit/quickwit-metastore/test-data/manifest/v0.7.expected.json @@ -70,7 +70,7 @@ "type": "stable_log" }, "resources": { - "heap_size": "2.0 GB" + "heap_size": 2000000000 }, "split_num_docs_target": 10000000 }, diff --git a/quickwit/quickwit-metastore/test-data/manifest/v0.8.expected.json b/quickwit/quickwit-metastore/test-data/manifest/v0.8.expected.json index caf1f679c7e..2b6819d6bbb 100644 --- a/quickwit/quickwit-metastore/test-data/manifest/v0.8.expected.json +++ b/quickwit/quickwit-metastore/test-data/manifest/v0.8.expected.json @@ -70,7 +70,7 @@ "type": "stable_log" }, "resources": { - "heap_size": "2.0 GB" + "heap_size": 2000000000 }, "split_num_docs_target": 10000000 }, diff --git a/quickwit/quickwit-metastore/test-data/manifest/v0.9.expected.json b/quickwit/quickwit-metastore/test-data/manifest/v0.9.expected.json index 669a1cb9dea..11f3c3287a0 100644 --- a/quickwit/quickwit-metastore/test-data/manifest/v0.9.expected.json +++ b/quickwit/quickwit-metastore/test-data/manifest/v0.9.expected.json @@ -70,7 +70,7 @@ "type": "stable_log" }, "resources": { - "heap_size": "2.0 GB" + "heap_size": 2000000000 }, "split_num_docs_target": 10000000 }, diff --git a/quickwit/quickwit-metastore/test-data/manifest/v0.9.json b/quickwit/quickwit-metastore/test-data/manifest/v0.9.json index b02695b2ddd..914047f5421 100644 --- a/quickwit/quickwit-metastore/test-data/manifest/v0.9.json +++ b/quickwit/quickwit-metastore/test-data/manifest/v0.9.json @@ -70,7 +70,7 @@ "type": "stable_log" }, "resources": { - "heap_size": "2.0 GB" + "heap_size": 2000000000 }, "split_num_docs_target": 10000000 }, diff --git a/quickwit/quickwit-opentelemetry/Cargo.toml b/quickwit/quickwit-opentelemetry/Cargo.toml index 3534f8ab88e..d6e897990a2 100644 --- a/quickwit/quickwit-opentelemetry/Cargo.toml +++ b/quickwit/quickwit-opentelemetry/Cargo.toml @@ -12,9 +12,10 @@ license.workspace = true [dependencies] anyhow = { workspace = true } +arrow = { workspace = true } async-trait = { workspace = true } -hex = { workspace = true } once_cell = { workspace = true } +parquet = { workspace = true } prost = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/quickwit/quickwit-opentelemetry/src/otlp/arrow_metrics.rs b/quickwit/quickwit-opentelemetry/src/otlp/arrow_metrics.rs new file mode 100644 index 00000000000..1811a63a909 --- /dev/null +++ b/quickwit/quickwit-opentelemetry/src/otlp/arrow_metrics.rs @@ -0,0 +1,724 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Arrow-based batch building for metrics with dictionary encoding. +//! +//! This module provides Arrow RecordBatch construction with dictionary-encoded +//! string columns for efficient storage of metrics with low cardinality tags. + +use std::io::Cursor; +use std::sync::Arc; + +use arrow::array::{ + ArrayBuilder, ArrayRef, Float64Builder, RecordBatch, StringBuilder, StringDictionaryBuilder, + UInt8Builder, UInt64Builder, +}; +use arrow::datatypes::{DataType, Field, Fields, Int32Type, Schema as ArrowSchema}; +use arrow::ipc::reader::StreamReader; +use arrow::ipc::writer::StreamWriter; +use parquet::variant::{VariantArrayBuilder, VariantBuilderExt, VariantType}; +use quickwit_proto::bytes::Bytes; +use quickwit_proto::ingest::{DocBatchV2, DocFormat}; +use quickwit_proto::types::DocUid; + +use super::otel_metrics::{MetricDataPoint, MetricType}; + +/// Creates the Arrow schema for metrics with dictionary-encoded string columns. +/// +/// Dictionary encoding stores unique string values once and references them by +/// integer index, providing significant compression for low cardinality tag values. +pub fn metrics_arrow_schema() -> ArrowSchema { + ArrowSchema::new(vec![ + // Dictionary-encoded string columns for low cardinality fields + Field::new( + "metric_name", + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + false, + ), + // MetricType enum stored as UInt8 (only ~5 possible values) + Field::new("metric_type", DataType::UInt8, false), + Field::new("metric_unit", DataType::Utf8, true), + // Measurement timestamp in seconds since Unix epoch. + Field::new("timestamp_secs", DataType::UInt64, false), + Field::new("start_timestamp_secs", DataType::UInt64, true), + Field::new("value", DataType::Float64, false), + // Dictionary-encoded tag columns (low cardinality expected) + Field::new( + "tag_service", + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + true, + ), + Field::new( + "tag_env", + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + true, + ), + Field::new( + "tag_datacenter", + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + true, + ), + Field::new( + "tag_region", + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + true, + ), + Field::new( + "tag_host", + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + true, + ), + // VARIANT fields for semi-structured attributes + // VariantArrayBuilder produces BinaryView fields, not Binary + Field::new( + "attributes", + DataType::Struct(Fields::from(vec![ + Field::new("metadata", DataType::BinaryView, false), + Field::new("value", DataType::BinaryView, false), + ])), + true, + ) + .with_extension_type(VariantType), + // Service name (low cardinality) + Field::new( + "service_name", + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + false, + ), + Field::new( + "resource_attributes", + DataType::Struct(Fields::from(vec![ + Field::new("metadata", DataType::BinaryView, false), + Field::new("value", DataType::BinaryView, false), + ])), + true, + ) + .with_extension_type(VariantType), + ]) +} + +/// Builder for creating Arrow RecordBatch from MetricDataPoints. +/// +/// Uses dictionary encoding for low cardinality string columns +/// (tags, service names, metric names) to achieve significant compression. +/// Uses VARIANT encoding for semi-structured attributes. +pub struct ArrowMetricsBatchBuilder { + metric_name: StringDictionaryBuilder, + metric_type: UInt8Builder, + metric_unit: StringBuilder, + timestamp_secs: UInt64Builder, + start_timestamp_secs: UInt64Builder, + value: Float64Builder, + tag_service: StringDictionaryBuilder, + tag_env: StringDictionaryBuilder, + tag_datacenter: StringDictionaryBuilder, + tag_region: StringDictionaryBuilder, + tag_host: StringDictionaryBuilder, + attributes: VariantArrayBuilder, + service_name: StringDictionaryBuilder, + resource_attributes: VariantArrayBuilder, +} + +impl ArrowMetricsBatchBuilder { + /// Creates a new builder with pre-allocated capacity. + pub fn with_capacity(capacity: usize) -> Self { + Self { + metric_name: StringDictionaryBuilder::new(), + metric_type: UInt8Builder::with_capacity(capacity), + metric_unit: StringBuilder::with_capacity(capacity, capacity * 8), + timestamp_secs: UInt64Builder::with_capacity(capacity), + start_timestamp_secs: UInt64Builder::with_capacity(capacity), + value: Float64Builder::with_capacity(capacity), + tag_service: StringDictionaryBuilder::new(), + tag_env: StringDictionaryBuilder::new(), + tag_datacenter: StringDictionaryBuilder::new(), + tag_region: StringDictionaryBuilder::new(), + tag_host: StringDictionaryBuilder::new(), + attributes: VariantArrayBuilder::new(capacity), + service_name: StringDictionaryBuilder::new(), + resource_attributes: VariantArrayBuilder::new(capacity), + } + } + + /// Appends a MetricDataPoint to the batch. + pub fn append(&mut self, data_point: &MetricDataPoint) { + self.metric_name.append_value(&data_point.metric_name); + self.metric_type.append_value(data_point.metric_type as u8); + + match &data_point.metric_unit { + Some(unit) => self.metric_unit.append_value(unit), + None => self.metric_unit.append_null(), + } + + self.timestamp_secs.append_value(data_point.timestamp_secs); + match data_point.start_timestamp_secs { + Some(ts) => self.start_timestamp_secs.append_value(ts), + None => self.start_timestamp_secs.append_null(), + } + self.value.append_value(data_point.value); + + append_optional_dict(&mut self.tag_service, &data_point.tag_service); + append_optional_dict(&mut self.tag_env, &data_point.tag_env); + append_optional_dict(&mut self.tag_datacenter, &data_point.tag_datacenter); + append_optional_dict(&mut self.tag_region, &data_point.tag_region); + append_optional_dict(&mut self.tag_host, &data_point.tag_host); + + if data_point.attributes.is_empty() { + self.attributes.append_null(); + } else { + append_variant_object(&mut self.attributes, &data_point.attributes); + } + + self.service_name.append_value(&data_point.service_name); + + if data_point.resource_attributes.is_empty() { + self.resource_attributes.append_null(); + } else { + append_variant_object( + &mut self.resource_attributes, + &data_point.resource_attributes, + ); + } + } + + /// Finalizes and returns the RecordBatch. + pub fn finish(mut self) -> RecordBatch { + // Build variant arrays and convert to ArrayRef + let attributes_array = self.attributes.build(); + let resource_attributes_array = self.resource_attributes.build(); + + let arrays: Vec = vec![ + Arc::new(self.metric_name.finish()), + Arc::new(self.metric_type.finish()), + Arc::new(self.metric_unit.finish()), + Arc::new(self.timestamp_secs.finish()), + Arc::new(self.start_timestamp_secs.finish()), + Arc::new(self.value.finish()), + Arc::new(self.tag_service.finish()), + Arc::new(self.tag_env.finish()), + Arc::new(self.tag_datacenter.finish()), + Arc::new(self.tag_region.finish()), + Arc::new(self.tag_host.finish()), + ArrayRef::from(attributes_array), + Arc::new(self.service_name.finish()), + ArrayRef::from(resource_attributes_array), + ]; + + RecordBatch::try_new(Arc::new(metrics_arrow_schema()), arrays) + .expect("record batch should match Arrow schema") + } + + /// Returns the number of rows appended so far. + pub fn len(&self) -> usize { + self.timestamp_secs.len() + } + + /// Returns true if no rows have been appended. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// Helper to append optional string values to dictionary builder. +fn append_optional_dict(builder: &mut StringDictionaryBuilder, value: &Option) { + match value { + Some(s) => builder.append_value(s), + None => builder.append_null(), + } +} + +/// Helper to append a HashMap as a VARIANT object to the builder. +fn append_variant_object( + builder: &mut VariantArrayBuilder, + map: &std::collections::HashMap, +) { + // Use a macro-like approach with fold to build the object + // We need to chain with_field calls which consume and return the builder + let obj_builder = builder.new_object(); + + // Build object by folding over the map entries + let final_builder = map.iter().fold(obj_builder, |b, (key, value)| { + match value { + serde_json::Value::Null => b.with_field(key.as_str(), ()), + serde_json::Value::Bool(v) => b.with_field(key.as_str(), *v), + serde_json::Value::Number(n) => { + if let Some(i) = n.as_i64() { + b.with_field(key.as_str(), i) + } else if let Some(f) = n.as_f64() { + b.with_field(key.as_str(), f) + } else { + b.with_field(key.as_str(), ()) + } + } + serde_json::Value::String(s) => b.with_field(key.as_str(), s.as_str()), + serde_json::Value::Array(arr) => { + // For arrays, serialize to JSON string as fallback + let json_str = serde_json::to_string(arr).unwrap_or_default(); + b.with_field(key.as_str(), json_str.as_str()) + } + serde_json::Value::Object(obj) => { + // For nested objects, serialize to JSON string as fallback + let json_str = serde_json::to_string(obj).unwrap_or_default(); + b.with_field(key.as_str(), json_str.as_str()) + } + } + }); + + final_builder.finish(); +} + +/// Error type for Arrow IPC operations. +#[derive(Debug, thiserror::Error)] +pub enum ArrowIpcError { + #[error("Arrow IPC error: {0}")] + Arrow(#[from] arrow::error::ArrowError), + #[error("Expected exactly 1 RecordBatch in IPC stream, got {0}")] + UnexpectedBatchCount(usize), +} + +/// Serializes a RecordBatch to Arrow IPC stream format. +pub fn record_batch_to_ipc(batch: &RecordBatch) -> Result, ArrowIpcError> { + let mut buffer = Vec::new(); + { + let mut writer = StreamWriter::try_new(&mut buffer, &batch.schema())?; + writer.write(batch)?; + writer.finish()?; + } + Ok(buffer) +} + +/// Deserializes Arrow IPC stream format to a RecordBatch. +pub fn ipc_to_record_batch(ipc_bytes: &[u8]) -> Result { + let cursor = Cursor::new(ipc_bytes); + let reader = StreamReader::try_new(cursor, None)?; + + let batches: Vec = reader.collect::, _>>()?; + + if batches.len() != 1 { + return Err(ArrowIpcError::UnexpectedBatchCount(batches.len())); + } + + let batch = batches + .into_iter() + .next() + .expect("batches should contain exactly one element"); + Ok(batch) +} + +/// Converts Arrow IPC bytes to JSON objects (one per row). +/// +/// Returns a Vec of (json_value, estimated_bytes) tuples. +pub fn ipc_to_json_values( + ipc_bytes: &[u8], + total_bytes: usize, +) -> Result, ArrowIpcError> { + use arrow::array::{Array, AsArray}; + use arrow::datatypes::DataType; + + let record_batch = ipc_to_record_batch(ipc_bytes)?; + let num_rows = record_batch.num_rows(); + let bytes_per_row = total_bytes / num_rows.max(1); + let schema = record_batch.schema(); + + let mut results = Vec::with_capacity(num_rows); + + for row_idx in 0..num_rows { + let mut json_obj = serde_json::Map::with_capacity(schema.fields().len()); + + for (col_idx, field) in schema.fields().iter().enumerate() { + let column = record_batch.column(col_idx); + let field_name = field.name().clone(); + + if column.is_null(row_idx) { + continue; + } + + let value = match field.data_type() { + DataType::Utf8 => { + let arr = column.as_string::(); + serde_json::Value::String(arr.value(row_idx).to_string()) + } + DataType::Dictionary(_, value_type) if **value_type == DataType::Utf8 => { + // Dictionary-encoded string + let dict_arr = column + .as_any() + .downcast_ref::>( + ) + .unwrap(); + let values = dict_arr.values().as_string::(); + let key = dict_arr.keys().value(row_idx) as usize; + serde_json::Value::String(values.value(key).to_string()) + } + DataType::UInt8 => { + // Special handling for metric_type enum + let arr = column.as_primitive::(); + let val = arr.value(row_idx); + if field_name == "metric_type" { + // Convert metric_type u8 to string + let type_str = MetricType::from_u8(val) + .map(|mt| mt.as_str()) + .unwrap_or("unknown"); + serde_json::Value::String(type_str.to_string()) + } else { + serde_json::Value::Number(val.into()) + } + } + DataType::UInt64 => { + let arr = column.as_primitive::(); + serde_json::Value::Number(arr.value(row_idx).into()) + } + DataType::Float64 => { + let arr = column.as_primitive::(); + let val = arr.value(row_idx); + serde_json::Number::from_f64(val) + .map(serde_json::Value::Number) + .unwrap_or(serde_json::Value::Null) + } + _ => { + // For other types, skip + continue; + } + }; + + json_obj.insert(field_name, value); + } + + results.push((serde_json::Value::Object(json_obj), bytes_per_row)); + } + + Ok(results) +} + +/// Builder for creating DocBatchV2 with Arrow IPC format. +/// +/// Unlike JsonDocBatchV2Builder which serializes each document separately, +/// this builder takes a complete RecordBatch and serializes it as a single +/// Arrow IPC stream. The doc_uids track individual rows within the batch. +pub struct ArrowDocBatchV2Builder { + doc_uids: Vec, + ipc_buffer: Vec, +} + +impl ArrowDocBatchV2Builder { + /// Creates a new builder from a RecordBatch and corresponding doc UIDs. + /// + /// The number of doc_uids must equal the number of rows in the RecordBatch. + pub fn from_record_batch( + batch: &RecordBatch, + doc_uids: Vec, + ) -> Result { + assert_eq!( + batch.num_rows(), + doc_uids.len(), + "doc_uids length must match RecordBatch row count" + ); + + let ipc_buffer = record_batch_to_ipc(batch)?; + + Ok(Self { + doc_uids, + ipc_buffer, + }) + } + + /// Builds the DocBatchV2 with Arrow IPC format. + /// + /// The doc_buffer contains the entire Arrow IPC stream. + /// doc_lengths contains a single entry with the total IPC buffer size. + /// doc_format is set to ArrowIpc. + pub fn build(self) -> DocBatchV2 { + let buffer_len = self.ipc_buffer.len() as u32; + DocBatchV2 { + doc_uids: self.doc_uids, + doc_buffer: Bytes::from(self.ipc_buffer), + doc_lengths: vec![buffer_len], + doc_format: DocFormat::ArrowIpc as i32, + } + } + + /// Returns the number of documents (rows) in this batch. + pub fn num_docs(&self) -> usize { + self.doc_uids.len() + } + + /// Returns the size of the IPC buffer in bytes. + pub fn ipc_size(&self) -> usize { + self.ipc_buffer.len() + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use serde_json::Value as JsonValue; + + use super::*; + + fn make_test_data_point() -> MetricDataPoint { + MetricDataPoint { + metric_name: "cpu.usage".to_string(), + metric_type: MetricType::Gauge, + metric_unit: Some("%".to_string()), + timestamp_secs: 1704067200, + start_timestamp_secs: Some(1704067190), + value: 85.5, + tag_service: Some("api".to_string()), + tag_env: Some("prod".to_string()), + tag_datacenter: Some("us-east-1a".to_string()), + tag_region: Some("us-east-1".to_string()), + tag_host: Some("server-001".to_string()), + attributes: HashMap::from([( + "endpoint".to_string(), + JsonValue::String("/health".to_string()), + )]), + service_name: "api-service".to_string(), + resource_attributes: HashMap::from([( + "k8s.pod".to_string(), + JsonValue::String("pod-123".to_string()), + )]), + } + } + + #[test] + fn test_arrow_batch_builder_single_row() { + let dp = make_test_data_point(); + let mut builder = ArrowMetricsBatchBuilder::with_capacity(1); + builder.append(&dp); + + assert_eq!(builder.len(), 1); + assert!(!builder.is_empty()); + + let batch = builder.finish(); + assert_eq!(batch.num_rows(), 1); + assert_eq!(batch.num_columns(), 14); + } + + #[test] + fn test_arrow_batch_builder_multiple_rows() { + let mut builder = ArrowMetricsBatchBuilder::with_capacity(100); + + for i in 0..100 { + let dp = MetricDataPoint { + metric_name: "test.metric".to_string(), + metric_type: MetricType::Gauge, + metric_unit: None, + timestamp_secs: 1704067200 + i as u64, + start_timestamp_secs: None, + value: i as f64 * 0.1, + tag_service: Some(format!("service-{}", i % 10)), // 10 unique values + tag_env: Some("prod".to_string()), // 1 unique value + tag_datacenter: None, + tag_region: None, + tag_host: Some(format!("host-{}", i % 5)), // 5 unique values + attributes: HashMap::new(), + service_name: "test-service".to_string(), + resource_attributes: HashMap::new(), + }; + builder.append(&dp); + } + + assert_eq!(builder.len(), 100); + let batch = builder.finish(); + assert_eq!(batch.num_rows(), 100); + } + + #[test] + fn test_dictionary_encoding_deduplication() { + let mut builder = ArrowMetricsBatchBuilder::with_capacity(1000); + + // Create 1000 data points with only 10 unique service values + for i in 0..1000 { + let dp = MetricDataPoint { + metric_name: "test.metric".to_string(), + metric_type: MetricType::Gauge, + metric_unit: None, + timestamp_secs: 1704067200 + i as u64, + start_timestamp_secs: None, + value: i as f64, + tag_service: Some(format!("service-{}", i % 10)), + tag_env: Some("prod".to_string()), + tag_datacenter: Some(format!("dc-{}", i % 4)), + tag_region: None, + tag_host: None, + attributes: HashMap::new(), + service_name: format!("svc-{}", i % 5), + resource_attributes: HashMap::new(), + }; + builder.append(&dp); + } + + let batch = builder.finish(); + assert_eq!(batch.num_rows(), 1000); + + // Verify the batch was created successfully with dictionary encoding + // The dictionary arrays should have far fewer unique values than rows + let schema = batch.schema(); + + // Check that tag_service uses dictionary encoding + let tag_service_field = schema.field_with_name("tag_service").unwrap(); + assert!(matches!( + tag_service_field.data_type(), + DataType::Dictionary(_, _) + )); + } + + #[test] + fn test_null_handling() { + let dp = MetricDataPoint { + metric_name: "minimal.metric".to_string(), + metric_type: MetricType::Gauge, + metric_unit: None, // null + timestamp_secs: 1704067200, + start_timestamp_secs: None, // null + value: 0.0, + tag_service: None, // null + tag_env: None, // null + tag_datacenter: None, + tag_region: None, + tag_host: None, + attributes: HashMap::new(), // empty -> null + service_name: "unknown".to_string(), + resource_attributes: HashMap::new(), // empty -> null + }; + + let mut builder = ArrowMetricsBatchBuilder::with_capacity(1); + builder.append(&dp); + let batch = builder.finish(); + + assert_eq!(batch.num_rows(), 1); + // The batch should handle nulls correctly + } + + #[test] + fn test_schema_field_count() { + let schema = metrics_arrow_schema(); + assert_eq!(schema.fields().len(), 14); + + // Verify field names + let field_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect(); + assert!(field_names.contains(&"metric_name")); + assert!(field_names.contains(&"metric_type")); + assert!(field_names.contains(&"metric_unit")); + assert!(field_names.contains(&"timestamp_secs")); + assert!(field_names.contains(&"start_timestamp_secs")); + assert!(field_names.contains(&"value")); + assert!(field_names.contains(&"tag_service")); + assert!(field_names.contains(&"tag_env")); + assert!(field_names.contains(&"tag_datacenter")); + assert!(field_names.contains(&"tag_region")); + assert!(field_names.contains(&"tag_host")); + assert!(field_names.contains(&"attributes")); + assert!(field_names.contains(&"service_name")); + assert!(field_names.contains(&"resource_attributes")); + } + + #[test] + fn test_empty_builder() { + let builder = ArrowMetricsBatchBuilder::with_capacity(0); + assert!(builder.is_empty()); + assert_eq!(builder.len(), 0); + + let batch = builder.finish(); + assert_eq!(batch.num_rows(), 0); + } + + #[test] + fn test_ipc_round_trip() { + // Build a RecordBatch + let mut builder = ArrowMetricsBatchBuilder::with_capacity(10); + for i in 0..10 { + let dp = MetricDataPoint { + metric_name: "test.metric".to_string(), + metric_type: MetricType::Gauge, + metric_unit: None, + timestamp_secs: 1704067200 + i as u64, + start_timestamp_secs: None, + value: i as f64 * 0.1, + tag_service: Some(format!("service-{}", i % 3)), + tag_env: Some("prod".to_string()), + tag_datacenter: None, + tag_region: None, + tag_host: None, + attributes: HashMap::new(), + service_name: "test-service".to_string(), + resource_attributes: HashMap::new(), + }; + builder.append(&dp); + } + let original_batch = builder.finish(); + + // Serialize to IPC + let ipc_bytes = record_batch_to_ipc(&original_batch).unwrap(); + assert!(!ipc_bytes.is_empty()); + + // Deserialize back + let recovered_batch = ipc_to_record_batch(&ipc_bytes).unwrap(); + + // Verify + assert_eq!(recovered_batch.num_rows(), original_batch.num_rows()); + assert_eq!(recovered_batch.num_columns(), original_batch.num_columns()); + assert_eq!(recovered_batch.schema(), original_batch.schema()); + } + + #[test] + fn test_arrow_doc_batch_v2_builder() { + use quickwit_proto::ingest::DocFormat; + use quickwit_proto::types::DocUidGenerator; + + // Build a RecordBatch + let mut builder = ArrowMetricsBatchBuilder::with_capacity(5); + let mut doc_uid_generator = DocUidGenerator::default(); + let mut doc_uids = Vec::new(); + + for i in 0..5 { + let dp = MetricDataPoint { + metric_name: "test.metric".to_string(), + metric_type: MetricType::Gauge, + metric_unit: None, + timestamp_secs: 1704067200 + i as u64, + start_timestamp_secs: None, + value: i as f64, + tag_service: None, + tag_env: None, + tag_datacenter: None, + tag_region: None, + tag_host: None, + attributes: HashMap::new(), + service_name: "test".to_string(), + resource_attributes: HashMap::new(), + }; + builder.append(&dp); + doc_uids.push(doc_uid_generator.next_doc_uid()); + } + let batch = builder.finish(); + + // Build DocBatchV2 with Arrow IPC + let arrow_builder = + ArrowDocBatchV2Builder::from_record_batch(&batch, doc_uids.clone()).unwrap(); + assert_eq!(arrow_builder.num_docs(), 5); + + let doc_batch = arrow_builder.build(); + + // Verify DocBatchV2 properties + assert_eq!(doc_batch.doc_uids.len(), 5); + assert_eq!(doc_batch.doc_format, DocFormat::ArrowIpc as i32); + assert_eq!(doc_batch.doc_lengths.len(), 1); // Single IPC buffer + assert!(!doc_batch.doc_buffer.is_empty()); + + // Verify we can recover the RecordBatch + let recovered = ipc_to_record_batch(&doc_batch.doc_buffer).unwrap(); + assert_eq!(recovered.num_rows(), 5); + } +} diff --git a/quickwit/quickwit-opentelemetry/src/otlp/metrics.rs b/quickwit/quickwit-opentelemetry/src/otlp/metrics.rs index 0d841211b48..14d85e746b6 100644 --- a/quickwit/quickwit-opentelemetry/src/otlp/metrics.rs +++ b/quickwit/quickwit-opentelemetry/src/otlp/metrics.rs @@ -23,6 +23,7 @@ pub struct OtlpServiceMetrics { pub request_duration_seconds: HistogramVec<5>, pub ingested_log_records_total: IntCounterVec<4>, pub ingested_spans_total: IntCounterVec<4>, + pub ingested_data_points_total: IntCounterVec<4>, pub ingested_bytes_total: IntCounterVec<4>, } @@ -65,6 +66,13 @@ impl Default for OtlpServiceMetrics { &[], ["service", "index", "transport", "format"], ), + ingested_data_points_total: new_counter_vec( + "ingested_data_points_total", + "Number of metric data points ingested", + "otlp", + &[], + ["service", "index", "transport", "format"], + ), ingested_bytes_total: new_counter_vec( "ingested_bytes_total", "Number of bytes ingested", diff --git a/quickwit/quickwit-opentelemetry/src/otlp/mod.rs b/quickwit/quickwit-opentelemetry/src/otlp/mod.rs index 95060232805..67abd6c1d68 100644 --- a/quickwit/quickwit-opentelemetry/src/otlp/mod.rs +++ b/quickwit/quickwit-opentelemetry/src/otlp/mod.rs @@ -27,23 +27,29 @@ use quickwit_proto::opentelemetry::proto::common::v1::{ }; use serde_json::{Number as JsonNumber, Value as JsonValue}; +mod arrow_metrics; mod logs; mod metrics; -mod span_id; +mod otel_metrics; #[cfg(any(test, feature = "testsuite"))] mod test_utils; -mod trace_id; mod traces; +pub use arrow_metrics::{ + ArrowDocBatchV2Builder, ArrowIpcError, ArrowMetricsBatchBuilder, ipc_to_json_values, + ipc_to_record_batch, metrics_arrow_schema, record_batch_to_ipc, +}; pub use logs::{ JsonLogIterator, OTEL_LOGS_INDEX_ID, OtlpGrpcLogsService, OtlpLogsError, parse_otlp_logs_json, parse_otlp_logs_protobuf, }; -pub use span_id::{SpanId, TryFromSpanIdError}; +pub use otel_metrics::{ + MetricDataPoint, MetricType, OTEL_METRICS_INDEX_ID, OtlpGrpcMetricsService, OtlpMetricsError, +}; +pub use quickwit_proto::search::{SpanId, TraceId, TryFromSpanIdError, TryFromTraceIdError}; #[cfg(any(test, feature = "testsuite"))] pub use test_utils::make_resource_spans_for_test; use tonic::Status; -pub use trace_id::{TraceId, TryFromTraceIdError}; pub use traces::{ Event, JsonSpanIterator, Link, OTEL_TRACES_INDEX_ID, OTEL_TRACES_INDEX_ID_PATTERN, OtlpGrpcTracesService, OtlpTracesError, Span, SpanFingerprint, SpanKind, SpanStatus, @@ -53,6 +59,7 @@ pub use traces::{ #[derive(Debug, Clone, Copy)] pub enum OtelSignal { Logs, + Metrics, Traces, } @@ -60,6 +67,7 @@ impl OtelSignal { pub fn header_name(&self) -> &'static str { match self { OtelSignal::Logs => "qw-otel-logs-index", + OtelSignal::Metrics => "qw-otel-metrics-index", OtelSignal::Traces => "qw-otel-traces-index", } } @@ -67,6 +75,7 @@ impl OtelSignal { pub fn default_index_id(&self) -> &'static str { match self { OtelSignal::Logs => OTEL_LOGS_INDEX_ID, + OtelSignal::Metrics => OTEL_METRICS_INDEX_ID, OtelSignal::Traces => OTEL_TRACES_INDEX_ID, } } @@ -84,18 +93,6 @@ impl From for tonic::Status { } } -impl From for tonic::Status { - fn from(error: TryFromSpanIdError) -> Self { - tonic::Status::invalid_argument(error.to_string()) - } -} - -impl From for tonic::Status { - fn from(error: TryFromTraceIdError) -> Self { - tonic::Status::invalid_argument(error.to_string()) - } -} - // An `Attribute` is a key-value pair, which MUST have the following properties: // - The attribute key MUST be a non-null and non-empty string. // - The attribute value is either: @@ -229,7 +226,7 @@ pub(crate) fn extract_otel_index_id_from_metadata( Ok(index_id.to_string()) } -async fn ingest_doc_batch_v2( +pub async fn ingest_doc_batch_v2( ingest_router: IngestRouterServiceClient, index_id: String, doc_batch: DocBatchV2, diff --git a/quickwit/quickwit-opentelemetry/src/otlp/otel_metrics.rs b/quickwit/quickwit-opentelemetry/src/otlp/otel_metrics.rs new file mode 100644 index 00000000000..bcb67c5fadb --- /dev/null +++ b/quickwit/quickwit-opentelemetry/src/otlp/otel_metrics.rs @@ -0,0 +1,924 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; + +use async_trait::async_trait; +use quickwit_common::thread_pool::run_cpu_intensive; +use quickwit_common::uri::Uri; +use quickwit_config::{ConfigFormat, IndexConfig, load_index_config_from_user_config}; +use quickwit_ingest::CommitType; +use quickwit_proto::ingest::DocBatchV2; +use quickwit_proto::ingest::router::IngestRouterServiceClient; +use quickwit_proto::opentelemetry::proto::collector::metrics::v1::metrics_service_server::MetricsService; +use quickwit_proto::opentelemetry::proto::collector::metrics::v1::{ + ExportMetricsPartialSuccess, ExportMetricsServiceRequest, ExportMetricsServiceResponse, +}; +use quickwit_proto::opentelemetry::proto::metrics::v1::{ + AggregationTemporality, Metric, NumberDataPoint, metric, +}; +use quickwit_proto::types::{DocUidGenerator, IndexId}; +use serde_json::Value as JsonValue; +use tonic::{Request, Response, Status}; +use tracing::field::Empty; +use tracing::{Span as RuntimeSpan, error, instrument, warn}; + +use super::arrow_metrics::{ArrowDocBatchV2Builder, ArrowMetricsBatchBuilder}; +use super::{OtelSignal, extract_otel_index_id_from_metadata, ingest_doc_batch_v2}; +use crate::otlp::extract_attributes; +use crate::otlp::metrics::OTLP_SERVICE_METRICS; + +pub const OTEL_METRICS_INDEX_ID: &str = "otel-metrics-v0_9"; + +/// Minimal index config for the metrics index. +/// +/// The parquet pipeline bypasses Tantivy entirely — it writes Arrow/Parquet data +/// directly and queries via DataFusion. The doc mapping is unused; this config +/// exists only so the index can be registered in the metastore for source +/// assignment and lifecycle management. +const OTEL_METRICS_INDEX_CONFIG: &str = r#" +version: 0.8 + +index_id: ${INDEX_ID} + +doc_mapping: + mode: dynamic + +indexing_settings: + commit_timeout_secs: 15 + +search_settings: + default_search_fields: [] +"#; + +/// Metric type enumeration for efficient storage. +/// +/// Stored as UInt8 in Arrow for compact representation. +/// Only a few metric types exist, so this is more efficient than +/// dictionary-encoded strings. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[repr(u8)] +pub enum MetricType { + /// Gauge metric - instantaneous value + Gauge = 0, + /// Sum metric - cumulative or delta sum + Sum = 1, + /// Histogram metric (not yet fully supported) + Histogram = 2, + /// Exponential histogram metric (not yet fully supported) + ExponentialHistogram = 3, + /// Summary metric (not yet fully supported) + Summary = 4, +} + +impl MetricType { + /// Converts from u8 value. + pub fn from_u8(value: u8) -> Option { + match value { + 0 => Some(Self::Gauge), + 1 => Some(Self::Sum), + 2 => Some(Self::Histogram), + 3 => Some(Self::ExponentialHistogram), + 4 => Some(Self::Summary), + _ => None, + } + } + + /// Gets the string representation for JSON/Tantivy. + pub fn as_str(&self) -> &'static str { + match self { + Self::Gauge => "gauge", + Self::Sum => "sum", + Self::Histogram => "histogram", + Self::ExponentialHistogram => "exponential_histogram", + Self::Summary => "summary", + } + } +} + +impl std::fmt::Display for MetricType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.as_str()) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum OtlpMetricsError { + #[error("failed to deserialize JSON metric records: `{0}`")] + Json(#[from] serde_json::Error), + #[error("failed to deserialize Protobuf metric records: `{0}`")] + Protobuf(#[from] prost::DecodeError), + #[error("invalid argument: {0}")] + InvalidArgument(String), +} + +impl From for tonic::Status { + fn from(err: OtlpMetricsError) -> Self { + match &err { + OtlpMetricsError::InvalidArgument(_) => { + tonic::Status::invalid_argument(err.to_string()) + } + _ => tonic::Status::internal(err.to_string()), + } + } +} + +/// Represents a single metric data point document +#[derive(Debug, Clone)] +pub struct MetricDataPoint { + // Metric identity + pub metric_name: String, + pub metric_type: MetricType, + pub metric_unit: Option, + + // Timestamps (seconds granularity) + pub timestamp_secs: u64, + pub start_timestamp_secs: Option, + + // Value (f64 only) + pub value: f64, + + // Explicit tag columns + pub tag_service: Option, + pub tag_env: Option, + pub tag_datacenter: Option, + pub tag_region: Option, + pub tag_host: Option, + + // Dynamic tags (remaining attributes) + pub attributes: HashMap, + + // Resource metadata + pub service_name: String, + pub resource_attributes: HashMap, +} + +struct ExplicitTags { + service: Option, + env: Option, + datacenter: Option, + region: Option, + host: Option, +} + +fn extract_string_tag(attributes: &mut HashMap, key: &str) -> Option { + attributes.remove(key).and_then(|v| match v { + JsonValue::String(s) => Some(s), + _ => None, + }) +} + +fn extract_explicit_tags(attributes: &mut HashMap) -> ExplicitTags { + ExplicitTags { + service: extract_string_tag(attributes, "service"), + env: extract_string_tag(attributes, "env"), + datacenter: extract_string_tag(attributes, "datacenter"), + region: extract_string_tag(attributes, "region"), + host: extract_string_tag(attributes, "host"), + } +} + +/// Convert nanoseconds to seconds +fn nanos_to_secs(nanos: u64) -> u64 { + nanos / 1_000_000_000 +} + +struct ParsedMetrics { + doc_batch: DocBatchV2, + num_data_points: u64, + num_parse_errors: u64, + error_message: String, +} + +#[derive(Clone)] +pub struct OtlpGrpcMetricsService { + ingest_router: IngestRouterServiceClient, +} + +impl OtlpGrpcMetricsService { + pub fn new(ingest_router: IngestRouterServiceClient) -> Self { + Self { ingest_router } + } + + pub fn index_config(default_index_root_uri: &Uri) -> anyhow::Result { + let index_config_str = + OTEL_METRICS_INDEX_CONFIG.replace("${INDEX_ID}", OTEL_METRICS_INDEX_ID); + let index_config = load_index_config_from_user_config( + ConfigFormat::Yaml, + index_config_str.as_bytes(), + default_index_root_uri, + )?; + Ok(index_config) + } + + async fn export_inner( + &mut self, + request: ExportMetricsServiceRequest, + index_id: IndexId, + labels: [&str; 4], + ) -> Result { + let ParsedMetrics { + doc_batch, + num_data_points, + num_parse_errors, + error_message, + } = run_cpu_intensive({ + let parent_span = RuntimeSpan::current(); + || Self::parse_metrics(request, parent_span) + }) + .await + .map_err(|join_error| { + error!(error=?join_error, "failed to parse metric records"); + Status::internal("failed to parse metric records") + })??; + + if num_data_points == num_parse_errors { + return Err(tonic::Status::internal(error_message)); + } + + let num_bytes = doc_batch.num_bytes() as u64; + self.store_metrics(index_id, doc_batch).await?; + + OTLP_SERVICE_METRICS + .ingested_data_points_total + .with_label_values(labels) + .inc_by(num_data_points - num_parse_errors); + OTLP_SERVICE_METRICS + .ingested_bytes_total + .with_label_values(labels) + .inc_by(num_bytes); + + let response = ExportMetricsServiceResponse { + partial_success: Some(ExportMetricsPartialSuccess { + rejected_data_points: num_parse_errors as i64, + error_message, + }), + }; + Ok(response) + } + + #[instrument(skip_all, parent = parent_span, fields(num_data_points = Empty, num_bytes = Empty, num_parse_errors = Empty))] + #[allow(clippy::result_large_err)] + fn parse_metrics( + request: ExportMetricsServiceRequest, + parent_span: RuntimeSpan, + ) -> tonic::Result { + let data_points = parse_otlp_metrics(request)?; + let num_data_points = data_points.len() as u64; + + // Build Arrow RecordBatch from data points + let mut arrow_builder = ArrowMetricsBatchBuilder::with_capacity(num_data_points as usize); + let mut doc_uid_generator = DocUidGenerator::default(); + let mut doc_uids = Vec::with_capacity(num_data_points as usize); + + for data_point in &data_points { + arrow_builder.append(data_point); + doc_uids.push(doc_uid_generator.next_doc_uid()); + } + + let record_batch = arrow_builder.finish(); + + // Serialize to Arrow IPC format + let doc_batch = ArrowDocBatchV2Builder::from_record_batch(&record_batch, doc_uids) + .map_err(|e| { + error!(error=?e, "failed to serialize Arrow IPC"); + tonic::Status::internal(format!("failed to serialize Arrow IPC: {e}")) + })? + .build(); + + let current_span = RuntimeSpan::current(); + current_span.record("num_data_points", num_data_points); + current_span.record("num_bytes", doc_batch.num_bytes()); + current_span.record("num_parse_errors", 0u64); + + let parsed_metrics = ParsedMetrics { + doc_batch, + num_data_points, + num_parse_errors: 0, + error_message: String::new(), + }; + Ok(parsed_metrics) + } + + #[instrument(skip_all, fields(num_bytes = doc_batch.num_bytes()))] + async fn store_metrics( + &mut self, + index_id: String, + doc_batch: DocBatchV2, + ) -> Result<(), tonic::Status> { + ingest_doc_batch_v2( + self.ingest_router.clone(), + index_id, + doc_batch, + CommitType::Auto, + ) + .await?; + Ok(()) + } + + async fn export_instrumented( + &mut self, + request: ExportMetricsServiceRequest, + index_id: IndexId, + ) -> Result { + let start = std::time::Instant::now(); + + let labels = ["metrics", &index_id, "grpc", "protobuf"]; + + OTLP_SERVICE_METRICS + .requests_total + .with_label_values(labels) + .inc(); + + let (export_res, is_error) = + match self.export_inner(request, index_id.clone(), labels).await { + ok @ Ok(_) => (ok, "false"), + err @ Err(_) => { + OTLP_SERVICE_METRICS + .request_errors_total + .with_label_values(labels) + .inc(); + (err, "true") + } + }; + + let elapsed = start.elapsed().as_secs_f64(); + let labels = ["metrics", &index_id, "grpc", "protobuf", is_error]; + OTLP_SERVICE_METRICS + .request_duration_seconds + .with_label_values(labels) + .observe(elapsed); + + export_res + } +} + +#[async_trait] +impl MetricsService for OtlpGrpcMetricsService { + #[instrument(name = "ingest_metrics", skip_all)] + async fn export( + &self, + request: Request, + ) -> Result, Status> { + let index_id = + extract_otel_index_id_from_metadata(request.metadata(), OtelSignal::Metrics)?; + let request = request.into_inner(); + self.clone() + .export_instrumented(request, index_id) + .await + .map(Response::new) + } +} + +fn parse_otlp_metrics( + request: ExportMetricsServiceRequest, +) -> Result, OtlpMetricsError> { + let mut data_points = Vec::new(); + + for resource_metrics in request.resource_metrics { + let mut resource_attributes = extract_attributes( + resource_metrics + .resource + .clone() + .map(|rsrc| rsrc.attributes) + .unwrap_or_default(), + ); + let service_name = match resource_attributes.remove("service.name") { + Some(JsonValue::String(value)) => value, + _ => "unknown_service".to_string(), + }; + + for scope_metrics in resource_metrics.scope_metrics { + for metric in scope_metrics.metrics { + parse_metric( + &metric, + &service_name, + &resource_attributes, + &mut data_points, + )?; + } + } + } + + Ok(data_points) +} + +fn parse_metric( + metric: &Metric, + service_name: &str, + resource_attributes: &HashMap, + data_points: &mut Vec, +) -> Result<(), OtlpMetricsError> { + let metric_name = metric.name.clone(); + let metric_unit = if metric.unit.is_empty() { + None + } else { + Some(metric.unit.clone()) + }; + + match &metric.data { + Some(metric::Data::Gauge(gauge)) => { + for dp in &gauge.data_points { + let data_point = create_number_data_point( + &metric_name, + MetricType::Gauge, + &metric_unit, + dp, + service_name, + resource_attributes, + )?; + data_points.push(data_point); + } + } + Some(metric::Data::Sum(sum)) => { + // Only support DELTA temporality + if sum.aggregation_temporality == AggregationTemporality::Cumulative as i32 { + return Err(OtlpMetricsError::InvalidArgument( + "cumulative aggregation temporality is not supported, only delta is supported" + .to_string(), + )); + } + + for dp in &sum.data_points { + let data_point = create_number_data_point( + &metric_name, + MetricType::Sum, + &metric_unit, + dp, + service_name, + resource_attributes, + )?; + data_points.push(data_point); + } + } + Some(metric::Data::Histogram(_)) => { + warn!("histogram metrics not supported, skipping"); + } + Some(metric::Data::ExponentialHistogram(_)) => { + warn!("exponential histogram metrics not supported, skipping"); + } + Some(metric::Data::Summary(_)) => { + warn!("summary metrics not supported, skipping"); + } + None => { + warn!("metric has no data, skipping"); + } + } + + Ok(()) +} + +fn create_number_data_point( + metric_name: &str, + metric_type: MetricType, + metric_unit: &Option, + dp: &NumberDataPoint, + service_name: &str, + resource_attributes: &HashMap, +) -> Result { + // Extract value as f64 + let value = match &dp.value { + Some( + quickwit_proto::opentelemetry::proto::metrics::v1::number_data_point::Value::AsDouble( + v, + ), + ) => *v, + Some( + quickwit_proto::opentelemetry::proto::metrics::v1::number_data_point::Value::AsInt(v), + ) => *v as f64, + None => 0.0, + }; + + // Extract attributes and explicit tags + let mut attributes = extract_attributes(dp.attributes.clone()); + let explicit_tags = extract_explicit_tags(&mut attributes); + + // Convert timestamps to seconds + let timestamp_secs = nanos_to_secs(dp.time_unix_nano); + let start_timestamp_secs = if dp.start_time_unix_nano != 0 { + Some(nanos_to_secs(dp.start_time_unix_nano)) + } else { + None + }; + + Ok(MetricDataPoint { + metric_name: metric_name.to_string(), + metric_type, + metric_unit: metric_unit.clone(), + timestamp_secs, + start_timestamp_secs, + value, + tag_service: explicit_tags.service, + tag_env: explicit_tags.env, + tag_datacenter: explicit_tags.datacenter, + tag_region: explicit_tags.region, + tag_host: explicit_tags.host, + attributes, + service_name: service_name.to_string(), + resource_attributes: resource_attributes.clone(), + }) +} + +#[cfg(test)] +mod tests { + use quickwit_metastore::{CreateIndexRequestExt, metastore_for_test}; + use quickwit_proto::metastore::{CreateIndexRequest, MetastoreService}; + + use super::*; + + #[test] + fn test_index_config_is_valid() { + let index_config = + OtlpGrpcMetricsService::index_config(&Uri::for_test("ram:///indexes")).unwrap(); + assert_eq!(index_config.index_id, OTEL_METRICS_INDEX_ID); + } + + #[tokio::test] + async fn test_create_index() { + let metastore = metastore_for_test(); + let index_config = + OtlpGrpcMetricsService::index_config(&Uri::for_test("ram:///indexes")).unwrap(); + let create_index_request = + CreateIndexRequest::try_from_index_config(&index_config).unwrap(); + metastore.create_index(create_index_request).await.unwrap(); + } + + #[test] + fn test_nanos_to_secs() { + assert_eq!(nanos_to_secs(0), 0); + assert_eq!(nanos_to_secs(1_000_000_000), 1); + assert_eq!(nanos_to_secs(1_500_000_000), 1); + assert_eq!(nanos_to_secs(2_000_000_000), 2); + } + + #[test] + fn test_extract_explicit_tags() { + let mut attributes = HashMap::from([ + ("service".to_string(), JsonValue::String("api".to_string())), + ("env".to_string(), JsonValue::String("prod".to_string())), + ( + "datacenter".to_string(), + JsonValue::String("us-east".to_string()), + ), + ( + "region".to_string(), + JsonValue::String("us-east-1".to_string()), + ), + ( + "host".to_string(), + JsonValue::String("server-1".to_string()), + ), + ( + "custom_tag".to_string(), + JsonValue::String("custom_value".to_string()), + ), + ]); + + let explicit_tags = extract_explicit_tags(&mut attributes); + + assert_eq!(explicit_tags.service, Some("api".to_string())); + assert_eq!(explicit_tags.env, Some("prod".to_string())); + assert_eq!(explicit_tags.datacenter, Some("us-east".to_string())); + assert_eq!(explicit_tags.region, Some("us-east-1".to_string())); + assert_eq!(explicit_tags.host, Some("server-1".to_string())); + + // custom_tag should remain in attributes + assert_eq!(attributes.len(), 1); + assert!(attributes.contains_key("custom_tag")); + } + + fn make_test_gauge_request() -> ExportMetricsServiceRequest { + use quickwit_proto::opentelemetry::proto::common::v1::{AnyValue, KeyValue, any_value}; + use quickwit_proto::opentelemetry::proto::metrics::v1::{ + Gauge, ResourceMetrics, ScopeMetrics, number_data_point, + }; + use quickwit_proto::opentelemetry::proto::resource::v1::Resource; + + ExportMetricsServiceRequest { + resource_metrics: vec![ResourceMetrics { + resource: Some(Resource { + attributes: vec![KeyValue { + key: "service.name".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue("test-service".to_string())), + }), + }], + dropped_attributes_count: 0, + }), + scope_metrics: vec![ScopeMetrics { + scope: None, + metrics: vec![Metric { + name: "cpu.usage".to_string(), + description: "CPU usage percentage".to_string(), + unit: "%".to_string(), + data: Some(metric::Data::Gauge(Gauge { + data_points: vec![NumberDataPoint { + attributes: vec![ + KeyValue { + key: "service".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue( + "api".to_string(), + )), + }), + }, + KeyValue { + key: "env".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue( + "prod".to_string(), + )), + }), + }, + ], + start_time_unix_nano: 1_000_000_000, + time_unix_nano: 2_000_000_000, + exemplars: Vec::new(), + flags: 0, + value: Some(number_data_point::Value::AsDouble(85.5)), + }], + })), + }], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + } + } + + fn make_test_sum_delta_request() -> ExportMetricsServiceRequest { + use quickwit_proto::opentelemetry::proto::common::v1::{AnyValue, KeyValue, any_value}; + use quickwit_proto::opentelemetry::proto::metrics::v1::{ + ResourceMetrics, ScopeMetrics, Sum, number_data_point, + }; + use quickwit_proto::opentelemetry::proto::resource::v1::Resource; + + ExportMetricsServiceRequest { + resource_metrics: vec![ResourceMetrics { + resource: Some(Resource { + attributes: vec![KeyValue { + key: "service.name".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue( + "counter-service".to_string(), + )), + }), + }], + dropped_attributes_count: 0, + }), + scope_metrics: vec![ScopeMetrics { + scope: None, + metrics: vec![Metric { + name: "http.requests".to_string(), + description: "HTTP request count".to_string(), + unit: "1".to_string(), + data: Some(metric::Data::Sum(Sum { + data_points: vec![NumberDataPoint { + attributes: vec![KeyValue { + key: "host".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue( + "server-1".to_string(), + )), + }), + }], + start_time_unix_nano: 1_000_000_000, + time_unix_nano: 2_000_000_000, + exemplars: Vec::new(), + flags: 0, + value: Some(number_data_point::Value::AsInt(100)), + }], + aggregation_temporality: AggregationTemporality::Delta as i32, + is_monotonic: true, + })), + }], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + } + } + + fn make_test_sum_cumulative_request() -> ExportMetricsServiceRequest { + use quickwit_proto::opentelemetry::proto::metrics::v1::{ + ResourceMetrics, ScopeMetrics, Sum, number_data_point, + }; + + ExportMetricsServiceRequest { + resource_metrics: vec![ResourceMetrics { + resource: None, + scope_metrics: vec![ScopeMetrics { + scope: None, + metrics: vec![Metric { + name: "cumulative.counter".to_string(), + description: String::new(), + unit: String::new(), + data: Some(metric::Data::Sum(Sum { + data_points: vec![NumberDataPoint { + attributes: Vec::new(), + start_time_unix_nano: 0, + time_unix_nano: 1_000_000_000, + exemplars: Vec::new(), + flags: 0, + value: Some(number_data_point::Value::AsDouble(50.0)), + }], + aggregation_temporality: AggregationTemporality::Cumulative as i32, + is_monotonic: true, + })), + }], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + } + } + + #[test] + fn test_parse_gauge_metrics() { + let request = make_test_gauge_request(); + let data_points = parse_otlp_metrics(request).unwrap(); + + assert_eq!(data_points.len(), 1); + let dp = &data_points[0]; + assert_eq!(dp.metric_name, "cpu.usage"); + assert_eq!(dp.metric_type, MetricType::Gauge); + assert_eq!(dp.metric_unit, Some("%".to_string())); + assert_eq!(dp.timestamp_secs, 2); + assert_eq!(dp.start_timestamp_secs, Some(1)); + assert_eq!(dp.value, 85.5); + assert_eq!(dp.tag_service, Some("api".to_string())); + assert_eq!(dp.tag_env, Some("prod".to_string())); + assert_eq!(dp.service_name, "test-service"); + } + + #[test] + fn test_parse_sum_delta_metrics() { + let request = make_test_sum_delta_request(); + let data_points = parse_otlp_metrics(request).unwrap(); + + assert_eq!(data_points.len(), 1); + let dp = &data_points[0]; + assert_eq!(dp.metric_name, "http.requests"); + assert_eq!(dp.metric_type, MetricType::Sum); + assert_eq!(dp.metric_unit, Some("1".to_string())); + assert_eq!(dp.timestamp_secs, 2); + assert_eq!(dp.value, 100.0); // int converted to f64 + assert_eq!(dp.tag_host, Some("server-1".to_string())); + assert_eq!(dp.service_name, "counter-service"); + } + + #[test] + fn test_reject_cumulative_temporality() { + let request = make_test_sum_cumulative_request(); + let result = parse_otlp_metrics(request); + + assert!(result.is_err()); + match result.unwrap_err() { + OtlpMetricsError::InvalidArgument(_) => {} + err => panic!("unexpected error type: {:?}", err), + } + } + + /// Test parsing metrics with various attribute types + #[test] + fn test_parse_metrics_with_various_attribute_types() { + use quickwit_proto::opentelemetry::proto::common::v1::{AnyValue, KeyValue, any_value}; + use quickwit_proto::opentelemetry::proto::metrics::v1::{ + Gauge, ResourceMetrics, ScopeMetrics, number_data_point, + }; + use quickwit_proto::opentelemetry::proto::resource::v1::Resource; + + let request = ExportMetricsServiceRequest { + resource_metrics: vec![ResourceMetrics { + resource: Some(Resource { + attributes: vec![ + KeyValue { + key: "service.name".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue("test".to_string())), + }), + }, + KeyValue { + key: "int_attr".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::IntValue(42)), + }), + }, + KeyValue { + key: "bool_attr".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::BoolValue(true)), + }), + }, + KeyValue { + key: "double_attr".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::DoubleValue(std::f64::consts::PI)), + }), + }, + ], + dropped_attributes_count: 0, + }), + scope_metrics: vec![ScopeMetrics { + scope: None, + metrics: vec![Metric { + name: "test.metric".to_string(), + description: String::new(), + unit: String::new(), + data: Some(metric::Data::Gauge(Gauge { + data_points: vec![NumberDataPoint { + attributes: vec![KeyValue { + key: "string_tag".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue( + "value".to_string(), + )), + }), + }], + start_time_unix_nano: 0, + time_unix_nano: 1_000_000_000, + exemplars: Vec::new(), + flags: 0, + value: Some(number_data_point::Value::AsDouble(1.0)), + }], + })), + }], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + }; + + let data_points = parse_otlp_metrics(request).unwrap(); + assert_eq!(data_points.len(), 1); + + let dp = &data_points[0]; + assert_eq!(dp.service_name, "test"); + + // Verify resource attributes contain the non-service.name attributes + assert!(dp.resource_attributes.contains_key("int_attr")); + assert!(dp.resource_attributes.contains_key("bool_attr")); + assert!(dp.resource_attributes.contains_key("double_attr")); + + // Verify data point attributes + assert!(dp.attributes.contains_key("string_tag")); + } + + /// Test metrics with empty and missing values + #[test] + fn test_parse_metrics_with_empty_values() { + use quickwit_proto::opentelemetry::proto::metrics::v1::{ + Gauge, ResourceMetrics, ScopeMetrics, number_data_point, + }; + + let request = ExportMetricsServiceRequest { + resource_metrics: vec![ResourceMetrics { + resource: None, // No resource + scope_metrics: vec![ScopeMetrics { + scope: None, + metrics: vec![Metric { + name: "minimal.metric".to_string(), + description: String::new(), + unit: String::new(), // Empty unit + data: Some(metric::Data::Gauge(Gauge { + data_points: vec![NumberDataPoint { + attributes: Vec::new(), // No attributes + start_time_unix_nano: 0, // No start time + time_unix_nano: 1_000_000_000, + exemplars: Vec::new(), + flags: 0, + value: Some(number_data_point::Value::AsDouble(0.0)), + }], + })), + }], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + }; + + let data_points = parse_otlp_metrics(request).unwrap(); + assert_eq!(data_points.len(), 1); + + let dp = &data_points[0]; + assert_eq!(dp.metric_name, "minimal.metric"); + assert_eq!(dp.service_name, "unknown_service"); // Default value + assert!(dp.metric_unit.is_none()); + assert!(dp.start_timestamp_secs.is_none()); + assert!(dp.tag_service.is_none()); + assert!(dp.tag_env.is_none()); + assert!(dp.attributes.is_empty()); + assert!(dp.resource_attributes.is_empty()); + } +} diff --git a/quickwit/quickwit-parquet-engine/Cargo.toml b/quickwit/quickwit-parquet-engine/Cargo.toml new file mode 100644 index 00000000000..f886083eded --- /dev/null +++ b/quickwit/quickwit-parquet-engine/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "quickwit-parquet-engine" +description = "Quickwit Parquet Engine - DataFusion/Parquet based metrics storage" + +version.workspace = true +edition.workspace = true +homepage.workspace = true +documentation.workspace = true +repository.workspace = true +authors.workspace = true +license.workspace = true + +[dependencies] +arrow = { workspace = true } +parquet = { workspace = true } +quickwit-common = { workspace = true } +sea-query = { workspace = true, optional = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } +ulid = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true } + +[features] +postgres = ["sea-query"] +testsuite = [] diff --git a/quickwit/quickwit-parquet-engine/src/index/accumulator.rs b/quickwit/quickwit-parquet-engine/src/index/accumulator.rs new file mode 100644 index 00000000000..29065847136 --- /dev/null +++ b/quickwit/quickwit-parquet-engine/src/index/accumulator.rs @@ -0,0 +1,360 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Batch accumulator for producing splits from RecordBatches. + +use std::time::Instant; + +use arrow::compute::concat_batches; +use arrow::record_batch::RecordBatch; +use tracing::{debug, info}; + +use super::config::ParquetIndexingConfig; +use crate::metrics::PARQUET_ENGINE_METRICS; +use crate::schema::ParquetSchema; + +/// Error type for index operations. +#[derive(Debug, thiserror::Error)] +pub enum IndexingError { + /// Arrow error during RecordBatch concatenation. + #[error("Arrow error: {0}")] + Arrow(#[from] arrow::error::ArrowError), +} + +/// Accumulator that buffers RecordBatches and produces concatenated batches when thresholds +/// are exceeded. +/// +/// Batches are accumulated until either `max_rows` or `max_bytes` threshold is reached, +/// at which point they are concatenated and returned for downstream processing (writing to +/// Parquet by the ParquetPackager actor). +pub struct ParquetBatchAccumulator { + /// Configuration for accumulation thresholds. + config: ParquetIndexingConfig, + /// Metrics schema for concatenation. + schema: ParquetSchema, + /// Pending batches waiting to be flushed. + pending_batches: Vec, + /// Total rows in pending batches. + pending_rows: usize, + /// Estimated bytes in pending batches. + pending_bytes: usize, +} + +impl ParquetBatchAccumulator { + /// Creates a new ParquetBatchAccumulator. + /// + /// # Arguments + /// * `config` - Configuration for accumulation thresholds + pub fn new(config: ParquetIndexingConfig) -> Self { + let schema = ParquetSchema::new(); + + Self { + config, + schema, + pending_batches: Vec::new(), + pending_rows: 0, + pending_bytes: 0, + } + } + + /// Adds a RecordBatch to the accumulator. + /// + /// If thresholds are exceeded after adding the batch, concatenates all pending batches + /// and returns the combined batch. Returns None if the threshold has not been reached. + pub fn add_batch(&mut self, batch: RecordBatch) -> Result, IndexingError> { + let start = Instant::now(); + let batch_rows = batch.num_rows(); + let batch_bytes = estimate_batch_bytes(&batch); + + // Record index metrics + PARQUET_ENGINE_METRICS.index_batches_total.inc(); + PARQUET_ENGINE_METRICS + .index_rows_total + .inc_by(batch_rows as u64); + + self.pending_batches.push(batch); + self.pending_rows += batch_rows; + self.pending_bytes += batch_bytes; + + debug!( + batch_rows, + batch_bytes, + total_pending_rows = self.pending_rows, + total_pending_bytes = self.pending_bytes, + "Added batch to accumulator" + ); + + let flushed = if self.should_flush() { + info!( + pending_rows = self.pending_rows, + pending_bytes = self.pending_bytes, + max_rows = self.config.max_rows, + max_bytes = self.config.max_bytes, + "Threshold exceeded, triggering flush" + ); + self.flush_internal()? + } else { + None + }; + + // Record batch processing duration + PARQUET_ENGINE_METRICS + .index_batch_duration_seconds + .observe(start.elapsed().as_secs_f64()); + + Ok(flushed) + } + + /// Discard all pending data without producing output. + pub fn discard(&mut self) { + self.pending_batches.clear(); + self.pending_rows = 0; + self.pending_bytes = 0; + } + + /// Force flush all pending batches. + /// + /// Returns None if no pending data. + pub fn flush(&mut self) -> Result, IndexingError> { + self.flush_internal() + } + + /// Internal flush implementation. + fn flush_internal(&mut self) -> Result, IndexingError> { + if self.pending_batches.is_empty() { + return Ok(None); + } + + // Concatenate all pending batches into one + let combined = concat_batches(self.schema.arrow_schema(), self.pending_batches.iter())?; + + let num_rows = combined.num_rows(); + info!(num_rows, "Flushed accumulated batches"); + + // Reset state + self.pending_batches.clear(); + self.pending_rows = 0; + self.pending_bytes = 0; + + Ok(Some(combined)) + } + + /// Checks if pending data exceeds thresholds. + fn should_flush(&self) -> bool { + !self.pending_batches.is_empty() + && (self.pending_rows >= self.config.max_rows + || self.pending_bytes >= self.config.max_bytes) + } + + /// Returns current pending row count. + pub fn pending_rows(&self) -> usize { + self.pending_rows + } + + /// Returns current pending byte estimate. + pub fn pending_bytes(&self) -> usize { + self.pending_bytes + } + + /// Returns the number of pending batches. + pub fn pending_batch_count(&self) -> usize { + self.pending_batches.len() + } +} + +/// Estimate the memory size of a RecordBatch. +fn estimate_batch_bytes(batch: &RecordBatch) -> usize { + batch + .columns() + .iter() + .map(|col| col.get_array_memory_size()) + .sum() +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow::array::{ + ArrayRef, DictionaryArray, Float64Array, Int32Array, StringArray, UInt8Array, UInt64Array, + }; + use arrow::datatypes::Int32Type; + use parquet::variant::{VariantArrayBuilder, VariantBuilderExt}; + + use super::*; + + /// Create dictionary array for string fields with Int32 keys. + fn create_dict_array(values: &[&str]) -> ArrayRef { + let keys: Vec = (0..values.len()).map(|i| i as i32).collect(); + let string_array = StringArray::from(values.to_vec()); + Arc::new( + DictionaryArray::::try_new(Int32Array::from(keys), Arc::new(string_array)) + .unwrap(), + ) + } + + /// Create nullable dictionary array for optional string fields. + fn create_nullable_dict_array(values: &[Option<&str>]) -> ArrayRef { + let keys: Vec> = values + .iter() + .enumerate() + .map(|(i, v)| v.map(|_| i as i32)) + .collect(); + let string_values: Vec<&str> = values.iter().filter_map(|v| *v).collect(); + let string_array = StringArray::from(string_values); + Arc::new( + DictionaryArray::::try_new(Int32Array::from(keys), Arc::new(string_array)) + .unwrap(), + ) + } + + /// Create a VARIANT array for testing with specified number of rows. + fn create_variant_array(num_rows: usize, fields: Option<&[(&str, &str)]>) -> ArrayRef { + let mut builder = VariantArrayBuilder::new(num_rows); + for _ in 0..num_rows { + match fields { + Some(kv_pairs) => { + let mut obj = builder.new_object(); + for (key, value) in kv_pairs { + obj = obj.with_field(key, *value); + } + obj.finish(); + } + None => { + builder.append_null(); + } + } + } + ArrayRef::from(builder.build()) + } + + /// Create a test batch matching the metrics schema. + fn create_test_batch(num_rows: usize) -> RecordBatch { + let schema = ParquetSchema::new(); + + let metric_names: Vec<&str> = vec!["cpu.usage"; num_rows]; + let metric_name: ArrayRef = create_dict_array(&metric_names); + let metric_type: ArrayRef = Arc::new(UInt8Array::from(vec![0u8; num_rows])); + let metric_unit: ArrayRef = Arc::new(StringArray::from(vec![Some("bytes"); num_rows])); + let timestamps: Vec = (0..num_rows).map(|i| 100 + i as u64).collect(); + let timestamp_secs: ArrayRef = Arc::new(UInt64Array::from(timestamps)); + let start_timestamp_secs: ArrayRef = + Arc::new(UInt64Array::from(vec![None::; num_rows])); + let values: Vec = (0..num_rows).map(|i| 42.0 + i as f64).collect(); + let value: ArrayRef = Arc::new(Float64Array::from(values)); + let tag_service: ArrayRef = create_nullable_dict_array(&vec![Some("web"); num_rows]); + let tag_env: ArrayRef = create_nullable_dict_array(&vec![Some("prod"); num_rows]); + let tag_datacenter: ArrayRef = + create_nullable_dict_array(&vec![Some("us-east-1"); num_rows]); + let tag_region: ArrayRef = create_nullable_dict_array(&vec![None; num_rows]); + let tag_host: ArrayRef = create_nullable_dict_array(&vec![Some("host-001"); num_rows]); + let attributes: ArrayRef = create_variant_array(num_rows, None); + let service_name: ArrayRef = create_dict_array(&vec!["my-service"; num_rows]); + let resource_attributes: ArrayRef = create_variant_array(num_rows, None); + + RecordBatch::try_new( + schema.arrow_schema().clone(), + vec![ + metric_name, + metric_type, + metric_unit, + timestamp_secs, + start_timestamp_secs, + value, + tag_service, + tag_env, + tag_datacenter, + tag_region, + tag_host, + attributes, + service_name, + resource_attributes, + ], + ) + .unwrap() + } + + #[test] + fn test_accumulator_below_threshold() { + let config = ParquetIndexingConfig::default().with_max_rows(1000); + let mut accumulator = ParquetBatchAccumulator::new(config); + + let batch = create_test_batch(100); + let flushed = accumulator.add_batch(batch).unwrap(); + + assert!(flushed.is_none()); + assert_eq!(accumulator.pending_rows(), 100); + assert_eq!(accumulator.pending_batch_count(), 1); + } + + #[test] + fn test_accumulator_row_threshold() { + let config = ParquetIndexingConfig::default().with_max_rows(150); + let mut accumulator = ParquetBatchAccumulator::new(config); + + let batch1 = create_test_batch(100); + let flushed1 = accumulator.add_batch(batch1).unwrap(); + assert!(flushed1.is_none()); + assert_eq!(accumulator.pending_rows(), 100); + + let batch2 = create_test_batch(100); + let flushed2 = accumulator.add_batch(batch2).unwrap(); + let combined = flushed2.expect("should have flushed"); + assert_eq!(combined.num_rows(), 200); + assert_eq!(accumulator.pending_rows(), 0); + } + + #[test] + fn test_accumulator_flush_produces_batch() { + let config = ParquetIndexingConfig::default().with_max_rows(1000); + let mut accumulator = ParquetBatchAccumulator::new(config); + + let batch = create_test_batch(50); + let _ = accumulator.add_batch(batch).unwrap(); + assert_eq!(accumulator.pending_rows(), 50); + + let flushed = accumulator.flush().unwrap(); + assert!(flushed.is_some()); + let combined = flushed.unwrap(); + assert_eq!(combined.num_rows(), 50); + assert_eq!(accumulator.pending_rows(), 0); + + let flushed2 = accumulator.flush().unwrap(); + assert!(flushed2.is_none()); + } + + #[test] + fn test_accumulator_multiple_batches_concatenated() { + let config = ParquetIndexingConfig::default().with_max_rows(1000); + let mut accumulator = ParquetBatchAccumulator::new(config); + + for _ in 0..5 { + let batch = create_test_batch(10); + let _ = accumulator.add_batch(batch).unwrap(); + } + + assert_eq!(accumulator.pending_rows(), 50); + assert_eq!(accumulator.pending_batch_count(), 5); + + let combined = accumulator.flush().unwrap().unwrap(); + assert_eq!(combined.num_rows(), 50); + } + + #[test] + fn test_estimate_batch_bytes() { + let batch = create_test_batch(100); + let bytes = estimate_batch_bytes(&batch); + assert!(bytes > 0); + } +} diff --git a/quickwit/quickwit-parquet-engine/src/index/config.rs b/quickwit/quickwit-parquet-engine/src/index/config.rs new file mode 100644 index 00000000000..adf59c992be --- /dev/null +++ b/quickwit/quickwit-parquet-engine/src/index/config.rs @@ -0,0 +1,123 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Parquet index configuration. + +use std::sync::OnceLock; + +use crate::storage::ParquetWriterConfig; + +/// Default maximum rows to accumulate before flushing to split. +const DEFAULT_MAX_ROWS: usize = 1_000_000; + +/// Default maximum bytes to accumulate before flushing (128MB). +const DEFAULT_MAX_BYTES: usize = 128 * 1024 * 1024; + +/// Get max_rows from environment variable or use default. +fn get_max_rows_from_env() -> usize { + static MAX_ROWS: OnceLock = OnceLock::new(); + *MAX_ROWS.get_or_init(|| { + std::env::var("QW_METRICS_MAX_ROWS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(DEFAULT_MAX_ROWS) + }) +} + +/// Get max_bytes from environment variable or use default. +fn get_max_bytes_from_env() -> usize { + static MAX_BYTES: OnceLock = OnceLock::new(); + *MAX_BYTES.get_or_init(|| { + std::env::var("QW_METRICS_MAX_BYTES") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(DEFAULT_MAX_BYTES) + }) +} + +/// Configuration for parquet indexing. +/// +/// Controls when batches are flushed to create new splits based on +/// row count and byte size thresholds. +#[derive(Debug, Clone)] +pub struct ParquetIndexingConfig { + /// Maximum rows to accumulate before flushing to split. + pub max_rows: usize, + /// Maximum bytes to accumulate before flushing (approximate). + pub max_bytes: usize, + /// Parquet writer configuration for split creation. + pub writer_config: ParquetWriterConfig, +} + +impl Default for ParquetIndexingConfig { + fn default() -> Self { + Self { + max_rows: get_max_rows_from_env(), + max_bytes: get_max_bytes_from_env(), + writer_config: ParquetWriterConfig::default(), + } + } +} + +impl ParquetIndexingConfig { + /// Set the maximum rows to accumulate before flushing. + pub fn with_max_rows(mut self, rows: usize) -> Self { + self.max_rows = rows; + self + } + + /// Set the maximum bytes to accumulate before flushing. + pub fn with_max_bytes(mut self, bytes: usize) -> Self { + self.max_bytes = bytes; + self + } + + /// Set the Parquet writer configuration. + pub fn with_writer_config(mut self, config: ParquetWriterConfig) -> Self { + self.writer_config = config; + self + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::storage::Compression; + + #[test] + fn test_default_config() { + let config = ParquetIndexingConfig::default(); + assert_eq!(config.max_rows, 1_000_000); + assert_eq!(config.max_bytes, 128 * 1024 * 1024); + } + + #[test] + fn test_builder_pattern() { + let config = ParquetIndexingConfig::default() + .with_max_rows(500_000) + .with_max_bytes(64 * 1024 * 1024); + + assert_eq!(config.max_rows, 500_000); + assert_eq!(config.max_bytes, 64 * 1024 * 1024); + } + + #[test] + fn test_with_writer_config() { + let writer_config = ParquetWriterConfig::new().with_compression(Compression::Snappy); + + let config = ParquetIndexingConfig::default().with_writer_config(writer_config); + + assert_eq!(config.writer_config.compression, Compression::Snappy); + } +} diff --git a/quickwit/quickwit-lambda/src/searcher/mod.rs b/quickwit/quickwit-parquet-engine/src/index/mod.rs similarity index 68% rename from quickwit/quickwit-lambda/src/searcher/mod.rs rename to quickwit/quickwit-parquet-engine/src/index/mod.rs index e00d9418a6d..5d0b2e47b1e 100644 --- a/quickwit/quickwit-lambda/src/searcher/mod.rs +++ b/quickwit/quickwit-parquet-engine/src/index/mod.rs @@ -12,8 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -mod api; -mod environment; -pub mod warp_lambda; +//! Parquet indexing pipeline. +//! +//! Buffers Arrow IPC batches into RecordBatches and flushes them to Parquet +//! splits when size/row thresholds are reached. -pub use api::setup_searcher_api; +mod accumulator; +mod config; + +pub use accumulator::ParquetBatchAccumulator; +pub use config::ParquetIndexingConfig; diff --git a/quickwit/quickwit-lambda/src/indexer/mod.rs b/quickwit/quickwit-parquet-engine/src/ingest/mod.rs similarity index 68% rename from quickwit/quickwit-lambda/src/indexer/mod.rs rename to quickwit/quickwit-parquet-engine/src/ingest/mod.rs index 308ec589c1a..41604a6d32f 100644 --- a/quickwit/quickwit-lambda/src/indexer/mod.rs +++ b/quickwit/quickwit-parquet-engine/src/ingest/mod.rs @@ -12,9 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -mod environment; -mod handler; -mod ingest; -mod model; +//! Parquet ingest pipeline. +//! +//! Provides high-throughput metrics ingestion by converting Arrow IPC batches +//! directly to Parquet splits, bypassing Tantivy entirely. -pub use handler::handler; +pub mod processor; + +pub use processor::{IngestError, ParquetIngestProcessor, record_batch_to_ipc}; diff --git a/quickwit/quickwit-parquet-engine/src/ingest/processor.rs b/quickwit/quickwit-parquet-engine/src/ingest/processor.rs new file mode 100644 index 00000000000..80dd4f322e5 --- /dev/null +++ b/quickwit/quickwit-parquet-engine/src/ingest/processor.rs @@ -0,0 +1,320 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Parquet ingest processor for Arrow IPC to RecordBatch conversion. + +use std::io::Cursor; + +use arrow::ipc::reader::StreamReader; +use arrow::record_batch::RecordBatch; +use tracing::{debug, instrument, warn}; + +use crate::metrics::PARQUET_ENGINE_METRICS; +use crate::schema::ParquetSchema; + +/// Error type for ingest operations. +#[derive(Debug, thiserror::Error)] +pub enum IngestError { + /// DocBatchV2 is not in ArrowIpc format. + #[error("Invalid format: expected ArrowIpc format")] + InvalidFormat, + + /// Arrow IPC deserialization failed. + #[error("IPC decode error: {0}")] + IpcDecode(#[from] arrow::error::ArrowError), + + /// RecordBatch schema doesn't match expected metrics schema. + #[error("Schema mismatch: expected {expected} fields, got {actual}")] + SchemaMismatch { expected: usize, actual: usize }, + + /// Expected exactly one RecordBatch in IPC stream. + #[error("Expected 1 RecordBatch in IPC stream, got {0}")] + UnexpectedBatchCount(usize), + + /// Storage error during split writing. + #[error("Storage error: {0}")] + Storage(#[from] crate::storage::ParquetWriteError), + + /// IO error. + #[error("IO error: {0}")] + Io(#[from] std::io::Error), +} + +/// Processor that converts Arrow IPC bytes to RecordBatch. +/// +/// Validates that the decoded batch matches the expected metrics schema. +pub struct ParquetIngestProcessor { + schema: ParquetSchema, +} + +impl ParquetIngestProcessor { + /// Create a new ParquetIngestProcessor. + pub fn new(schema: ParquetSchema) -> Self { + Self { schema } + } + + /// Convert Arrow IPC bytes to RecordBatch. + /// + /// Returns error if IPC is malformed or schema doesn't match. + #[instrument(skip(self, ipc_bytes), fields(bytes_len = ipc_bytes.len()))] + pub fn process_ipc(&self, ipc_bytes: &[u8]) -> Result { + // Record bytes ingested + PARQUET_ENGINE_METRICS + .ingest_bytes_total + .inc_by(ipc_bytes.len() as u64); + + let batch = match ipc_to_record_batch(ipc_bytes) { + Ok(batch) => batch, + Err(e) => { + PARQUET_ENGINE_METRICS + .errors_total + .with_label_values(["ingest"]) + .inc(); + return Err(e); + } + }; + + if let Err(e) = self.validate_schema(&batch) { + PARQUET_ENGINE_METRICS + .errors_total + .with_label_values(["ingest"]) + .inc(); + return Err(e); + } + + debug!( + num_rows = batch.num_rows(), + "Successfully decoded IPC to RecordBatch" + ); + Ok(batch) + } + + /// Validate that the RecordBatch schema matches expected metrics schema. + fn validate_schema(&self, batch: &RecordBatch) -> Result<(), IngestError> { + let expected_fields = self.schema.num_fields(); + let actual_fields = batch.schema().fields().len(); + + if expected_fields != actual_fields { + warn!( + expected = expected_fields, + actual = actual_fields, + "Schema mismatch: field count differs" + ); + return Err(IngestError::SchemaMismatch { + expected: expected_fields, + actual: actual_fields, + }); + } + + // Verify field names match (in order) + let schema_fields = self.schema.arrow_schema().fields(); + let batch_schema = batch.schema(); + let batch_fields = batch_schema.fields(); + + for (expected, actual) in schema_fields.iter().zip(batch_fields.iter()) { + if expected.name() != actual.name() { + warn!( + expected_name = expected.name(), + actual_name = actual.name(), + "Schema mismatch: field name differs" + ); + return Err(IngestError::SchemaMismatch { + expected: expected_fields, + actual: actual_fields, + }); + } + if expected.data_type() != actual.data_type() { + warn!( + field = expected.name(), + expected_type = ?expected.data_type(), + actual_type = ?actual.data_type(), + "Schema mismatch: field type differs" + ); + return Err(IngestError::SchemaMismatch { + expected: expected_fields, + actual: actual_fields, + }); + } + } + + Ok(()) + } + + /// Get a reference to the schema. + pub fn schema(&self) -> &ParquetSchema { + &self.schema + } +} + +/// Deserialize Arrow IPC stream format to a RecordBatch. +fn ipc_to_record_batch(ipc_bytes: &[u8]) -> Result { + let cursor = Cursor::new(ipc_bytes); + let reader = StreamReader::try_new(cursor, None)?; + + let batches: Vec = reader.collect::, _>>()?; + + if batches.len() != 1 { + return Err(IngestError::UnexpectedBatchCount(batches.len())); + } + + // Safe: we verified exactly 1 batch above, but use ok_or for defensive programming + batches + .into_iter() + .next() + .ok_or(IngestError::UnexpectedBatchCount(0)) +} + +/// Serialize a RecordBatch to Arrow IPC stream format. +pub fn record_batch_to_ipc(batch: &RecordBatch) -> Result, IngestError> { + use arrow::ipc::writer::StreamWriter; + + let mut buffer = Vec::new(); + { + let mut writer = StreamWriter::try_new(&mut buffer, &batch.schema())?; + writer.write(batch)?; + writer.finish()?; + } + Ok(buffer) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow::array::{ + ArrayRef, DictionaryArray, Float64Array, Int32Array, StringArray, UInt8Array, UInt64Array, + }; + use arrow::datatypes::Int32Type; + use parquet::variant::VariantArrayBuilder; + + use super::*; + + /// Create dictionary array for string fields with Int32 keys. + fn create_dict_array(values: &[&str]) -> ArrayRef { + let keys: Vec = (0..values.len()).map(|i| i as i32).collect(); + let string_array = StringArray::from(values.to_vec()); + Arc::new( + DictionaryArray::::try_new(Int32Array::from(keys), Arc::new(string_array)) + .unwrap(), + ) + } + + /// Create nullable dictionary array for optional string fields. + fn create_nullable_dict_array(values: &[Option<&str>]) -> ArrayRef { + let keys: Vec> = values + .iter() + .enumerate() + .map(|(i, v)| v.map(|_| i as i32)) + .collect(); + let string_values: Vec<&str> = values.iter().filter_map(|v| *v).collect(); + let string_array = StringArray::from(string_values); + Arc::new( + DictionaryArray::::try_new(Int32Array::from(keys), Arc::new(string_array)) + .unwrap(), + ) + } + + /// Create a VARIANT array for testing with specified number of rows (all nulls). + fn create_variant_array(num_rows: usize) -> ArrayRef { + let mut builder = VariantArrayBuilder::new(num_rows); + for _ in 0..num_rows { + builder.append_null(); + } + ArrayRef::from(builder.build()) + } + + /// Create a test batch matching the metrics schema. + fn create_test_batch(num_rows: usize) -> RecordBatch { + let schema = ParquetSchema::new(); + + let metric_names: Vec<&str> = vec!["cpu.usage"; num_rows]; + let metric_name: ArrayRef = create_dict_array(&metric_names); + let metric_type: ArrayRef = Arc::new(UInt8Array::from(vec![0u8; num_rows])); + let metric_unit: ArrayRef = Arc::new(StringArray::from(vec![Some("bytes"); num_rows])); + let timestamps: Vec = (0..num_rows).map(|i| 100 + i as u64).collect(); + let timestamp_secs: ArrayRef = Arc::new(UInt64Array::from(timestamps)); + let start_timestamp_secs: ArrayRef = + Arc::new(UInt64Array::from(vec![None::; num_rows])); + let values: Vec = (0..num_rows).map(|i| 42.0 + i as f64).collect(); + let value: ArrayRef = Arc::new(Float64Array::from(values)); + let tag_service: ArrayRef = create_nullable_dict_array(&vec![Some("web"); num_rows]); + let tag_env: ArrayRef = create_nullable_dict_array(&vec![Some("prod"); num_rows]); + let tag_datacenter: ArrayRef = + create_nullable_dict_array(&vec![Some("us-east-1"); num_rows]); + let tag_region: ArrayRef = create_nullable_dict_array(&vec![None; num_rows]); + let tag_host: ArrayRef = create_nullable_dict_array(&vec![Some("host-001"); num_rows]); + let attributes: ArrayRef = create_variant_array(num_rows); + let service_name: ArrayRef = create_dict_array(&vec!["my-service"; num_rows]); + let resource_attributes: ArrayRef = create_variant_array(num_rows); + + RecordBatch::try_new( + schema.arrow_schema().clone(), + vec![ + metric_name, + metric_type, + metric_unit, + timestamp_secs, + start_timestamp_secs, + value, + tag_service, + tag_env, + tag_datacenter, + tag_region, + tag_host, + attributes, + service_name, + resource_attributes, + ], + ) + .unwrap() + } + + #[test] + fn test_process_ipc() { + let schema = ParquetSchema::new(); + let processor = ParquetIngestProcessor::new(schema); + + // Create a valid batch + let batch = create_test_batch(10); + let ipc_bytes = record_batch_to_ipc(&batch).unwrap(); + + // Process the IPC bytes + let result = processor.process_ipc(&ipc_bytes); + assert!(result.is_ok()); + + let recovered = result.unwrap(); + assert_eq!(recovered.num_rows(), 10); + assert_eq!(recovered.num_columns(), 14); + } + + #[test] + fn test_process_ipc_invalid_bytes() { + let schema = ParquetSchema::new(); + let processor = ParquetIngestProcessor::new(schema); + + let result = processor.process_ipc(&[0u8; 10]); + assert!(result.is_err()); + } + + #[test] + fn test_ipc_round_trip() { + let batch = create_test_batch(5); + let ipc_bytes = record_batch_to_ipc(&batch).unwrap(); + let recovered = ipc_to_record_batch(&ipc_bytes).unwrap(); + + assert_eq!(recovered.num_rows(), batch.num_rows()); + assert_eq!(recovered.num_columns(), batch.num_columns()); + assert_eq!(recovered.schema(), batch.schema()); + } +} diff --git a/quickwit/quickwit-parquet-engine/src/lib.rs b/quickwit/quickwit-parquet-engine/src/lib.rs new file mode 100644 index 00000000000..eac11bfa8e9 --- /dev/null +++ b/quickwit/quickwit-parquet-engine/src/lib.rs @@ -0,0 +1,28 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Pomsky Parquet Engine +//! +//! High-throughput metrics storage and query engine using DataFusion/Parquet. +//! Replaces Tantivy for metrics workloads while maintaining full compatibility +//! with Tantivy for logs/traces. + +#![deny(clippy::disallowed_methods)] + +pub mod index; +pub mod ingest; +pub mod metrics; +pub mod schema; +pub mod split; +pub mod storage; diff --git a/quickwit/quickwit-parquet-engine/src/metrics.rs b/quickwit/quickwit-parquet-engine/src/metrics.rs new file mode 100644 index 00000000000..de55f72deab --- /dev/null +++ b/quickwit/quickwit-parquet-engine/src/metrics.rs @@ -0,0 +1,124 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Prometheus metrics for the Pomsky Parquet Engine. +//! +//! Provides counters, histograms, and gauges for monitoring ingest throughput, +//! query performance, and error rates in production. + +use std::sync::LazyLock; + +use quickwit_common::metrics::{ + Histogram, IntCounter, IntCounterVec, new_counter, new_counter_vec, new_histogram, +}; + +/// Subsystem name for all metrics engine metrics. +const SUBSYSTEM: &str = "metrics_engine"; + +/// Histogram buckets for duration measurements (in seconds). +/// Covers sub-millisecond to multi-second operations. +fn duration_buckets() -> Vec { + vec![ + 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, + ] +} + +/// Metrics for the Pomsky Parquet Engine. +#[derive(Clone)] +pub struct ParquetEngineMetrics { + /// Total number of batches accumulated during indexing. + pub index_batches_total: IntCounter, + /// Total number of rows accumulated during indexing. + pub index_rows_total: IntCounter, + /// Total number of bytes received from IPC payloads during ingestion. + pub ingest_bytes_total: IntCounter, + /// Histogram of add_batch durations (seconds), including any triggered flush. + pub index_batch_duration_seconds: Histogram, + /// Total number of splits written to storage. + pub splits_written_total: IntCounter, + /// Total bytes written to split files. + pub splits_bytes_written: IntCounter, + /// Histogram of query execution durations (seconds). + pub query_duration_seconds: Histogram, + /// Total number of rows returned from queries. + pub query_rows_returned: IntCounter, + /// Errors by operation type: ingest, query, storage. + pub errors_total: IntCounterVec<1>, +} + +impl Default for ParquetEngineMetrics { + fn default() -> Self { + Self { + index_batches_total: new_counter( + "index_batches_total", + "Total number of batches accumulated during indexing.", + SUBSYSTEM, + &[], + ), + index_rows_total: new_counter( + "index_rows_total", + "Total number of rows accumulated during indexing.", + SUBSYSTEM, + &[], + ), + ingest_bytes_total: new_counter( + "ingest_bytes_total", + "Total number of bytes received from IPC payloads during ingestion.", + SUBSYSTEM, + &[], + ), + index_batch_duration_seconds: new_histogram( + "index_batch_duration_seconds", + "Histogram of add_batch durations in seconds, including any triggered flush.", + SUBSYSTEM, + duration_buckets(), + ), + splits_written_total: new_counter( + "splits_written_total", + "Total number of splits written to storage.", + SUBSYSTEM, + &[], + ), + splits_bytes_written: new_counter( + "splits_bytes_written", + "Total bytes written to split files.", + SUBSYSTEM, + &[], + ), + query_duration_seconds: new_histogram( + "query_duration_seconds", + "Histogram of query execution durations in seconds.", + SUBSYSTEM, + duration_buckets(), + ), + query_rows_returned: new_counter( + "query_rows_returned", + "Total number of rows returned from queries.", + SUBSYSTEM, + &[], + ), + errors_total: new_counter_vec( + "errors_total", + "Total errors by operation type.", + SUBSYSTEM, + &[], + ["operation"], + ), + } + } +} + +/// Global metrics instance for the metrics engine. +pub static PARQUET_ENGINE_METRICS: LazyLock = + LazyLock::new(ParquetEngineMetrics::default); diff --git a/quickwit/quickwit-parquet-engine/src/schema/fields.rs b/quickwit/quickwit-parquet-engine/src/schema/fields.rs new file mode 100644 index 00000000000..b6a52c738aa --- /dev/null +++ b/quickwit/quickwit-parquet-engine/src/schema/fields.rs @@ -0,0 +1,162 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Parquet field definitions with column metadata. + +use arrow::datatypes::{DataType, Field, Fields}; +use parquet::variant::VariantType; + +/// All fields in the parquet schema. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ParquetField { + MetricName, + MetricType, + MetricUnit, + TimestampSecs, + StartTimestampSecs, + Value, + TagService, + TagEnv, + TagDatacenter, + TagRegion, + TagHost, + Attributes, + ServiceName, + ResourceAttributes, +} + +impl ParquetField { + /// Field name as stored in Parquet. + pub fn name(&self) -> &'static str { + match self { + Self::MetricName => "metric_name", + Self::MetricType => "metric_type", + Self::MetricUnit => "metric_unit", + Self::TimestampSecs => "timestamp_secs", + Self::StartTimestampSecs => "start_timestamp_secs", + Self::Value => "value", + Self::TagService => "tag_service", + Self::TagEnv => "tag_env", + Self::TagDatacenter => "tag_datacenter", + Self::TagRegion => "tag_region", + Self::TagHost => "tag_host", + Self::Attributes => "attributes", + Self::ServiceName => "service_name", + Self::ResourceAttributes => "resource_attributes", + } + } + + /// Whether this field is nullable. + pub fn is_nullable(&self) -> bool { + matches!( + self, + Self::MetricUnit + | Self::StartTimestampSecs + | Self::TagService + | Self::TagEnv + | Self::TagDatacenter + | Self::TagRegion + | Self::TagHost + | Self::Attributes + | Self::ResourceAttributes + ) + } + + /// Arrow DataType for this field. + /// Use dictionary encoding for high-cardinality strings. + pub fn arrow_type(&self) -> DataType { + match self { + // Dictionary-encoded strings for high cardinality + Self::MetricName | Self::ServiceName => { + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)) + } + // Dictionary-encoded optional tags + Self::TagService + | Self::TagEnv + | Self::TagDatacenter + | Self::TagRegion + | Self::TagHost => { + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)) + } + // Enum stored as UInt8 + Self::MetricType => DataType::UInt8, + // Timestamps as UInt64 seconds + Self::TimestampSecs | Self::StartTimestampSecs => DataType::UInt64, + // Metric value + Self::Value => DataType::Float64, + // Plain string for metric unit + Self::MetricUnit => DataType::Utf8, + // VARIANT type for semi-structured attributes + // Uses the Parquet Variant binary encoding format + Self::Attributes | Self::ResourceAttributes => { + // VARIANT is stored as a struct with metadata and value BinaryView fields + // VariantArrayBuilder produces BinaryView, not Binary + DataType::Struct(Fields::from(vec![ + Field::new("metadata", DataType::BinaryView, false), + Field::new("value", DataType::BinaryView, false), + ])) + } + } + } + + /// Convert to Arrow Field. + pub fn to_arrow_field(&self) -> Field { + let field = Field::new(self.name(), self.arrow_type(), self.is_nullable()); + + // Add VARIANT extension type metadata for attributes fields + match self { + Self::Attributes | Self::ResourceAttributes => field.with_extension_type(VariantType), + _ => field, + } + } + + /// All fields in schema order. + pub fn all() -> &'static [ParquetField] { + &[ + Self::MetricName, + Self::MetricType, + Self::MetricUnit, + Self::TimestampSecs, + Self::StartTimestampSecs, + Self::Value, + Self::TagService, + Self::TagEnv, + Self::TagDatacenter, + Self::TagRegion, + Self::TagHost, + Self::Attributes, + Self::ServiceName, + Self::ResourceAttributes, + ] + } + + /// Sort order for metrics data (used for pruning). + /// Order: metric_name, common tags (service, env, datacenter, region, host), timestamp. + pub fn sort_order() -> &'static [ParquetField] { + &[ + Self::MetricName, + Self::TagService, + Self::TagEnv, + Self::TagDatacenter, + Self::TagRegion, + Self::TagHost, + Self::TimestampSecs, + ] + } + + /// Get the column index in the schema. + pub fn column_index(&self) -> usize { + Self::all().iter().position(|f| f == self).unwrap() + } +} diff --git a/quickwit/quickwit-lambda/src/lib.rs b/quickwit/quickwit-parquet-engine/src/schema/mod.rs similarity index 67% rename from quickwit/quickwit-lambda/src/lib.rs rename to quickwit/quickwit-parquet-engine/src/schema/mod.rs index e7efc4287b4..4cbe31d357d 100644 --- a/quickwit/quickwit-lambda/src/lib.rs +++ b/quickwit/quickwit-parquet-engine/src/schema/mod.rs @@ -12,8 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -mod environment; -pub mod indexer; -pub mod logger; -pub mod searcher; -mod utils; +//! Parquet schema definitions for storage. +//! +//! Extends the existing Arrow schema from `arrow_metrics.rs` with Parquet-specific +//! column types optimized for storage and query efficiency. + +mod fields; +mod parquet; + +pub use fields::ParquetField; +pub use parquet::ParquetSchema; diff --git a/quickwit/quickwit-parquet-engine/src/schema/parquet.rs b/quickwit/quickwit-parquet-engine/src/schema/parquet.rs new file mode 100644 index 00000000000..83ef99e351c --- /dev/null +++ b/quickwit/quickwit-parquet-engine/src/schema/parquet.rs @@ -0,0 +1,93 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Parquet schema construction for metrics. + +use std::sync::Arc; + +use arrow::datatypes::{Schema as ArrowSchema, SchemaRef}; +use parquet::arrow::ArrowSchemaConverter; +use parquet::schema::types::SchemaDescriptor; + +use super::fields::ParquetField; + +/// Parquet schema for storage. +#[derive(Debug, Clone)] +pub struct ParquetSchema { + arrow_schema: SchemaRef, +} + +impl ParquetSchema { + /// Create a new ParquetSchema. + pub fn new() -> Self { + let fields: Vec<_> = ParquetField::all() + .iter() + .map(|f| f.to_arrow_field()) + .collect(); + + let arrow_schema = Arc::new(ArrowSchema::new(fields)); + Self { arrow_schema } + } + + /// Get the Arrow schema. + pub fn arrow_schema(&self) -> &SchemaRef { + &self.arrow_schema + } + + /// Convert to Parquet schema descriptor. + pub fn parquet_schema(&self) -> Result { + ArrowSchemaConverter::new().convert(&self.arrow_schema) + } + + /// Get field by name. + pub fn field(&self, name: &str) -> Option<&arrow::datatypes::Field> { + self.arrow_schema.field_with_name(name).ok() + } + + /// Number of fields in schema. + pub fn num_fields(&self) -> usize { + self.arrow_schema.fields().len() + } +} + +impl Default for ParquetSchema { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_schema_creation() { + let schema = ParquetSchema::new(); + assert_eq!(schema.num_fields(), 14); + } + + #[test] + fn test_field_lookup() { + let schema = ParquetSchema::new(); + let field = schema.field("metric_name").unwrap(); + assert!(!field.is_nullable()); + } + + #[test] + fn test_parquet_conversion() { + let schema = ParquetSchema::new(); + let parquet_schema = schema.parquet_schema(); + assert!(parquet_schema.is_ok()); + } +} diff --git a/quickwit/quickwit-parquet-engine/src/split/format.rs b/quickwit/quickwit-parquet-engine/src/split/format.rs new file mode 100644 index 00000000000..d5722623815 --- /dev/null +++ b/quickwit/quickwit-parquet-engine/src/split/format.rs @@ -0,0 +1,130 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Parquet split format and serialization. + +use serde::{Deserialize, Serialize}; + +use super::metadata::{MetricsSplitMetadata, SplitId, TimeRange}; + +/// A parquet split - the storage unit for metrics data. +/// +/// Analogous to Tantivy's Split but contains Parquet file(s) instead +/// of Tantivy segments. Immutable once created. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ParquetSplit { + /// Split metadata. + pub metadata: MetricsSplitMetadata, + + /// Format version for forward compatibility. + pub format_version: u32, +} + +/// Current format version. +pub const CURRENT_FORMAT_VERSION: u32 = 1; + +impl ParquetSplit { + /// Create a new ParquetSplit. + pub fn new(metadata: MetricsSplitMetadata) -> Self { + Self { + metadata, + format_version: CURRENT_FORMAT_VERSION, + } + } + + /// Get the split ID. + pub fn id(&self) -> &SplitId { + &self.metadata.split_id + } + + /// Get the time range. + pub fn time_range(&self) -> &TimeRange { + &self.metadata.time_range + } + + /// Check if this split might contain data for the given time range. + pub fn overlaps_time_range(&self, query_range: &TimeRange) -> bool { + self.metadata.time_range.overlaps(query_range) + } + + /// Check if this split might contain the given metric. + pub fn might_contain_metric(&self, metric_name: &str) -> bool { + self.metadata.metric_names.is_empty() || self.metadata.metric_names.contains(metric_name) + } + + /// Serialize to JSON bytes. + pub fn to_json(&self) -> Result, serde_json::Error> { + serde_json::to_vec(self) + } + + /// Deserialize from JSON bytes. + pub fn from_json(bytes: &[u8]) -> Result { + serde_json::from_slice(bytes) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn create_test_split() -> ParquetSplit { + let metadata = MetricsSplitMetadata::builder() + .index_uid("test-index:00000000000000000000000000") + .time_range(TimeRange::new(1000, 2000)) + .num_rows(10000) + .size_bytes(1024 * 1024) + .add_metric_name("cpu.usage") + .add_metric_name("memory.used") + .add_low_cardinality_tag(crate::split::TAG_SERVICE, "web-server") + .build(); + + ParquetSplit::new(metadata) + } + + #[test] + fn test_split_creation() { + let split = create_test_split(); + assert_eq!(split.format_version, CURRENT_FORMAT_VERSION); + assert_eq!(split.metadata.num_rows, 10000); + } + + #[test] + fn test_time_range_overlap() { + let split = create_test_split(); + + // Overlapping range + assert!(split.overlaps_time_range(&TimeRange::new(1500, 2500))); + + // Non-overlapping range + assert!(!split.overlaps_time_range(&TimeRange::new(3000, 4000))); + } + + #[test] + fn test_metric_pruning() { + let split = create_test_split(); + + assert!(split.might_contain_metric("cpu.usage")); + assert!(!split.might_contain_metric("disk.io")); + } + + #[test] + fn test_json_roundtrip() { + let split = create_test_split(); + let json = split.to_json().unwrap(); + let restored = ParquetSplit::from_json(&json).unwrap(); + + assert_eq!(split.id().as_str(), restored.id().as_str()); + assert_eq!(split.metadata.num_rows, restored.metadata.num_rows); + } +} diff --git a/quickwit/quickwit-parquet-engine/src/split/metadata.rs b/quickwit/quickwit-parquet-engine/src/split/metadata.rs new file mode 100644 index 00000000000..992775be862 --- /dev/null +++ b/quickwit/quickwit-parquet-engine/src/split/metadata.rs @@ -0,0 +1,404 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Metrics split metadata definitions. + +use std::collections::{HashMap, HashSet}; +use std::time::SystemTime; + +use serde::{Deserialize, Serialize}; + +// Well-known tag key constants +pub const TAG_SERVICE: &str = "service"; +pub const TAG_ENV: &str = "env"; +pub const TAG_DATACENTER: &str = "datacenter"; +pub const TAG_REGION: &str = "region"; +pub const TAG_HOST: &str = "host"; + +/// Unique identifier for a metrics split. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct SplitId(String); + +impl SplitId { + /// Create a new SplitId from a string. + pub fn new(id: impl Into) -> Self { + Self(id.into()) + } + + /// Generates a new unique SplitId using a ULID (timestamp + randomness). + pub fn generate() -> Self { + Self(format!( + "metrics_{}", + ulid::Ulid::new().to_string().to_lowercase() + )) + } + + /// Get the string representation. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for SplitId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +/// Time range covered by a metrics split. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct TimeRange { + /// Start timestamp in seconds (inclusive). + pub start_secs: u64, + /// End timestamp in seconds (exclusive). + pub end_secs: u64, +} + +impl TimeRange { + /// Create a new time range. + pub fn new(start_secs: u64, end_secs: u64) -> Self { + debug_assert!(start_secs <= end_secs); + Self { + start_secs, + end_secs, + } + } + + /// Check if a timestamp falls within this range. + pub fn contains(&self, timestamp_secs: u64) -> bool { + timestamp_secs >= self.start_secs && timestamp_secs < self.end_secs + } + + /// Check if this range overlaps with another. + pub fn overlaps(&self, other: &TimeRange) -> bool { + self.start_secs < other.end_secs && other.start_secs < self.end_secs + } + + /// Duration of this range in seconds. + pub fn duration_secs(&self) -> u64 { + self.end_secs - self.start_secs + } +} + +/// State of a metrics split in the metastore. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum MetricsSplitState { + /// Split is staged (being written, not yet queryable). + Staged, + /// Split is published (queryable). + Published, + /// Split is marked for deletion. + MarkedForDeletion, +} + +impl MetricsSplitState { + /// Returns a string representation for database storage. + pub fn as_str(&self) -> &'static str { + match self { + MetricsSplitState::Staged => "Staged", + MetricsSplitState::Published => "Published", + MetricsSplitState::MarkedForDeletion => "MarkedForDeletion", + } + } +} + +impl std::fmt::Display for MetricsSplitState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.as_str()) + } +} + +/// Metadata for a metrics split. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MetricsSplitMetadata { + /// Unique split identifier. + pub split_id: SplitId, + + /// Index unique identifier for Postgres foreign key relationship. + pub index_uid: String, + + /// Time range covered by this split. + pub time_range: TimeRange, + + /// Number of data points in this split. + pub num_rows: u64, + + /// Size of Parquet file(s) in bytes. + pub size_bytes: u64, + + /// Distinct metric names in this split (for pruning). + pub metric_names: HashSet, + + /// Low-cardinality tag values by key (for Postgres pruning). + /// Tags with cardinality < CARDINALITY_THRESHOLD are stored here. + /// Format: HashMap> + /// Example: {"service": {"web", "api"}, "env": {"prod", "staging"}} + pub low_cardinality_tags: HashMap>, + + /// High-cardinality tag keys (for Parquet bloom filter). + /// Tags with cardinality >= CARDINALITY_THRESHOLD have only their keys stored here. + /// The actual values are stored in Parquet bloom filters. + pub high_cardinality_tag_keys: HashSet, + + /// When this split was created. + pub created_at: SystemTime, +} + +impl MetricsSplitMetadata { + /// Returns the parquet filename for this split, relative to the storage root. + /// Always `{split_id}.parquet`. + pub fn parquet_filename(&self) -> String { + format!("{}.parquet", self.split_id) + } + + /// Cardinality threshold for routing tags to Postgres vs Parquet. + /// Tags with < CARDINALITY_THRESHOLD unique values go to Postgres. + /// Tags with >= CARDINALITY_THRESHOLD unique values use Parquet bloom filters. + pub const CARDINALITY_THRESHOLD: usize = 1000; + + /// Create a new MetricsSplitMetadata builder. + pub fn builder() -> MetricsSplitMetadataBuilder { + MetricsSplitMetadataBuilder::default() + } + + /// Check if a tag key exceeds the cardinality threshold. + pub fn is_high_cardinality(&self, tag_key: &str) -> bool { + self.high_cardinality_tag_keys.contains(tag_key) + } + + /// Get all tag values for a low-cardinality tag key. + pub fn get_tag_values(&self, tag_key: &str) -> Option<&HashSet> { + self.low_cardinality_tags.get(tag_key) + } + + /// Get service names (convenience method for common query pattern). + pub fn service_names(&self) -> Option<&HashSet> { + self.get_tag_values(TAG_SERVICE) + } + + /// Promote a tag key from low to high cardinality storage. + /// Called when tag value count exceeds CARDINALITY_THRESHOLD during split finalization. + pub fn promote_to_high_cardinality(&mut self, tag_key: &str) { + self.low_cardinality_tags.remove(tag_key); + self.high_cardinality_tag_keys.insert(tag_key.to_string()); + } + + /// Finalize tag cardinality by checking thresholds. + /// Tags exceeding CARDINALITY_THRESHOLD are promoted to high cardinality. + pub fn finalize_tag_cardinality(&mut self) { + let keys_to_promote: Vec = self + .low_cardinality_tags + .iter() + .filter(|(_, values)| values.len() >= Self::CARDINALITY_THRESHOLD) + .map(|(key, _)| key.clone()) + .collect(); + + for key in keys_to_promote { + self.promote_to_high_cardinality(&key); + } + } +} + +/// Builder for MetricsSplitMetadata. +#[derive(Default)] +pub struct MetricsSplitMetadataBuilder { + split_id: Option, + index_uid: Option, + time_range: Option, + num_rows: u64, + size_bytes: u64, + metric_names: HashSet, + low_cardinality_tags: HashMap>, + high_cardinality_tag_keys: HashSet, +} + +impl MetricsSplitMetadataBuilder { + pub fn split_id(mut self, id: SplitId) -> Self { + self.split_id = Some(id); + self + } + + pub fn time_range(mut self, range: TimeRange) -> Self { + self.time_range = Some(range); + self + } + + pub fn num_rows(mut self, count: u64) -> Self { + self.num_rows = count; + self + } + + pub fn size_bytes(mut self, size: u64) -> Self { + self.size_bytes = size; + self + } + + pub fn add_metric_name(mut self, name: impl Into) -> Self { + self.metric_names.insert(name.into()); + self + } + + pub fn index_uid(mut self, uid: impl Into) -> Self { + self.index_uid = Some(uid.into()); + self + } + + pub fn add_low_cardinality_tag( + mut self, + key: impl Into, + value: impl Into, + ) -> Self { + self.low_cardinality_tags + .entry(key.into()) + .or_default() + .insert(value.into()); + self + } + + pub fn add_high_cardinality_tag_key(mut self, key: impl Into) -> Self { + self.high_cardinality_tag_keys.insert(key.into()); + self + } + + /// Add a tag, automatically routing to low cardinality storage. + /// For initial ingestion, always routes to low cardinality; threshold check happens at split + /// finalization. + pub fn add_tag(mut self, key: impl Into, value: impl Into) -> Self { + let key = key.into(); + let value = value.into(); + self.low_cardinality_tags + .entry(key) + .or_default() + .insert(value); + self + } + + pub fn build(self) -> MetricsSplitMetadata { + MetricsSplitMetadata { + split_id: self.split_id.unwrap_or_else(SplitId::generate), + index_uid: self.index_uid.expect("index_uid is required"), + time_range: self.time_range.expect("time_range is required"), + num_rows: self.num_rows, + size_bytes: self.size_bytes, + metric_names: self.metric_names, + low_cardinality_tags: self.low_cardinality_tags, + high_cardinality_tag_keys: self.high_cardinality_tag_keys, + created_at: SystemTime::now(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_split_id_generation() { + let id1 = SplitId::generate(); + // Sleep 1ms to ensure different timestamp + std::thread::sleep(std::time::Duration::from_millis(1)); + let id2 = SplitId::generate(); + assert_ne!(id1.as_str(), id2.as_str()); + assert!(id1.as_str().starts_with("metrics_")); + } + + #[test] + fn test_time_range_overlap() { + let range1 = TimeRange::new(100, 200); + let range2 = TimeRange::new(150, 250); + let range3 = TimeRange::new(200, 300); + let range4 = TimeRange::new(50, 100); + + assert!(range1.overlaps(&range2)); + assert!(!range1.overlaps(&range3)); // Adjacent, not overlapping + assert!(!range1.overlaps(&range4)); // Adjacent, not overlapping + } + + #[test] + fn test_metadata_builder_with_tags() { + let metadata = MetricsSplitMetadata::builder() + .index_uid("test-index:00000000000000000000000000") + .time_range(TimeRange::new(1000, 2000)) + .add_metric_name("cpu.usage") + .add_low_cardinality_tag(TAG_SERVICE, "web") + .add_low_cardinality_tag(TAG_SERVICE, "api") + .add_low_cardinality_tag(TAG_ENV, "prod") + .add_high_cardinality_tag_key(TAG_HOST) + .build(); + + assert_eq!(metadata.index_uid, "test-index:00000000000000000000000000"); + assert!(metadata.metric_names.contains("cpu.usage")); + assert_eq!(metadata.get_tag_values(TAG_SERVICE).unwrap().len(), 2); + assert!( + metadata + .get_tag_values(TAG_SERVICE) + .unwrap() + .contains("web") + ); + assert!(metadata.get_tag_values(TAG_ENV).unwrap().contains("prod")); + assert!(metadata.is_high_cardinality(TAG_HOST)); + assert!(!metadata.is_high_cardinality(TAG_SERVICE)); + } + + #[test] + fn test_cardinality_promotion() { + let mut metadata = MetricsSplitMetadata::builder() + .index_uid("test-index:00000000000000000000000000") + .time_range(TimeRange::new(1000, 2000)) + .build(); + + // Add 1001 unique values for "host" tag + for i in 0..1001 { + metadata + .low_cardinality_tags + .entry(TAG_HOST.to_string()) + .or_default() + .insert(format!("host-{}", i)); + } + + assert!(!metadata.is_high_cardinality(TAG_HOST)); + assert_eq!(metadata.get_tag_values(TAG_HOST).unwrap().len(), 1001); + + metadata.finalize_tag_cardinality(); + + assert!(metadata.is_high_cardinality(TAG_HOST)); + assert!(metadata.get_tag_values(TAG_HOST).is_none()); + } + + #[test] + fn test_service_names_convenience() { + let metadata = MetricsSplitMetadata::builder() + .index_uid("test-index:00000000000000000000000000") + .time_range(TimeRange::new(1000, 2000)) + .add_low_cardinality_tag(TAG_SERVICE, "web") + .add_low_cardinality_tag(TAG_SERVICE, "api") + .build(); + + let services = metadata.service_names().unwrap(); + assert_eq!(services.len(), 2); + assert!(services.contains("web")); + assert!(services.contains("api")); + } + + #[test] + fn test_metrics_split_state() { + assert_eq!(MetricsSplitState::Staged.as_str(), "Staged"); + assert_eq!(MetricsSplitState::Published.as_str(), "Published"); + assert_eq!( + MetricsSplitState::MarkedForDeletion.as_str(), + "MarkedForDeletion" + ); + assert_eq!(format!("{}", MetricsSplitState::Published), "Published"); + } +} diff --git a/quickwit/quickwit-ui/src/reportWebVitals.ts b/quickwit/quickwit-parquet-engine/src/split/mod.rs similarity index 51% rename from quickwit/quickwit-ui/src/reportWebVitals.ts rename to quickwit/quickwit-parquet-engine/src/split/mod.rs index 8124164f404..51a49dfff1c 100644 --- a/quickwit/quickwit-ui/src/reportWebVitals.ts +++ b/quickwit/quickwit-parquet-engine/src/split/mod.rs @@ -12,18 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { ReportHandler } from 'web-vitals'; +//! Parquet split format replacing Tantivy segments. +//! +//! A ParquetSplit is the storage unit for metrics data, containing +//! Parquet file(s) and metadata for efficient query pruning. -const reportWebVitals = (onPerfEntry?: ReportHandler) => { - if (onPerfEntry && onPerfEntry instanceof Function) { - import('web-vitals').then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => { - getCLS(onPerfEntry); - getFID(onPerfEntry); - getFCP(onPerfEntry); - getLCP(onPerfEntry); - getTTFB(onPerfEntry); - }); - } -}; +mod format; +mod metadata; +pub mod postgres; -export default reportWebVitals; +pub use format::{CURRENT_FORMAT_VERSION, ParquetSplit}; +pub use metadata::{ + MetricsSplitMetadata, MetricsSplitMetadataBuilder, MetricsSplitState, SplitId, TAG_DATACENTER, + TAG_ENV, TAG_HOST, TAG_REGION, TAG_SERVICE, TimeRange, +}; +pub use postgres::{InsertableMetricsSplit, MetricsSplitRecord, MetricsSplits, PgMetricsSplit}; diff --git a/quickwit/quickwit-parquet-engine/src/split/postgres.rs b/quickwit/quickwit-parquet-engine/src/split/postgres.rs new file mode 100644 index 00000000000..8b2c8767846 --- /dev/null +++ b/quickwit/quickwit-parquet-engine/src/split/postgres.rs @@ -0,0 +1,270 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! PostgreSQL model for metrics_splits table. +//! +//! This module provides the database model and conversion logic for storing +//! MetricsSplitMetadata in the Postgres metastore for Tier 1 pruning. + +use std::collections::{HashMap, HashSet}; + +#[cfg(feature = "postgres")] +use sea_query::Iden; + +use super::metadata::{ + MetricsSplitMetadata, MetricsSplitState, TAG_DATACENTER, TAG_ENV, TAG_HOST, TAG_REGION, + TAG_SERVICE, +}; + +/// Sea-query table identifier for metrics_splits. +#[cfg_attr(feature = "postgres", derive(Iden))] +#[derive(Clone, Copy)] +pub enum MetricsSplits { + Table, + SplitId, + SplitState, + IndexId, + TimeRangeStart, + TimeRangeEnd, + MetricNames, + TagService, + TagEnv, + TagDatacenter, + TagRegion, + TagHost, + HighCardinalityTagKeys, + NumRows, + SizeBytes, + SplitMetadataJson, + CreateTimestamp, + UpdateTimestamp, +} + +/// PostgreSQL row model for metrics_splits table. +/// Used for reading rows from the database. +#[derive(Debug, Clone)] +pub struct PgMetricsSplit { + pub split_id: String, + pub split_state: String, + pub index_uid: String, + pub time_range_start: i64, + pub time_range_end: i64, + pub metric_names: Vec, + pub tag_service: Option>, + pub tag_env: Option>, + pub tag_datacenter: Option>, + pub tag_region: Option>, + pub tag_host: Option>, + pub high_cardinality_tag_keys: Vec, + pub num_rows: i64, + pub size_bytes: i64, + pub split_metadata_json: String, + pub update_timestamp: i64, +} + +/// Insertable row for metrics_splits table. +/// Used for writing rows to the database. +#[derive(Debug, Clone)] +pub struct InsertableMetricsSplit { + pub split_id: String, + pub split_state: String, + pub index_uid: String, + pub time_range_start: i64, + pub time_range_end: i64, + pub metric_names: Vec, + pub tag_service: Option>, + pub tag_env: Option>, + pub tag_datacenter: Option>, + pub tag_region: Option>, + pub tag_host: Option>, + pub high_cardinality_tag_keys: Vec, + pub num_rows: i64, + pub size_bytes: i64, + pub split_metadata_json: String, +} + +impl InsertableMetricsSplit { + /// Convert MetricsSplitMetadata to an insertable row. + pub fn from_metadata( + metadata: &MetricsSplitMetadata, + state: MetricsSplitState, + ) -> Result { + let split_metadata_json = serde_json::to_string(metadata)?; + + Ok(Self { + split_id: metadata.split_id.as_str().to_string(), + split_state: state.as_str().to_string(), + index_uid: metadata.index_uid.clone(), + time_range_start: metadata.time_range.start_secs as i64, + time_range_end: metadata.time_range.end_secs as i64, + metric_names: metadata.metric_names.iter().cloned().collect(), + tag_service: extract_tag_values(&metadata.low_cardinality_tags, TAG_SERVICE), + tag_env: extract_tag_values(&metadata.low_cardinality_tags, TAG_ENV), + tag_datacenter: extract_tag_values(&metadata.low_cardinality_tags, TAG_DATACENTER), + tag_region: extract_tag_values(&metadata.low_cardinality_tags, TAG_REGION), + tag_host: extract_tag_values(&metadata.low_cardinality_tags, TAG_HOST), + high_cardinality_tag_keys: metadata.high_cardinality_tag_keys.iter().cloned().collect(), + num_rows: metadata.num_rows as i64, + size_bytes: metadata.size_bytes as i64, + split_metadata_json, + }) + } +} + +/// Extract tag values for a specific key, returning None if key doesn't exist. +fn extract_tag_values(tags: &HashMap>, key: &str) -> Option> { + tags.get(key).map(|values| values.iter().cloned().collect()) +} + +impl PgMetricsSplit { + /// Convert database row to MetricsSplitMetadata. + /// Falls back to deserializing from JSON if row data is incomplete. + pub fn to_metadata(&self) -> Result { + // Primary path: deserialize from JSON (authoritative) + let metadata: MetricsSplitMetadata = serde_json::from_str(&self.split_metadata_json)?; + + // Overlay database columns (for consistency verification in debug builds) + debug_assert_eq!(metadata.split_id.as_str(), self.split_id); + debug_assert_eq!(metadata.time_range.start_secs, self.time_range_start as u64); + debug_assert_eq!(metadata.time_range.end_secs, self.time_range_end as u64); + + Ok(metadata) + } + + /// Parse the split state from the database string. + pub fn split_state(&self) -> Option { + match self.split_state.as_str() { + "Staged" => Some(MetricsSplitState::Staged), + "Published" => Some(MetricsSplitState::Published), + "MarkedForDeletion" => Some(MetricsSplitState::MarkedForDeletion), + _ => None, + } + } +} + +/// A complete metrics split record from the database. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct MetricsSplitRecord { + /// The split's state. + pub state: MetricsSplitState, + /// Timestamp when the split was last updated. + pub update_timestamp: i64, + /// The split's immutable metadata. + pub metadata: MetricsSplitMetadata, +} + +impl TryFrom for MetricsSplitRecord { + type Error = String; + + fn try_from(row: PgMetricsSplit) -> Result { + let state = row + .split_state() + .ok_or_else(|| format!("unknown split state: {}", row.split_state))?; + let metadata = row + .to_metadata() + .map_err(|e| format!("failed to deserialize metadata: {}", e))?; + + Ok(Self { + state, + update_timestamp: row.update_timestamp, + metadata, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::split::metadata::{MetricsSplitMetadata, SplitId, TimeRange}; + + #[test] + fn test_insertable_from_metadata() { + let metadata = MetricsSplitMetadata::builder() + .split_id(SplitId::new("test-split-001")) + .index_uid("otel-metrics-v0_1:00000000000000000000000000") + .time_range(TimeRange::new(1700000000, 1700003600)) + .num_rows(50000) + .size_bytes(1024 * 1024) + .add_metric_name("cpu.usage") + .add_metric_name("memory.used") + .add_low_cardinality_tag(TAG_SERVICE, "web") + .add_low_cardinality_tag(TAG_SERVICE, "api") + .add_low_cardinality_tag(TAG_ENV, "prod") + .add_high_cardinality_tag_key(TAG_HOST) + .build(); + + let insertable = + InsertableMetricsSplit::from_metadata(&metadata, MetricsSplitState::Staged) + .expect("conversion should succeed"); + + assert_eq!(insertable.split_id, "test-split-001"); + assert_eq!(insertable.split_state, "Staged"); + assert_eq!( + insertable.index_uid, + "otel-metrics-v0_1:00000000000000000000000000" + ); + assert_eq!(insertable.time_range_start, 1700000000); + assert_eq!(insertable.time_range_end, 1700003600); + assert_eq!(insertable.metric_names.len(), 2); + assert!(insertable.metric_names.contains(&"cpu.usage".to_string())); + assert_eq!(insertable.tag_service.as_ref().unwrap().len(), 2); + assert_eq!(insertable.tag_env.as_ref().unwrap().len(), 1); + assert!(insertable.tag_datacenter.is_none()); + assert_eq!(insertable.high_cardinality_tag_keys, vec!["host"]); + assert_eq!(insertable.num_rows, 50000); + assert_eq!(insertable.size_bytes, 1024 * 1024); + } + + #[test] + fn test_pg_split_to_metadata_roundtrip() { + let original = MetricsSplitMetadata::builder() + .split_id(SplitId::new("roundtrip-test")) + .index_uid("test-index:00000000000000000000000000") + .time_range(TimeRange::new(1000, 2000)) + .num_rows(100) + .size_bytes(500) + .add_metric_name("test.metric") + .add_low_cardinality_tag(TAG_SERVICE, "test-service") + .build(); + + let insertable = + InsertableMetricsSplit::from_metadata(&original, MetricsSplitState::Published) + .expect("conversion should succeed"); + + let pg_row = PgMetricsSplit { + split_id: insertable.split_id, + split_state: insertable.split_state, + index_uid: insertable.index_uid, + time_range_start: insertable.time_range_start, + time_range_end: insertable.time_range_end, + metric_names: insertable.metric_names, + tag_service: insertable.tag_service, + tag_env: insertable.tag_env, + tag_datacenter: insertable.tag_datacenter, + tag_region: insertable.tag_region, + tag_host: insertable.tag_host, + high_cardinality_tag_keys: insertable.high_cardinality_tag_keys, + num_rows: insertable.num_rows, + size_bytes: insertable.size_bytes, + split_metadata_json: insertable.split_metadata_json, + update_timestamp: 1704067200, + }; + + let recovered = pg_row.to_metadata().expect("should deserialize"); + assert_eq!(recovered.split_id.as_str(), original.split_id.as_str()); + assert_eq!(recovered.index_uid, original.index_uid); + assert_eq!(recovered.time_range, original.time_range); + assert_eq!(recovered.num_rows, original.num_rows); + } +} diff --git a/quickwit/quickwit-parquet-engine/src/storage/config.rs b/quickwit/quickwit-parquet-engine/src/storage/config.rs new file mode 100644 index 00000000000..f87845d2ba9 --- /dev/null +++ b/quickwit/quickwit-parquet-engine/src/storage/config.rs @@ -0,0 +1,407 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Parquet writer configuration for metrics storage. + +use parquet::basic::Compression as ParquetCompression; +use parquet::file::metadata::SortingColumn; +use parquet::file::properties::{EnabledStatistics, WriterProperties, WriterPropertiesBuilder}; +use parquet::schema::types::ColumnPath; + +use crate::schema::ParquetField; + +/// Default row group size: 128K rows for efficient columnar scans. +const DEFAULT_ROW_GROUP_SIZE: usize = 128 * 1024; + +/// Default data page size: 1MB for good compression/random access balance. +const DEFAULT_DATA_PAGE_SIZE: usize = 1024 * 1024; + +/// Default write batch size: 64K rows per batch. +const DEFAULT_WRITE_BATCH_SIZE: usize = 64 * 1024; + +/// Default zstd compression level: 3 balances speed and ratio. +const DEFAULT_ZSTD_LEVEL: i32 = 3; + +/// Default bloom filter false positive probability (5%). +const BLOOM_FILTER_FPP: f64 = 0.05; + +/// Default NDV estimate for tag columns. +const BLOOM_FILTER_NDV_TAGS: u64 = 10_000; + +/// Default NDV estimate for metric_name column (higher cardinality). +const BLOOM_FILTER_NDV_METRIC_NAME: u64 = 100_000; + +/// Compression algorithm for Parquet files. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum Compression { + /// Zstd compression (default) - best compression ratio. + #[default] + Zstd, + /// Snappy compression - faster but lower ratio. + Snappy, + /// No compression. + Uncompressed, +} + +/// Configuration for Parquet writer. +#[derive(Debug, Clone)] +pub struct ParquetWriterConfig { + /// Compression algorithm to use. + pub compression: Compression, + /// Compression level for zstd (1-22). Ignored for other algorithms. + pub compression_level: Option, + /// Number of rows per row group. + pub row_group_size: usize, + /// Target size in bytes for data pages. + pub data_page_size: usize, + /// Number of rows per write batch. + pub write_batch_size: usize, +} + +impl Default for ParquetWriterConfig { + fn default() -> Self { + Self { + compression: Compression::default(), + compression_level: Some(DEFAULT_ZSTD_LEVEL), + row_group_size: DEFAULT_ROW_GROUP_SIZE, + data_page_size: DEFAULT_DATA_PAGE_SIZE, + write_batch_size: DEFAULT_WRITE_BATCH_SIZE, + } + } +} + +impl ParquetWriterConfig { + /// Create a new ParquetWriterConfig with default settings. + pub fn new() -> Self { + Self::default() + } + + /// Set the compression algorithm. + pub fn with_compression(mut self, compression: Compression) -> Self { + self.compression = compression; + self + } + + /// Set the compression level for zstd. + pub fn with_compression_level(mut self, level: i32) -> Self { + self.compression_level = Some(level); + self + } + + /// Set the row group size. + pub fn with_row_group_size(mut self, size: usize) -> Self { + self.row_group_size = size; + self + } + + /// Set the data page size. + pub fn with_data_page_size(mut self, size: usize) -> Self { + self.data_page_size = size; + self + } + + /// Set the write batch size. + pub fn with_write_batch_size(mut self, size: usize) -> Self { + self.write_batch_size = size; + self + } + + /// Convert to Parquet WriterProperties. + pub fn to_writer_properties(&self) -> WriterProperties { + let mut builder = WriterProperties::builder() + .set_max_row_group_size(self.row_group_size) + .set_data_page_size_limit(self.data_page_size) + .set_write_batch_size(self.write_batch_size) + // Enable column index for efficient pruning on sorted data (64 bytes default) + .set_column_index_truncate_length(Some(64)) + // Set sorting columns metadata for readers to use during pruning + .set_sorting_columns(Some(Self::sorting_columns())) + // Enable row group level statistics (min/max/null_count) for query pruning + // This allows DataFusion to skip row groups based on timestamp ranges + .set_statistics_enabled(EnabledStatistics::Chunk); + + builder = match self.compression { + Compression::Zstd => { + let level = self.compression_level.unwrap_or(DEFAULT_ZSTD_LEVEL); + builder.set_compression(ParquetCompression::ZSTD( + parquet::basic::ZstdLevel::try_new(level).unwrap_or_default(), + )) + } + Compression::Snappy => builder.set_compression(ParquetCompression::SNAPPY), + Compression::Uncompressed => builder.set_compression(ParquetCompression::UNCOMPRESSED), + }; + + // Apply RLE_DICTIONARY encoding and bloom filters for dictionary columns + builder = Self::configure_dictionary_columns(builder); + + builder.build() + } + + /// Configure dictionary encoding and bloom filters for high-cardinality columns. + /// + /// Dictionary-encoded columns benefit from: + /// - Dictionary encoding: Enabled by default, uses RLE for dictionary indices + /// - Bloom filters: Enable efficient equality filtering without scanning + /// + /// Note: Dictionary encoding is ON by default in Parquet. When enabled, the dictionary + /// indices are automatically encoded using RLE (run-length encoding), which efficiently + /// compresses runs of repeated values. This is ideal for sorted data where consecutive + /// rows often share the same dictionary index. + fn configure_dictionary_columns( + mut builder: WriterPropertiesBuilder, + ) -> WriterPropertiesBuilder { + // Dictionary-encoded columns - ensure dictionary encoding is explicitly enabled + // (default is true, but being explicit documents intent) + let dictionary_columns = [ + ParquetField::MetricName, + ParquetField::TagService, + ParquetField::TagEnv, + ParquetField::TagDatacenter, + ParquetField::TagRegion, + ParquetField::TagHost, + ParquetField::ServiceName, + ]; + + // Columns that benefit from bloom filters (used in WHERE clauses) + // Note: We enable bloom filters on filtering columns, not timestamp_secs or value + let bloom_filter_columns = [ + (ParquetField::MetricName, BLOOM_FILTER_NDV_METRIC_NAME), + (ParquetField::TagService, BLOOM_FILTER_NDV_TAGS), + (ParquetField::TagEnv, BLOOM_FILTER_NDV_TAGS), + (ParquetField::TagDatacenter, BLOOM_FILTER_NDV_TAGS), + (ParquetField::TagHost, BLOOM_FILTER_NDV_TAGS), + (ParquetField::ServiceName, BLOOM_FILTER_NDV_TAGS), + ]; + + // Ensure dictionary encoding is enabled on dictionary columns + // (dictionary encoding uses RLE for indices automatically) + for field in dictionary_columns { + let col_path = ColumnPath::new(vec![field.name().to_string()]); + builder = builder.set_column_dictionary_enabled(col_path, true); + } + + // Enable bloom filters on filtering columns + for (field, ndv) in bloom_filter_columns { + let col_path = ColumnPath::new(vec![field.name().to_string()]); + builder = builder + .set_column_bloom_filter_enabled(col_path.clone(), true) + .set_column_bloom_filter_fpp(col_path.clone(), BLOOM_FILTER_FPP) + .set_column_bloom_filter_ndv(col_path, ndv); + } + + builder + } + + /// Get the sorting columns for parquet metadata. + /// Order: metric_name, tag_service, tag_env, tag_datacenter, tag_region, tag_host, + /// timestamp_secs. + fn sorting_columns() -> Vec { + ParquetField::sort_order() + .iter() + .map(|field| SortingColumn { + column_idx: field.column_index() as i32, + descending: false, + nulls_first: true, + }) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_default_config() { + let config = ParquetWriterConfig::default(); + assert_eq!(config.compression, Compression::Zstd); + assert_eq!(config.compression_level, Some(3)); + assert_eq!(config.row_group_size, 128 * 1024); + assert_eq!(config.data_page_size, 1024 * 1024); + assert_eq!(config.write_batch_size, 64 * 1024); + } + + #[test] + fn test_builder_pattern() { + let config = ParquetWriterConfig::new() + .with_compression(Compression::Snappy) + .with_row_group_size(256 * 1024); + + assert_eq!(config.compression, Compression::Snappy); + assert_eq!(config.row_group_size, 256 * 1024); + } + + #[test] + fn test_to_writer_properties_zstd() { + let config = ParquetWriterConfig::default(); + let props = config.to_writer_properties(); + // WriterProperties doesn't expose compression directly, but we can verify it builds + assert!(props.max_row_group_size() == 128 * 1024); + } + + #[test] + fn test_to_writer_properties_snappy() { + let config = ParquetWriterConfig::new().with_compression(Compression::Snappy); + let props = config.to_writer_properties(); + assert!(props.max_row_group_size() == 128 * 1024); + } + + #[test] + fn test_to_writer_properties_uncompressed() { + let config = ParquetWriterConfig::new().with_compression(Compression::Uncompressed); + let props = config.to_writer_properties(); + assert!(props.max_row_group_size() == 128 * 1024); + } + + #[test] + fn test_bloom_filter_configuration() { + let config = ParquetWriterConfig::default(); + let props = config.to_writer_properties(); + + // Verify bloom filter is enabled on metric_name column + let metric_name_path = ColumnPath::new(vec!["metric_name".to_string()]); + let bloom_props = props.bloom_filter_properties(&metric_name_path); + assert!( + bloom_props.is_some(), + "Bloom filter should be enabled for metric_name" + ); + + let bloom_props = bloom_props.unwrap(); + assert!( + (bloom_props.fpp - BLOOM_FILTER_FPP).abs() < 0.001, + "Bloom filter FPP should be {}", + BLOOM_FILTER_FPP + ); + assert_eq!( + bloom_props.ndv, BLOOM_FILTER_NDV_METRIC_NAME, + "Bloom filter NDV for metric_name should be {}", + BLOOM_FILTER_NDV_METRIC_NAME + ); + + // Verify bloom filter is enabled on tag columns + let tag_service_path = ColumnPath::new(vec!["tag_service".to_string()]); + let bloom_props = props.bloom_filter_properties(&tag_service_path); + assert!( + bloom_props.is_some(), + "Bloom filter should be enabled for tag_service" + ); + + let bloom_props = bloom_props.unwrap(); + assert_eq!( + bloom_props.ndv, BLOOM_FILTER_NDV_TAGS, + "Bloom filter NDV for tag columns should be {}", + BLOOM_FILTER_NDV_TAGS + ); + + // Verify bloom filter is NOT enabled on timestamp_secs (not a filtering column) + let timestamp_path = ColumnPath::new(vec!["timestamp_secs".to_string()]); + let bloom_props = props.bloom_filter_properties(×tamp_path); + assert!( + bloom_props.is_none(), + "Bloom filter should NOT be enabled for timestamp_secs" + ); + + // Verify bloom filter is NOT enabled on value column + let value_path = ColumnPath::new(vec!["value".to_string()]); + let bloom_props = props.bloom_filter_properties(&value_path); + assert!( + bloom_props.is_none(), + "Bloom filter should NOT be enabled for value" + ); + } + + #[test] + fn test_statistics_enabled() { + let config = ParquetWriterConfig::default(); + let props = config.to_writer_properties(); + + // Verify statistics are enabled at Chunk (row group) level + let metric_name_path = ColumnPath::new(vec!["metric_name".to_string()]); + assert_eq!( + props.statistics_enabled(&metric_name_path), + EnabledStatistics::Chunk, + "Statistics should be enabled at Chunk level" + ); + + // Verify for timestamp column as well (important for time range pruning) + let timestamp_path = ColumnPath::new(vec!["timestamp_secs".to_string()]); + assert_eq!( + props.statistics_enabled(×tamp_path), + EnabledStatistics::Chunk, + "Statistics should be enabled at Chunk level for timestamp" + ); + } + + #[test] + fn test_dictionary_encoding_enabled() { + let config = ParquetWriterConfig::default(); + let props = config.to_writer_properties(); + + // Verify dictionary encoding is enabled for dictionary columns + let dictionary_columns = [ + "metric_name", + "tag_service", + "tag_env", + "tag_datacenter", + "tag_region", + "tag_host", + "service_name", + ]; + + for col_name in dictionary_columns { + let col_path = ColumnPath::new(vec![col_name.to_string()]); + assert!( + props.dictionary_enabled(&col_path), + "Dictionary encoding should be enabled for {}", + col_name + ); + } + } + + #[test] + fn test_sorting_columns_order() { + let sorting_cols = ParquetWriterConfig::sorting_columns(); + + // Verify we have the expected number of sorting columns + assert_eq!( + sorting_cols.len(), + 7, + "Should have 7 sorting columns: metric_name, 5 tags, timestamp" + ); + + // Verify sort order matches expected: metric_name, tag_service, tag_env, + // tag_datacenter, tag_region, tag_host, timestamp_secs + let expected_order = [ + ParquetField::MetricName, + ParquetField::TagService, + ParquetField::TagEnv, + ParquetField::TagDatacenter, + ParquetField::TagRegion, + ParquetField::TagHost, + ParquetField::TimestampSecs, + ]; + + for (i, expected_field) in expected_order.iter().enumerate() { + assert_eq!( + sorting_cols[i].column_idx, + expected_field.column_index() as i32, + "Sorting column {} should be {}", + i, + expected_field.name() + ); + assert!(!sorting_cols[i].descending, "Sorting should be ascending"); + assert!(sorting_cols[i].nulls_first, "Nulls should be first"); + } + } +} diff --git a/quickwit/quickwit-lambda/src/bin/indexer.rs b/quickwit/quickwit-parquet-engine/src/storage/mod.rs similarity index 63% rename from quickwit/quickwit-lambda/src/bin/indexer.rs rename to quickwit/quickwit-parquet-engine/src/storage/mod.rs index 7af8d250dc0..0eb7a96391e 100644 --- a/quickwit/quickwit-lambda/src/bin/indexer.rs +++ b/quickwit/quickwit-parquet-engine/src/storage/mod.rs @@ -12,15 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -use lambda_runtime::service_fn; -use quickwit_lambda::indexer::handler; -use quickwit_lambda::logger; +//! Storage layer for metrics Parquet files. +//! +//! Provides configurable Parquet writer with compression options +//! optimized for metrics workloads. -#[tokio::main] -async fn main() -> anyhow::Result<()> { - logger::setup_lambda_tracer(tracing::Level::INFO)?; - let func = service_fn(handler); - lambda_runtime::run(func) - .await - .map_err(|e| anyhow::anyhow!(e)) -} +mod config; +mod split_writer; +mod writer; + +pub use config::{Compression, ParquetWriterConfig}; +pub use split_writer::ParquetSplitWriter; +pub use writer::{ParquetWriteError, ParquetWriter}; diff --git a/quickwit/quickwit-parquet-engine/src/storage/split_writer.rs b/quickwit/quickwit-parquet-engine/src/storage/split_writer.rs new file mode 100644 index 00000000000..8505eb0f75c --- /dev/null +++ b/quickwit/quickwit-parquet-engine/src/storage/split_writer.rs @@ -0,0 +1,517 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Split writer that orchestrates Parquet file writing with metadata tracking. + +use std::collections::HashSet; +use std::path::PathBuf; + +use arrow::array::{Array, AsArray}; +use arrow::compute::{max, min}; +use arrow::datatypes::UInt64Type; +use arrow::record_batch::RecordBatch; +use tracing::{debug, info, instrument}; + +use super::config::ParquetWriterConfig; +use super::writer::{ParquetWriteError, ParquetWriter}; +use crate::schema::{ParquetField, ParquetSchema}; +use crate::split::{MetricsSplitMetadata, ParquetSplit, SplitId, TAG_SERVICE, TimeRange}; + +/// Writer that produces complete ParquetSplit with metadata from RecordBatch data. +pub struct ParquetSplitWriter { + /// The underlying Parquet writer. + writer: ParquetWriter, + /// Base directory for split files. + base_path: PathBuf, +} + +impl ParquetSplitWriter { + /// Create a new ParquetSplitWriter. + /// + /// # Arguments + /// * `schema` - The metrics schema for validation + /// * `config` - Parquet writer configuration + /// * `base_path` - Directory where split files will be written + pub fn new( + schema: ParquetSchema, + config: ParquetWriterConfig, + base_path: impl Into, + ) -> Self { + Self { + writer: ParquetWriter::new(schema, config), + base_path: base_path.into(), + } + } + + /// Get the base path for split files. + pub fn base_path(&self) -> &PathBuf { + &self.base_path + } + + /// Write a RecordBatch to a Parquet file and return a ParquetSplit with metadata. + /// + /// # Arguments + /// * `batch` - The RecordBatch to write + /// * `index_uid` - The index unique identifier for the split metadata + /// + /// # Returns + /// A ParquetSplit containing metadata extracted from the batch and the file path. + #[instrument(skip(self, batch), fields(batch_rows = batch.num_rows()))] + pub fn write_split( + &self, + batch: &RecordBatch, + index_uid: &str, + ) -> Result { + // Generate unique split ID + let split_id = SplitId::generate(); + + let file_path = self.base_path.join(format!("{}.parquet", split_id)); + + // Ensure the base directory exists + std::fs::create_dir_all(&self.base_path)?; + + // Write batch to file + let size_bytes = self.writer.write_to_file(batch, &file_path)?; + + // Extract time range from batch + let time_range = extract_time_range(batch)?; + debug!( + start_secs = time_range.start_secs, + end_secs = time_range.end_secs, + "Extracted time range from batch" + ); + + // Extract distinct metric names from batch + let metric_names = extract_metric_names(batch)?; + + // Extract distinct service names from batch + let service_names = extract_service_names(batch)?; + + // Build metadata + let metadata = MetricsSplitMetadata::builder() + .split_id(split_id.clone()) + .index_uid(index_uid) + .time_range(time_range) + .num_rows(batch.num_rows() as u64) + .size_bytes(size_bytes); + + // Add metric names + let metadata = metric_names + .into_iter() + .fold(metadata, |m, name| m.add_metric_name(name)); + + // Add service names as low-cardinality tags + let metadata = service_names.into_iter().fold(metadata, |m, name| { + m.add_low_cardinality_tag(TAG_SERVICE, name) + }); + + let metadata = metadata.build(); + + info!( + split_id = %split_id, + file_path = %file_path.display(), + size_bytes, + "Split file written successfully" + ); + + Ok(ParquetSplit::new(metadata)) + } +} + +/// Extracts the time range (min/max timestamp_secs) from a RecordBatch. +fn extract_time_range(batch: &RecordBatch) -> Result { + let timestamp_col = batch.column(ParquetField::TimestampSecs.column_index()); + let timestamp_array = timestamp_col.as_primitive::(); + + let min_val = min(timestamp_array); + let max_val = max(timestamp_array); + + match (min_val, max_val) { + (Some(start), Some(end)) => { + // End is exclusive, so add 1 + Ok(TimeRange::new(start, end + 1)) + } + _ => { + // Empty batch or all nulls - use sentinel values + Ok(TimeRange::new(0, 0)) + } + } +} + +/// Extracts distinct metric names from a RecordBatch. +fn extract_metric_names(batch: &RecordBatch) -> Result, ParquetWriteError> { + let metric_col = batch.column(ParquetField::MetricName.column_index()); + let mut names = HashSet::new(); + + // The column is Dictionary(Int32, Utf8) + if let Some(dict_array) = metric_col + .as_any() + .downcast_ref::>() + { + let values = dict_array.values(); + if let Some(string_values) = values.as_any().downcast_ref::() { + // Get all dictionary values that are actually used + for i in 0..dict_array.len() { + if !dict_array.is_null(i) + && let Ok(key) = dict_array.keys().value(i).try_into() + { + let key: usize = key; + if key < string_values.len() && !string_values.is_null(key) { + names.insert(string_values.value(key).to_string()); + } + } + } + } + } + + Ok(names) +} + +/// Extracts distinct service names from a RecordBatch. +fn extract_service_names(batch: &RecordBatch) -> Result, ParquetWriteError> { + let service_col = batch.column(ParquetField::ServiceName.column_index()); + let mut names = HashSet::new(); + + // The column is Dictionary(Int32, Utf8) + if let Some(dict_array) = service_col + .as_any() + .downcast_ref::>() + { + let values = dict_array.values(); + if let Some(string_values) = values.as_any().downcast_ref::() { + // Get all dictionary values that are actually used + for i in 0..dict_array.len() { + if !dict_array.is_null(i) + && let Ok(key) = dict_array.keys().value(i).try_into() + { + let key: usize = key; + if key < string_values.len() && !string_values.is_null(key) { + names.insert(string_values.value(key).to_string()); + } + } + } + } + } + + Ok(names) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow::array::{ + ArrayRef, DictionaryArray, Float64Array, Int32Array, StringArray, UInt8Array, UInt64Array, + }; + use arrow::datatypes::Int32Type; + use parquet::variant::{VariantArrayBuilder, VariantBuilderExt}; + + use super::*; + + #[test] + fn test_column_indices_match_schema() { + let schema = ParquetSchema::new(); + for field in ParquetField::all() { + let expected: usize = + schema + .arrow_schema() + .index_of(field.name()) + .unwrap_or_else(|_| { + panic!("field {:?} should exist in arrow schema", field.name()) + }); + assert_eq!( + field.column_index(), + expected, + "column_index() for {:?} does not match arrow schema position", + field.name() + ); + } + } + + /// Create a VARIANT array for testing with specified number of rows. + fn create_variant_array(num_rows: usize, fields: Option<&[(&str, &str)]>) -> ArrayRef { + let mut builder = VariantArrayBuilder::new(num_rows); + for _ in 0..num_rows { + match fields { + Some(kv_pairs) => { + let mut obj = builder.new_object(); + for (key, value) in kv_pairs { + obj = obj.with_field(key, *value); + } + obj.finish(); + } + None => { + builder.append_null(); + } + } + } + ArrayRef::from(builder.build()) + } + + /// Create dictionary array for string fields with Int32 keys. + fn create_dict_array(values: &[&str]) -> ArrayRef { + let keys: Vec = (0..values.len()).map(|i| i as i32).collect(); + let string_array = StringArray::from(values.to_vec()); + Arc::new( + DictionaryArray::::try_new(Int32Array::from(keys), Arc::new(string_array)) + .unwrap(), + ) + } + + /// Create nullable dictionary array for optional string fields. + fn create_nullable_dict_array(values: &[Option<&str>]) -> ArrayRef { + let keys: Vec> = values + .iter() + .enumerate() + .map(|(i, v)| v.map(|_| i as i32)) + .collect(); + let string_values: Vec<&str> = values.iter().filter_map(|v| *v).collect(); + let string_array = StringArray::from(string_values); + Arc::new( + DictionaryArray::::try_new(Int32Array::from(keys), Arc::new(string_array)) + .unwrap(), + ) + } + + /// Create a test batch with specified number of rows and test data. + fn create_test_batch(num_rows: usize) -> RecordBatch { + let schema = ParquetSchema::new(); + + // Generate test data + let metric_names: Vec<&str> = (0..num_rows) + .map(|i| { + if i % 2 == 0 { + "cpu.usage" + } else { + "memory.used" + } + }) + .collect(); + let timestamps: Vec = (0..num_rows).map(|i| 100 + i as u64 * 10).collect(); + + // MetricName: Dictionary(Int32, Utf8) + let metric_name: ArrayRef = create_dict_array(&metric_names); + + // MetricType: UInt8 + let metric_type: ArrayRef = Arc::new(UInt8Array::from(vec![0u8; num_rows])); + + // MetricUnit: Utf8 (nullable) + let metric_unit: ArrayRef = Arc::new(StringArray::from(vec![Some("bytes"); num_rows])); + + // TimestampSecs: UInt64 + let timestamp_secs: ArrayRef = Arc::new(UInt64Array::from(timestamps)); + + // StartTimestampSecs: UInt64 (nullable) + let start_timestamp_secs: ArrayRef = + Arc::new(UInt64Array::from(vec![None::; num_rows])); + + // Value: Float64 + let values: Vec = (0..num_rows).map(|i| 42.0 + i as f64).collect(); + let value: ArrayRef = Arc::new(Float64Array::from(values)); + + // TagService: Dictionary(Int32, Utf8) (nullable) + let tag_service: ArrayRef = create_nullable_dict_array(&vec![Some("web"); num_rows]); + + // TagEnv: Dictionary(Int32, Utf8) (nullable) + let tag_env: ArrayRef = create_nullable_dict_array(&vec![Some("prod"); num_rows]); + + // TagDatacenter: Dictionary(Int32, Utf8) (nullable) + let tag_datacenter: ArrayRef = + create_nullable_dict_array(&vec![Some("us-east-1"); num_rows]); + + // TagRegion: Dictionary(Int32, Utf8) (nullable) + let tag_region: ArrayRef = create_nullable_dict_array(&vec![None; num_rows]); + + // TagHost: Dictionary(Int32, Utf8) (nullable) + let tag_host: ArrayRef = create_nullable_dict_array(&vec![Some("host-001"); num_rows]); + + // Attributes: VARIANT (nullable) + let attributes: ArrayRef = create_variant_array(num_rows, Some(&[("key", "value")])); + + // ServiceName: Dictionary(Int32, Utf8) + let service_names: Vec<&str> = (0..num_rows).map(|_| "my-service").collect(); + let service_name: ArrayRef = create_dict_array(&service_names); + + // ResourceAttributes: VARIANT (nullable) + let resource_attributes: ArrayRef = create_variant_array(num_rows, None); + + RecordBatch::try_new( + schema.arrow_schema().clone(), + vec![ + metric_name, + metric_type, + metric_unit, + timestamp_secs, + start_timestamp_secs, + value, + tag_service, + tag_env, + tag_datacenter, + tag_region, + tag_host, + attributes, + service_name, + resource_attributes, + ], + ) + .unwrap() + } + + #[test] + fn test_write_split_creates_file() { + let schema = ParquetSchema::new(); + let config = ParquetWriterConfig::default(); + let temp_dir = tempfile::tempdir().unwrap(); + + let writer = ParquetSplitWriter::new(schema, config, temp_dir.path()); + + let batch = create_test_batch(10); + let split = writer.write_split(&batch, "test-index").unwrap(); + + // Verify file exists + let file_path = temp_dir.path().join(split.metadata.parquet_filename()); + assert!( + std::fs::metadata(&file_path).is_ok(), + "Parquet file should exist" + ); + + // Verify metadata + assert_eq!(split.metadata.num_rows, 10); + assert!(split.metadata.size_bytes > 0); + } + + #[test] + fn test_write_split_extracts_time_range() { + let schema = ParquetSchema::new(); + let config = ParquetWriterConfig::default(); + let temp_dir = tempfile::tempdir().unwrap(); + + let writer = ParquetSplitWriter::new(schema, config, temp_dir.path()); + + // Create batch with timestamps [100, 150, 200] + let batch = create_test_batch_with_timestamps(&[100, 150, 200]); + let split = writer.write_split(&batch, "test-index").unwrap(); + + // Verify time range + assert_eq!(split.metadata.time_range.start_secs, 100); + assert_eq!(split.metadata.time_range.end_secs, 201); // exclusive + } + + #[test] + fn test_write_split_extracts_metric_names() { + let schema = ParquetSchema::new(); + let config = ParquetWriterConfig::default(); + let temp_dir = tempfile::tempdir().unwrap(); + + let writer = ParquetSplitWriter::new(schema, config, temp_dir.path()); + + // Create batch with specific metric names + let batch = create_test_batch_with_metric_names(&["cpu.usage", "memory.used", "cpu.usage"]); + let split = writer.write_split(&batch, "test-index").unwrap(); + + // Verify metric names (distinct values) + assert!(split.metadata.metric_names.contains("cpu.usage")); + assert!(split.metadata.metric_names.contains("memory.used")); + assert_eq!(split.metadata.metric_names.len(), 2); + } + + /// Create a test batch with specific timestamps. + fn create_test_batch_with_timestamps(timestamps: &[u64]) -> RecordBatch { + let schema = ParquetSchema::new(); + let num_rows = timestamps.len(); + + let metric_names: Vec<&str> = vec!["test.metric"; num_rows]; + let metric_name: ArrayRef = create_dict_array(&metric_names); + let metric_type: ArrayRef = Arc::new(UInt8Array::from(vec![0u8; num_rows])); + let metric_unit: ArrayRef = Arc::new(StringArray::from(vec![Some("bytes"); num_rows])); + let timestamp_secs: ArrayRef = Arc::new(UInt64Array::from(timestamps.to_vec())); + let start_timestamp_secs: ArrayRef = + Arc::new(UInt64Array::from(vec![None::; num_rows])); + let value: ArrayRef = Arc::new(Float64Array::from(vec![42.0; num_rows])); + let tag_service: ArrayRef = create_nullable_dict_array(&vec![Some("web"); num_rows]); + let tag_env: ArrayRef = create_nullable_dict_array(&vec![Some("prod"); num_rows]); + let tag_datacenter: ArrayRef = + create_nullable_dict_array(&vec![Some("us-east-1"); num_rows]); + let tag_region: ArrayRef = create_nullable_dict_array(&vec![None; num_rows]); + let tag_host: ArrayRef = create_nullable_dict_array(&vec![Some("host-001"); num_rows]); + let attributes: ArrayRef = create_variant_array(num_rows, None); + let service_name: ArrayRef = create_dict_array(&vec!["my-service"; num_rows]); + let resource_attributes: ArrayRef = create_variant_array(num_rows, None); + + RecordBatch::try_new( + schema.arrow_schema().clone(), + vec![ + metric_name, + metric_type, + metric_unit, + timestamp_secs, + start_timestamp_secs, + value, + tag_service, + tag_env, + tag_datacenter, + tag_region, + tag_host, + attributes, + service_name, + resource_attributes, + ], + ) + .unwrap() + } + + /// Create a test batch with specific metric names. + fn create_test_batch_with_metric_names(metric_names: &[&str]) -> RecordBatch { + let schema = ParquetSchema::new(); + let num_rows = metric_names.len(); + + let metric_name: ArrayRef = create_dict_array(metric_names); + let metric_type: ArrayRef = Arc::new(UInt8Array::from(vec![0u8; num_rows])); + let metric_unit: ArrayRef = Arc::new(StringArray::from(vec![Some("bytes"); num_rows])); + let timestamp_secs: ArrayRef = Arc::new(UInt64Array::from(vec![100u64; num_rows])); + let start_timestamp_secs: ArrayRef = + Arc::new(UInt64Array::from(vec![None::; num_rows])); + let value: ArrayRef = Arc::new(Float64Array::from(vec![42.0; num_rows])); + let tag_service: ArrayRef = create_nullable_dict_array(&vec![Some("web"); num_rows]); + let tag_env: ArrayRef = create_nullable_dict_array(&vec![Some("prod"); num_rows]); + let tag_datacenter: ArrayRef = + create_nullable_dict_array(&vec![Some("us-east-1"); num_rows]); + let tag_region: ArrayRef = create_nullable_dict_array(&vec![None; num_rows]); + let tag_host: ArrayRef = create_nullable_dict_array(&vec![Some("host-001"); num_rows]); + let attributes: ArrayRef = create_variant_array(num_rows, None); + let service_name: ArrayRef = create_dict_array(&vec!["my-service"; num_rows]); + let resource_attributes: ArrayRef = create_variant_array(num_rows, None); + + RecordBatch::try_new( + schema.arrow_schema().clone(), + vec![ + metric_name, + metric_type, + metric_unit, + timestamp_secs, + start_timestamp_secs, + value, + tag_service, + tag_env, + tag_datacenter, + tag_region, + tag_host, + attributes, + service_name, + resource_attributes, + ], + ) + .unwrap() + } +} diff --git a/quickwit/quickwit-parquet-engine/src/storage/writer.rs b/quickwit/quickwit-parquet-engine/src/storage/writer.rs new file mode 100644 index 00000000000..50a375f42b8 --- /dev/null +++ b/quickwit/quickwit-parquet-engine/src/storage/writer.rs @@ -0,0 +1,568 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Parquet writer for metrics RecordBatch data. + +use std::fs::File; +use std::io::Cursor; +use std::path::Path; +use std::sync::Arc; + +use arrow::array::RecordBatch; +use arrow::compute::{SortColumn, SortOptions, lexsort_to_indices, take_record_batch}; +use parquet::arrow::ArrowWriter; +use parquet::errors::ParquetError; +use thiserror::Error; +use tracing::{debug, instrument}; + +use super::config::ParquetWriterConfig; +use crate::schema::{ParquetField, ParquetSchema}; + +/// Errors that can occur during parquet writing. +#[derive(Debug, Error)] +pub enum ParquetWriteError { + /// Parquet write error. + #[error("Parquet error: {0}")] + ParquetError(#[from] ParquetError), + + /// IO error. + #[error("IO error: {0}")] + IoError(#[from] std::io::Error), + + /// Arrow compute error during sorting. + #[error("Arrow error: {0}")] + ArrowError(#[from] arrow::error::ArrowError), + + /// Schema mismatch between RecordBatch and ParquetSchema. + #[error("Schema mismatch: expected {expected} fields, got {got}")] + SchemaMismatch { expected: usize, got: usize }, +} + +/// Writer for metrics data to Parquet format. +pub struct ParquetWriter { + config: ParquetWriterConfig, + schema: ParquetSchema, +} + +impl ParquetWriter { + /// Create a new ParquetWriter. + pub fn new(schema: ParquetSchema, config: ParquetWriterConfig) -> Self { + Self { config, schema } + } + + /// Get the writer configuration. + pub fn config(&self) -> &ParquetWriterConfig { + &self.config + } + + /// Get the metrics schema. + pub fn schema(&self) -> &ParquetSchema { + &self.schema + } + + /// Validate that a RecordBatch matches the expected schema. + fn validate_batch(&self, batch: &RecordBatch) -> Result<(), ParquetWriteError> { + let expected = self.schema.num_fields(); + let got = batch.num_columns(); + if expected != got { + return Err(ParquetWriteError::SchemaMismatch { expected, got }); + } + Ok(()) + } + + /// Sort a RecordBatch by the metrics sort order. + /// Order: metric_name, tag_service, tag_env, tag_datacenter, tag_region, tag_host, + /// timestamp_secs. This sorting enables efficient pruning during query execution. + fn sort_batch(&self, batch: &RecordBatch) -> Result { + // Build sort columns from the defined sort order + let sort_columns: Vec = ParquetField::sort_order() + .iter() + .map(|field| { + let col_idx = field.column_index(); + SortColumn { + values: Arc::clone(batch.column(col_idx)), + options: Some(SortOptions { + descending: false, + nulls_first: true, + }), + } + }) + .collect(); + + // Compute sorted indices + let indices = lexsort_to_indices(&sort_columns, None)?; + + // Reorder the batch using the sorted indices + let sorted_batch = take_record_batch(batch, &indices)?; + Ok(sorted_batch) + } + + /// Write a RecordBatch to Parquet bytes in memory. + /// The batch is sorted before writing by: metric_name, common tags, timestamp. + #[instrument(skip(self, batch), fields(batch_rows = batch.num_rows()))] + pub fn write_to_bytes(&self, batch: &RecordBatch) -> Result, ParquetWriteError> { + self.validate_batch(batch)?; + + // Sort the batch before writing for efficient pruning + let sorted_batch = self.sort_batch(batch)?; + + let props = self.config.to_writer_properties(); + let buffer = Cursor::new(Vec::new()); + + let mut writer = ArrowWriter::try_new(buffer, sorted_batch.schema(), Some(props))?; + writer.write(&sorted_batch)?; + let buffer = writer.into_inner()?; + + let bytes = buffer.into_inner(); + debug!(bytes_written = bytes.len(), "Completed write to bytes"); + Ok(bytes) + } + + /// Write a RecordBatch to a Parquet file. + /// The batch is sorted before writing by: metric_name, common tags, timestamp. + /// + /// Returns the number of bytes written. + #[instrument(skip(self, batch), fields(batch_rows = batch.num_rows(), path = %path.display()))] + pub fn write_to_file( + &self, + batch: &RecordBatch, + path: &Path, + ) -> Result { + self.validate_batch(batch)?; + + // Sort the batch before writing for efficient pruning + let sorted_batch = self.sort_batch(batch)?; + + let props = self.config.to_writer_properties(); + let file = File::create(path)?; + + let mut writer = ArrowWriter::try_new(file, sorted_batch.schema(), Some(props))?; + writer.write(&sorted_batch)?; + let file = writer.into_inner()?; + + let bytes_written = file.metadata()?.len(); + debug!(bytes_written, "Completed write to file"); + Ok(bytes_written) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow::array::{ + ArrayRef, DictionaryArray, Float64Array, StringArray, UInt8Array, UInt64Array, + }; + use arrow::datatypes::{DataType, Field, Int32Type, Schema}; + use parquet::variant::{VariantArrayBuilder, VariantBuilderExt}; + + use super::*; + + /// Create dictionary array for string fields with Int32 keys. + fn create_dict_array(values: Vec<&str>) -> ArrayRef { + let string_array = StringArray::from(values); + Arc::new( + DictionaryArray::::try_new( + arrow::array::Int32Array::from(vec![0i32]), + Arc::new(string_array), + ) + .unwrap(), + ) + } + + /// Create nullable dictionary array for optional string fields. + fn create_nullable_dict_array(value: Option<&str>) -> ArrayRef { + match value { + Some(v) => { + let string_array = StringArray::from(vec![v]); + Arc::new( + DictionaryArray::::try_new( + arrow::array::Int32Array::from(vec![0i32]), + Arc::new(string_array), + ) + .unwrap(), + ) + } + None => { + let string_array = StringArray::from(vec![None::<&str>]); + Arc::new( + DictionaryArray::::try_new( + arrow::array::Int32Array::from(vec![None::]), + Arc::new(string_array), + ) + .unwrap(), + ) + } + } + } + + /// Create a VARIANT array for testing. + fn create_variant_array(fields: Option<&[(&str, &str)]>) -> ArrayRef { + let mut builder = VariantArrayBuilder::new(1); + match fields { + Some(kv_pairs) => { + let mut obj = builder.new_object(); + for (key, value) in kv_pairs { + obj = obj.with_field(key, *value); + } + obj.finish(); + } + None => { + builder.append_null(); + } + } + ArrayRef::from(builder.build()) + } + + fn create_test_batch() -> RecordBatch { + let schema = ParquetSchema::new(); + + // Create arrays for all 14 fields in ParquetSchema matching fields.rs: + // MetricName: Dictionary(Int32, Utf8) + let metric_name: ArrayRef = create_dict_array(vec!["test.metric"]); + + // MetricType: UInt8 + let metric_type: ArrayRef = Arc::new(UInt8Array::from(vec![0u8])); // gauge + + // MetricUnit: Utf8 (nullable) + let metric_unit: ArrayRef = Arc::new(StringArray::from(vec![Some("bytes")])); + + // TimestampSecs: UInt64 + let timestamp_secs: ArrayRef = Arc::new(UInt64Array::from(vec![1704067200u64])); + + // StartTimestampSecs: UInt64 (nullable) + let start_timestamp_secs: ArrayRef = Arc::new(UInt64Array::from(vec![None::])); + + // Value: Float64 + let value: ArrayRef = Arc::new(Float64Array::from(vec![42.0])); + + // TagService: Dictionary(Int32, Utf8) (nullable) + let tag_service: ArrayRef = create_nullable_dict_array(Some("web")); + + // TagEnv: Dictionary(Int32, Utf8) (nullable) + let tag_env: ArrayRef = create_nullable_dict_array(Some("prod")); + + // TagDatacenter: Dictionary(Int32, Utf8) (nullable) + let tag_datacenter: ArrayRef = create_nullable_dict_array(Some("us-east-1")); + + // TagRegion: Dictionary(Int32, Utf8) (nullable) + let tag_region: ArrayRef = create_nullable_dict_array(None); + + // TagHost: Dictionary(Int32, Utf8) (nullable) + let tag_host: ArrayRef = create_nullable_dict_array(Some("host-001")); + + // Attributes: VARIANT (nullable) + let attributes: ArrayRef = create_variant_array(Some(&[("key", "value")])); + + // ServiceName: Dictionary(Int32, Utf8) + let service_name: ArrayRef = create_dict_array(vec!["my-service"]); + + // ResourceAttributes: VARIANT (nullable) + let resource_attributes: ArrayRef = create_variant_array(None); + + RecordBatch::try_new( + schema.arrow_schema().clone(), + vec![ + metric_name, + metric_type, + metric_unit, + timestamp_secs, + start_timestamp_secs, + value, + tag_service, + tag_env, + tag_datacenter, + tag_region, + tag_host, + attributes, + service_name, + resource_attributes, + ], + ) + .unwrap() + } + + #[test] + fn test_writer_creation() { + let schema = ParquetSchema::new(); + let config = ParquetWriterConfig::default(); + let writer = ParquetWriter::new(schema, config); + + assert_eq!(writer.schema().num_fields(), 14); + } + + #[test] + fn test_write_to_bytes() { + let schema = ParquetSchema::new(); + let config = ParquetWriterConfig::default(); + let writer = ParquetWriter::new(schema, config); + + let batch = create_test_batch(); + let bytes = writer.write_to_bytes(&batch).unwrap(); + + // Parquet files start with PAR1 magic bytes + assert!(bytes.len() > 4); + assert_eq!(&bytes[0..4], b"PAR1"); + } + + #[test] + fn test_write_to_file() { + let schema = ParquetSchema::new(); + let config = ParquetWriterConfig::default(); + let writer = ParquetWriter::new(schema, config); + + let batch = create_test_batch(); + let temp_dir = std::env::temp_dir(); + let path = temp_dir.join("test_metrics.parquet"); + + let bytes_written = writer.write_to_file(&batch, &path).unwrap(); + assert!(bytes_written > 0); + + // Clean up + std::fs::remove_file(&path).ok(); + } + + #[test] + fn test_schema_mismatch() { + let schema = ParquetSchema::new(); + let config = ParquetWriterConfig::default(); + let writer = ParquetWriter::new(schema, config); + + // Create a batch with wrong number of columns + let wrong_schema = Arc::new(Schema::new(vec![Field::new( + "single_field", + DataType::Utf8, + false, + )])); + let wrong_batch = RecordBatch::try_new( + wrong_schema, + vec![Arc::new(StringArray::from(vec!["test"]))], + ) + .unwrap(); + + let result = writer.write_to_bytes(&wrong_batch); + assert!(matches!( + result, + Err(ParquetWriteError::SchemaMismatch { + expected: 14, + got: 1 + }) + )); + } + + #[test] + fn test_write_with_snappy_compression() { + use super::super::config::Compression; + + let schema = ParquetSchema::new(); + let config = ParquetWriterConfig::new().with_compression(Compression::Snappy); + let writer = ParquetWriter::new(schema, config); + + let batch = create_test_batch(); + let bytes = writer.write_to_bytes(&batch).unwrap(); + + assert!(bytes.len() > 4); + assert_eq!(&bytes[0..4], b"PAR1"); + } + + #[test] + fn test_write_sorts_data() { + use std::fs::File; + + use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; + + let schema = ParquetSchema::new(); + let config = ParquetWriterConfig::default(); + let writer = ParquetWriter::new(schema.clone(), config); + + // Create unsorted batch with multiple rows: + // Row 0: metric_b, service_a, timestamp=300 + // Row 1: metric_a, service_b, timestamp=100 + // Row 2: metric_a, service_a, timestamp=200 + // Expected sorted order: metric_a/service_a/200, metric_a/service_b/100, + // metric_b/service_a/300 + + // Build arrays for 3 rows (original unsorted order in comments above) + let timestamps = [300u64, 100u64, 200u64]; + let values = [1.0, 2.0, 3.0]; + + // metric_name: Dictionary(Int32, Utf8) + let metric_name: ArrayRef = { + let keys = arrow::array::Int32Array::from(vec![0i32, 1, 1]); + let values = StringArray::from(vec!["metric_b", "metric_a"]); + Arc::new(DictionaryArray::::try_new(keys, Arc::new(values)).unwrap()) + }; + + let metric_type: ArrayRef = Arc::new(UInt8Array::from(vec![0u8, 0, 0])); + let metric_unit: ArrayRef = Arc::new(StringArray::from(vec![ + Some("bytes"), + Some("bytes"), + Some("bytes"), + ])); + let timestamp_secs: ArrayRef = Arc::new(UInt64Array::from(timestamps.to_vec())); + let start_timestamp_secs: ArrayRef = + Arc::new(UInt64Array::from(vec![None::, None, None])); + let value: ArrayRef = Arc::new(Float64Array::from(values.to_vec())); + + // tag_service: Dictionary(Int32, Utf8) (nullable) + let tag_service: ArrayRef = { + let keys = arrow::array::Int32Array::from(vec![Some(0i32), Some(1), Some(0)]); + let values = StringArray::from(vec!["service_a", "service_b"]); + Arc::new(DictionaryArray::::try_new(keys, Arc::new(values)).unwrap()) + }; + + let tag_env: ArrayRef = { + let keys = arrow::array::Int32Array::from(vec![Some(0i32), Some(0), Some(0)]); + let values = StringArray::from(vec!["prod"]); + Arc::new(DictionaryArray::::try_new(keys, Arc::new(values)).unwrap()) + }; + + let tag_datacenter: ArrayRef = { + let keys = arrow::array::Int32Array::from(vec![None::, None, None]); + let values = StringArray::from(vec![None::<&str>]); + Arc::new(DictionaryArray::::try_new(keys, Arc::new(values)).unwrap()) + }; + + let tag_region: ArrayRef = { + let keys = arrow::array::Int32Array::from(vec![None::, None, None]); + let values = StringArray::from(vec![None::<&str>]); + Arc::new(DictionaryArray::::try_new(keys, Arc::new(values)).unwrap()) + }; + + let tag_host: ArrayRef = { + let keys = arrow::array::Int32Array::from(vec![None::, None, None]); + let values = StringArray::from(vec![None::<&str>]); + Arc::new(DictionaryArray::::try_new(keys, Arc::new(values)).unwrap()) + }; + + // Build VARIANT arrays for 3 rows + let attributes: ArrayRef = { + let mut builder = VariantArrayBuilder::new(3); + for _ in 0..3 { + builder.append_null(); + } + ArrayRef::from(builder.build()) + }; + + let service_name: ArrayRef = { + let keys = arrow::array::Int32Array::from(vec![0i32, 1, 0]); + let values = StringArray::from(vec!["service_a", "service_b"]); + Arc::new(DictionaryArray::::try_new(keys, Arc::new(values)).unwrap()) + }; + + let resource_attributes: ArrayRef = { + let mut builder = VariantArrayBuilder::new(3); + for _ in 0..3 { + builder.append_null(); + } + ArrayRef::from(builder.build()) + }; + + let batch = RecordBatch::try_new( + schema.arrow_schema().clone(), + vec![ + metric_name, + metric_type, + metric_unit, + timestamp_secs, + start_timestamp_secs, + value, + tag_service, + tag_env, + tag_datacenter, + tag_region, + tag_host, + attributes, + service_name, + resource_attributes, + ], + ) + .unwrap(); + + // Write to file (will be sorted) + let temp_dir = std::env::temp_dir(); + let path = temp_dir.join("test_sorting.parquet"); + writer.write_to_file(&batch, &path).unwrap(); + + // Read back and verify sort order + let file = File::open(&path).unwrap(); + let reader = ParquetRecordBatchReaderBuilder::try_new(file) + .unwrap() + .build() + .unwrap(); + + let batches: Vec<_> = reader.collect::, _>>().unwrap(); + assert_eq!(batches.len(), 1); + + let result = &batches[0]; + assert_eq!(result.num_rows(), 3); + + // Extract metric names and timestamps to verify sort order + let metric_col = result + .column(ParquetField::MetricName.column_index()) + .as_any() + .downcast_ref::>() + .unwrap(); + let ts_col = result + .column(ParquetField::TimestampSecs.column_index()) + .as_any() + .downcast_ref::() + .unwrap(); + let service_col = result + .column(ParquetField::TagService.column_index()) + .as_any() + .downcast_ref::>() + .unwrap(); + + // Get string values from dictionary + let get_metric = |i: usize| -> &str { + let key = metric_col.keys().value(i); + metric_col + .values() + .as_any() + .downcast_ref::() + .unwrap() + .value(key as usize) + }; + let get_service = |i: usize| -> &str { + let key = service_col.keys().value(i); + service_col + .values() + .as_any() + .downcast_ref::() + .unwrap() + .value(key as usize) + }; + + // Expected sort order: metric_name ASC, tag_service ASC, timestamp ASC + // Row 0: metric_a, service_a, 200 (original row 2) + // Row 1: metric_a, service_b, 100 (original row 1) + // Row 2: metric_b, service_a, 300 (original row 0) + + assert_eq!(get_metric(0), "metric_a"); + assert_eq!(get_service(0), "service_a"); + assert_eq!(ts_col.value(0), 200); + + assert_eq!(get_metric(1), "metric_a"); + assert_eq!(get_service(1), "service_b"); + assert_eq!(ts_col.value(1), 100); + + assert_eq!(get_metric(2), "metric_b"); + assert_eq!(get_service(2), "service_a"); + assert_eq!(ts_col.value(2), 300); + + // Clean up + std::fs::remove_file(&path).ok(); + } +} diff --git a/quickwit/quickwit-proto/Cargo.toml b/quickwit/quickwit-proto/Cargo.toml index 1d68bc539a0..fa52a1b3fea 100644 --- a/quickwit/quickwit-proto/Cargo.toml +++ b/quickwit/quickwit-proto/Cargo.toml @@ -16,6 +16,7 @@ bytes = { workspace = true } bytesize = { workspace = true } bytestring = { workspace = true } futures = { workspace = true, optional = true } +hex = { workspace = true } http = { workspace = true } mockall = { workspace = true, optional = true } opentelemetry = { workspace = true } @@ -32,6 +33,7 @@ tonic = { workspace = true, features = [ "server", "channel", ] } +tonic-prost = { workspace = true } tower = { workspace = true } tracing = { workspace = true } tracing-opentelemetry = { workspace = true } @@ -50,6 +52,7 @@ mockall = { workspace = true } glob = "0.3" prost-build = { workspace = true } tonic-build = { workspace = true } +tonic-prost-build = { workspace = true } quickwit-codegen = { workspace = true } diff --git a/quickwit/quickwit-proto/build.rs b/quickwit/quickwit-proto/build.rs index 2dd109cf533..569d9b5315b 100644 --- a/quickwit/quickwit-proto/build.rs +++ b/quickwit/quickwit-proto/build.rs @@ -147,7 +147,6 @@ fn main() -> Result<(), Box> { .extern_path(".quickwit.common.IndexUid", "crate::types::IndexUid") .extern_path(".quickwit.ingest.Position", "crate::types::Position") .extern_path(".quickwit.ingest.ShardId", "crate::types::ShardId") - .type_attribute("Shard", "#[derive(Eq)]") .field_attribute( "Shard.follower_id", "#[serde(default, skip_serializing_if = \"Option::is_none\")]", @@ -190,33 +189,38 @@ fn main() -> Result<(), Box> { .file_descriptor_set_path("src/codegen/quickwit/search_descriptor.bin") .protoc_arg("--experimental_allow_proto3_optional"); - tonic_build::configure() + tonic_prost_build::configure() .enum_attribute(".", "#[serde(rename_all=\"snake_case\")]") .type_attribute( ".", "#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)]", ) - .type_attribute("PartialHit", "#[derive(Eq, Hash)]") .type_attribute("PartialHit.sort_value", "#[derive(Copy)]") - .type_attribute("SearchRequest", "#[derive(Eq, Hash)]") - .type_attribute("ListFieldSerialized", "#[derive(Eq)]") .type_attribute("SortByValue", "#[derive(Ord, PartialOrd)]") - .type_attribute("SortField", "#[derive(Eq, Hash)]") + .type_attribute("SearchRequest", "#[derive(Hash, Eq)]") + .type_attribute("PartialHit", "#[derive(Hash, Eq)]") .out_dir("src/codegen/quickwit") - .compile_protos_with_config(prost_config, &["protos/quickwit/search.proto"], &["protos"])?; + .compile_with_config( + prost_config, + &[std::path::PathBuf::from("protos/quickwit/search.proto")], + &[std::path::PathBuf::from("protos")], + )?; // Jaeger proto let protos = find_protos("protos/third-party/jaeger"); let mut prost_config = prost_build::Config::default(); - prost_config.type_attribute("Operation", "#[derive(Eq, Ord, PartialOrd)]"); + prost_config.type_attribute("Operation", "#[derive(Ord, PartialOrd)]"); - tonic_build::configure() + tonic_prost_build::configure() .out_dir("src/codegen/jaeger") - .compile_protos_with_config( + .compile_with_config( prost_config, &protos, - &["protos/third-party/jaeger", "protos/third-party"], + &[ + std::path::PathBuf::from("protos/third-party/jaeger"), + std::path::PathBuf::from("protos/third-party"), + ], )?; // OTEL proto @@ -224,7 +228,7 @@ fn main() -> Result<(), Box> { prost_config.protoc_arg("--experimental_allow_proto3_optional"); let protos = find_protos("protos/third-party/opentelemetry"); - tonic_build::configure() + tonic_prost_build::configure() .type_attribute(".", "#[derive(serde::Serialize, serde::Deserialize)]") .type_attribute("StatusCode", r#"#[serde(rename_all = "snake_case")]"#) .type_attribute( @@ -232,7 +236,11 @@ fn main() -> Result<(), Box> { r#"#[derive(utoipa::ToSchema)]"#, ) .out_dir("src/codegen/opentelemetry") - .compile_protos_with_config(prost_config, &protos, &["protos/third-party"])?; + .compile_with_config( + prost_config, + &protos, + &[std::path::PathBuf::from("protos/third-party")], + )?; Ok(()) } diff --git a/quickwit/quickwit-proto/protos/quickwit/control_plane.proto b/quickwit/quickwit-proto/protos/quickwit/control_plane.proto index 453b469fa77..d0850091280 100644 --- a/quickwit/quickwit-proto/protos/quickwit/control_plane.proto +++ b/quickwit/quickwit-proto/protos/quickwit/control_plane.proto @@ -118,6 +118,7 @@ message GetOrCreateOpenShardsFailure { message AdviseResetShardsRequest { repeated quickwit.ingest.ShardIds shard_ids = 1; + string ingester_id = 2; } message AdviseResetShardsResponse { diff --git a/quickwit/quickwit-proto/protos/quickwit/ingest.proto b/quickwit/quickwit-proto/protos/quickwit/ingest.proto index d7b32b6504e..349490ce8b5 100644 --- a/quickwit/quickwit-proto/protos/quickwit/ingest.proto +++ b/quickwit/quickwit-proto/protos/quickwit/ingest.proto @@ -48,10 +48,20 @@ enum CommitTypeV2 { COMMIT_TYPE_V2_FORCE = 3; } +// Format of documents in DocBatchV2.doc_buffer +enum DocFormat { + // JSON documents, one per entry in doc_lengths (default for backward compatibility) + DOC_FORMAT_JSON = 0; + // Arrow IPC stream format containing a single RecordBatch + DOC_FORMAT_ARROW_IPC = 1; +} + message DocBatchV2 { bytes doc_buffer = 1; repeated uint32 doc_lengths = 2; repeated quickwit.common.DocUid doc_uids = 3; + // Format of the documents in doc_buffer. Defaults to JSON for backward compatibility. + DocFormat doc_format = 4; } message MRecordBatch { diff --git a/quickwit/quickwit-proto/protos/quickwit/ingester.proto b/quickwit/quickwit-proto/protos/quickwit/ingester.proto index a5b651d94d8..a76daa912d7 100644 --- a/quickwit/quickwit-proto/protos/quickwit/ingester.proto +++ b/quickwit/quickwit-proto/protos/quickwit/ingester.proto @@ -73,14 +73,27 @@ message PersistSubrequest { uint32 subrequest_id = 1; quickwit.common.IndexUid index_uid = 2; string source_id = 3; - quickwit.ingest.ShardId shard_id = 4; quickwit.ingest.DocBatchV2 doc_batch = 5; + reserved 4; } message PersistResponse { string leader_id = 1; repeated PersistSuccess successes = 2; repeated PersistFailure failures = 3; + RoutingUpdate routing_update = 4; +} + +message RoutingUpdate { + uint32 capacity_score = 1; + repeated SourceShardUpdate source_shard_updates = 2; + repeated quickwit.ingest.ShardIds closed_shards = 3; +} + +message SourceShardUpdate { + quickwit.common.IndexUid index_uid = 1; + string source_id = 2; + uint32 open_shard_count = 3; } message PersistSuccess { @@ -96,19 +109,21 @@ message PersistSuccess { enum PersistFailureReason { PERSIST_FAILURE_REASON_UNSPECIFIED = 0; - PERSIST_FAILURE_REASON_SHARD_NOT_FOUND = 1; - PERSIST_FAILURE_REASON_SHARD_CLOSED = 2; - PERSIST_FAILURE_REASON_SHARD_RATE_LIMITED = 3; PERSIST_FAILURE_REASON_WAL_FULL = 4; PERSIST_FAILURE_REASON_TIMEOUT = 5; + PERSIST_FAILURE_REASON_NO_SHARDS_AVAILABLE = 6; + PERSIST_FAILURE_REASON_NODE_UNAVAILABLE = 7; + reserved 1; + reserved 2; + reserved 3; } message PersistFailure { uint32 subrequest_id = 1; quickwit.common.IndexUid index_uid = 2; string source_id = 3; - quickwit.ingest.ShardId shard_id = 4; PersistFailureReason reason = 5; + reserved 4; } message SynReplicationMessage { @@ -300,6 +315,8 @@ enum IngesterStatus { INGESTER_STATUS_INITIALIZING = 1; // The ingester is ready and accepts read and write requests. INGESTER_STATUS_READY = 2; + // The ingester is about to be decommissioned. It still accepts read and write requests, but will not accept write requests in a few seconds and should be avoided by future write requests. + INGESTER_STATUS_RETIRING = 6; // The ingester is being decommissioned. It accepts read requests but rejects write requests // (open shards, persist, and replicate requests). It will transition to `Decommissioned` once // all shards are fully indexed. diff --git a/quickwit/quickwit-proto/protos/quickwit/metastore.proto b/quickwit/quickwit-proto/protos/quickwit/metastore.proto index 38da75e4f56..206d835537c 100644 --- a/quickwit/quickwit-proto/protos/quickwit/metastore.proto +++ b/quickwit/quickwit-proto/protos/quickwit/metastore.proto @@ -107,6 +107,9 @@ service MetastoreService { // Deletes an index rpc DeleteIndex(DeleteIndexRequest) returns (EmptyResponse); + // Returns a list of size info for each index. + rpc ListIndexStats(ListIndexStatsRequest) returns (ListIndexStatsResponse); + // Streams splits from index. rpc ListSplits(ListSplitsRequest) returns (stream ListSplitsResponse); @@ -198,6 +201,29 @@ service MetastoreService { // Deletes index templates. rpc DeleteIndexTemplates(DeleteIndexTemplatesRequest) returns (EmptyResponse); + + // Get cluster identity + rpc GetClusterIdentity(GetClusterIdentityRequest) returns (GetClusterIdentityResponse); + + // ======================================== + // Metrics Split API + // ======================================== + // Parallel lifecycle to Tantivy splits, operates on metrics_splits table. + + // Stages metrics splits in the metastore. + rpc StageMetricsSplits(StageMetricsSplitsRequest) returns (EmptyResponse); + + // Publishes metrics splits. + rpc PublishMetricsSplits(PublishMetricsSplitsRequest) returns (EmptyResponse); + + // Lists metrics splits. + rpc ListMetricsSplits(ListMetricsSplitsRequest) returns (ListMetricsSplitsResponse); + + // Marks metrics splits for deletion. + rpc MarkMetricsSplitsForDeletion(MarkMetricsSplitsForDeletionRequest) returns (EmptyResponse); + + // Deletes metrics splits. + rpc DeleteMetricsSplits(DeleteMetricsSplitsRequest) returns (EmptyResponse); } message EmptyResponse { @@ -283,6 +309,30 @@ enum IndexMetadataFailureReason { INDEX_METADATA_FAILURE_REASON_INTERNAL = 2; } +message ListIndexStatsRequest { + // List of patterns an index should match or not match to get considered + // An index must match at least one positive pattern (a pattern not starting + // with a '-'), and no negative pattern (a pattern starting with a '-'). + repeated string index_id_patterns = 1; +} + +message ListIndexStatsResponse { + // list of IndexStats. each one has the index id, the number of splits and the total size. + repeated IndexStats index_stats = 1; +} + +message IndexStats { + quickwit.common.IndexUid index_uid = 1; + SplitStats staged = 2; + SplitStats published = 3; + SplitStats marked_for_deletion = 4; +} + +message SplitStats { + uint64 num_splits = 1; + uint64 total_size_bytes = 2; +} + message ListSplitsRequest { // Predicate used to filter splits. // The predicate is expressed as a JSON serialized @@ -530,3 +580,57 @@ message ListIndexTemplatesResponse { message DeleteIndexTemplatesRequest { repeated string template_ids = 1; } + +message GetClusterIdentityRequest { +} + +message GetClusterIdentityResponse { + string uuid = 1; +} + +// ======================================== +// Metrics Split Messages +// ======================================== +// Metrics splits use a separate table (metrics_splits) with different +// schema optimized for time-series queries (GIN indexes on tags, etc.) +// Messages mirror Tantivy split messages and use IndexUid consistently. + +// Stage metrics splits - first step after MetricsSplitWriter creates Parquet +message StageMetricsSplitsRequest { + quickwit.common.IndexUid index_uid = 1; + // Each element is a MetricsSplitMetadata serialized as JSON + repeated string splits_metadata_json = 2; +} + +// Publish metrics splits - make staged splits queryable +message PublishMetricsSplitsRequest { + quickwit.common.IndexUid index_uid = 1; + repeated string staged_split_ids = 2; + repeated string replaced_split_ids = 3; // For merge operations (future) + optional string index_checkpoint_delta_json_opt = 4; + optional string publish_token_opt = 5; +} + +// List metrics splits - for query planning and Tier 1 pruning +message ListMetricsSplitsRequest { + quickwit.common.IndexUid index_uid = 1; + // Query predicate as JSON (ListMetricsSplitsQuery) + string query_json = 2; +} + +message ListMetricsSplitsResponse { + // Each element is a MetricsSplit serialized as JSON + repeated string splits_serialized_json = 1; +} + +// Mark metrics splits for deletion - cleanup flow +message MarkMetricsSplitsForDeletionRequest { + quickwit.common.IndexUid index_uid = 1; + repeated string split_ids = 2; +} + +// Delete metrics splits - after storage cleanup complete +message DeleteMetricsSplitsRequest { + quickwit.common.IndexUid index_uid = 1; + repeated string split_ids = 2; +} diff --git a/quickwit/quickwit-proto/protos/quickwit/search.proto b/quickwit/quickwit-proto/protos/quickwit/search.proto index ca96ad13e60..a49b951b3ca 100644 --- a/quickwit/quickwit-proto/protos/quickwit/search.proto +++ b/quickwit/quickwit-proto/protos/quickwit/search.proto @@ -37,9 +37,6 @@ service SearchService { /// This methods takes `PartialHit`s and returns `Hit`s. rpc FetchDocs(FetchDocsRequest) returns (FetchDocsResponse); - // Perform a leaf stream on a given set of splits. - rpc LeafSearchStream(LeafSearchStreamRequest) returns (stream LeafSearchStreamResponse); - // Root list terms API. // This RPC identifies the set of splits on which the query should run on, // and dispatches the several calls to `LeafListTerms`. @@ -128,6 +125,10 @@ message ListFieldsRequest { optional int64 start_timestamp = 3; optional int64 end_timestamp = 4; + // JSON-serialized QueryAst for index_filter support. + // When provided, only fields from documents matching this query are returned. + optional string query_ast = 5; + // Control if the request will fail if split_ids contains a split that does not exist. // optional bool fail_on_missing_index = 6; } @@ -144,7 +145,6 @@ message LeafListFieldsRequest { // Optional limit query to a list of fields // Wildcard expressions are supported. repeated string fields = 4; - } message ListFieldsResponse { @@ -201,6 +201,9 @@ message SearchRequest { // Time filter, expressed in seconds since epoch. // That filter is to be interpreted as the semi-open interval: // [start_timestamp, end_timestamp). + // If the query AST contains a range query over the timestamp field, + // then the the bounds of the range query are used directly and + // these two fields are ignored. optional int64 start_timestamp = 4; optional int64 end_timestamp = 5; @@ -243,6 +246,14 @@ message SearchRequest { optional PartialHit search_after = 16; CountHits count_hits = 17; + + // When an exact index ID is provided (not a pattern), the query fails only if + // that index is not found and this parameter is set to `false`. + bool ignore_missing_indexes = 18; + + // When true, skip finalization of aggregation results and return + // the raw IntermediateAggregationResults bytes instead. + bool skip_aggregation_finalization = 19; } enum CountHits { @@ -325,8 +336,7 @@ message SplitSearchError { bool retryable_error = 3; } -/// A LeafSearchRequest can span multiple indices. -/// +// A LeafSearchRequest can span multiple indices. message LeafSearchRequest { // Search request. This is a perfect copy of the original search request // that was sent to root apart from the start_offset, max_hits params and index_id_patterns. @@ -353,7 +363,7 @@ message ResourceStats { uint64 cpu_microsecs = 5; } -/// LeafRequestRef references data in LeafSearchRequest to deduplicate data. +// LeafRequestRef references data in LeafSearchRequest to deduplicate data. message LeafRequestRef { // The ordinal of the doc_mapper in `LeafSearchRequest.doc_mappers` uint32 doc_mapper_ord = 1; @@ -489,6 +499,25 @@ message LeafSearchResponse { ResourceStats resource_stats = 8; } +// The result of searching a single split in a Lambda invocation. +// Each result is tagged with its split_id so that ordering is irrelevant. +message LambdaSingleSplitResult { + // The split that was searched. + string split_id = 1; + oneof outcome { + // On success, the leaf search response for this split. + LeafSearchResponse response = 2; + // On failure, the error message. + string error = 3; + } +} + +// Wrapper for per-split results from a Lambda invocation. +message LambdaSearchResponses { + reserved 1; // was: repeated LeafSearchResponse responses + repeated LambdaSingleSplitResult split_results = 2; +} + message SnippetRequest { repeated string snippet_fields = 1; string query_ast_resolved = 2; @@ -578,73 +607,3 @@ message LeafListTermsResponse { // Total number of single split search attempted. uint64 num_attempted_splits = 4; } - -// -- Stream ------------------- - -enum OutputFormat { - // Comma Separated Values format (https://datatracker.ietf.org/doc/html/rfc4180). - // The delimiter is `,`. - CSV = 0; //< This will be the default value - // Format data by row in ClickHouse binary format. - // https://clickhouse.tech/docs/en/interfaces/formats/#rowbinary - CLICK_HOUSE_ROW_BINARY = 1; -} - -message SearchStreamRequest { - // Index ID - string index_id = 1; - - // deprecated `query`. - reserved 2; - - // Quickwit Query AST encoded in Json - string query_ast = 11; - - // deprecated `search_fields` - reserved 3; - - // The time filter is interpreted as a semi-open interval. [start, end) - optional int64 start_timestamp = 4; - optional int64 end_timestamp = 5; - - // Name of the fast field to extract - string fast_field = 6; - - // The output format - OutputFormat output_format = 7; - - reserved 8; // deprecated field: tags - - // The field by which we want to partition - optional string partition_by_field = 9; - - // Fields to extract snippet on. - repeated string snippet_fields = 10; -} - -message LeafSearchStreamRequest { - // Stream request. This is a perfect copy of the original stream request, - // that was sent to root. - SearchStreamRequest request = 1; - - // Index split ids to apply the query on. - // This ids are resolved from the index_uri defined in the stream request. - repeated SplitIdAndFooterOffsets split_offsets = 2; - - // `DocMapper` as json serialized trait. - string doc_mapper = 5; - - // Index URI. The index URI defines the location of the storage that contains the - // split files. - string index_uri = 6; - -} - - -message LeafSearchStreamResponse { - // Row of data serialized in bytes. - bytes data = 1; - - // Split id. - string split_id = 2; -} diff --git a/quickwit/quickwit-proto/protos/third-party/jaeger/storage/v2/trace_storage.proto b/quickwit/quickwit-proto/protos/third-party/jaeger/storage/v2/trace_storage.proto new file mode 100644 index 00000000000..47b366aa7ac --- /dev/null +++ b/quickwit/quickwit-proto/protos/third-party/jaeger/storage/v2/trace_storage.proto @@ -0,0 +1,177 @@ +// Copyright (c) 2024 The Jaeger Authors. +// SPDX-License-Identifier: Apache-2.0 + +syntax = "proto3"; + +package jaeger.storage.v2; + +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; +import "opentelemetry/proto/trace/v1/trace.proto"; + +option go_package = "storage"; + +// GetTraceParams represents the query for a single trace from the storage backend. +message GetTraceParams { + // trace_id is a 16 byte array containing the unique identifier for the trace to query. + bytes trace_id = 1; + + // start_time is the start of the time interval to search for the trace_id. + // + // This field is optional. + google.protobuf.Timestamp start_time = 2; + + // end_time is the end of the time interval to search for the trace_id. + // + // This field is optional. + google.protobuf.Timestamp end_time = 3; +} + +// GetTracesRequest represents a request to retrieve multiple traces. +message GetTracesRequest { + repeated GetTraceParams query = 1; +} + +// GetServicesRequest represents a request to get service names. +message GetServicesRequest {} + +// GetServicesResponse represents the response for GetServicesRequest. +message GetServicesResponse { + repeated string services = 1; +} + +// GetOperationsRequest represents a request to get operation names. +message GetOperationsRequest { + // service is the name of the service for which to get operation names. + // + // This field is required. + string service = 1; + + // span_kind is the type of span which is used to distinguish between + // spans generated in a particular context. + // + // This field is optional. + string span_kind = 2; +} + +// Operation contains information about an operation for a given service. +message Operation { + string name = 1; + string span_kind = 2; +} + +// GetOperationsResponse represents the response for GetOperationsRequest. +message GetOperationsResponse { + repeated Operation operations = 1; +} + +// KeyValue and all its associated types are copied from opentelemetry-proto/common/v1/common.proto +// (https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/common/v1/common.proto). +// This type is used to store attributes in traces. +message KeyValue { + string key = 1; + AnyValue value = 2; +} + +message AnyValue { + oneof value { + string string_value = 1; + bool bool_value = 2; + int64 int_value = 3; + double double_value = 4; + ArrayValue array_value = 5; + KeyValueList kvlist_value = 6; + bytes bytes_value = 7; + } +} + +message KeyValueList { + repeated KeyValue values = 1; +} + +message ArrayValue { + repeated AnyValue values = 1; +} + +// TraceQueryParameters contains query parameters to find traces. For a detailed +// definition of each field in this message, refer to `TraceQueryParameters` in `jaeger.api_v3` +// (https://github.com/jaegertracing/jaeger-idl/blob/main/proto/api_v3/query_service.proto). +message TraceQueryParameters { + string service_name = 1; + string operation_name = 2; + repeated KeyValue attributes = 3; + google.protobuf.Timestamp start_time_min = 4; + google.protobuf.Timestamp start_time_max = 5; + google.protobuf.Duration duration_min = 6; + google.protobuf.Duration duration_max = 7; + int32 search_depth = 8; +} + +// FindTracesRequest represents a request to find traces. +// It can be used to retrieve the traces (FindTraces) or simply +// the trace IDs (FindTraceIDs). +message FindTracesRequest { + TraceQueryParameters query = 1; +} + +// FoundTraceID is a wrapper around trace ID returned from FindTraceIDs +// with an optional time range that may be used in GetTraces calls. +// +// The time range is provided as an optimization hint for some storage backends +// that can perform more efficient queries when they know the approximate time range. +// The value should not be used for precise time-based filtering or assumptions. +// It is meant as a rough boundary and may not be populated in all cases. +message FoundTraceID { + bytes trace_id = 1; + google.protobuf.Timestamp start = 2; + google.protobuf.Timestamp end = 3; +} + +// FindTraceIDsResponse represents the response for FindTracesRequest. +message FindTraceIDsResponse { + repeated FoundTraceID trace_ids = 1; +} + +// TraceReader is a service that allows reading traces from storage. +// Note that if you implement this service, you should also implement +// OTEL's TraceService in package opentelemetry.proto.collector.trace.v1 +// to allow pushing traces to the storage backend +// () +service TraceReader { + // GetTraces returns a stream that retrieves all traces with given IDs. + // + // Chunking requirements: + // - A single TracesData chunk MUST NOT contain spans from multiple traces. + // - Large traces MAY be split across multiple, *consecutive* TracesData chunks. + // - Each returned TracesData object MUST NOT be empty. + // + // Edge cases: + // - If no spans are found for any given trace ID, the ID is ignored. + // - If none of the trace IDs are found in the storage, an empty response is returned. + rpc GetTraces(GetTracesRequest) returns (stream opentelemetry.proto.trace.v1.TracesData) {} + + // GetServices returns all service names known to the backend from traces + // within its retention period. + rpc GetServices(GetServicesRequest) returns (GetServicesResponse) {} + + // GetOperations returns all operation names for a given service + // known to the backend from traces within its retention period. + rpc GetOperations(GetOperationsRequest) returns (GetOperationsResponse) {} + + // FindTraces returns a stream that retrieves traces matching query parameters. + // + // The chunking rules are the same as for GetTraces. + // + // If no matching traces are found, an empty stream is returned. + rpc FindTraces(FindTracesRequest) returns (stream opentelemetry.proto.trace.v1.TracesData) {} + + // FindTraceIDs returns a stream that retrieves IDs of traces matching query parameters. + // + // If no matching traces are found, an empty stream is returned. + // + // This call behaves identically to FindTraces, except that it returns only the list + // of matching trace IDs. This is useful in some contexts, such as batch jobs, where a + // large list of trace IDs may be queried first and then the full traces are loaded + // in batches. + rpc FindTraceIDs(FindTracesRequest) returns (FindTraceIDsResponse) {} +} diff --git a/quickwit/quickwit-proto/src/codegen/jaeger/jaeger.api_v2.rs b/quickwit/quickwit-proto/src/codegen/jaeger/jaeger.api_v2.rs index 5afca901e56..62c517f5232 100644 --- a/quickwit/quickwit-proto/src/codegen/jaeger/jaeger.api_v2.rs +++ b/quickwit/quickwit-proto/src/codegen/jaeger/jaeger.api_v2.rs @@ -23,7 +23,7 @@ pub struct Log { #[prost(message, repeated, tag = "2")] pub fields: ::prost::alloc::vec::Vec, } -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SpanRef { #[prost(bytes = "vec", tag = "1")] pub trace_id: ::prost::alloc::vec::Vec, @@ -101,7 +101,7 @@ pub struct Batch { #[prost(message, optional, tag = "2")] pub process: ::core::option::Option, } -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DependencyLink { #[prost(string, tag = "1")] pub parent: ::prost::alloc::string::String, diff --git a/quickwit/quickwit-proto/src/codegen/jaeger/jaeger.storage.v1.rs b/quickwit/quickwit-proto/src/codegen/jaeger/jaeger.storage.v1.rs index 70bbd1d3a46..aa3807fbb32 100644 --- a/quickwit/quickwit-proto/src/codegen/jaeger/jaeger.storage.v1.rs +++ b/quickwit/quickwit-proto/src/codegen/jaeger/jaeger.storage.v1.rs @@ -1,5 +1,5 @@ // This file is @generated by prost-build. -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetDependenciesRequest { #[prost(message, optional, tag = "1")] pub start_time: ::core::option::Option<::prost_types::Timestamp>, @@ -17,35 +17,35 @@ pub struct WriteSpanRequest { pub span: ::core::option::Option, } /// empty; extensible in the future -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct WriteSpanResponse {} /// empty; extensible in the future -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct CloseWriterRequest {} /// empty; extensible in the future -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct CloseWriterResponse {} -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetTraceRequest { #[prost(bytes = "vec", tag = "1")] pub trace_id: ::prost::alloc::vec::Vec, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetServicesRequest {} -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetServicesResponse { #[prost(string, repeated, tag = "1")] pub services: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetOperationsRequest { #[prost(string, tag = "1")] pub service: ::prost::alloc::string::String, #[prost(string, tag = "2")] pub span_kind: ::prost::alloc::string::String, } -#[derive(Eq, Ord, PartialOrd)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Ord, PartialOrd)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct Operation { #[prost(string, tag = "1")] pub name: ::prost::alloc::string::String, @@ -97,15 +97,15 @@ pub struct FindTraceIDsRequest { #[prost(message, optional, tag = "1")] pub query: ::core::option::Option, } -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct FindTraceIDsResponse { #[prost(bytes = "vec", repeated, tag = "1")] pub trace_ids: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, } /// empty; extensible in the future -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct CapabilitiesRequest {} -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct CapabilitiesResponse { #[prost(bool, tag = "1")] pub archive_span_reader: bool, @@ -221,7 +221,7 @@ pub mod span_writer_plugin_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/jaeger.storage.v1.SpanWriterPlugin/WriteSpan", ); @@ -247,7 +247,7 @@ pub mod span_writer_plugin_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/jaeger.storage.v1.SpanWriterPlugin/Close", ); @@ -258,6 +258,241 @@ pub mod span_writer_plugin_client { } } } +/// Generated server implementations. +pub mod span_writer_plugin_server { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with SpanWriterPluginServer. + #[async_trait] + pub trait SpanWriterPlugin: std::marker::Send + std::marker::Sync + 'static { + /// spanstore/Writer + async fn write_span( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + async fn close( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + } + #[derive(Debug)] + pub struct SpanWriterPluginServer { + inner: Arc, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + impl SpanWriterPluginServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for SpanWriterPluginServer + where + T: SpanWriterPlugin, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + match req.uri().path() { + "/jaeger.storage.v1.SpanWriterPlugin/WriteSpan" => { + #[allow(non_camel_case_types)] + struct WriteSpanSvc(pub Arc); + impl< + T: SpanWriterPlugin, + > tonic::server::UnaryService + for WriteSpanSvc { + type Response = super::WriteSpanResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::write_span(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = WriteSpanSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/jaeger.storage.v1.SpanWriterPlugin/Close" => { + #[allow(non_camel_case_types)] + struct CloseSvc(pub Arc); + impl< + T: SpanWriterPlugin, + > tonic::server::UnaryService + for CloseSvc { + type Response = super::CloseWriterResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::close(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = CloseSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => { + Box::pin(async move { + let mut response = http::Response::new( + tonic::body::Body::default(), + ); + let headers = response.headers_mut(); + headers + .insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers + .insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) + }) + } + } + } + } + impl Clone for SpanWriterPluginServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "jaeger.storage.v1.SpanWriterPlugin"; + impl tonic::server::NamedService for SpanWriterPluginServer { + const NAME: &'static str = SERVICE_NAME; + } +} /// Generated client implementations. pub mod streaming_span_writer_plugin_client { #![allow( @@ -366,7 +601,7 @@ pub mod streaming_span_writer_plugin_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/jaeger.storage.v1.StreamingSpanWriterPlugin/WriteSpanStream", ); @@ -382,8 +617,8 @@ pub mod streaming_span_writer_plugin_client { } } } -/// Generated client implementations. -pub mod span_reader_plugin_client { +/// Generated server implementations. +pub mod streaming_span_writer_plugin_server { #![allow( unused_variables, dead_code, @@ -392,10 +627,199 @@ pub mod span_reader_plugin_client { clippy::let_unit_value, )] use tonic::codegen::*; - use tonic::codegen::http::Uri; - #[derive(Debug, Clone)] - pub struct SpanReaderPluginClient { - inner: tonic::client::Grpc, + /// Generated trait containing gRPC methods that should be implemented for use with StreamingSpanWriterPluginServer. + #[async_trait] + pub trait StreamingSpanWriterPlugin: std::marker::Send + std::marker::Sync + 'static { + async fn write_span_stream( + &self, + request: tonic::Request>, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + } + #[derive(Debug)] + pub struct StreamingSpanWriterPluginServer { + inner: Arc, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + impl StreamingSpanWriterPluginServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> + for StreamingSpanWriterPluginServer + where + T: StreamingSpanWriterPlugin, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + match req.uri().path() { + "/jaeger.storage.v1.StreamingSpanWriterPlugin/WriteSpanStream" => { + #[allow(non_camel_case_types)] + struct WriteSpanStreamSvc(pub Arc); + impl< + T: StreamingSpanWriterPlugin, + > tonic::server::ClientStreamingService + for WriteSpanStreamSvc { + type Response = super::WriteSpanResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request< + tonic::Streaming, + >, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::write_span_stream( + &inner, + request, + ) + .await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = WriteSpanStreamSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.client_streaming(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => { + Box::pin(async move { + let mut response = http::Response::new( + tonic::body::Body::default(), + ); + let headers = response.headers_mut(); + headers + .insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers + .insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) + }) + } + } + } + } + impl Clone for StreamingSpanWriterPluginServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "jaeger.storage.v1.StreamingSpanWriterPlugin"; + impl tonic::server::NamedService for StreamingSpanWriterPluginServer { + const NAME: &'static str = SERVICE_NAME; + } +} +/// Generated client implementations. +pub mod span_reader_plugin_client { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] + use tonic::codegen::*; + use tonic::codegen::http::Uri; + #[derive(Debug, Clone)] + pub struct SpanReaderPluginClient { + inner: tonic::client::Grpc, } impl SpanReaderPluginClient { /// Attempt to create a new client by connecting to a given endpoint. @@ -489,7 +913,7 @@ pub mod span_reader_plugin_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/jaeger.storage.v1.SpanReaderPlugin/GetTrace", ); @@ -515,7 +939,7 @@ pub mod span_reader_plugin_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/jaeger.storage.v1.SpanReaderPlugin/GetServices", ); @@ -541,7 +965,7 @@ pub mod span_reader_plugin_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/jaeger.storage.v1.SpanReaderPlugin/GetOperations", ); @@ -570,7 +994,7 @@ pub mod span_reader_plugin_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/jaeger.storage.v1.SpanReaderPlugin/FindTraces", ); @@ -596,939 +1020,18 @@ pub mod span_reader_plugin_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/jaeger.storage.v1.SpanReaderPlugin/FindTraceIDs", ); let mut req = request.into_request(); req.extensions_mut() - .insert( - GrpcMethod::new("jaeger.storage.v1.SpanReaderPlugin", "FindTraceIDs"), - ); - self.inner.unary(req, path, codec).await - } - } -} -/// Generated client implementations. -pub mod archive_span_writer_plugin_client { - #![allow( - unused_variables, - dead_code, - missing_docs, - clippy::wildcard_imports, - clippy::let_unit_value, - )] - use tonic::codegen::*; - use tonic::codegen::http::Uri; - #[derive(Debug, Clone)] - pub struct ArchiveSpanWriterPluginClient { - inner: tonic::client::Grpc, - } - impl ArchiveSpanWriterPluginClient { - /// Attempt to create a new client by connecting to a given endpoint. - pub async fn connect(dst: D) -> Result - where - D: TryInto, - D::Error: Into, - { - let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; - Ok(Self::new(conn)) - } - } - impl ArchiveSpanWriterPluginClient - where - T: tonic::client::GrpcService, - T::Error: Into, - T::ResponseBody: Body + std::marker::Send + 'static, - ::Error: Into + std::marker::Send, - { - pub fn new(inner: T) -> Self { - let inner = tonic::client::Grpc::new(inner); - Self { inner } - } - pub fn with_origin(inner: T, origin: Uri) -> Self { - let inner = tonic::client::Grpc::with_origin(inner, origin); - Self { inner } - } - pub fn with_interceptor( - inner: T, - interceptor: F, - ) -> ArchiveSpanWriterPluginClient> - where - F: tonic::service::Interceptor, - T::ResponseBody: Default, - T: tonic::codegen::Service< - http::Request, - Response = http::Response< - >::ResponseBody, - >, - >, - , - >>::Error: Into + std::marker::Send + std::marker::Sync, - { - ArchiveSpanWriterPluginClient::new( - InterceptedService::new(inner, interceptor), - ) - } - /// Compress requests with the given encoding. - /// - /// This requires the server to support it otherwise it might respond with an - /// error. - #[must_use] - pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { - self.inner = self.inner.send_compressed(encoding); - self - } - /// Enable decompressing responses. - #[must_use] - pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { - self.inner = self.inner.accept_compressed(encoding); - self - } - /// Limits the maximum size of a decoded message. - /// - /// Default: `4MB` - #[must_use] - pub fn max_decoding_message_size(mut self, limit: usize) -> Self { - self.inner = self.inner.max_decoding_message_size(limit); - self - } - /// Limits the maximum size of an encoded message. - /// - /// Default: `usize::MAX` - #[must_use] - pub fn max_encoding_message_size(mut self, limit: usize) -> Self { - self.inner = self.inner.max_encoding_message_size(limit); - self - } - /// spanstore/Writer - pub async fn write_archive_span( - &mut self, - request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/jaeger.storage.v1.ArchiveSpanWriterPlugin/WriteArchiveSpan", - ); - let mut req = request.into_request(); - req.extensions_mut() - .insert( - GrpcMethod::new( - "jaeger.storage.v1.ArchiveSpanWriterPlugin", - "WriteArchiveSpan", - ), - ); - self.inner.unary(req, path, codec).await - } - } -} -/// Generated client implementations. -pub mod archive_span_reader_plugin_client { - #![allow( - unused_variables, - dead_code, - missing_docs, - clippy::wildcard_imports, - clippy::let_unit_value, - )] - use tonic::codegen::*; - use tonic::codegen::http::Uri; - #[derive(Debug, Clone)] - pub struct ArchiveSpanReaderPluginClient { - inner: tonic::client::Grpc, - } - impl ArchiveSpanReaderPluginClient { - /// Attempt to create a new client by connecting to a given endpoint. - pub async fn connect(dst: D) -> Result - where - D: TryInto, - D::Error: Into, - { - let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; - Ok(Self::new(conn)) - } - } - impl ArchiveSpanReaderPluginClient - where - T: tonic::client::GrpcService, - T::Error: Into, - T::ResponseBody: Body + std::marker::Send + 'static, - ::Error: Into + std::marker::Send, - { - pub fn new(inner: T) -> Self { - let inner = tonic::client::Grpc::new(inner); - Self { inner } - } - pub fn with_origin(inner: T, origin: Uri) -> Self { - let inner = tonic::client::Grpc::with_origin(inner, origin); - Self { inner } - } - pub fn with_interceptor( - inner: T, - interceptor: F, - ) -> ArchiveSpanReaderPluginClient> - where - F: tonic::service::Interceptor, - T::ResponseBody: Default, - T: tonic::codegen::Service< - http::Request, - Response = http::Response< - >::ResponseBody, - >, - >, - , - >>::Error: Into + std::marker::Send + std::marker::Sync, - { - ArchiveSpanReaderPluginClient::new( - InterceptedService::new(inner, interceptor), - ) - } - /// Compress requests with the given encoding. - /// - /// This requires the server to support it otherwise it might respond with an - /// error. - #[must_use] - pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { - self.inner = self.inner.send_compressed(encoding); - self - } - /// Enable decompressing responses. - #[must_use] - pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { - self.inner = self.inner.accept_compressed(encoding); - self - } - /// Limits the maximum size of a decoded message. - /// - /// Default: `4MB` - #[must_use] - pub fn max_decoding_message_size(mut self, limit: usize) -> Self { - self.inner = self.inner.max_decoding_message_size(limit); - self - } - /// Limits the maximum size of an encoded message. - /// - /// Default: `usize::MAX` - #[must_use] - pub fn max_encoding_message_size(mut self, limit: usize) -> Self { - self.inner = self.inner.max_encoding_message_size(limit); - self - } - /// spanstore/Reader - pub async fn get_archive_trace( - &mut self, - request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response>, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/jaeger.storage.v1.ArchiveSpanReaderPlugin/GetArchiveTrace", - ); - let mut req = request.into_request(); - req.extensions_mut() - .insert( - GrpcMethod::new( - "jaeger.storage.v1.ArchiveSpanReaderPlugin", - "GetArchiveTrace", - ), - ); - self.inner.server_streaming(req, path, codec).await - } - } -} -/// Generated client implementations. -pub mod dependencies_reader_plugin_client { - #![allow( - unused_variables, - dead_code, - missing_docs, - clippy::wildcard_imports, - clippy::let_unit_value, - )] - use tonic::codegen::*; - use tonic::codegen::http::Uri; - #[derive(Debug, Clone)] - pub struct DependenciesReaderPluginClient { - inner: tonic::client::Grpc, - } - impl DependenciesReaderPluginClient { - /// Attempt to create a new client by connecting to a given endpoint. - pub async fn connect(dst: D) -> Result - where - D: TryInto, - D::Error: Into, - { - let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; - Ok(Self::new(conn)) - } - } - impl DependenciesReaderPluginClient - where - T: tonic::client::GrpcService, - T::Error: Into, - T::ResponseBody: Body + std::marker::Send + 'static, - ::Error: Into + std::marker::Send, - { - pub fn new(inner: T) -> Self { - let inner = tonic::client::Grpc::new(inner); - Self { inner } - } - pub fn with_origin(inner: T, origin: Uri) -> Self { - let inner = tonic::client::Grpc::with_origin(inner, origin); - Self { inner } - } - pub fn with_interceptor( - inner: T, - interceptor: F, - ) -> DependenciesReaderPluginClient> - where - F: tonic::service::Interceptor, - T::ResponseBody: Default, - T: tonic::codegen::Service< - http::Request, - Response = http::Response< - >::ResponseBody, - >, - >, - , - >>::Error: Into + std::marker::Send + std::marker::Sync, - { - DependenciesReaderPluginClient::new( - InterceptedService::new(inner, interceptor), - ) - } - /// Compress requests with the given encoding. - /// - /// This requires the server to support it otherwise it might respond with an - /// error. - #[must_use] - pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { - self.inner = self.inner.send_compressed(encoding); - self - } - /// Enable decompressing responses. - #[must_use] - pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { - self.inner = self.inner.accept_compressed(encoding); - self - } - /// Limits the maximum size of a decoded message. - /// - /// Default: `4MB` - #[must_use] - pub fn max_decoding_message_size(mut self, limit: usize) -> Self { - self.inner = self.inner.max_decoding_message_size(limit); - self - } - /// Limits the maximum size of an encoded message. - /// - /// Default: `usize::MAX` - #[must_use] - pub fn max_encoding_message_size(mut self, limit: usize) -> Self { - self.inner = self.inner.max_encoding_message_size(limit); - self - } - /// dependencystore/Reader - pub async fn get_dependencies( - &mut self, - request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/jaeger.storage.v1.DependenciesReaderPlugin/GetDependencies", - ); - let mut req = request.into_request(); - req.extensions_mut() - .insert( - GrpcMethod::new( - "jaeger.storage.v1.DependenciesReaderPlugin", - "GetDependencies", - ), - ); - self.inner.unary(req, path, codec).await - } - } -} -/// Generated client implementations. -pub mod plugin_capabilities_client { - #![allow( - unused_variables, - dead_code, - missing_docs, - clippy::wildcard_imports, - clippy::let_unit_value, - )] - use tonic::codegen::*; - use tonic::codegen::http::Uri; - #[derive(Debug, Clone)] - pub struct PluginCapabilitiesClient { - inner: tonic::client::Grpc, - } - impl PluginCapabilitiesClient { - /// Attempt to create a new client by connecting to a given endpoint. - pub async fn connect(dst: D) -> Result - where - D: TryInto, - D::Error: Into, - { - let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; - Ok(Self::new(conn)) - } - } - impl PluginCapabilitiesClient - where - T: tonic::client::GrpcService, - T::Error: Into, - T::ResponseBody: Body + std::marker::Send + 'static, - ::Error: Into + std::marker::Send, - { - pub fn new(inner: T) -> Self { - let inner = tonic::client::Grpc::new(inner); - Self { inner } - } - pub fn with_origin(inner: T, origin: Uri) -> Self { - let inner = tonic::client::Grpc::with_origin(inner, origin); - Self { inner } - } - pub fn with_interceptor( - inner: T, - interceptor: F, - ) -> PluginCapabilitiesClient> - where - F: tonic::service::Interceptor, - T::ResponseBody: Default, - T: tonic::codegen::Service< - http::Request, - Response = http::Response< - >::ResponseBody, - >, - >, - , - >>::Error: Into + std::marker::Send + std::marker::Sync, - { - PluginCapabilitiesClient::new(InterceptedService::new(inner, interceptor)) - } - /// Compress requests with the given encoding. - /// - /// This requires the server to support it otherwise it might respond with an - /// error. - #[must_use] - pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { - self.inner = self.inner.send_compressed(encoding); - self - } - /// Enable decompressing responses. - #[must_use] - pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { - self.inner = self.inner.accept_compressed(encoding); - self - } - /// Limits the maximum size of a decoded message. - /// - /// Default: `4MB` - #[must_use] - pub fn max_decoding_message_size(mut self, limit: usize) -> Self { - self.inner = self.inner.max_decoding_message_size(limit); - self - } - /// Limits the maximum size of an encoded message. - /// - /// Default: `usize::MAX` - #[must_use] - pub fn max_encoding_message_size(mut self, limit: usize) -> Self { - self.inner = self.inner.max_encoding_message_size(limit); - self - } - pub async fn capabilities( - &mut self, - request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/jaeger.storage.v1.PluginCapabilities/Capabilities", - ); - let mut req = request.into_request(); - req.extensions_mut() - .insert( - GrpcMethod::new( - "jaeger.storage.v1.PluginCapabilities", - "Capabilities", - ), - ); - self.inner.unary(req, path, codec).await - } - } -} -/// Generated server implementations. -pub mod span_writer_plugin_server { - #![allow( - unused_variables, - dead_code, - missing_docs, - clippy::wildcard_imports, - clippy::let_unit_value, - )] - use tonic::codegen::*; - /// Generated trait containing gRPC methods that should be implemented for use with SpanWriterPluginServer. - #[async_trait] - pub trait SpanWriterPlugin: std::marker::Send + std::marker::Sync + 'static { - /// spanstore/Writer - async fn write_span( - &self, - request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; - async fn close( - &self, - request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; - } - #[derive(Debug)] - pub struct SpanWriterPluginServer { - inner: Arc, - accept_compression_encodings: EnabledCompressionEncodings, - send_compression_encodings: EnabledCompressionEncodings, - max_decoding_message_size: Option, - max_encoding_message_size: Option, - } - impl SpanWriterPluginServer { - pub fn new(inner: T) -> Self { - Self::from_arc(Arc::new(inner)) - } - pub fn from_arc(inner: Arc) -> Self { - Self { - inner, - accept_compression_encodings: Default::default(), - send_compression_encodings: Default::default(), - max_decoding_message_size: None, - max_encoding_message_size: None, - } - } - pub fn with_interceptor( - inner: T, - interceptor: F, - ) -> InterceptedService - where - F: tonic::service::Interceptor, - { - InterceptedService::new(Self::new(inner), interceptor) - } - /// Enable decompressing requests with the given encoding. - #[must_use] - pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { - self.accept_compression_encodings.enable(encoding); - self - } - /// Compress responses with the given encoding, if the client supports it. - #[must_use] - pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { - self.send_compression_encodings.enable(encoding); - self - } - /// Limits the maximum size of a decoded message. - /// - /// Default: `4MB` - #[must_use] - pub fn max_decoding_message_size(mut self, limit: usize) -> Self { - self.max_decoding_message_size = Some(limit); - self - } - /// Limits the maximum size of an encoded message. - /// - /// Default: `usize::MAX` - #[must_use] - pub fn max_encoding_message_size(mut self, limit: usize) -> Self { - self.max_encoding_message_size = Some(limit); - self - } - } - impl tonic::codegen::Service> for SpanWriterPluginServer - where - T: SpanWriterPlugin, - B: Body + std::marker::Send + 'static, - B::Error: Into + std::marker::Send + 'static, - { - type Response = http::Response; - type Error = std::convert::Infallible; - type Future = BoxFuture; - fn poll_ready( - &mut self, - _cx: &mut Context<'_>, - ) -> Poll> { - Poll::Ready(Ok(())) - } - fn call(&mut self, req: http::Request) -> Self::Future { - match req.uri().path() { - "/jaeger.storage.v1.SpanWriterPlugin/WriteSpan" => { - #[allow(non_camel_case_types)] - struct WriteSpanSvc(pub Arc); - impl< - T: SpanWriterPlugin, - > tonic::server::UnaryService - for WriteSpanSvc { - type Response = super::WriteSpanResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { - let inner = Arc::clone(&self.0); - let fut = async move { - ::write_span(&inner, request).await - }; - Box::pin(fut) - } - } - let accept_compression_encodings = self.accept_compression_encodings; - let send_compression_encodings = self.send_compression_encodings; - let max_decoding_message_size = self.max_decoding_message_size; - let max_encoding_message_size = self.max_encoding_message_size; - let inner = self.inner.clone(); - let fut = async move { - let method = WriteSpanSvc(inner); - let codec = tonic::codec::ProstCodec::default(); - let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); - let res = grpc.unary(method, req).await; - Ok(res) - }; - Box::pin(fut) - } - "/jaeger.storage.v1.SpanWriterPlugin/Close" => { - #[allow(non_camel_case_types)] - struct CloseSvc(pub Arc); - impl< - T: SpanWriterPlugin, - > tonic::server::UnaryService - for CloseSvc { - type Response = super::CloseWriterResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { - let inner = Arc::clone(&self.0); - let fut = async move { - ::close(&inner, request).await - }; - Box::pin(fut) - } - } - let accept_compression_encodings = self.accept_compression_encodings; - let send_compression_encodings = self.send_compression_encodings; - let max_decoding_message_size = self.max_decoding_message_size; - let max_encoding_message_size = self.max_encoding_message_size; - let inner = self.inner.clone(); - let fut = async move { - let method = CloseSvc(inner); - let codec = tonic::codec::ProstCodec::default(); - let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); - let res = grpc.unary(method, req).await; - Ok(res) - }; - Box::pin(fut) - } - _ => { - Box::pin(async move { - let mut response = http::Response::new( - tonic::body::Body::default(), - ); - let headers = response.headers_mut(); - headers - .insert( - tonic::Status::GRPC_STATUS, - (tonic::Code::Unimplemented as i32).into(), - ); - headers - .insert( - http::header::CONTENT_TYPE, - tonic::metadata::GRPC_CONTENT_TYPE, - ); - Ok(response) - }) - } - } - } - } - impl Clone for SpanWriterPluginServer { - fn clone(&self) -> Self { - let inner = self.inner.clone(); - Self { - inner, - accept_compression_encodings: self.accept_compression_encodings, - send_compression_encodings: self.send_compression_encodings, - max_decoding_message_size: self.max_decoding_message_size, - max_encoding_message_size: self.max_encoding_message_size, - } - } - } - /// Generated gRPC service name - pub const SERVICE_NAME: &str = "jaeger.storage.v1.SpanWriterPlugin"; - impl tonic::server::NamedService for SpanWriterPluginServer { - const NAME: &'static str = SERVICE_NAME; - } -} -/// Generated server implementations. -pub mod streaming_span_writer_plugin_server { - #![allow( - unused_variables, - dead_code, - missing_docs, - clippy::wildcard_imports, - clippy::let_unit_value, - )] - use tonic::codegen::*; - /// Generated trait containing gRPC methods that should be implemented for use with StreamingSpanWriterPluginServer. - #[async_trait] - pub trait StreamingSpanWriterPlugin: std::marker::Send + std::marker::Sync + 'static { - async fn write_span_stream( - &self, - request: tonic::Request>, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; - } - #[derive(Debug)] - pub struct StreamingSpanWriterPluginServer { - inner: Arc, - accept_compression_encodings: EnabledCompressionEncodings, - send_compression_encodings: EnabledCompressionEncodings, - max_decoding_message_size: Option, - max_encoding_message_size: Option, - } - impl StreamingSpanWriterPluginServer { - pub fn new(inner: T) -> Self { - Self::from_arc(Arc::new(inner)) - } - pub fn from_arc(inner: Arc) -> Self { - Self { - inner, - accept_compression_encodings: Default::default(), - send_compression_encodings: Default::default(), - max_decoding_message_size: None, - max_encoding_message_size: None, - } - } - pub fn with_interceptor( - inner: T, - interceptor: F, - ) -> InterceptedService - where - F: tonic::service::Interceptor, - { - InterceptedService::new(Self::new(inner), interceptor) - } - /// Enable decompressing requests with the given encoding. - #[must_use] - pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { - self.accept_compression_encodings.enable(encoding); - self - } - /// Compress responses with the given encoding, if the client supports it. - #[must_use] - pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { - self.send_compression_encodings.enable(encoding); - self - } - /// Limits the maximum size of a decoded message. - /// - /// Default: `4MB` - #[must_use] - pub fn max_decoding_message_size(mut self, limit: usize) -> Self { - self.max_decoding_message_size = Some(limit); - self - } - /// Limits the maximum size of an encoded message. - /// - /// Default: `usize::MAX` - #[must_use] - pub fn max_encoding_message_size(mut self, limit: usize) -> Self { - self.max_encoding_message_size = Some(limit); - self - } - } - impl tonic::codegen::Service> - for StreamingSpanWriterPluginServer - where - T: StreamingSpanWriterPlugin, - B: Body + std::marker::Send + 'static, - B::Error: Into + std::marker::Send + 'static, - { - type Response = http::Response; - type Error = std::convert::Infallible; - type Future = BoxFuture; - fn poll_ready( - &mut self, - _cx: &mut Context<'_>, - ) -> Poll> { - Poll::Ready(Ok(())) - } - fn call(&mut self, req: http::Request) -> Self::Future { - match req.uri().path() { - "/jaeger.storage.v1.StreamingSpanWriterPlugin/WriteSpanStream" => { - #[allow(non_camel_case_types)] - struct WriteSpanStreamSvc(pub Arc); - impl< - T: StreamingSpanWriterPlugin, - > tonic::server::ClientStreamingService - for WriteSpanStreamSvc { - type Response = super::WriteSpanResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request< - tonic::Streaming, - >, - ) -> Self::Future { - let inner = Arc::clone(&self.0); - let fut = async move { - ::write_span_stream( - &inner, - request, - ) - .await - }; - Box::pin(fut) - } - } - let accept_compression_encodings = self.accept_compression_encodings; - let send_compression_encodings = self.send_compression_encodings; - let max_decoding_message_size = self.max_decoding_message_size; - let max_encoding_message_size = self.max_encoding_message_size; - let inner = self.inner.clone(); - let fut = async move { - let method = WriteSpanStreamSvc(inner); - let codec = tonic::codec::ProstCodec::default(); - let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); - let res = grpc.client_streaming(method, req).await; - Ok(res) - }; - Box::pin(fut) - } - _ => { - Box::pin(async move { - let mut response = http::Response::new( - tonic::body::Body::default(), - ); - let headers = response.headers_mut(); - headers - .insert( - tonic::Status::GRPC_STATUS, - (tonic::Code::Unimplemented as i32).into(), - ); - headers - .insert( - http::header::CONTENT_TYPE, - tonic::metadata::GRPC_CONTENT_TYPE, - ); - Ok(response) - }) - } - } - } - } - impl Clone for StreamingSpanWriterPluginServer { - fn clone(&self) -> Self { - let inner = self.inner.clone(); - Self { - inner, - accept_compression_encodings: self.accept_compression_encodings, - send_compression_encodings: self.send_compression_encodings, - max_decoding_message_size: self.max_decoding_message_size, - max_encoding_message_size: self.max_encoding_message_size, - } + .insert( + GrpcMethod::new("jaeger.storage.v1.SpanReaderPlugin", "FindTraceIDs"), + ); + self.inner.unary(req, path, codec).await } } - /// Generated gRPC service name - pub const SERVICE_NAME: &str = "jaeger.storage.v1.StreamingSpanWriterPlugin"; - impl tonic::server::NamedService for StreamingSpanWriterPluginServer { - const NAME: &'static str = SERVICE_NAME; - } } /// Generated server implementations. pub mod span_reader_plugin_server { @@ -1693,7 +1196,7 @@ pub mod span_reader_plugin_server { let inner = self.inner.clone(); let fut = async move { let method = GetTraceSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -1738,7 +1241,7 @@ pub mod span_reader_plugin_server { let inner = self.inner.clone(); let fut = async move { let method = GetServicesSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -1784,7 +1287,7 @@ pub mod span_reader_plugin_server { let inner = self.inner.clone(); let fut = async move { let method = GetOperationsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -1830,7 +1333,7 @@ pub mod span_reader_plugin_server { let inner = self.inner.clone(); let fut = async move { let method = FindTracesSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -1876,7 +1379,7 @@ pub mod span_reader_plugin_server { let inner = self.inner.clone(); let fut = async move { let method = FindTraceIDsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -1931,6 +1434,131 @@ pub mod span_reader_plugin_server { const NAME: &'static str = SERVICE_NAME; } } +/// Generated client implementations. +pub mod archive_span_writer_plugin_client { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] + use tonic::codegen::*; + use tonic::codegen::http::Uri; + #[derive(Debug, Clone)] + pub struct ArchiveSpanWriterPluginClient { + inner: tonic::client::Grpc, + } + impl ArchiveSpanWriterPluginClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl ArchiveSpanWriterPluginClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> ArchiveSpanWriterPluginClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + , + >>::Error: Into + std::marker::Send + std::marker::Sync, + { + ArchiveSpanWriterPluginClient::new( + InterceptedService::new(inner, interceptor), + ) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + /// spanstore/Writer + pub async fn write_archive_span( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/jaeger.storage.v1.ArchiveSpanWriterPlugin/WriteArchiveSpan", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "jaeger.storage.v1.ArchiveSpanWriterPlugin", + "WriteArchiveSpan", + ), + ); + self.inner.unary(req, path, codec).await + } + } +} /// Generated server implementations. pub mod archive_span_writer_plugin_server { #![allow( @@ -2064,7 +1692,7 @@ pub mod archive_span_writer_plugin_server { let inner = self.inner.clone(); let fut = async move { let method = WriteArchiveSpanSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -2113,11 +1741,136 @@ pub mod archive_span_writer_plugin_server { } } } - /// Generated gRPC service name - pub const SERVICE_NAME: &str = "jaeger.storage.v1.ArchiveSpanWriterPlugin"; - impl tonic::server::NamedService for ArchiveSpanWriterPluginServer { - const NAME: &'static str = SERVICE_NAME; - } + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "jaeger.storage.v1.ArchiveSpanWriterPlugin"; + impl tonic::server::NamedService for ArchiveSpanWriterPluginServer { + const NAME: &'static str = SERVICE_NAME; + } +} +/// Generated client implementations. +pub mod archive_span_reader_plugin_client { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] + use tonic::codegen::*; + use tonic::codegen::http::Uri; + #[derive(Debug, Clone)] + pub struct ArchiveSpanReaderPluginClient { + inner: tonic::client::Grpc, + } + impl ArchiveSpanReaderPluginClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl ArchiveSpanReaderPluginClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> ArchiveSpanReaderPluginClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + , + >>::Error: Into + std::marker::Send + std::marker::Sync, + { + ArchiveSpanReaderPluginClient::new( + InterceptedService::new(inner, interceptor), + ) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + /// spanstore/Reader + pub async fn get_archive_trace( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response>, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/jaeger.storage.v1.ArchiveSpanReaderPlugin/GetArchiveTrace", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "jaeger.storage.v1.ArchiveSpanReaderPlugin", + "GetArchiveTrace", + ), + ); + self.inner.server_streaming(req, path, codec).await + } + } } /// Generated server implementations. pub mod archive_span_reader_plugin_server { @@ -2259,7 +2012,7 @@ pub mod archive_span_reader_plugin_server { let inner = self.inner.clone(); let fut = async move { let method = GetArchiveTraceSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -2314,6 +2067,131 @@ pub mod archive_span_reader_plugin_server { const NAME: &'static str = SERVICE_NAME; } } +/// Generated client implementations. +pub mod dependencies_reader_plugin_client { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] + use tonic::codegen::*; + use tonic::codegen::http::Uri; + #[derive(Debug, Clone)] + pub struct DependenciesReaderPluginClient { + inner: tonic::client::Grpc, + } + impl DependenciesReaderPluginClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl DependenciesReaderPluginClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> DependenciesReaderPluginClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + , + >>::Error: Into + std::marker::Send + std::marker::Sync, + { + DependenciesReaderPluginClient::new( + InterceptedService::new(inner, interceptor), + ) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + /// dependencystore/Reader + pub async fn get_dependencies( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/jaeger.storage.v1.DependenciesReaderPlugin/GetDependencies", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "jaeger.storage.v1.DependenciesReaderPlugin", + "GetDependencies", + ), + ); + self.inner.unary(req, path, codec).await + } + } +} /// Generated server implementations. pub mod dependencies_reader_plugin_server { #![allow( @@ -2447,7 +2325,7 @@ pub mod dependencies_reader_plugin_server { let inner = self.inner.clone(); let fut = async move { let method = GetDependenciesSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -2502,6 +2380,128 @@ pub mod dependencies_reader_plugin_server { const NAME: &'static str = SERVICE_NAME; } } +/// Generated client implementations. +pub mod plugin_capabilities_client { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] + use tonic::codegen::*; + use tonic::codegen::http::Uri; + #[derive(Debug, Clone)] + pub struct PluginCapabilitiesClient { + inner: tonic::client::Grpc, + } + impl PluginCapabilitiesClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl PluginCapabilitiesClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> PluginCapabilitiesClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + , + >>::Error: Into + std::marker::Send + std::marker::Sync, + { + PluginCapabilitiesClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + pub async fn capabilities( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/jaeger.storage.v1.PluginCapabilities/Capabilities", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "jaeger.storage.v1.PluginCapabilities", + "Capabilities", + ), + ); + self.inner.unary(req, path, codec).await + } + } +} /// Generated server implementations. pub mod plugin_capabilities_server { #![allow( @@ -2630,7 +2630,7 @@ pub mod plugin_capabilities_server { let inner = self.inner.clone(); let fut = async move { let method = CapabilitiesSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, diff --git a/quickwit/quickwit-proto/src/codegen/jaeger/jaeger.storage.v2.rs b/quickwit/quickwit-proto/src/codegen/jaeger/jaeger.storage.v2.rs new file mode 100644 index 00000000000..9adef233a95 --- /dev/null +++ b/quickwit/quickwit-proto/src/codegen/jaeger/jaeger.storage.v2.rs @@ -0,0 +1,857 @@ +// This file is @generated by prost-build. +/// GetTraceParams represents the query for a single trace from the storage backend. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetTraceParams { + /// trace_id is a 16 byte array containing the unique identifier for the trace to query. + #[prost(bytes = "vec", tag = "1")] + pub trace_id: ::prost::alloc::vec::Vec, + /// start_time is the start of the time interval to search for the trace_id. + /// + /// This field is optional. + #[prost(message, optional, tag = "2")] + pub start_time: ::core::option::Option<::prost_types::Timestamp>, + /// end_time is the end of the time interval to search for the trace_id. + /// + /// This field is optional. + #[prost(message, optional, tag = "3")] + pub end_time: ::core::option::Option<::prost_types::Timestamp>, +} +/// GetTracesRequest represents a request to retrieve multiple traces. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetTracesRequest { + #[prost(message, repeated, tag = "1")] + pub query: ::prost::alloc::vec::Vec, +} +/// GetServicesRequest represents a request to get service names. +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetServicesRequest {} +/// GetServicesResponse represents the response for GetServicesRequest. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetServicesResponse { + #[prost(string, repeated, tag = "1")] + pub services: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +/// GetOperationsRequest represents a request to get operation names. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetOperationsRequest { + /// service is the name of the service for which to get operation names. + /// + /// This field is required. + #[prost(string, tag = "1")] + pub service: ::prost::alloc::string::String, + /// span_kind is the type of span which is used to distinguish between + /// spans generated in a particular context. + /// + /// This field is optional. + #[prost(string, tag = "2")] + pub span_kind: ::prost::alloc::string::String, +} +/// Operation contains information about an operation for a given service. +#[derive(Ord, PartialOrd)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Operation { + #[prost(string, tag = "1")] + pub name: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub span_kind: ::prost::alloc::string::String, +} +/// GetOperationsResponse represents the response for GetOperationsRequest. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetOperationsResponse { + #[prost(message, repeated, tag = "1")] + pub operations: ::prost::alloc::vec::Vec, +} +/// KeyValue and all its associated types are copied from opentelemetry-proto/common/v1/common.proto +/// (). +/// This type is used to store attributes in traces. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct KeyValue { + #[prost(string, tag = "1")] + pub key: ::prost::alloc::string::String, + #[prost(message, optional, tag = "2")] + pub value: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AnyValue { + #[prost(oneof = "any_value::Value", tags = "1, 2, 3, 4, 5, 6, 7")] + pub value: ::core::option::Option, +} +/// Nested message and enum types in `AnyValue`. +pub mod any_value { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Value { + #[prost(string, tag = "1")] + StringValue(::prost::alloc::string::String), + #[prost(bool, tag = "2")] + BoolValue(bool), + #[prost(int64, tag = "3")] + IntValue(i64), + #[prost(double, tag = "4")] + DoubleValue(f64), + #[prost(message, tag = "5")] + ArrayValue(super::ArrayValue), + #[prost(message, tag = "6")] + KvlistValue(super::KeyValueList), + #[prost(bytes, tag = "7")] + BytesValue(::prost::alloc::vec::Vec), + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct KeyValueList { + #[prost(message, repeated, tag = "1")] + pub values: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ArrayValue { + #[prost(message, repeated, tag = "1")] + pub values: ::prost::alloc::vec::Vec, +} +/// TraceQueryParameters contains query parameters to find traces. For a detailed +/// definition of each field in this message, refer to `TraceQueryParameters` in `jaeger.api_v3` +/// (). +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TraceQueryParameters { + #[prost(string, tag = "1")] + pub service_name: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub operation_name: ::prost::alloc::string::String, + #[prost(message, repeated, tag = "3")] + pub attributes: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag = "4")] + pub start_time_min: ::core::option::Option<::prost_types::Timestamp>, + #[prost(message, optional, tag = "5")] + pub start_time_max: ::core::option::Option<::prost_types::Timestamp>, + #[prost(message, optional, tag = "6")] + pub duration_min: ::core::option::Option<::prost_types::Duration>, + #[prost(message, optional, tag = "7")] + pub duration_max: ::core::option::Option<::prost_types::Duration>, + #[prost(int32, tag = "8")] + pub search_depth: i32, +} +/// FindTracesRequest represents a request to find traces. +/// It can be used to retrieve the traces (FindTraces) or simply +/// the trace IDs (FindTraceIDs). +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FindTracesRequest { + #[prost(message, optional, tag = "1")] + pub query: ::core::option::Option, +} +/// FoundTraceID is a wrapper around trace ID returned from FindTraceIDs +/// with an optional time range that may be used in GetTraces calls. +/// +/// The time range is provided as an optimization hint for some storage backends +/// that can perform more efficient queries when they know the approximate time range. +/// The value should not be used for precise time-based filtering or assumptions. +/// It is meant as a rough boundary and may not be populated in all cases. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct FoundTraceId { + #[prost(bytes = "vec", tag = "1")] + pub trace_id: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag = "2")] + pub start: ::core::option::Option<::prost_types::Timestamp>, + #[prost(message, optional, tag = "3")] + pub end: ::core::option::Option<::prost_types::Timestamp>, +} +/// FindTraceIDsResponse represents the response for FindTracesRequest. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FindTraceIDsResponse { + #[prost(message, repeated, tag = "1")] + pub trace_ids: ::prost::alloc::vec::Vec, +} +/// Generated client implementations. +pub mod trace_reader_client { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] + use tonic::codegen::*; + use tonic::codegen::http::Uri; + /// TraceReader is a service that allows reading traces from storage. + /// Note that if you implement this service, you should also implement + /// OTEL's TraceService in package opentelemetry.proto.collector.trace.v1 + /// to allow pushing traces to the storage backend + /// () + #[derive(Debug, Clone)] + pub struct TraceReaderClient { + inner: tonic::client::Grpc, + } + impl TraceReaderClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl TraceReaderClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> TraceReaderClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + , + >>::Error: Into + std::marker::Send + std::marker::Sync, + { + TraceReaderClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + /// GetTraces returns a stream that retrieves all traces with given IDs. + /// + /// Chunking requirements: + /// + /// * A single TracesData chunk MUST NOT contain spans from multiple traces. + /// * Large traces MAY be split across multiple, *consecutive* TracesData chunks. + /// * Each returned TracesData object MUST NOT be empty. + /// + /// Edge cases: + /// + /// * If no spans are found for any given trace ID, the ID is ignored. + /// * If none of the trace IDs are found in the storage, an empty response is returned. + pub async fn get_traces( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response< + tonic::codec::Streaming< + super::super::super::super::opentelemetry::proto::trace::v1::TracesData, + >, + >, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/jaeger.storage.v2.TraceReader/GetTraces", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("jaeger.storage.v2.TraceReader", "GetTraces")); + self.inner.server_streaming(req, path, codec).await + } + /// GetServices returns all service names known to the backend from traces + /// within its retention period. + pub async fn get_services( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/jaeger.storage.v2.TraceReader/GetServices", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("jaeger.storage.v2.TraceReader", "GetServices")); + self.inner.unary(req, path, codec).await + } + /// GetOperations returns all operation names for a given service + /// known to the backend from traces within its retention period. + pub async fn get_operations( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/jaeger.storage.v2.TraceReader/GetOperations", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new("jaeger.storage.v2.TraceReader", "GetOperations"), + ); + self.inner.unary(req, path, codec).await + } + /// FindTraces returns a stream that retrieves traces matching query parameters. + /// + /// The chunking rules are the same as for GetTraces. + /// + /// If no matching traces are found, an empty stream is returned. + pub async fn find_traces( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response< + tonic::codec::Streaming< + super::super::super::super::opentelemetry::proto::trace::v1::TracesData, + >, + >, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/jaeger.storage.v2.TraceReader/FindTraces", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("jaeger.storage.v2.TraceReader", "FindTraces")); + self.inner.server_streaming(req, path, codec).await + } + /// FindTraceIDs returns a stream that retrieves IDs of traces matching query parameters. + /// + /// If no matching traces are found, an empty stream is returned. + /// + /// This call behaves identically to FindTraces, except that it returns only the list + /// of matching trace IDs. This is useful in some contexts, such as batch jobs, where a + /// large list of trace IDs may be queried first and then the full traces are loaded + /// in batches. + pub async fn find_trace_i_ds( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/jaeger.storage.v2.TraceReader/FindTraceIDs", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new("jaeger.storage.v2.TraceReader", "FindTraceIDs"), + ); + self.inner.unary(req, path, codec).await + } + } +} +/// Generated server implementations. +pub mod trace_reader_server { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with TraceReaderServer. + #[async_trait] + pub trait TraceReader: std::marker::Send + std::marker::Sync + 'static { + /// Server streaming response type for the GetTraces method. + type GetTracesStream: tonic::codegen::tokio_stream::Stream< + Item = std::result::Result< + super::super::super::super::opentelemetry::proto::trace::v1::TracesData, + tonic::Status, + >, + > + + std::marker::Send + + 'static; + /// GetTraces returns a stream that retrieves all traces with given IDs. + /// + /// Chunking requirements: + /// + /// * A single TracesData chunk MUST NOT contain spans from multiple traces. + /// * Large traces MAY be split across multiple, *consecutive* TracesData chunks. + /// * Each returned TracesData object MUST NOT be empty. + /// + /// Edge cases: + /// + /// * If no spans are found for any given trace ID, the ID is ignored. + /// * If none of the trace IDs are found in the storage, an empty response is returned. + async fn get_traces( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + /// GetServices returns all service names known to the backend from traces + /// within its retention period. + async fn get_services( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + /// GetOperations returns all operation names for a given service + /// known to the backend from traces within its retention period. + async fn get_operations( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + /// Server streaming response type for the FindTraces method. + type FindTracesStream: tonic::codegen::tokio_stream::Stream< + Item = std::result::Result< + super::super::super::super::opentelemetry::proto::trace::v1::TracesData, + tonic::Status, + >, + > + + std::marker::Send + + 'static; + /// FindTraces returns a stream that retrieves traces matching query parameters. + /// + /// The chunking rules are the same as for GetTraces. + /// + /// If no matching traces are found, an empty stream is returned. + async fn find_traces( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + /// FindTraceIDs returns a stream that retrieves IDs of traces matching query parameters. + /// + /// If no matching traces are found, an empty stream is returned. + /// + /// This call behaves identically to FindTraces, except that it returns only the list + /// of matching trace IDs. This is useful in some contexts, such as batch jobs, where a + /// large list of trace IDs may be queried first and then the full traces are loaded + /// in batches. + async fn find_trace_i_ds( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + } + /// TraceReader is a service that allows reading traces from storage. + /// Note that if you implement this service, you should also implement + /// OTEL's TraceService in package opentelemetry.proto.collector.trace.v1 + /// to allow pushing traces to the storage backend + /// () + #[derive(Debug)] + pub struct TraceReaderServer { + inner: Arc, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + impl TraceReaderServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for TraceReaderServer + where + T: TraceReader, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + match req.uri().path() { + "/jaeger.storage.v2.TraceReader/GetTraces" => { + #[allow(non_camel_case_types)] + struct GetTracesSvc(pub Arc); + impl< + T: TraceReader, + > tonic::server::ServerStreamingService + for GetTracesSvc { + type Response = super::super::super::super::opentelemetry::proto::trace::v1::TracesData; + type ResponseStream = T::GetTracesStream; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::get_traces(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = GetTracesSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.server_streaming(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/jaeger.storage.v2.TraceReader/GetServices" => { + #[allow(non_camel_case_types)] + struct GetServicesSvc(pub Arc); + impl< + T: TraceReader, + > tonic::server::UnaryService + for GetServicesSvc { + type Response = super::GetServicesResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::get_services(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = GetServicesSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/jaeger.storage.v2.TraceReader/GetOperations" => { + #[allow(non_camel_case_types)] + struct GetOperationsSvc(pub Arc); + impl< + T: TraceReader, + > tonic::server::UnaryService + for GetOperationsSvc { + type Response = super::GetOperationsResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::get_operations(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = GetOperationsSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/jaeger.storage.v2.TraceReader/FindTraces" => { + #[allow(non_camel_case_types)] + struct FindTracesSvc(pub Arc); + impl< + T: TraceReader, + > tonic::server::ServerStreamingService + for FindTracesSvc { + type Response = super::super::super::super::opentelemetry::proto::trace::v1::TracesData; + type ResponseStream = T::FindTracesStream; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::find_traces(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = FindTracesSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.server_streaming(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/jaeger.storage.v2.TraceReader/FindTraceIDs" => { + #[allow(non_camel_case_types)] + struct FindTraceIDsSvc(pub Arc); + impl< + T: TraceReader, + > tonic::server::UnaryService + for FindTraceIDsSvc { + type Response = super::FindTraceIDsResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::find_trace_i_ds(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = FindTraceIDsSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => { + Box::pin(async move { + let mut response = http::Response::new( + tonic::body::Body::default(), + ); + let headers = response.headers_mut(); + headers + .insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers + .insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) + }) + } + } + } + } + impl Clone for TraceReaderServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "jaeger.storage.v2.TraceReader"; + impl tonic::server::NamedService for TraceReaderServer { + const NAME: &'static str = SERVICE_NAME; + } +} diff --git a/quickwit/quickwit-proto/src/codegen/jaeger/opentelemetry.proto.common.v1.rs b/quickwit/quickwit-proto/src/codegen/jaeger/opentelemetry.proto.common.v1.rs new file mode 100644 index 00000000000..e0c4b431d7b --- /dev/null +++ b/quickwit/quickwit-proto/src/codegen/jaeger/opentelemetry.proto.common.v1.rs @@ -0,0 +1,78 @@ +// This file is @generated by prost-build. +/// AnyValue is used to represent any type of attribute value. AnyValue may contain a +/// primitive value such as a string or integer or it may contain an arbitrary nested +/// object containing arrays, key-value lists and primitives. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AnyValue { + /// The value is one of the listed fields. It is valid for all values to be unspecified + /// in which case this AnyValue is considered to be "empty". + #[prost(oneof = "any_value::Value", tags = "1, 2, 3, 4, 5, 6, 7")] + pub value: ::core::option::Option, +} +/// Nested message and enum types in `AnyValue`. +pub mod any_value { + /// The value is one of the listed fields. It is valid for all values to be unspecified + /// in which case this AnyValue is considered to be "empty". + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Value { + #[prost(string, tag = "1")] + StringValue(::prost::alloc::string::String), + #[prost(bool, tag = "2")] + BoolValue(bool), + #[prost(int64, tag = "3")] + IntValue(i64), + #[prost(double, tag = "4")] + DoubleValue(f64), + #[prost(message, tag = "5")] + ArrayValue(super::ArrayValue), + #[prost(message, tag = "6")] + KvlistValue(super::KeyValueList), + #[prost(bytes, tag = "7")] + BytesValue(::prost::alloc::vec::Vec), + } +} +/// ArrayValue is a list of AnyValue messages. We need ArrayValue as a message +/// since oneof in AnyValue does not allow repeated fields. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ArrayValue { + /// Array of values. The array may be empty (contain 0 elements). + #[prost(message, repeated, tag = "1")] + pub values: ::prost::alloc::vec::Vec, +} +/// KeyValueList is a list of KeyValue messages. We need KeyValueList as a message +/// since `oneof` in AnyValue does not allow repeated fields. Everywhere else where we need +/// a list of KeyValue messages (e.g. in Span) we use `repeated KeyValue` directly to +/// avoid unnecessary extra wrapping (which slows down the protocol). The 2 approaches +/// are semantically equivalent. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct KeyValueList { + /// A collection of key/value pairs of key-value pairs. The list may be empty (may + /// contain 0 elements). + /// The keys MUST be unique (it is not allowed to have more than one + /// value with the same key). + #[prost(message, repeated, tag = "1")] + pub values: ::prost::alloc::vec::Vec, +} +/// KeyValue is a key-value pair that is used to store Span attributes, Link +/// attributes, etc. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct KeyValue { + #[prost(string, tag = "1")] + pub key: ::prost::alloc::string::String, + #[prost(message, optional, tag = "2")] + pub value: ::core::option::Option, +} +/// InstrumentationScope is a message representing the instrumentation scope information +/// such as the fully qualified name and version. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct InstrumentationScope { + /// An empty instrumentation scope name means the name is unknown. + #[prost(string, tag = "1")] + pub name: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub version: ::prost::alloc::string::String, + #[prost(message, repeated, tag = "3")] + pub attributes: ::prost::alloc::vec::Vec, + #[prost(uint32, tag = "4")] + pub dropped_attributes_count: u32, +} diff --git a/quickwit/quickwit-proto/src/codegen/jaeger/opentelemetry.proto.resource.v1.rs b/quickwit/quickwit-proto/src/codegen/jaeger/opentelemetry.proto.resource.v1.rs new file mode 100644 index 00000000000..0daa6740b11 --- /dev/null +++ b/quickwit/quickwit-proto/src/codegen/jaeger/opentelemetry.proto.resource.v1.rs @@ -0,0 +1,14 @@ +// This file is @generated by prost-build. +/// Resource information. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Resource { + /// Set of attributes that describe the resource. + /// Attribute keys MUST be unique (it is not allowed to have more than one + /// attribute with the same key). + #[prost(message, repeated, tag = "1")] + pub attributes: ::prost::alloc::vec::Vec, + /// dropped_attributes_count is the number of dropped attributes. If the value is 0, then + /// no attributes were dropped. + #[prost(uint32, tag = "2")] + pub dropped_attributes_count: u32, +} diff --git a/quickwit/quickwit-proto/src/codegen/jaeger/opentelemetry.proto.trace.v1.rs b/quickwit/quickwit-proto/src/codegen/jaeger/opentelemetry.proto.trace.v1.rs new file mode 100644 index 00000000000..afa08ca3c9d --- /dev/null +++ b/quickwit/quickwit-proto/src/codegen/jaeger/opentelemetry.proto.trace.v1.rs @@ -0,0 +1,337 @@ +// This file is @generated by prost-build. +/// TracesData represents the traces data that can be stored in a persistent storage, +/// OR can be embedded by other protocols that transfer OTLP traces data but do +/// not implement the OTLP protocol. +/// +/// The main difference between this message and collector protocol is that +/// in this message there will not be any "control" or "metadata" specific to +/// OTLP protocol. +/// +/// When new fields are added into this message, the OTLP request MUST be updated +/// as well. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TracesData { + /// An array of ResourceSpans. + /// For data coming from a single resource this array will typically contain + /// one element. Intermediary nodes that receive data from multiple origins + /// typically batch the data before forwarding further and in that case this + /// array will contain multiple elements. + #[prost(message, repeated, tag = "1")] + pub resource_spans: ::prost::alloc::vec::Vec, +} +/// A collection of ScopeSpans from a Resource. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ResourceSpans { + /// The resource for the spans in this message. + /// If this field is not set then no resource info is known. + #[prost(message, optional, tag = "1")] + pub resource: ::core::option::Option, + /// A list of ScopeSpans that originate from a resource. + #[prost(message, repeated, tag = "2")] + pub scope_spans: ::prost::alloc::vec::Vec, + /// This schema_url applies to the data in the "resource" field. It does not apply + /// to the data in the "scope_spans" field which have their own schema_url field. + #[prost(string, tag = "3")] + pub schema_url: ::prost::alloc::string::String, +} +/// A collection of Spans produced by an InstrumentationScope. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ScopeSpans { + /// The instrumentation scope information for the spans in this message. + /// Semantically when InstrumentationScope isn't set, it is equivalent with + /// an empty instrumentation scope name (unknown). + #[prost(message, optional, tag = "1")] + pub scope: ::core::option::Option, + /// A list of Spans that originate from an instrumentation scope. + #[prost(message, repeated, tag = "2")] + pub spans: ::prost::alloc::vec::Vec, + /// This schema_url applies to all spans and span events in the "spans" field. + #[prost(string, tag = "3")] + pub schema_url: ::prost::alloc::string::String, +} +/// A Span represents a single operation performed by a single component of the system. +/// +/// The next available field id is 17. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Span { + /// A unique identifier for a trace. All spans from the same trace share + /// the same `trace_id`. The ID is a 16-byte array. An ID with all zeroes + /// is considered invalid. + /// + /// This field is semantically required. Receiver should generate new + /// random trace_id if empty or invalid trace_id was received. + /// + /// This field is required. + #[prost(bytes = "vec", tag = "1")] + pub trace_id: ::prost::alloc::vec::Vec, + /// A unique identifier for a span within a trace, assigned when the span + /// is created. The ID is an 8-byte array. An ID with all zeroes is considered + /// invalid. + /// + /// This field is semantically required. Receiver should generate new + /// random span_id if empty or invalid span_id was received. + /// + /// This field is required. + #[prost(bytes = "vec", tag = "2")] + pub span_id: ::prost::alloc::vec::Vec, + /// trace_state conveys information about request position in multiple distributed tracing graphs. + /// It is a trace_state in w3c-trace-context format: + /// See also for more details about this field. + #[prost(string, tag = "3")] + pub trace_state: ::prost::alloc::string::String, + /// The `span_id` of this span's parent span. If this is a root span, then this + /// field must be empty. The ID is an 8-byte array. + #[prost(bytes = "vec", tag = "4")] + pub parent_span_id: ::prost::alloc::vec::Vec, + /// A description of the span's operation. + /// + /// For example, the name can be a qualified method name or a file name + /// and a line number where the operation is called. A best practice is to use + /// the same display name at the same call point in an application. + /// This makes it easier to correlate spans in different traces. + /// + /// This field is semantically required to be set to non-empty string. + /// Empty value is equivalent to an unknown span name. + /// + /// This field is required. + #[prost(string, tag = "5")] + pub name: ::prost::alloc::string::String, + /// Distinguishes between spans generated in a particular context. For example, + /// two spans with the same name may be distinguished using `CLIENT` (caller) + /// and `SERVER` (callee) to identify queueing latency associated with the span. + #[prost(enumeration = "span::SpanKind", tag = "6")] + pub kind: i32, + /// start_time_unix_nano is the start time of the span. On the client side, this is the time + /// kept by the local machine where the span execution starts. On the server side, this + /// is the time when the server's application handler starts running. + /// Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + /// + /// This field is semantically required and it is expected that end_time >= start_time. + #[prost(fixed64, tag = "7")] + pub start_time_unix_nano: u64, + /// end_time_unix_nano is the end time of the span. On the client side, this is the time + /// kept by the local machine where the span execution ends. On the server side, this + /// is the time when the server application handler stops running. + /// Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + /// + /// This field is semantically required and it is expected that end_time >= start_time. + #[prost(fixed64, tag = "8")] + pub end_time_unix_nano: u64, + /// attributes is a collection of key/value pairs. Note, global attributes + /// like server name can be set using the resource API. Examples of attributes: + /// + /// ```text + /// "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" + /// "/http/server_latency": 300 + /// "abc.com/myattribute": true + /// "abc.com/score": 10.239 + /// ``` + /// + /// The OpenTelemetry API specification further restricts the allowed value types: + /// + /// Attribute keys MUST be unique (it is not allowed to have more than one + /// attribute with the same key). + #[prost(message, repeated, tag = "9")] + pub attributes: ::prost::alloc::vec::Vec, + /// dropped_attributes_count is the number of attributes that were discarded. Attributes + /// can be discarded because their keys are too long or because there are too many + /// attributes. If this value is 0, then no attributes were dropped. + #[prost(uint32, tag = "10")] + pub dropped_attributes_count: u32, + /// events is a collection of Event items. + #[prost(message, repeated, tag = "11")] + pub events: ::prost::alloc::vec::Vec, + /// dropped_events_count is the number of dropped events. If the value is 0, then no + /// events were dropped. + #[prost(uint32, tag = "12")] + pub dropped_events_count: u32, + /// links is a collection of Links, which are references from this span to a span + /// in the same or different trace. + #[prost(message, repeated, tag = "13")] + pub links: ::prost::alloc::vec::Vec, + /// dropped_links_count is the number of dropped links after the maximum size was + /// enforced. If this value is 0, then no links were dropped. + #[prost(uint32, tag = "14")] + pub dropped_links_count: u32, + /// An optional final status for this span. Semantically when Status isn't set, it means + /// span's status code is unset, i.e. assume STATUS_CODE_UNSET (code = 0). + #[prost(message, optional, tag = "15")] + pub status: ::core::option::Option, +} +/// Nested message and enum types in `Span`. +pub mod span { + /// Event is a time-stamped annotation of the span, consisting of user-supplied + /// text description and key-value pairs. + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Event { + /// time_unix_nano is the time the event occurred. + #[prost(fixed64, tag = "1")] + pub time_unix_nano: u64, + /// name of the event. + /// This field is semantically required to be set to non-empty string. + #[prost(string, tag = "2")] + pub name: ::prost::alloc::string::String, + /// attributes is a collection of attribute key/value pairs on the event. + /// Attribute keys MUST be unique (it is not allowed to have more than one + /// attribute with the same key). + #[prost(message, repeated, tag = "3")] + pub attributes: ::prost::alloc::vec::Vec< + super::super::super::common::v1::KeyValue, + >, + /// dropped_attributes_count is the number of dropped attributes. If the value is 0, + /// then no attributes were dropped. + #[prost(uint32, tag = "4")] + pub dropped_attributes_count: u32, + } + /// A pointer from the current span to another span in the same trace or in a + /// different trace. For example, this can be used in batching operations, + /// where a single batch handler processes multiple requests from different + /// traces or when the handler receives a request from a different project. + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Link { + /// A unique identifier of a trace that this linked span is part of. The ID is a + /// 16-byte array. + #[prost(bytes = "vec", tag = "1")] + pub trace_id: ::prost::alloc::vec::Vec, + /// A unique identifier for the linked span. The ID is an 8-byte array. + #[prost(bytes = "vec", tag = "2")] + pub span_id: ::prost::alloc::vec::Vec, + /// The trace_state associated with the link. + #[prost(string, tag = "3")] + pub trace_state: ::prost::alloc::string::String, + /// attributes is a collection of attribute key/value pairs on the link. + /// Attribute keys MUST be unique (it is not allowed to have more than one + /// attribute with the same key). + #[prost(message, repeated, tag = "4")] + pub attributes: ::prost::alloc::vec::Vec< + super::super::super::common::v1::KeyValue, + >, + /// dropped_attributes_count is the number of dropped attributes. If the value is 0, + /// then no attributes were dropped. + #[prost(uint32, tag = "5")] + pub dropped_attributes_count: u32, + } + /// SpanKind is the type of span. Can be used to specify additional relationships between spans + /// in addition to a parent/child relationship. + #[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + ::prost::Enumeration + )] + #[repr(i32)] + pub enum SpanKind { + /// Unspecified. Do NOT use as default. + /// Implementations MAY assume SpanKind to be INTERNAL when receiving UNSPECIFIED. + Unspecified = 0, + /// Indicates that the span represents an internal operation within an application, + /// as opposed to an operation happening at the boundaries. Default value. + Internal = 1, + /// Indicates that the span covers server-side handling of an RPC or other + /// remote network request. + Server = 2, + /// Indicates that the span describes a request to some remote service. + Client = 3, + /// Indicates that the span describes a producer sending a message to a broker. + /// Unlike CLIENT and SERVER, there is often no direct critical path latency relationship + /// between producer and consumer spans. A PRODUCER span ends when the message was accepted + /// by the broker while the logical processing of the message might span a much longer time. + Producer = 4, + /// Indicates that the span describes consumer receiving a message from a broker. + /// Like the PRODUCER kind, there is often no direct critical path latency relationship + /// between producer and consumer spans. + Consumer = 5, + } + impl SpanKind { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unspecified => "SPAN_KIND_UNSPECIFIED", + Self::Internal => "SPAN_KIND_INTERNAL", + Self::Server => "SPAN_KIND_SERVER", + Self::Client => "SPAN_KIND_CLIENT", + Self::Producer => "SPAN_KIND_PRODUCER", + Self::Consumer => "SPAN_KIND_CONSUMER", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "SPAN_KIND_UNSPECIFIED" => Some(Self::Unspecified), + "SPAN_KIND_INTERNAL" => Some(Self::Internal), + "SPAN_KIND_SERVER" => Some(Self::Server), + "SPAN_KIND_CLIENT" => Some(Self::Client), + "SPAN_KIND_PRODUCER" => Some(Self::Producer), + "SPAN_KIND_CONSUMER" => Some(Self::Consumer), + _ => None, + } + } + } +} +/// The Status type defines a logical error model that is suitable for different +/// programming environments, including REST APIs and RPC APIs. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Status { + /// A developer-facing human readable error message. + #[prost(string, tag = "2")] + pub message: ::prost::alloc::string::String, + /// The status code. + #[prost(enumeration = "status::StatusCode", tag = "3")] + pub code: i32, +} +/// Nested message and enum types in `Status`. +pub mod status { + /// For the semantics of status codes see + /// + #[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + ::prost::Enumeration + )] + #[repr(i32)] + pub enum StatusCode { + /// The default status. + Unset = 0, + /// The Span has been validated by an Application developer or Operator to + /// have completed successfully. + Ok = 1, + /// The Span contains an error. + Error = 2, + } + impl StatusCode { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unset => "STATUS_CODE_UNSET", + Self::Ok => "STATUS_CODE_OK", + Self::Error => "STATUS_CODE_ERROR", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "STATUS_CODE_UNSET" => Some(Self::Unset), + "STATUS_CODE_OK" => Some(Self::Ok), + "STATUS_CODE_ERROR" => Some(Self::Error), + _ => None, + } + } + } +} diff --git a/quickwit/quickwit-proto/src/codegen/opentelemetry/opentelemetry.proto.collector.logs.v1.rs b/quickwit/quickwit-proto/src/codegen/opentelemetry/opentelemetry.proto.collector.logs.v1.rs index 4716de6fbf9..6160453b8ea 100644 --- a/quickwit/quickwit-proto/src/codegen/opentelemetry/opentelemetry.proto.collector.logs.v1.rs +++ b/quickwit/quickwit-proto/src/codegen/opentelemetry/opentelemetry.proto.collector.logs.v1.rs @@ -14,7 +14,7 @@ pub struct ExportLogsServiceRequest { } #[derive(serde::Serialize, serde::Deserialize)] #[derive(utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ExportLogsServiceResponse { /// The details of a partially successful export request. /// @@ -28,14 +28,13 @@ pub struct ExportLogsServiceResponse { /// In such cases, the `rejected_` MUST have a value of `0` and /// the `error_message` MUST be non-empty. /// - /// A `partial_success` message with an empty value (`rejected_1 = 0 and - /// `error_message` = "") is equivalent to it not being set/present. Senders + /// A `partial_success` message with an empty value (`rejected_1 = 0 and `error_message\` = "") is equivalent to it not being set/present. Senders /// SHOULD interpret it the same way as in the full success case. #[prost(message, optional, tag = "1")] pub partial_success: ::core::option::Option, } #[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ExportLogsPartialSuccess { /// The number of rejected log records. /// @@ -164,7 +163,7 @@ pub mod logs_service_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/opentelemetry.proto.collector.logs.v1.LogsService/Export", ); @@ -312,7 +311,7 @@ pub mod logs_service_server { let inner = self.inner.clone(); let fut = async move { let method = ExportSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, diff --git a/quickwit/quickwit-proto/src/codegen/opentelemetry/opentelemetry.proto.collector.metrics.v1.rs b/quickwit/quickwit-proto/src/codegen/opentelemetry/opentelemetry.proto.collector.metrics.v1.rs index 6b899c43ba6..fab8cd4de8f 100644 --- a/quickwit/quickwit-proto/src/codegen/opentelemetry/opentelemetry.proto.collector.metrics.v1.rs +++ b/quickwit/quickwit-proto/src/codegen/opentelemetry/opentelemetry.proto.collector.metrics.v1.rs @@ -13,7 +13,7 @@ pub struct ExportMetricsServiceRequest { >, } #[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ExportMetricsServiceResponse { /// The details of a partially successful export request. /// @@ -27,14 +27,14 @@ pub struct ExportMetricsServiceResponse { /// In such cases, the `rejected_` MUST have a value of `0` and /// the `error_message` MUST be non-empty. /// - /// A `partial_success` message with an empty value (rejected_ = 0 and + /// A `partial_success` message with an empty value (rejected\_ = 0 and /// `error_message` = "") is equivalent to it not being set/present. Senders /// SHOULD interpret it the same way as in the full success case. #[prost(message, optional, tag = "1")] pub partial_success: ::core::option::Option, } #[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ExportMetricsPartialSuccess { /// The number of rejected data points. /// @@ -163,7 +163,7 @@ pub mod metrics_service_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/opentelemetry.proto.collector.metrics.v1.MetricsService/Export", ); @@ -311,7 +311,7 @@ pub mod metrics_service_server { let inner = self.inner.clone(); let fut = async move { let method = ExportSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, diff --git a/quickwit/quickwit-proto/src/codegen/opentelemetry/opentelemetry.proto.collector.trace.v1.rs b/quickwit/quickwit-proto/src/codegen/opentelemetry/opentelemetry.proto.collector.trace.v1.rs index dfadd5f6e79..2ba3e83cb9e 100644 --- a/quickwit/quickwit-proto/src/codegen/opentelemetry/opentelemetry.proto.collector.trace.v1.rs +++ b/quickwit/quickwit-proto/src/codegen/opentelemetry/opentelemetry.proto.collector.trace.v1.rs @@ -13,7 +13,7 @@ pub struct ExportTraceServiceRequest { >, } #[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ExportTraceServiceResponse { /// The details of a partially successful export request. /// @@ -27,14 +27,14 @@ pub struct ExportTraceServiceResponse { /// In such cases, the `rejected_` MUST have a value of `0` and /// the `error_message` MUST be non-empty. /// - /// A `partial_success` message with an empty value (rejected_ = 0 and + /// A `partial_success` message with an empty value (rejected\_ = 0 and /// `error_message` = "") is equivalent to it not being set/present. Senders /// SHOULD interpret it the same way as in the full success case. #[prost(message, optional, tag = "1")] pub partial_success: ::core::option::Option, } #[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ExportTracePartialSuccess { /// The number of rejected spans. /// @@ -163,7 +163,7 @@ pub mod trace_service_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/opentelemetry.proto.collector.trace.v1.TraceService/Export", ); @@ -311,7 +311,7 @@ pub mod trace_service_server { let inner = self.inner.clone(); let fut = async move { let method = ExportSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, diff --git a/quickwit/quickwit-proto/src/codegen/opentelemetry/opentelemetry.proto.logs.v1.rs b/quickwit/quickwit-proto/src/codegen/opentelemetry/opentelemetry.proto.logs.v1.rs index 0ffab8a5334..2a4af4d5139 100644 --- a/quickwit/quickwit-proto/src/codegen/opentelemetry/opentelemetry.proto.logs.v1.rs +++ b/quickwit/quickwit-proto/src/codegen/opentelemetry/opentelemetry.proto.logs.v1.rs @@ -73,7 +73,8 @@ pub struct LogRecord { /// For converting OpenTelemetry log data to formats that support only one timestamp or /// when receiving OpenTelemetry log data by recipients that support only one timestamp /// internally the following logic is recommended: - /// - Use time_unix_nano if it is present, otherwise use observed_time_unix_nano. + /// + /// * Use time_unix_nano if it is present, otherwise use observed_time_unix_nano. /// /// Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. /// Value of 0 indicates unknown or missing timestamp. diff --git a/quickwit/quickwit-proto/src/codegen/opentelemetry/opentelemetry.proto.metrics.v1.rs b/quickwit/quickwit-proto/src/codegen/opentelemetry/opentelemetry.proto.metrics.v1.rs index 10dee3a6b8a..cf4cdece793 100644 --- a/quickwit/quickwit-proto/src/codegen/opentelemetry/opentelemetry.proto.metrics.v1.rs +++ b/quickwit/quickwit-proto/src/codegen/opentelemetry/opentelemetry.proto.metrics.v1.rs @@ -55,66 +55,69 @@ pub struct ScopeMetrics { /// Defines a Metric which has one or more timeseries. The following is a /// brief summary of the Metric data model. For more details, see: /// -/// -/// +/// /// /// The data model and relation between entities is shown in the /// diagram below. Here, "DataPoint" is the term used to refer to any /// one of the specific data point value types, and "points" is the term used /// to refer to any one of the lists of points contained in the Metric. /// -/// - Metric is composed of a metadata and data. -/// - Metadata part contains a name, description, unit. -/// - Data is one of the possible types (Sum, Gauge, Histogram, Summary). -/// - DataPoint contains timestamps, attributes, and one of the possible value type -/// fields. +/// * Metric is composed of a metadata and data. +/// +/// * Metadata part contains a name, description, unit. +/// +/// * Data is one of the possible types (Sum, Gauge, Histogram, Summary). +/// +/// * DataPoint contains timestamps, attributes, and one of the possible value type +/// fields. /// -/// Metric +/// Metric /// +------------+ -/// |name | -/// |description | -/// |unit | +------------------------------------+ -/// |data |---> |Gauge, Sum, Histogram, Summary, ... | +/// \|name | +/// \|description | +/// \|unit | +------------------------------------+ +/// \|data |---> |Gauge, Sum, Histogram, Summary, ... | /// +------------+ +------------------------------------+ /// -/// Data \[One of Gauge, Sum, Histogram, Summary, ...\] +/// Data \[One of Gauge, Sum, Histogram, Summary, ...\] /// +-----------+ -/// |... | // Metadata about the Data. -/// |points |--+ +/// \|... | // Metadata about the Data. +/// \|points |--+ /// +-----------+ | -/// | +---------------------------+ -/// | |DataPoint 1 | -/// v |+------+------+ +------+ | -/// +-----+ ||label |label |...|label | | -/// | 1 |-->||value1|value2|...|valueN| | -/// +-----+ |+------+------+ +------+ | -/// | . | |+-----+ | -/// | . | ||value| | -/// | . | |+-----+ | -/// | . | +---------------------------+ -/// | . | . -/// | . | . -/// | . | . -/// | . | +---------------------------+ -/// | . | |DataPoint M | -/// +-----+ |+------+------+ +------+ | -/// | M |-->||label |label |...|label | | -/// +-----+ ||value1|value2|...|valueN| | -/// |+------+------+ +------+ | -/// |+-----+ | -/// ||value| | -/// |+-----+ | -/// +---------------------------+ +/// \| +---------------------------+ +/// \| |DataPoint 1 | +/// v |+------+------+ +------+ | +/// +-----+ ||label |label |...|label | | +/// \| 1 |-->||value1|value2|...|valueN| | +/// +-----+ |+------+------+ +------+ | +/// \| . | |+-----+ | +/// \| . | ||value| | +/// \| . | |+-----+ | +/// \| . | +---------------------------+ +/// \| . | . +/// \| . | . +/// \| . | . +/// \| . | +---------------------------+ +/// \| . | |DataPoint M | +/// +-----+ |+------+------+ +------+ | +/// \| M |-->||label |label |...|label | | +/// +-----+ ||value1|value2|...|valueN| | +/// \|+------+------+ +------+ | +/// \|+-----+ | +/// \||value| | +/// \|+-----+ | +/// +---------------------------+ /// /// Each distinct type of DataPoint represents the output of a specific /// aggregation function, the result of applying the DataPoint's /// associated function of to one or more measurements. /// /// All DataPoint types have three common fields: -/// - Attributes includes key-value pairs associated with the data point -/// - TimeUnixNano is required, set to the end time of the aggregation -/// - StartTimeUnixNano is optional, but strongly encouraged for DataPoints -/// having an AggregationTemporality field, as discussed below. +/// +/// * Attributes includes key-value pairs associated with the data point +/// * TimeUnixNano is required, set to the end time of the aggregation +/// * StartTimeUnixNano is optional, but strongly encouraged for DataPoints +/// having an AggregationTemporality field, as discussed below. /// /// Both TimeUnixNano and StartTimeUnixNano values are expressed as /// UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. @@ -352,8 +355,8 @@ pub struct HistogramDataPoint { /// /// The boundaries for bucket at index i are: /// - /// (-infinity, explicit_bounds\[i]\] for i == 0 - /// (explicit_bounds\[i-1\], explicit_bounds\[i]\] for 0 < i < size(explicit_bounds) + /// (-infinity, explicit_bounds\[i\]\] for i == 0 + /// (explicit_bounds\[i-1\], explicit_bounds\[i\]\] for 0 \< i \< size(explicit_bounds) /// (explicit_bounds\[i-1\], +infinity) for i == size(explicit_bounds) /// /// The values in the explicit_bounds array must be strictly increasing. @@ -371,10 +374,10 @@ pub struct HistogramDataPoint { /// for the available flags and their meaning. #[prost(uint32, tag = "10")] pub flags: u32, - /// min is the minimum value over (start_time, end_time]. + /// min is the minimum value over (start_time, end_time\]. #[prost(double, optional, tag = "11")] pub min: ::core::option::Option, - /// max is the maximum value over (start_time, end_time]. + /// max is the maximum value over (start_time, end_time\]. #[prost(double, optional, tag = "12")] pub max: ::core::option::Option, } @@ -382,7 +385,6 @@ pub struct HistogramDataPoint { /// time-varying values of a ExponentialHistogram of double values. A ExponentialHistogram contains /// summary statistics for a population of values, it may optionally contain the /// distribution of those values across a set of buckets. -/// #[derive(serde::Serialize, serde::Deserialize)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ExponentialHistogramDataPoint { @@ -423,7 +425,7 @@ pub struct ExponentialHistogramDataPoint { /// scale describes the resolution of the histogram. Boundaries are /// located at powers of the base, where: /// - /// base = (2^(2^-scale)) + /// base = (2^(2^-scale)) /// /// The histogram bucket identified by `index`, a signed integer, /// contains values that are greater than (base^index) and @@ -461,10 +463,10 @@ pub struct ExponentialHistogramDataPoint { /// measurements that were used to form the data point #[prost(message, repeated, tag = "11")] pub exemplars: ::prost::alloc::vec::Vec, - /// min is the minimum value over (start_time, end_time]. + /// min is the minimum value over (start_time, end_time\]. #[prost(double, optional, tag = "12")] pub min: ::core::option::Option, - /// max is the maximum value over (start_time, end_time]. + /// max is the maximum value over (start_time, end_time\]. #[prost(double, optional, tag = "13")] pub max: ::core::option::Option, } @@ -473,7 +475,7 @@ pub mod exponential_histogram_data_point { /// Buckets are a set of bucket counts, encoded in a contiguous array /// of counts. #[derive(serde::Serialize, serde::Deserialize)] - #[derive(Clone, PartialEq, ::prost::Message)] + #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct Buckets { /// Offset is the bucket index of the first entry in the bucket_counts array. /// @@ -544,8 +546,9 @@ pub mod summary_data_point { /// Represents the value at a given quantile of a distribution. /// /// To record Min and Max values following conventions are used: - /// - The 1.0 quantile is equivalent to the maximum value observed. - /// - The 0.0 quantile is equivalent to the minimum value observed. + /// + /// * The 1.0 quantile is equivalent to the maximum value observed. + /// * The 0.0 quantile is equivalent to the minimum value observed. /// /// See the following issue for more context: /// @@ -634,18 +637,18 @@ pub enum AggregationTemporality { /// it receives and reports the sum of these requests every second as a /// DELTA metric: /// - /// 1. The system starts receiving at time=t_0. - /// 2. A request is received, the system measures 1 request. - /// 3. A request is received, the system measures 1 request. - /// 4. A request is received, the system measures 1 request. - /// 5. The 1 second collection cycle ends. A metric is exported for the - /// number of requests received over the interval of time t_0 to - /// t_0+1 with a value of 3. - /// 6. A request is received, the system measures 1 request. - /// 7. A request is received, the system measures 1 request. - /// 8. The 1 second collection cycle ends. A metric is exported for the - /// number of requests received over the interval of time t_0+1 to - /// t_0+2 with a value of 2. + /// 1. The system starts receiving at time=t_0. + /// 1. A request is received, the system measures 1 request. + /// 1. A request is received, the system measures 1 request. + /// 1. A request is received, the system measures 1 request. + /// 1. The 1 second collection cycle ends. A metric is exported for the + /// number of requests received over the interval of time t_0 to + /// t_0+1 with a value of 3. + /// 1. A request is received, the system measures 1 request. + /// 1. A request is received, the system measures 1 request. + /// 1. The 1 second collection cycle ends. A metric is exported for the + /// number of requests received over the interval of time t_0+1 to + /// t_0+2 with a value of 2. Delta = 1, /// CUMULATIVE is an AggregationTemporality for a metric aggregator which /// reports changes since a fixed start time. This means that current values @@ -659,24 +662,24 @@ pub enum AggregationTemporality { /// it receives and reports the sum of these requests every second as a /// CUMULATIVE metric: /// - /// 1. The system starts receiving at time=t_0. - /// 2. A request is received, the system measures 1 request. - /// 3. A request is received, the system measures 1 request. - /// 4. A request is received, the system measures 1 request. - /// 5. The 1 second collection cycle ends. A metric is exported for the - /// number of requests received over the interval of time t_0 to - /// t_0+1 with a value of 3. - /// 6. A request is received, the system measures 1 request. - /// 7. A request is received, the system measures 1 request. - /// 8. The 1 second collection cycle ends. A metric is exported for the - /// number of requests received over the interval of time t_0 to - /// t_0+2 with a value of 5. - /// 9. The system experiences a fault and loses state. - /// 10. The system recovers and resumes receiving at time=t_1. - /// 11. A request is received, the system measures 1 request. - /// 12. The 1 second collection cycle ends. A metric is exported for the - /// number of requests received over the interval of time t_1 to - /// t_0+1 with a value of 1. + /// 1. The system starts receiving at time=t_0. + /// 1. A request is received, the system measures 1 request. + /// 1. A request is received, the system measures 1 request. + /// 1. A request is received, the system measures 1 request. + /// 1. The 1 second collection cycle ends. A metric is exported for the + /// number of requests received over the interval of time t_0 to + /// t_0+1 with a value of 3. + /// 1. A request is received, the system measures 1 request. + /// 1. A request is received, the system measures 1 request. + /// 1. The 1 second collection cycle ends. A metric is exported for the + /// number of requests received over the interval of time t_0 to + /// t_0+2 with a value of 5. + /// 1. The system experiences a fault and loses state. + /// 1. The system recovers and resumes receiving at time=t_1. + /// 1. A request is received, the system measures 1 request. + /// 1. The 1 second collection cycle ends. A metric is exported for the + /// number of requests received over the interval of time t_1 to + /// t_0+1 with a value of 1. /// /// Note: Even though, when reporting changes since last report time, using /// CUMULATIVE is valid, it is not recommended. This may cause problems for @@ -711,8 +714,7 @@ impl AggregationTemporality { /// enum is a bit-mask. To test the presence of a single flag in the flags of /// a data point, for example, use an expression like: /// -/// (point.flags & FLAG_NO_RECORDED_VALUE) == FLAG_NO_RECORDED_VALUE -/// +/// (point.flags & FLAG_NO_RECORDED_VALUE) == FLAG_NO_RECORDED_VALUE #[derive(serde::Serialize, serde::Deserialize)] #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] diff --git a/quickwit/quickwit-proto/src/codegen/opentelemetry/opentelemetry.proto.trace.v1.rs b/quickwit/quickwit-proto/src/codegen/opentelemetry/opentelemetry.proto.trace.v1.rs index e9b1c46e185..7159014a96c 100644 --- a/quickwit/quickwit-proto/src/codegen/opentelemetry/opentelemetry.proto.trace.v1.rs +++ b/quickwit/quickwit-proto/src/codegen/opentelemetry/opentelemetry.proto.trace.v1.rs @@ -124,10 +124,12 @@ pub struct Span { /// attributes is a collection of key/value pairs. Note, global attributes /// like server name can be set using the resource API. Examples of attributes: /// - /// "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" - /// "/http/server_latency": 300 - /// "abc.com/myattribute": true - /// "abc.com/score": 10.239 + /// ```text + /// "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" + /// "/http/server_latency": 300 + /// "abc.com/myattribute": true + /// "abc.com/score": 10.239 + /// ``` /// /// The OpenTelemetry API specification further restricts the allowed value types: /// @@ -284,7 +286,7 @@ pub mod span { /// The Status type defines a logical error model that is suitable for different /// programming environments, including REST APIs and RPC APIs. #[derive(serde::Serialize, serde::Deserialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct Status { /// A developer-facing human readable error message. #[prost(string, tag = "2")] diff --git a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.cluster.rs b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.cluster.rs index 8c8549d7730..f326b8e7c29 100644 --- a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.cluster.rs +++ b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.cluster.rs @@ -1,6 +1,6 @@ // This file is @generated by prost-build. #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ChitchatId { #[prost(string, tag = "1")] pub node_id: ::prost::alloc::string::String, @@ -10,7 +10,7 @@ pub struct ChitchatId { pub gossip_advertise_addr: ::prost::alloc::string::String, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct VersionedKeyValue { #[prost(string, tag = "1")] pub key: ::prost::alloc::string::String, @@ -34,7 +34,7 @@ pub struct NodeState { pub last_gc_version: u64, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct FetchClusterStateRequest { #[prost(string, tag = "1")] pub cluster_id: ::prost::alloc::string::String, @@ -678,7 +678,7 @@ pub mod cluster_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.cluster.ClusterService/FetchClusterState", ); @@ -825,7 +825,7 @@ pub mod cluster_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = FetchClusterStateSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, diff --git a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.common.rs b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.common.rs index b757cebb6d0..82aeb311280 100644 --- a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.common.rs +++ b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.common.rs @@ -4,7 +4,7 @@ /// /// Modify at your own risk. #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DocUid { /// ULID encoded as a sequence of 16 bytes (big-endian u128). #[prost(bytes = "vec", tag = "1")] diff --git a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.control_plane.rs b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.control_plane.rs index 311aad39b6f..09cfbdebf58 100644 --- a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.control_plane.rs +++ b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.control_plane.rs @@ -15,7 +15,7 @@ pub struct GetOrCreateOpenShardsRequest { pub unavailable_leaders: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetOrCreateOpenShardsSubrequest { #[prost(uint32, tag = "1")] pub subrequest_id: u32, @@ -45,7 +45,7 @@ pub struct GetOrCreateOpenShardsSuccess { pub open_shards: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetOrCreateOpenShardsFailure { #[prost(uint32, tag = "1")] pub subrequest_id: u32, @@ -61,6 +61,8 @@ pub struct GetOrCreateOpenShardsFailure { pub struct AdviseResetShardsRequest { #[prost(message, repeated, tag = "1")] pub shard_ids: ::prost::alloc::vec::Vec, + #[prost(string, tag = "2")] + pub ingester_id: ::prost::alloc::string::String, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -125,55 +127,55 @@ use tower::{Layer, Service, ServiceExt}; #[cfg_attr(any(test, feature = "testsuite"), mockall::automock)] #[async_trait::async_trait] pub trait ControlPlaneService: std::fmt::Debug + Send + Sync + 'static { - /// Creates a new index. + ///Creates a new index. async fn create_index( &self, request: super::metastore::CreateIndexRequest, ) -> crate::control_plane::ControlPlaneResult; - /// Updates an index. + ///Updates an index. async fn update_index( &self, request: super::metastore::UpdateIndexRequest, ) -> crate::control_plane::ControlPlaneResult< super::metastore::IndexMetadataResponse, >; - /// Deletes an index. + ///Deletes an index. async fn delete_index( &self, request: super::metastore::DeleteIndexRequest, ) -> crate::control_plane::ControlPlaneResult; - /// Adds a source to an index. + ///Adds a source to an index. async fn add_source( &self, request: super::metastore::AddSourceRequest, ) -> crate::control_plane::ControlPlaneResult; - /// Update a source. + ///Update a source. async fn update_source( &self, request: super::metastore::UpdateSourceRequest, ) -> crate::control_plane::ControlPlaneResult; - /// Enables or disables a source. + ///Enables or disables a source. async fn toggle_source( &self, request: super::metastore::ToggleSourceRequest, ) -> crate::control_plane::ControlPlaneResult; - /// Removes a source from an index. + ///Removes a source from an index. async fn delete_source( &self, request: super::metastore::DeleteSourceRequest, ) -> crate::control_plane::ControlPlaneResult; - /// Returns the list of open shards for one or several sources. If the control plane is not able to find any - /// for a source, it will pick a pair of leader-follower ingesters and will open a new shard. + ///Returns the list of open shards for one or several sources. If the control plane is not able to find any + ///for a source, it will pick a pair of leader-follower ingesters and will open a new shard. async fn get_or_create_open_shards( &self, request: GetOrCreateOpenShardsRequest, ) -> crate::control_plane::ControlPlaneResult; - /// Asks the control plane whether the shards listed in the request should be deleted or truncated. + ///Asks the control plane whether the shards listed in the request should be deleted or truncated. async fn advise_reset_shards( &self, request: AdviseResetShardsRequest, ) -> crate::control_plane::ControlPlaneResult; - /// Performs a debounced shard pruning request to the metastore. + ///Performs a debounced shard pruning request to the metastore. async fn prune_shards( &self, request: super::metastore::PruneShardsRequest, @@ -2157,7 +2159,7 @@ pub mod control_plane_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.control_plane.ControlPlaneService/CreateIndex", ); @@ -2187,7 +2189,7 @@ pub mod control_plane_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.control_plane.ControlPlaneService/UpdateIndex", ); @@ -2217,7 +2219,7 @@ pub mod control_plane_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.control_plane.ControlPlaneService/DeleteIndex", ); @@ -2247,7 +2249,7 @@ pub mod control_plane_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.control_plane.ControlPlaneService/AddSource", ); @@ -2279,7 +2281,7 @@ pub mod control_plane_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.control_plane.ControlPlaneService/UpdateSource", ); @@ -2311,7 +2313,7 @@ pub mod control_plane_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.control_plane.ControlPlaneService/ToggleSource", ); @@ -2343,7 +2345,7 @@ pub mod control_plane_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.control_plane.ControlPlaneService/DeleteSource", ); @@ -2374,7 +2376,7 @@ pub mod control_plane_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.control_plane.ControlPlaneService/GetOrCreateOpenShards", ); @@ -2404,7 +2406,7 @@ pub mod control_plane_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.control_plane.ControlPlaneService/AdviseResetShards", ); @@ -2434,7 +2436,7 @@ pub mod control_plane_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.control_plane.ControlPlaneService/PruneShards", ); @@ -2659,7 +2661,7 @@ pub mod control_plane_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = CreateIndexSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -2711,7 +2713,7 @@ pub mod control_plane_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = UpdateIndexSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -2763,7 +2765,7 @@ pub mod control_plane_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = DeleteIndexSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -2812,7 +2814,7 @@ pub mod control_plane_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = AddSourceSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -2864,7 +2866,7 @@ pub mod control_plane_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = UpdateSourceSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -2916,7 +2918,7 @@ pub mod control_plane_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = ToggleSourceSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -2968,7 +2970,7 @@ pub mod control_plane_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = DeleteSourceSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -3019,7 +3021,7 @@ pub mod control_plane_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = GetOrCreateOpenShardsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -3068,7 +3070,7 @@ pub mod control_plane_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = AdviseResetShardsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -3120,7 +3122,7 @@ pub mod control_plane_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = PruneShardsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, diff --git a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.developer.rs b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.developer.rs index 4956b0c2aae..38520a259ec 100644 --- a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.developer.rs +++ b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.developer.rs @@ -1,13 +1,13 @@ // This file is @generated by prost-build. #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetDebugInfoRequest { /// Restricts the debug info to the given roles. #[prost(string, repeated, tag = "1")] pub roles: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetDebugInfoResponse { #[prost(bytes = "bytes", tag = "1")] pub debug_info_json: ::prost::bytes::Bytes, @@ -611,7 +611,7 @@ pub mod developer_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.developer.DeveloperService/GetDebugInfo", ); @@ -756,7 +756,7 @@ pub mod developer_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = GetDebugInfoSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, diff --git a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.indexing.rs b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.indexing.rs index ede0c73c9bd..c22235f36ec 100644 --- a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.indexing.rs +++ b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.indexing.rs @@ -26,7 +26,7 @@ pub struct IndexingTask { pub params_fingerprint: u64, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct ApplyIndexingPlanResponse {} /// BEGIN quickwit-codegen #[allow(unused_imports)] @@ -35,7 +35,7 @@ use tower::{Layer, Service, ServiceExt}; #[cfg_attr(any(test, feature = "testsuite"), mockall::automock)] #[async_trait::async_trait] pub trait IndexingService: std::fmt::Debug + Send + Sync + 'static { - /// Apply an indexing plan on the node. + ///Apply an indexing plan on the node. async fn apply_indexing_plan( &self, request: ApplyIndexingPlanRequest, @@ -624,7 +624,7 @@ pub mod indexing_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.indexing.IndexingService/ApplyIndexingPlan", ); @@ -772,7 +772,7 @@ pub mod indexing_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = ApplyIndexingPlanSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, diff --git a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.ingester.rs b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.ingester.rs index 353abb15301..c472e2ea3ec 100644 --- a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.ingester.rs +++ b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.ingester.rs @@ -16,7 +16,7 @@ pub struct RetainShardsRequest { pub retain_shards_for_sources: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct RetainShardsResponse {} #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -37,8 +37,6 @@ pub struct PersistSubrequest { pub index_uid: ::core::option::Option, #[prost(string, tag = "3")] pub source_id: ::prost::alloc::string::String, - #[prost(message, optional, tag = "4")] - pub shard_id: ::core::option::Option, #[prost(message, optional, tag = "5")] pub doc_batch: ::core::option::Option, } @@ -51,6 +49,28 @@ pub struct PersistResponse { pub successes: ::prost::alloc::vec::Vec, #[prost(message, repeated, tag = "3")] pub failures: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag = "4")] + pub routing_update: ::core::option::Option, +} +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RoutingUpdate { + #[prost(uint32, tag = "1")] + pub capacity_score: u32, + #[prost(message, repeated, tag = "2")] + pub source_shard_updates: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "3")] + pub closed_shards: ::prost::alloc::vec::Vec, +} +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct SourceShardUpdate { + #[prost(message, optional, tag = "1")] + pub index_uid: ::core::option::Option, + #[prost(string, tag = "2")] + pub source_id: ::prost::alloc::string::String, + #[prost(uint32, tag = "3")] + pub open_shard_count: u32, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -71,7 +91,7 @@ pub struct PersistSuccess { pub parse_failures: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PersistFailure { #[prost(uint32, tag = "1")] pub subrequest_id: u32, @@ -79,8 +99,6 @@ pub struct PersistFailure { pub index_uid: ::core::option::Option, #[prost(string, tag = "3")] pub source_id: ::prost::alloc::string::String, - #[prost(message, optional, tag = "4")] - pub shard_id: ::core::option::Option, #[prost(enumeration = "PersistFailureReason", tag = "5")] pub reason: i32, } @@ -125,7 +143,7 @@ pub mod ack_replication_message { } } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct OpenReplicationStreamRequest { #[prost(string, tag = "1")] pub leader_id: ::prost::alloc::string::String, @@ -136,14 +154,14 @@ pub struct OpenReplicationStreamRequest { pub replication_seqno: u64, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct OpenReplicationStreamResponse { /// Position of the response in the replication stream. It should match the position of the request. #[prost(uint64, tag = "1")] pub replication_seqno: u64, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct InitReplicaRequest { #[prost(message, optional, tag = "1")] pub replica_shard: ::core::option::Option, @@ -151,7 +169,7 @@ pub struct InitReplicaRequest { pub replication_seqno: u64, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct InitReplicaResponse { #[prost(uint64, tag = "1")] pub replication_seqno: u64, @@ -201,7 +219,7 @@ pub struct ReplicateResponse { pub replication_seqno: u64, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReplicateSuccess { #[prost(uint32, tag = "1")] pub subrequest_id: u32, @@ -215,7 +233,7 @@ pub struct ReplicateSuccess { pub replication_position_inclusive: ::core::option::Option, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReplicateFailure { #[prost(uint32, tag = "1")] pub subrequest_id: u32, @@ -237,7 +255,7 @@ pub struct TruncateShardsRequest { pub subrequests: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct TruncateShardsSubrequest { #[prost(message, optional, tag = "1")] pub index_uid: ::core::option::Option, @@ -253,10 +271,10 @@ pub struct TruncateShardsSubrequest { } /// TODO #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct TruncateShardsResponse {} #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct OpenFetchStreamRequest { #[prost(string, tag = "1")] pub client_id: ::prost::alloc::string::String, @@ -270,7 +288,7 @@ pub struct OpenFetchStreamRequest { pub from_position_exclusive: ::core::option::Option, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct FetchMessage { #[prost(oneof = "fetch_message::Message", tags = "1, 2")] pub message: ::core::option::Option, @@ -279,7 +297,7 @@ pub struct FetchMessage { pub mod fetch_message { #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] - #[derive(Clone, PartialEq, ::prost::Oneof)] + #[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)] pub enum Message { #[prost(message, tag = "1")] Payload(super::FetchPayload), @@ -288,7 +306,7 @@ pub mod fetch_message { } } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct FetchPayload { #[prost(message, optional, tag = "1")] pub index_uid: ::core::option::Option, @@ -304,7 +322,7 @@ pub struct FetchPayload { pub to_position_inclusive: ::core::option::Option, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct FetchEof { #[prost(message, optional, tag = "1")] pub index_uid: ::core::option::Option, @@ -322,7 +340,7 @@ pub struct InitShardsRequest { pub subrequests: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct InitShardSubrequest { #[prost(uint32, tag = "1")] pub subrequest_id: u32, @@ -342,7 +360,7 @@ pub struct InitShardsResponse { pub failures: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct InitShardSuccess { #[prost(uint32, tag = "1")] pub subrequest_id: u32, @@ -350,7 +368,7 @@ pub struct InitShardSuccess { pub shard: ::core::option::Option, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct InitShardFailure { #[prost(uint32, tag = "1")] pub subrequest_id: u32, @@ -375,16 +393,16 @@ pub struct CloseShardsResponse { pub successes: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct DecommissionRequest {} #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct DecommissionResponse {} #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct OpenObservationStreamRequest {} #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ObservationMessage { #[prost(string, tag = "1")] pub node_id: ::prost::alloc::string::String, @@ -397,11 +415,10 @@ pub struct ObservationMessage { #[repr(i32)] pub enum PersistFailureReason { Unspecified = 0, - ShardNotFound = 1, - ShardClosed = 2, - ShardRateLimited = 3, WalFull = 4, Timeout = 5, + NoShardsAvailable = 6, + NodeUnavailable = 7, } impl PersistFailureReason { /// String value of the enum field names used in the ProtoBuf definition. @@ -411,22 +428,20 @@ impl PersistFailureReason { pub fn as_str_name(&self) -> &'static str { match self { Self::Unspecified => "PERSIST_FAILURE_REASON_UNSPECIFIED", - Self::ShardNotFound => "PERSIST_FAILURE_REASON_SHARD_NOT_FOUND", - Self::ShardClosed => "PERSIST_FAILURE_REASON_SHARD_CLOSED", - Self::ShardRateLimited => "PERSIST_FAILURE_REASON_SHARD_RATE_LIMITED", Self::WalFull => "PERSIST_FAILURE_REASON_WAL_FULL", Self::Timeout => "PERSIST_FAILURE_REASON_TIMEOUT", + Self::NoShardsAvailable => "PERSIST_FAILURE_REASON_NO_SHARDS_AVAILABLE", + Self::NodeUnavailable => "PERSIST_FAILURE_REASON_NODE_UNAVAILABLE", } } /// Creates an enum from field names used in the ProtoBuf definition. pub fn from_str_name(value: &str) -> ::core::option::Option { match value { "PERSIST_FAILURE_REASON_UNSPECIFIED" => Some(Self::Unspecified), - "PERSIST_FAILURE_REASON_SHARD_NOT_FOUND" => Some(Self::ShardNotFound), - "PERSIST_FAILURE_REASON_SHARD_CLOSED" => Some(Self::ShardClosed), - "PERSIST_FAILURE_REASON_SHARD_RATE_LIMITED" => Some(Self::ShardRateLimited), "PERSIST_FAILURE_REASON_WAL_FULL" => Some(Self::WalFull), "PERSIST_FAILURE_REASON_TIMEOUT" => Some(Self::Timeout), + "PERSIST_FAILURE_REASON_NO_SHARDS_AVAILABLE" => Some(Self::NoShardsAvailable), + "PERSIST_FAILURE_REASON_NODE_UNAVAILABLE" => Some(Self::NodeUnavailable), _ => None, } } @@ -475,6 +490,8 @@ pub enum IngesterStatus { Initializing = 1, /// The ingester is ready and accepts read and write requests. Ready = 2, + /// The ingester is about to be decommissioned. It still accepts read and write requests, but will not accept write requests in a few seconds and should be avoided by future write requests. + Retiring = 6, /// The ingester is being decommissioned. It accepts read requests but rejects write requests /// (open shards, persist, and replicate requests). It will transition to `Decommissioned` once /// all shards are fully indexed. @@ -495,6 +512,7 @@ impl IngesterStatus { Self::Unspecified => "INGESTER_STATUS_UNSPECIFIED", Self::Initializing => "INGESTER_STATUS_INITIALIZING", Self::Ready => "INGESTER_STATUS_READY", + Self::Retiring => "INGESTER_STATUS_RETIRING", Self::Decommissioning => "INGESTER_STATUS_DECOMMISSIONING", Self::Decommissioned => "INGESTER_STATUS_DECOMMISSIONED", Self::Failed => "INGESTER_STATUS_FAILED", @@ -506,6 +524,7 @@ impl IngesterStatus { "INGESTER_STATUS_UNSPECIFIED" => Some(Self::Unspecified), "INGESTER_STATUS_INITIALIZING" => Some(Self::Initializing), "INGESTER_STATUS_READY" => Some(Self::Ready), + "INGESTER_STATUS_RETIRING" => Some(Self::Retiring), "INGESTER_STATUS_DECOMMISSIONING" => Some(Self::Decommissioning), "INGESTER_STATUS_DECOMMISSIONED" => Some(Self::Decommissioned), "INGESTER_STATUS_FAILED" => Some(Self::Failed), @@ -569,49 +588,49 @@ pub type IngesterServiceStream = quickwit_common::ServiceStream< #[cfg_attr(any(test, feature = "testsuite"), mockall::automock)] #[async_trait::async_trait] pub trait IngesterService: std::fmt::Debug + Send + Sync + 'static { - /// Persists batches of documents to primary shards hosted on a leader. + ///Persists batches of documents to primary shards hosted on a leader. async fn persist( &self, request: PersistRequest, ) -> crate::ingest::IngestV2Result; - /// Opens a replication stream from a leader to a follower. + ///Opens a replication stream from a leader to a follower. async fn open_replication_stream( &self, request: quickwit_common::ServiceStream, ) -> crate::ingest::IngestV2Result>; - /// Streams records from a leader or a follower. The client can optionally specify a range of positions to fetch, - /// otherwise the stream will go indefinitely or until the shard is closed. + ///Streams records from a leader or a follower. The client can optionally specify a range of positions to fetch, + ///otherwise the stream will go indefinitely or until the shard is closed. async fn open_fetch_stream( &self, request: OpenFetchStreamRequest, ) -> crate::ingest::IngestV2Result>; - /// Streams status updates, called "observations", from an ingester. + ///Streams status updates, called "observations", from an ingester. async fn open_observation_stream( &self, request: OpenObservationStreamRequest, ) -> crate::ingest::IngestV2Result>; - /// Creates and initializes a set of newly opened shards. This RPC is called by the control plane on leaders. + ///Creates and initializes a set of newly opened shards. This RPC is called by the control plane on leaders. async fn init_shards( &self, request: InitShardsRequest, ) -> crate::ingest::IngestV2Result; - /// Only retain the shards that are listed in the request. - /// Other shards are deleted. + ///Only retain the shards that are listed in the request. + ///Other shards are deleted. async fn retain_shards( &self, request: RetainShardsRequest, ) -> crate::ingest::IngestV2Result; - /// Truncates a set of shards at the given positions. This RPC is called by indexers on leaders AND followers. + ///Truncates a set of shards at the given positions. This RPC is called by indexers on leaders AND followers. async fn truncate_shards( &self, request: TruncateShardsRequest, ) -> crate::ingest::IngestV2Result; - /// Closes a set of shards. This RPC is called by the control plane. + ///Closes a set of shards. This RPC is called by the control plane. async fn close_shards( &self, request: CloseShardsRequest, ) -> crate::ingest::IngestV2Result; - /// Decommissions the ingester. + ///Decommissions the ingester. async fn decommission( &self, request: DecommissionRequest, @@ -2403,7 +2422,7 @@ pub mod ingester_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.ingest.ingester.IngesterService/Persist", ); @@ -2435,7 +2454,7 @@ pub mod ingester_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.ingest.ingester.IngesterService/OpenReplicationStream", ); @@ -2466,7 +2485,7 @@ pub mod ingester_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.ingest.ingester.IngesterService/OpenFetchStream", ); @@ -2496,7 +2515,7 @@ pub mod ingester_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.ingest.ingester.IngesterService/OpenObservationStream", ); @@ -2526,7 +2545,7 @@ pub mod ingester_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.ingest.ingester.IngesterService/InitShards", ); @@ -2557,7 +2576,7 @@ pub mod ingester_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.ingest.ingester.IngesterService/RetainShards", ); @@ -2587,7 +2606,7 @@ pub mod ingester_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.ingest.ingester.IngesterService/TruncateShards", ); @@ -2617,7 +2636,7 @@ pub mod ingester_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.ingest.ingester.IngesterService/CloseShards", ); @@ -2647,7 +2666,7 @@ pub mod ingester_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.ingest.ingester.IngesterService/Decommission", ); @@ -2872,7 +2891,7 @@ pub mod ingester_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = PersistSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -2924,7 +2943,7 @@ pub mod ingester_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = OpenReplicationStreamSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -2975,7 +2994,7 @@ pub mod ingester_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = OpenFetchStreamSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -3026,7 +3045,7 @@ pub mod ingester_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = OpenObservationStreamSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -3072,7 +3091,7 @@ pub mod ingester_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = InitShardsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -3118,7 +3137,7 @@ pub mod ingester_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = RetainShardsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -3164,7 +3183,7 @@ pub mod ingester_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = TruncateShardsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -3210,7 +3229,7 @@ pub mod ingester_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = CloseShardsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -3256,7 +3275,7 @@ pub mod ingester_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = DecommissionSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, diff --git a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.router.rs b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.router.rs index b540175fcec..f502783e038 100644 --- a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.router.rs +++ b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.router.rs @@ -51,7 +51,7 @@ pub struct IngestSuccess { pub parse_failures: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct IngestFailure { #[prost(uint32, tag = "1")] pub subrequest_id: u32, @@ -132,8 +132,8 @@ impl RpcName for IngestRequestV2 { #[cfg_attr(any(test, feature = "testsuite"), mockall::automock)] #[async_trait::async_trait] pub trait IngestRouterService: std::fmt::Debug + Send + Sync + 'static { - /// Ingests batches of documents for one or multiple indexes. - /// TODO: Describe error cases and how to handle them. + ///Ingests batches of documents for one or multiple indexes. + ///TODO: Describe error cases and how to handle them. async fn ingest( &self, request: IngestRequestV2, @@ -726,7 +726,7 @@ pub mod ingest_router_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.ingest.router.IngestRouterService/Ingest", ); @@ -873,7 +873,7 @@ pub mod ingest_router_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = IngestSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, diff --git a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.rs b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.rs index e127f1320f9..7bd2757d00a 100644 --- a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.rs +++ b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.rs @@ -1,7 +1,7 @@ // This file is @generated by prost-build. /// Shard primary key. #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ShardPKey { #[prost(message, optional, tag = "1")] pub index_uid: ::core::option::Option, @@ -19,9 +19,12 @@ pub struct DocBatchV2 { pub doc_lengths: ::prost::alloc::vec::Vec, #[prost(message, repeated, tag = "3")] pub doc_uids: ::prost::alloc::vec::Vec, + /// Format of the documents in doc_buffer. Defaults to JSON for backward compatibility. + #[prost(enumeration = "DocFormat", tag = "4")] + pub doc_format: i32, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct MRecordBatch { /// Buffer of encoded and then concatenated mrecords. #[prost(bytes = "bytes", tag = "1")] @@ -30,9 +33,8 @@ pub struct MRecordBatch { #[prost(uint32, repeated, tag = "2")] pub mrecord_lengths: ::prost::alloc::vec::Vec, } -#[derive(Eq)] #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct Shard { /// Immutable fields #[prost(message, optional, tag = "1")] @@ -91,7 +93,7 @@ pub struct ShardIdPositions { pub shard_positions: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ShardIdPosition { #[prost(message, optional, tag = "1")] pub shard_id: ::core::option::Option, @@ -99,7 +101,7 @@ pub struct ShardIdPosition { pub publish_position_inclusive: ::core::option::Option, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ParseFailure { #[prost(message, optional, tag = "1")] pub doc_uid: ::core::option::Option, @@ -142,6 +144,37 @@ impl CommitTypeV2 { } } } +/// Format of documents in DocBatchV2.doc_buffer +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum DocFormat { + /// JSON documents, one per entry in doc_lengths (default for backward compatibility) + Json = 0, + /// Arrow IPC stream format containing a single RecordBatch + ArrowIpc = 1, +} +impl DocFormat { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Json => "DOC_FORMAT_JSON", + Self::ArrowIpc => "DOC_FORMAT_ARROW_IPC", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "DOC_FORMAT_JSON" => Some(Self::Json), + "DOC_FORMAT_ARROW_IPC" => Some(Self::ArrowIpc), + _ => None, + } + } +} #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] diff --git a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.metastore.rs b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.metastore.rs index ea2bf132549..2ce790ebbf9 100644 --- a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.metastore.rs +++ b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.metastore.rs @@ -1,9 +1,9 @@ // This file is @generated by prost-build. #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct EmptyResponse {} #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct CreateIndexRequest { #[prost(string, tag = "2")] pub index_config_json: ::prost::alloc::string::String, @@ -11,7 +11,7 @@ pub struct CreateIndexRequest { pub source_configs_json: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct CreateIndexResponse { #[prost(message, optional, tag = "1")] pub index_uid: ::core::option::Option, @@ -19,7 +19,7 @@ pub struct CreateIndexResponse { pub index_metadata_json: ::prost::alloc::string::String, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct UpdateIndexRequest { #[prost(message, optional, tag = "1")] pub index_uid: ::core::option::Option, @@ -37,7 +37,7 @@ pub struct UpdateIndexRequest { >, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ListIndexesMetadataRequest { /// List of patterns an index should match or not match to get considered /// An index must match at least one positive pattern (a pattern not starting @@ -46,7 +46,7 @@ pub struct ListIndexesMetadataRequest { pub index_id_patterns: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ListIndexesMetadataResponse { /// Deprecated (v0.9.0), use `indexes_metadata_json_zstd` instead. #[prost(string, optional, tag = "1")] @@ -59,7 +59,7 @@ pub struct ListIndexesMetadataResponse { pub indexes_metadata_json_zstd: ::prost::bytes::Bytes, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeleteIndexRequest { #[prost(message, optional, tag = "1")] pub index_uid: ::core::option::Option, @@ -69,7 +69,7 @@ pub struct DeleteIndexRequest { /// /// If both are supplied, `index_uid` is used. #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct IndexMetadataRequest { #[prost(string, optional, tag = "1")] pub index_id: ::core::option::Option<::prost::alloc::string::String>, @@ -77,7 +77,7 @@ pub struct IndexMetadataRequest { pub index_uid: ::core::option::Option, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct IndexMetadataResponse { #[prost(string, tag = "1")] pub index_metadata_serialized_json: ::prost::alloc::string::String, @@ -89,7 +89,7 @@ pub struct IndexesMetadataRequest { pub subrequests: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct IndexMetadataSubrequest { #[prost(string, optional, tag = "1")] pub index_id: ::core::option::Option<::prost::alloc::string::String>, @@ -107,7 +107,7 @@ pub struct IndexesMetadataResponse { pub failures: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct IndexMetadataFailure { #[prost(string, optional, tag = "1")] pub index_id: ::core::option::Option<::prost::alloc::string::String>, @@ -117,7 +117,43 @@ pub struct IndexMetadataFailure { pub reason: i32, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ListIndexStatsRequest { + /// List of patterns an index should match or not match to get considered + /// An index must match at least one positive pattern (a pattern not starting + /// with a '-'), and no negative pattern (a pattern starting with a '-'). + #[prost(string, repeated, tag = "1")] + pub index_id_patterns: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListIndexStatsResponse { + /// list of IndexStats. each one has the index id, the number of splits and the total size. + #[prost(message, repeated, tag = "1")] + pub index_stats: ::prost::alloc::vec::Vec, +} +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct IndexStats { + #[prost(message, optional, tag = "1")] + pub index_uid: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub staged: ::core::option::Option, + #[prost(message, optional, tag = "3")] + pub published: ::core::option::Option, + #[prost(message, optional, tag = "4")] + pub marked_for_deletion: ::core::option::Option, +} +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct SplitStats { + #[prost(uint64, tag = "1")] + pub num_splits: u64, + #[prost(uint64, tag = "2")] + pub total_size_bytes: u64, +} +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ListSplitsRequest { /// Predicate used to filter splits. /// The predicate is expressed as a JSON serialized @@ -126,14 +162,14 @@ pub struct ListSplitsRequest { pub query_json: ::prost::alloc::string::String, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ListSplitsResponse { /// TODO use repeated and encode splits json individually. #[prost(string, tag = "1")] pub splits_serialized_json: ::prost::alloc::string::String, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct StageSplitsRequest { #[prost(message, optional, tag = "1")] pub index_uid: ::core::option::Option, @@ -141,7 +177,7 @@ pub struct StageSplitsRequest { pub split_metadata_list_serialized_json: ::prost::alloc::string::String, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PublishSplitsRequest { #[prost(message, optional, tag = "1")] pub index_uid: ::core::option::Option, @@ -157,7 +193,7 @@ pub struct PublishSplitsRequest { pub publish_token_opt: ::core::option::Option<::prost::alloc::string::String>, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct MarkSplitsForDeletionRequest { #[prost(message, optional, tag = "2")] pub index_uid: ::core::option::Option, @@ -165,7 +201,7 @@ pub struct MarkSplitsForDeletionRequest { pub split_ids: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeleteSplitsRequest { #[prost(message, optional, tag = "2")] pub index_uid: ::core::option::Option, @@ -173,7 +209,7 @@ pub struct DeleteSplitsRequest { pub split_ids: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct AddSourceRequest { #[prost(message, optional, tag = "1")] pub index_uid: ::core::option::Option, @@ -181,7 +217,7 @@ pub struct AddSourceRequest { pub source_config_json: ::prost::alloc::string::String, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct UpdateSourceRequest { #[prost(message, optional, tag = "1")] pub index_uid: ::core::option::Option, @@ -189,7 +225,7 @@ pub struct UpdateSourceRequest { pub source_config_json: ::prost::alloc::string::String, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ToggleSourceRequest { #[prost(message, optional, tag = "1")] pub index_uid: ::core::option::Option, @@ -199,7 +235,7 @@ pub struct ToggleSourceRequest { pub enable: bool, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeleteSourceRequest { #[prost(message, optional, tag = "1")] pub index_uid: ::core::option::Option, @@ -207,7 +243,7 @@ pub struct DeleteSourceRequest { pub source_id: ::prost::alloc::string::String, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ResetSourceCheckpointRequest { #[prost(message, optional, tag = "1")] pub index_uid: ::core::option::Option, @@ -215,7 +251,7 @@ pub struct ResetSourceCheckpointRequest { pub source_id: ::prost::alloc::string::String, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeleteTask { #[prost(int64, tag = "1")] pub create_timestamp: i64, @@ -225,7 +261,7 @@ pub struct DeleteTask { pub delete_query: ::core::option::Option, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeleteQuery { /// Index UID. #[prost(message, optional, tag = "1")] @@ -236,7 +272,7 @@ pub struct DeleteQuery { #[prost(int64, optional, tag = "2")] #[serde(skip_serializing_if = "Option::is_none")] pub start_timestamp: ::core::option::Option, - /// If set, restrict search to documents with a `timestamp < end_timestamp``. + /// If set, restrict search to documents with a \`timestamp \< end_timestamp\``. #[prost(int64, optional, tag = "3")] #[serde(skip_serializing_if = "Option::is_none")] pub end_timestamp: ::core::option::Option, @@ -246,7 +282,7 @@ pub struct DeleteQuery { pub query_ast: ::prost::alloc::string::String, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct UpdateSplitsDeleteOpstampRequest { #[prost(message, optional, tag = "1")] pub index_uid: ::core::option::Option, @@ -256,22 +292,22 @@ pub struct UpdateSplitsDeleteOpstampRequest { pub delete_opstamp: u64, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct UpdateSplitsDeleteOpstampResponse {} #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct LastDeleteOpstampRequest { #[prost(message, optional, tag = "1")] pub index_uid: ::core::option::Option, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct LastDeleteOpstampResponse { #[prost(uint64, tag = "1")] pub last_delete_opstamp: u64, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ListStaleSplitsRequest { #[prost(message, optional, tag = "1")] pub index_uid: ::core::option::Option, @@ -281,7 +317,7 @@ pub struct ListStaleSplitsRequest { pub num_splits: u64, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ListDeleteTasksRequest { #[prost(message, optional, tag = "1")] pub index_uid: ::core::option::Option, @@ -301,7 +337,7 @@ pub struct OpenShardsRequest { pub subrequests: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct OpenShardSubrequest { #[prost(uint32, tag = "1")] pub subrequest_id: u32, @@ -327,7 +363,7 @@ pub struct OpenShardsResponse { pub subresponses: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct OpenShardSubresponse { #[prost(uint32, tag = "1")] pub subrequest_id: u32, @@ -382,7 +418,7 @@ pub struct DeleteShardsResponse { pub failures: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PruneShardsRequest { #[prost(message, optional, tag = "1")] pub index_uid: ::core::option::Option, @@ -405,7 +441,7 @@ pub struct ListShardsRequest { pub subrequests: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ListShardsSubrequest { #[prost(message, optional, tag = "1")] pub index_uid: ::core::option::Option, @@ -431,7 +467,7 @@ pub struct ListShardsSubresponse { pub shards: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct CreateIndexTemplateRequest { #[prost(string, tag = "1")] pub index_template_json: ::prost::alloc::string::String, @@ -439,19 +475,19 @@ pub struct CreateIndexTemplateRequest { pub overwrite: bool, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetIndexTemplateRequest { #[prost(string, tag = "1")] pub template_id: ::prost::alloc::string::String, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetIndexTemplateResponse { #[prost(string, tag = "1")] pub index_template_json: ::prost::alloc::string::String, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct FindIndexTemplateMatchesRequest { #[prost(string, repeated, tag = "1")] pub index_ids: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, @@ -463,7 +499,7 @@ pub struct FindIndexTemplateMatchesResponse { pub matches: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct IndexTemplateMatch { #[prost(string, tag = "1")] pub index_id: ::prost::alloc::string::String, @@ -473,21 +509,93 @@ pub struct IndexTemplateMatch { pub index_template_json: ::prost::alloc::string::String, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct ListIndexTemplatesRequest {} #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ListIndexTemplatesResponse { #[prost(string, repeated, tag = "1")] pub index_templates_json: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeleteIndexTemplatesRequest { #[prost(string, repeated, tag = "1")] pub template_ids: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetClusterIdentityRequest {} +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetClusterIdentityResponse { + #[prost(string, tag = "1")] + pub uuid: ::prost::alloc::string::String, +} +/// Stage metrics splits - first step after MetricsSplitWriter creates Parquet +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct StageMetricsSplitsRequest { + #[prost(message, optional, tag = "1")] + pub index_uid: ::core::option::Option, + /// Each element is a MetricsSplitMetadata serialized as JSON + #[prost(string, repeated, tag = "2")] + pub splits_metadata_json: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +/// Publish metrics splits - make staged splits queryable +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PublishMetricsSplitsRequest { + #[prost(message, optional, tag = "1")] + pub index_uid: ::core::option::Option, + #[prost(string, repeated, tag = "2")] + pub staged_split_ids: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + /// For merge operations (future) + #[prost(string, repeated, tag = "3")] + pub replaced_split_ids: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(string, optional, tag = "4")] + pub index_checkpoint_delta_json_opt: ::core::option::Option< + ::prost::alloc::string::String, + >, + #[prost(string, optional, tag = "5")] + pub publish_token_opt: ::core::option::Option<::prost::alloc::string::String>, +} +/// List metrics splits - for query planning and Tier 1 pruning +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ListMetricsSplitsRequest { + #[prost(message, optional, tag = "1")] + pub index_uid: ::core::option::Option, + /// Query predicate as JSON (ListMetricsSplitsQuery) + #[prost(string, tag = "2")] + pub query_json: ::prost::alloc::string::String, +} +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ListMetricsSplitsResponse { + /// Each element is a MetricsSplit serialized as JSON + #[prost(string, repeated, tag = "1")] + pub splits_serialized_json: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +/// Mark metrics splits for deletion - cleanup flow +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct MarkMetricsSplitsForDeletionRequest { + #[prost(message, optional, tag = "1")] + pub index_uid: ::core::option::Option, + #[prost(string, repeated, tag = "2")] + pub split_ids: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +/// Delete metrics splits - after storage cleanup complete +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct DeleteMetricsSplitsRequest { + #[prost(message, optional, tag = "1")] + pub index_uid: ::core::option::Option, + #[prost(string, repeated, tag = "2")] + pub split_ids: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] @@ -618,6 +726,11 @@ impl RpcName for DeleteIndexRequest { "delete_index" } } +impl RpcName for ListIndexStatsRequest { + fn rpc_name() -> &'static str { + "list_index_stats" + } +} impl RpcName for ListSplitsRequest { fn rpc_name() -> &'static str { "list_splits" @@ -743,150 +856,185 @@ impl RpcName for DeleteIndexTemplatesRequest { "delete_index_templates" } } +impl RpcName for GetClusterIdentityRequest { + fn rpc_name() -> &'static str { + "get_cluster_identity" + } +} +impl RpcName for StageMetricsSplitsRequest { + fn rpc_name() -> &'static str { + "stage_metrics_splits" + } +} +impl RpcName for PublishMetricsSplitsRequest { + fn rpc_name() -> &'static str { + "publish_metrics_splits" + } +} +impl RpcName for ListMetricsSplitsRequest { + fn rpc_name() -> &'static str { + "list_metrics_splits" + } +} +impl RpcName for MarkMetricsSplitsForDeletionRequest { + fn rpc_name() -> &'static str { + "mark_metrics_splits_for_deletion" + } +} +impl RpcName for DeleteMetricsSplitsRequest { + fn rpc_name() -> &'static str { + "delete_metrics_splits" + } +} pub type MetastoreServiceStream = quickwit_common::ServiceStream< crate::metastore::MetastoreResult, >; #[cfg_attr(any(test, feature = "testsuite"), mockall::automock)] #[async_trait::async_trait] pub trait MetastoreService: std::fmt::Debug + Send + Sync + 'static { - /// Creates an index. + ///Creates an index. /// - /// This API creates a new index in the metastore. - /// An error will occur if an index that already exists in the storage is specified. + ///This API creates a new index in the metastore. + ///An error will occur if an index that already exists in the storage is specified. async fn create_index( &self, request: CreateIndexRequest, ) -> crate::metastore::MetastoreResult; - /// Update an index. + ///Update an index. async fn update_index( &self, request: UpdateIndexRequest, ) -> crate::metastore::MetastoreResult; - /// Returns the `IndexMetadata` of an index identified by its IndexID or its IndexUID. + ///Returns the `IndexMetadata` of an index identified by its IndexID or its IndexUID. async fn index_metadata( &self, request: IndexMetadataRequest, ) -> crate::metastore::MetastoreResult; - /// Fetches the metadata of a list of indexes identified by their Index IDs or UIDs. + ///Fetches the metadata of a list of indexes identified by their Index IDs or UIDs. async fn indexes_metadata( &self, request: IndexesMetadataRequest, ) -> crate::metastore::MetastoreResult; - /// Gets an indexes metadatas. + ///Gets an indexes metadatas. async fn list_indexes_metadata( &self, request: ListIndexesMetadataRequest, ) -> crate::metastore::MetastoreResult; - /// Deletes an index + ///Deletes an index async fn delete_index( &self, request: DeleteIndexRequest, ) -> crate::metastore::MetastoreResult; - /// Streams splits from index. + ///Returns a list of size info for each index. + async fn list_index_stats( + &self, + request: ListIndexStatsRequest, + ) -> crate::metastore::MetastoreResult; + ///Streams splits from index. async fn list_splits( &self, request: ListSplitsRequest, ) -> crate::metastore::MetastoreResult>; - /// Stages several splits. + ///Stages several splits. async fn stage_splits( &self, request: StageSplitsRequest, ) -> crate::metastore::MetastoreResult; - /// Publishes split. + ///Publishes split. async fn publish_splits( &self, request: PublishSplitsRequest, ) -> crate::metastore::MetastoreResult; - /// Marks splits for deletion. + ///Marks splits for deletion. async fn mark_splits_for_deletion( &self, request: MarkSplitsForDeletionRequest, ) -> crate::metastore::MetastoreResult; - /// Deletes splits. + ///Deletes splits. async fn delete_splits( &self, request: DeleteSplitsRequest, ) -> crate::metastore::MetastoreResult; - /// Adds a source. + ///Adds a source. async fn add_source( &self, request: AddSourceRequest, ) -> crate::metastore::MetastoreResult; - /// Updates a source. + ///Updates a source. async fn update_source( &self, request: UpdateSourceRequest, ) -> crate::metastore::MetastoreResult; - /// Toggles (turns on or off) source. + ///Toggles (turns on or off) source. async fn toggle_source( &self, request: ToggleSourceRequest, ) -> crate::metastore::MetastoreResult; - /// Removes source. + ///Removes source. async fn delete_source( &self, request: DeleteSourceRequest, ) -> crate::metastore::MetastoreResult; - /// Resets source checkpoint. + ///Resets source checkpoint. async fn reset_source_checkpoint( &self, request: ResetSourceCheckpointRequest, ) -> crate::metastore::MetastoreResult; - /// Gets last opstamp for a given `index_id`. + ///Gets last opstamp for a given `index_id`. async fn last_delete_opstamp( &self, request: LastDeleteOpstampRequest, ) -> crate::metastore::MetastoreResult; - /// Creates a delete task. + ///Creates a delete task. async fn create_delete_task( &self, request: DeleteQuery, ) -> crate::metastore::MetastoreResult; - /// Updates splits `delete_opstamp`. + ///Updates splits `delete_opstamp`. async fn update_splits_delete_opstamp( &self, request: UpdateSplitsDeleteOpstampRequest, ) -> crate::metastore::MetastoreResult; - /// Lists delete tasks with `delete_task.opstamp` > `opstamp_start` for a given `index_id`. + ///Lists delete tasks with `delete_task.opstamp` > `opstamp_start` for a given `index_id`. async fn list_delete_tasks( &self, request: ListDeleteTasksRequest, ) -> crate::metastore::MetastoreResult; - /// Lists splits with `split.delete_opstamp` < `delete_opstamp` for a given `index_id`. + ///Lists splits with `split.delete_opstamp` \< `delete_opstamp` for a given `index_id`. async fn list_stale_splits( &self, request: ListStaleSplitsRequest, ) -> crate::metastore::MetastoreResult; - /// Shard API + ///Shard API /// - /// Note that for the file-backed metastore implementation, the requests are not processed atomically. - /// Indeed, each request comprises one or more subrequests that target different indexes and sources processed - /// independently. Responses list the requests that succeeded or failed in the fields `successes` and - /// `failures`. + ///Note that for the file-backed metastore implementation, the requests are not processed atomically. + ///Indeed, each request comprises one or more subrequests that target different indexes and sources processed + ///independently. Responses list the requests that succeeded or failed in the fields `successes` and + ///`failures`. async fn open_shards( &self, request: OpenShardsRequest, ) -> crate::metastore::MetastoreResult; - /// Acquires a set of shards for indexing. This RPC locks the shards for publishing thanks to a publish token and only - /// the last indexer that has acquired the shards is allowed to publish. The response returns for each subrequest the - /// list of acquired shards along with the positions to index from. + ///Acquires a set of shards for indexing. This RPC locks the shards for publishing thanks to a publish token and only + ///the last indexer that has acquired the shards is allowed to publish. The response returns for each subrequest the + ///list of acquired shards along with the positions to index from. /// - /// If a requested shard is missing, this method does not return an error. It should simply return the list of - /// shards that were actually acquired. + ///If a requested shard is missing, this method does not return an error. It should simply return the list of + ///shards that were actually acquired. /// - /// For this reason, AcquireShards.acquire_shards may return less subresponse than there was in the request. - /// Also they may be returned in any order. + ///For this reason, AcquireShards.acquire_shards may return less subresponse than there was in the request. + ///Also they may be returned in any order. async fn acquire_shards( &self, request: AcquireShardsRequest, ) -> crate::metastore::MetastoreResult; - /// Deletes a set of shards. This RPC deletes the shards from the metastore. - /// If the shard did not exist to begin with, the operation is successful and does not return any error. + ///Deletes a set of shards. This RPC deletes the shards from the metastore. + ///If the shard did not exist to begin with, the operation is successful and does not return any error. async fn delete_shards( &self, request: DeleteShardsRequest, ) -> crate::metastore::MetastoreResult; - /// Deletes outdated shards. This RPC deletes the shards from the metastore. + ///Deletes outdated shards. This RPC deletes the shards from the metastore. async fn prune_shards( &self, request: PruneShardsRequest, @@ -895,31 +1043,61 @@ pub trait MetastoreService: std::fmt::Debug + Send + Sync + 'static { &self, request: ListShardsRequest, ) -> crate::metastore::MetastoreResult; - /// Creates an index template. + ///Creates an index template. async fn create_index_template( &self, request: CreateIndexTemplateRequest, ) -> crate::metastore::MetastoreResult; - /// Fetches an index template. + ///Fetches an index template. async fn get_index_template( &self, request: GetIndexTemplateRequest, ) -> crate::metastore::MetastoreResult; - /// Finds matching index templates. + ///Finds matching index templates. async fn find_index_template_matches( &self, request: FindIndexTemplateMatchesRequest, ) -> crate::metastore::MetastoreResult; - /// Returns all the index templates. + ///Returns all the index templates. async fn list_index_templates( &self, request: ListIndexTemplatesRequest, ) -> crate::metastore::MetastoreResult; - /// Deletes index templates. + ///Deletes index templates. async fn delete_index_templates( &self, request: DeleteIndexTemplatesRequest, ) -> crate::metastore::MetastoreResult; + ///Get cluster identity + async fn get_cluster_identity( + &self, + request: GetClusterIdentityRequest, + ) -> crate::metastore::MetastoreResult; + ///Stages metrics splits in the metastore. + async fn stage_metrics_splits( + &self, + request: StageMetricsSplitsRequest, + ) -> crate::metastore::MetastoreResult; + ///Publishes metrics splits. + async fn publish_metrics_splits( + &self, + request: PublishMetricsSplitsRequest, + ) -> crate::metastore::MetastoreResult; + ///Lists metrics splits. + async fn list_metrics_splits( + &self, + request: ListMetricsSplitsRequest, + ) -> crate::metastore::MetastoreResult; + ///Marks metrics splits for deletion. + async fn mark_metrics_splits_for_deletion( + &self, + request: MarkMetricsSplitsForDeletionRequest, + ) -> crate::metastore::MetastoreResult; + ///Deletes metrics splits. + async fn delete_metrics_splits( + &self, + request: DeleteMetricsSplitsRequest, + ) -> crate::metastore::MetastoreResult; async fn check_connectivity(&self) -> anyhow::Result<()>; fn endpoints(&self) -> Vec; } @@ -1066,6 +1244,12 @@ impl MetastoreService for MetastoreServiceClient { ) -> crate::metastore::MetastoreResult { self.inner.0.delete_index(request).await } + async fn list_index_stats( + &self, + request: ListIndexStatsRequest, + ) -> crate::metastore::MetastoreResult { + self.inner.0.list_index_stats(request).await + } async fn list_splits( &self, request: ListSplitsRequest, @@ -1216,6 +1400,42 @@ impl MetastoreService for MetastoreServiceClient { ) -> crate::metastore::MetastoreResult { self.inner.0.delete_index_templates(request).await } + async fn get_cluster_identity( + &self, + request: GetClusterIdentityRequest, + ) -> crate::metastore::MetastoreResult { + self.inner.0.get_cluster_identity(request).await + } + async fn stage_metrics_splits( + &self, + request: StageMetricsSplitsRequest, + ) -> crate::metastore::MetastoreResult { + self.inner.0.stage_metrics_splits(request).await + } + async fn publish_metrics_splits( + &self, + request: PublishMetricsSplitsRequest, + ) -> crate::metastore::MetastoreResult { + self.inner.0.publish_metrics_splits(request).await + } + async fn list_metrics_splits( + &self, + request: ListMetricsSplitsRequest, + ) -> crate::metastore::MetastoreResult { + self.inner.0.list_metrics_splits(request).await + } + async fn mark_metrics_splits_for_deletion( + &self, + request: MarkMetricsSplitsForDeletionRequest, + ) -> crate::metastore::MetastoreResult { + self.inner.0.mark_metrics_splits_for_deletion(request).await + } + async fn delete_metrics_splits( + &self, + request: DeleteMetricsSplitsRequest, + ) -> crate::metastore::MetastoreResult { + self.inner.0.delete_metrics_splits(request).await + } async fn check_connectivity(&self) -> anyhow::Result<()> { self.inner.0.check_connectivity().await } @@ -1268,6 +1488,12 @@ pub mod mock_metastore_service { ) -> crate::metastore::MetastoreResult { self.inner.lock().await.delete_index(request).await } + async fn list_index_stats( + &self, + request: super::ListIndexStatsRequest, + ) -> crate::metastore::MetastoreResult { + self.inner.lock().await.list_index_stats(request).await + } async fn list_splits( &self, request: super::ListSplitsRequest, @@ -1422,6 +1648,42 @@ pub mod mock_metastore_service { ) -> crate::metastore::MetastoreResult { self.inner.lock().await.delete_index_templates(request).await } + async fn get_cluster_identity( + &self, + request: super::GetClusterIdentityRequest, + ) -> crate::metastore::MetastoreResult { + self.inner.lock().await.get_cluster_identity(request).await + } + async fn stage_metrics_splits( + &self, + request: super::StageMetricsSplitsRequest, + ) -> crate::metastore::MetastoreResult { + self.inner.lock().await.stage_metrics_splits(request).await + } + async fn publish_metrics_splits( + &self, + request: super::PublishMetricsSplitsRequest, + ) -> crate::metastore::MetastoreResult { + self.inner.lock().await.publish_metrics_splits(request).await + } + async fn list_metrics_splits( + &self, + request: super::ListMetricsSplitsRequest, + ) -> crate::metastore::MetastoreResult { + self.inner.lock().await.list_metrics_splits(request).await + } + async fn mark_metrics_splits_for_deletion( + &self, + request: super::MarkMetricsSplitsForDeletionRequest, + ) -> crate::metastore::MetastoreResult { + self.inner.lock().await.mark_metrics_splits_for_deletion(request).await + } + async fn delete_metrics_splits( + &self, + request: super::DeleteMetricsSplitsRequest, + ) -> crate::metastore::MetastoreResult { + self.inner.lock().await.delete_metrics_splits(request).await + } async fn check_connectivity(&self) -> anyhow::Result<()> { self.inner.lock().await.check_connectivity().await } @@ -1529,6 +1791,22 @@ impl tower::Service for InnerMetastoreServiceClient { Box::pin(fut) } } +impl tower::Service for InnerMetastoreServiceClient { + type Response = ListIndexStatsResponse; + type Error = crate::metastore::MetastoreError; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + fn call(&mut self, request: ListIndexStatsRequest) -> Self::Future { + let svc = self.clone(); + let fut = async move { svc.0.list_index_stats(request).await }; + Box::pin(fut) + } +} impl tower::Service for InnerMetastoreServiceClient { type Response = MetastoreServiceStream; type Error = crate::metastore::MetastoreError; @@ -1929,79 +2207,181 @@ impl tower::Service for InnerMetastoreServiceClient Box::pin(fut) } } -/// A tower service stack is a set of tower services. -#[derive(Debug)] -struct MetastoreServiceTowerServiceStack { - #[allow(dead_code)] - inner: InnerMetastoreServiceClient, - create_index_svc: quickwit_common::tower::BoxService< - CreateIndexRequest, - CreateIndexResponse, - crate::metastore::MetastoreError, - >, - update_index_svc: quickwit_common::tower::BoxService< - UpdateIndexRequest, - IndexMetadataResponse, - crate::metastore::MetastoreError, - >, - index_metadata_svc: quickwit_common::tower::BoxService< - IndexMetadataRequest, - IndexMetadataResponse, - crate::metastore::MetastoreError, - >, - indexes_metadata_svc: quickwit_common::tower::BoxService< - IndexesMetadataRequest, - IndexesMetadataResponse, - crate::metastore::MetastoreError, - >, - list_indexes_metadata_svc: quickwit_common::tower::BoxService< - ListIndexesMetadataRequest, - ListIndexesMetadataResponse, - crate::metastore::MetastoreError, - >, - delete_index_svc: quickwit_common::tower::BoxService< - DeleteIndexRequest, - EmptyResponse, - crate::metastore::MetastoreError, - >, - list_splits_svc: quickwit_common::tower::BoxService< - ListSplitsRequest, - MetastoreServiceStream, - crate::metastore::MetastoreError, - >, - stage_splits_svc: quickwit_common::tower::BoxService< - StageSplitsRequest, - EmptyResponse, - crate::metastore::MetastoreError, - >, - publish_splits_svc: quickwit_common::tower::BoxService< - PublishSplitsRequest, - EmptyResponse, - crate::metastore::MetastoreError, - >, - mark_splits_for_deletion_svc: quickwit_common::tower::BoxService< - MarkSplitsForDeletionRequest, - EmptyResponse, - crate::metastore::MetastoreError, - >, - delete_splits_svc: quickwit_common::tower::BoxService< - DeleteSplitsRequest, - EmptyResponse, - crate::metastore::MetastoreError, - >, - add_source_svc: quickwit_common::tower::BoxService< - AddSourceRequest, - EmptyResponse, - crate::metastore::MetastoreError, - >, - update_source_svc: quickwit_common::tower::BoxService< - UpdateSourceRequest, - EmptyResponse, - crate::metastore::MetastoreError, - >, - toggle_source_svc: quickwit_common::tower::BoxService< - ToggleSourceRequest, - EmptyResponse, +impl tower::Service for InnerMetastoreServiceClient { + type Response = GetClusterIdentityResponse; + type Error = crate::metastore::MetastoreError; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + fn call(&mut self, request: GetClusterIdentityRequest) -> Self::Future { + let svc = self.clone(); + let fut = async move { svc.0.get_cluster_identity(request).await }; + Box::pin(fut) + } +} +impl tower::Service for InnerMetastoreServiceClient { + type Response = EmptyResponse; + type Error = crate::metastore::MetastoreError; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + fn call(&mut self, request: StageMetricsSplitsRequest) -> Self::Future { + let svc = self.clone(); + let fut = async move { svc.0.stage_metrics_splits(request).await }; + Box::pin(fut) + } +} +impl tower::Service for InnerMetastoreServiceClient { + type Response = EmptyResponse; + type Error = crate::metastore::MetastoreError; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + fn call(&mut self, request: PublishMetricsSplitsRequest) -> Self::Future { + let svc = self.clone(); + let fut = async move { svc.0.publish_metrics_splits(request).await }; + Box::pin(fut) + } +} +impl tower::Service for InnerMetastoreServiceClient { + type Response = ListMetricsSplitsResponse; + type Error = crate::metastore::MetastoreError; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + fn call(&mut self, request: ListMetricsSplitsRequest) -> Self::Future { + let svc = self.clone(); + let fut = async move { svc.0.list_metrics_splits(request).await }; + Box::pin(fut) + } +} +impl tower::Service +for InnerMetastoreServiceClient { + type Response = EmptyResponse; + type Error = crate::metastore::MetastoreError; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + fn call(&mut self, request: MarkMetricsSplitsForDeletionRequest) -> Self::Future { + let svc = self.clone(); + let fut = async move { svc.0.mark_metrics_splits_for_deletion(request).await }; + Box::pin(fut) + } +} +impl tower::Service for InnerMetastoreServiceClient { + type Response = EmptyResponse; + type Error = crate::metastore::MetastoreError; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + fn call(&mut self, request: DeleteMetricsSplitsRequest) -> Self::Future { + let svc = self.clone(); + let fut = async move { svc.0.delete_metrics_splits(request).await }; + Box::pin(fut) + } +} +/// A tower service stack is a set of tower services. +#[derive(Debug)] +struct MetastoreServiceTowerServiceStack { + #[allow(dead_code)] + inner: InnerMetastoreServiceClient, + create_index_svc: quickwit_common::tower::BoxService< + CreateIndexRequest, + CreateIndexResponse, + crate::metastore::MetastoreError, + >, + update_index_svc: quickwit_common::tower::BoxService< + UpdateIndexRequest, + IndexMetadataResponse, + crate::metastore::MetastoreError, + >, + index_metadata_svc: quickwit_common::tower::BoxService< + IndexMetadataRequest, + IndexMetadataResponse, + crate::metastore::MetastoreError, + >, + indexes_metadata_svc: quickwit_common::tower::BoxService< + IndexesMetadataRequest, + IndexesMetadataResponse, + crate::metastore::MetastoreError, + >, + list_indexes_metadata_svc: quickwit_common::tower::BoxService< + ListIndexesMetadataRequest, + ListIndexesMetadataResponse, + crate::metastore::MetastoreError, + >, + delete_index_svc: quickwit_common::tower::BoxService< + DeleteIndexRequest, + EmptyResponse, + crate::metastore::MetastoreError, + >, + list_index_stats_svc: quickwit_common::tower::BoxService< + ListIndexStatsRequest, + ListIndexStatsResponse, + crate::metastore::MetastoreError, + >, + list_splits_svc: quickwit_common::tower::BoxService< + ListSplitsRequest, + MetastoreServiceStream, + crate::metastore::MetastoreError, + >, + stage_splits_svc: quickwit_common::tower::BoxService< + StageSplitsRequest, + EmptyResponse, + crate::metastore::MetastoreError, + >, + publish_splits_svc: quickwit_common::tower::BoxService< + PublishSplitsRequest, + EmptyResponse, + crate::metastore::MetastoreError, + >, + mark_splits_for_deletion_svc: quickwit_common::tower::BoxService< + MarkSplitsForDeletionRequest, + EmptyResponse, + crate::metastore::MetastoreError, + >, + delete_splits_svc: quickwit_common::tower::BoxService< + DeleteSplitsRequest, + EmptyResponse, + crate::metastore::MetastoreError, + >, + add_source_svc: quickwit_common::tower::BoxService< + AddSourceRequest, + EmptyResponse, + crate::metastore::MetastoreError, + >, + update_source_svc: quickwit_common::tower::BoxService< + UpdateSourceRequest, + EmptyResponse, + crate::metastore::MetastoreError, + >, + toggle_source_svc: quickwit_common::tower::BoxService< + ToggleSourceRequest, + EmptyResponse, crate::metastore::MetastoreError, >, delete_source_svc: quickwit_common::tower::BoxService< @@ -2089,6 +2469,36 @@ struct MetastoreServiceTowerServiceStack { EmptyResponse, crate::metastore::MetastoreError, >, + get_cluster_identity_svc: quickwit_common::tower::BoxService< + GetClusterIdentityRequest, + GetClusterIdentityResponse, + crate::metastore::MetastoreError, + >, + stage_metrics_splits_svc: quickwit_common::tower::BoxService< + StageMetricsSplitsRequest, + EmptyResponse, + crate::metastore::MetastoreError, + >, + publish_metrics_splits_svc: quickwit_common::tower::BoxService< + PublishMetricsSplitsRequest, + EmptyResponse, + crate::metastore::MetastoreError, + >, + list_metrics_splits_svc: quickwit_common::tower::BoxService< + ListMetricsSplitsRequest, + ListMetricsSplitsResponse, + crate::metastore::MetastoreError, + >, + mark_metrics_splits_for_deletion_svc: quickwit_common::tower::BoxService< + MarkMetricsSplitsForDeletionRequest, + EmptyResponse, + crate::metastore::MetastoreError, + >, + delete_metrics_splits_svc: quickwit_common::tower::BoxService< + DeleteMetricsSplitsRequest, + EmptyResponse, + crate::metastore::MetastoreError, + >, } #[async_trait::async_trait] impl MetastoreService for MetastoreServiceTowerServiceStack { @@ -2128,6 +2538,12 @@ impl MetastoreService for MetastoreServiceTowerServiceStack { ) -> crate::metastore::MetastoreResult { self.delete_index_svc.clone().ready().await?.call(request).await } + async fn list_index_stats( + &self, + request: ListIndexStatsRequest, + ) -> crate::metastore::MetastoreResult { + self.list_index_stats_svc.clone().ready().await?.call(request).await + } async fn list_splits( &self, request: ListSplitsRequest, @@ -2278,6 +2694,47 @@ impl MetastoreService for MetastoreServiceTowerServiceStack { ) -> crate::metastore::MetastoreResult { self.delete_index_templates_svc.clone().ready().await?.call(request).await } + async fn get_cluster_identity( + &self, + request: GetClusterIdentityRequest, + ) -> crate::metastore::MetastoreResult { + self.get_cluster_identity_svc.clone().ready().await?.call(request).await + } + async fn stage_metrics_splits( + &self, + request: StageMetricsSplitsRequest, + ) -> crate::metastore::MetastoreResult { + self.stage_metrics_splits_svc.clone().ready().await?.call(request).await + } + async fn publish_metrics_splits( + &self, + request: PublishMetricsSplitsRequest, + ) -> crate::metastore::MetastoreResult { + self.publish_metrics_splits_svc.clone().ready().await?.call(request).await + } + async fn list_metrics_splits( + &self, + request: ListMetricsSplitsRequest, + ) -> crate::metastore::MetastoreResult { + self.list_metrics_splits_svc.clone().ready().await?.call(request).await + } + async fn mark_metrics_splits_for_deletion( + &self, + request: MarkMetricsSplitsForDeletionRequest, + ) -> crate::metastore::MetastoreResult { + self.mark_metrics_splits_for_deletion_svc + .clone() + .ready() + .await? + .call(request) + .await + } + async fn delete_metrics_splits( + &self, + request: DeleteMetricsSplitsRequest, + ) -> crate::metastore::MetastoreResult { + self.delete_metrics_splits_svc.clone().ready().await?.call(request).await + } async fn check_connectivity(&self) -> anyhow::Result<()> { self.inner.0.check_connectivity().await } @@ -2345,6 +2802,16 @@ type DeleteIndexLayer = quickwit_common::tower::BoxLayer< EmptyResponse, crate::metastore::MetastoreError, >; +type ListIndexStatsLayer = quickwit_common::tower::BoxLayer< + quickwit_common::tower::BoxService< + ListIndexStatsRequest, + ListIndexStatsResponse, + crate::metastore::MetastoreError, + >, + ListIndexStatsRequest, + ListIndexStatsResponse, + crate::metastore::MetastoreError, +>; type ListSplitsLayer = quickwit_common::tower::BoxLayer< quickwit_common::tower::BoxService< ListSplitsRequest, @@ -2595,6 +3062,66 @@ type DeleteIndexTemplatesLayer = quickwit_common::tower::BoxLayer< EmptyResponse, crate::metastore::MetastoreError, >; +type GetClusterIdentityLayer = quickwit_common::tower::BoxLayer< + quickwit_common::tower::BoxService< + GetClusterIdentityRequest, + GetClusterIdentityResponse, + crate::metastore::MetastoreError, + >, + GetClusterIdentityRequest, + GetClusterIdentityResponse, + crate::metastore::MetastoreError, +>; +type StageMetricsSplitsLayer = quickwit_common::tower::BoxLayer< + quickwit_common::tower::BoxService< + StageMetricsSplitsRequest, + EmptyResponse, + crate::metastore::MetastoreError, + >, + StageMetricsSplitsRequest, + EmptyResponse, + crate::metastore::MetastoreError, +>; +type PublishMetricsSplitsLayer = quickwit_common::tower::BoxLayer< + quickwit_common::tower::BoxService< + PublishMetricsSplitsRequest, + EmptyResponse, + crate::metastore::MetastoreError, + >, + PublishMetricsSplitsRequest, + EmptyResponse, + crate::metastore::MetastoreError, +>; +type ListMetricsSplitsLayer = quickwit_common::tower::BoxLayer< + quickwit_common::tower::BoxService< + ListMetricsSplitsRequest, + ListMetricsSplitsResponse, + crate::metastore::MetastoreError, + >, + ListMetricsSplitsRequest, + ListMetricsSplitsResponse, + crate::metastore::MetastoreError, +>; +type MarkMetricsSplitsForDeletionLayer = quickwit_common::tower::BoxLayer< + quickwit_common::tower::BoxService< + MarkMetricsSplitsForDeletionRequest, + EmptyResponse, + crate::metastore::MetastoreError, + >, + MarkMetricsSplitsForDeletionRequest, + EmptyResponse, + crate::metastore::MetastoreError, +>; +type DeleteMetricsSplitsLayer = quickwit_common::tower::BoxLayer< + quickwit_common::tower::BoxService< + DeleteMetricsSplitsRequest, + EmptyResponse, + crate::metastore::MetastoreError, + >, + DeleteMetricsSplitsRequest, + EmptyResponse, + crate::metastore::MetastoreError, +>; #[derive(Debug, Default)] pub struct MetastoreServiceTowerLayerStack { create_index_layers: Vec, @@ -2603,6 +3130,7 @@ pub struct MetastoreServiceTowerLayerStack { indexes_metadata_layers: Vec, list_indexes_metadata_layers: Vec, delete_index_layers: Vec, + list_index_stats_layers: Vec, list_splits_layers: Vec, stage_splits_layers: Vec, publish_splits_layers: Vec, @@ -2628,6 +3156,12 @@ pub struct MetastoreServiceTowerLayerStack { find_index_template_matches_layers: Vec, list_index_templates_layers: Vec, delete_index_templates_layers: Vec, + get_cluster_identity_layers: Vec, + stage_metrics_splits_layers: Vec, + publish_metrics_splits_layers: Vec, + list_metrics_splits_layers: Vec, + mark_metrics_splits_for_deletion_layers: Vec, + delete_metrics_splits_layers: Vec, } impl MetastoreServiceTowerLayerStack { pub fn stack_layer(mut self, layer: L) -> Self @@ -2784,6 +3318,31 @@ impl MetastoreServiceTowerLayerStack { crate::metastore::MetastoreError, >, >>::Service as tower::Service>::Future: Send + 'static, + L: tower::Layer< + quickwit_common::tower::BoxService< + ListIndexStatsRequest, + ListIndexStatsResponse, + crate::metastore::MetastoreError, + >, + > + Clone + Send + Sync + 'static, + , + >>::Service: tower::Service< + ListIndexStatsRequest, + Response = ListIndexStatsResponse, + Error = crate::metastore::MetastoreError, + > + Clone + Send + Sync + 'static, + <, + >>::Service as tower::Service>::Future: Send + 'static, L: tower::Layer< quickwit_common::tower::BoxService< ListSplitsRequest, @@ -3423,19 +3982,181 @@ impl MetastoreServiceTowerLayerStack { >>::Service as tower::Service< DeleteIndexTemplatesRequest, >>::Future: Send + 'static, - { - self.create_index_layers - .push(quickwit_common::tower::BoxLayer::new(layer.clone())); - self.update_index_layers - .push(quickwit_common::tower::BoxLayer::new(layer.clone())); - self.index_metadata_layers - .push(quickwit_common::tower::BoxLayer::new(layer.clone())); - self.indexes_metadata_layers - .push(quickwit_common::tower::BoxLayer::new(layer.clone())); - self.list_indexes_metadata_layers + L: tower::Layer< + quickwit_common::tower::BoxService< + GetClusterIdentityRequest, + GetClusterIdentityResponse, + crate::metastore::MetastoreError, + >, + > + Clone + Send + Sync + 'static, + , + >>::Service: tower::Service< + GetClusterIdentityRequest, + Response = GetClusterIdentityResponse, + Error = crate::metastore::MetastoreError, + > + Clone + Send + Sync + 'static, + <, + >>::Service as tower::Service< + GetClusterIdentityRequest, + >>::Future: Send + 'static, + L: tower::Layer< + quickwit_common::tower::BoxService< + StageMetricsSplitsRequest, + EmptyResponse, + crate::metastore::MetastoreError, + >, + > + Clone + Send + Sync + 'static, + , + >>::Service: tower::Service< + StageMetricsSplitsRequest, + Response = EmptyResponse, + Error = crate::metastore::MetastoreError, + > + Clone + Send + Sync + 'static, + <, + >>::Service as tower::Service< + StageMetricsSplitsRequest, + >>::Future: Send + 'static, + L: tower::Layer< + quickwit_common::tower::BoxService< + PublishMetricsSplitsRequest, + EmptyResponse, + crate::metastore::MetastoreError, + >, + > + Clone + Send + Sync + 'static, + , + >>::Service: tower::Service< + PublishMetricsSplitsRequest, + Response = EmptyResponse, + Error = crate::metastore::MetastoreError, + > + Clone + Send + Sync + 'static, + <, + >>::Service as tower::Service< + PublishMetricsSplitsRequest, + >>::Future: Send + 'static, + L: tower::Layer< + quickwit_common::tower::BoxService< + ListMetricsSplitsRequest, + ListMetricsSplitsResponse, + crate::metastore::MetastoreError, + >, + > + Clone + Send + Sync + 'static, + , + >>::Service: tower::Service< + ListMetricsSplitsRequest, + Response = ListMetricsSplitsResponse, + Error = crate::metastore::MetastoreError, + > + Clone + Send + Sync + 'static, + <, + >>::Service as tower::Service>::Future: Send + 'static, + L: tower::Layer< + quickwit_common::tower::BoxService< + MarkMetricsSplitsForDeletionRequest, + EmptyResponse, + crate::metastore::MetastoreError, + >, + > + Clone + Send + Sync + 'static, + , + >>::Service: tower::Service< + MarkMetricsSplitsForDeletionRequest, + Response = EmptyResponse, + Error = crate::metastore::MetastoreError, + > + Clone + Send + Sync + 'static, + <, + >>::Service as tower::Service< + MarkMetricsSplitsForDeletionRequest, + >>::Future: Send + 'static, + L: tower::Layer< + quickwit_common::tower::BoxService< + DeleteMetricsSplitsRequest, + EmptyResponse, + crate::metastore::MetastoreError, + >, + > + Clone + Send + Sync + 'static, + , + >>::Service: tower::Service< + DeleteMetricsSplitsRequest, + Response = EmptyResponse, + Error = crate::metastore::MetastoreError, + > + Clone + Send + Sync + 'static, + <, + >>::Service as tower::Service< + DeleteMetricsSplitsRequest, + >>::Future: Send + 'static, + { + self.create_index_layers + .push(quickwit_common::tower::BoxLayer::new(layer.clone())); + self.update_index_layers + .push(quickwit_common::tower::BoxLayer::new(layer.clone())); + self.index_metadata_layers + .push(quickwit_common::tower::BoxLayer::new(layer.clone())); + self.indexes_metadata_layers + .push(quickwit_common::tower::BoxLayer::new(layer.clone())); + self.list_indexes_metadata_layers .push(quickwit_common::tower::BoxLayer::new(layer.clone())); self.delete_index_layers .push(quickwit_common::tower::BoxLayer::new(layer.clone())); + self.list_index_stats_layers + .push(quickwit_common::tower::BoxLayer::new(layer.clone())); self.list_splits_layers .push(quickwit_common::tower::BoxLayer::new(layer.clone())); self.stage_splits_layers @@ -3486,6 +4207,18 @@ impl MetastoreServiceTowerLayerStack { .push(quickwit_common::tower::BoxLayer::new(layer.clone())); self.delete_index_templates_layers .push(quickwit_common::tower::BoxLayer::new(layer.clone())); + self.get_cluster_identity_layers + .push(quickwit_common::tower::BoxLayer::new(layer.clone())); + self.stage_metrics_splits_layers + .push(quickwit_common::tower::BoxLayer::new(layer.clone())); + self.publish_metrics_splits_layers + .push(quickwit_common::tower::BoxLayer::new(layer.clone())); + self.list_metrics_splits_layers + .push(quickwit_common::tower::BoxLayer::new(layer.clone())); + self.mark_metrics_splits_for_deletion_layers + .push(quickwit_common::tower::BoxLayer::new(layer.clone())); + self.delete_metrics_splits_layers + .push(quickwit_common::tower::BoxLayer::new(layer.clone())); self } pub fn stack_create_index_layer(mut self, layer: L) -> Self @@ -3605,6 +4338,25 @@ impl MetastoreServiceTowerLayerStack { self.delete_index_layers.push(quickwit_common::tower::BoxLayer::new(layer)); self } + pub fn stack_list_index_stats_layer(mut self, layer: L) -> Self + where + L: tower::Layer< + quickwit_common::tower::BoxService< + ListIndexStatsRequest, + ListIndexStatsResponse, + crate::metastore::MetastoreError, + >, + > + Send + Sync + 'static, + L::Service: tower::Service< + ListIndexStatsRequest, + Response = ListIndexStatsResponse, + Error = crate::metastore::MetastoreError, + > + Clone + Send + Sync + 'static, + >::Future: Send + 'static, + { + self.list_index_stats_layers.push(quickwit_common::tower::BoxLayer::new(layer)); + self + } pub fn stack_list_splits_layer(mut self, layer: L) -> Self where L: tower::Layer< @@ -4104,6 +4856,136 @@ impl MetastoreServiceTowerLayerStack { .push(quickwit_common::tower::BoxLayer::new(layer)); self } + pub fn stack_get_cluster_identity_layer(mut self, layer: L) -> Self + where + L: tower::Layer< + quickwit_common::tower::BoxService< + GetClusterIdentityRequest, + GetClusterIdentityResponse, + crate::metastore::MetastoreError, + >, + > + Send + Sync + 'static, + L::Service: tower::Service< + GetClusterIdentityRequest, + Response = GetClusterIdentityResponse, + Error = crate::metastore::MetastoreError, + > + Clone + Send + Sync + 'static, + >::Future: Send + 'static, + { + self.get_cluster_identity_layers + .push(quickwit_common::tower::BoxLayer::new(layer)); + self + } + pub fn stack_stage_metrics_splits_layer(mut self, layer: L) -> Self + where + L: tower::Layer< + quickwit_common::tower::BoxService< + StageMetricsSplitsRequest, + EmptyResponse, + crate::metastore::MetastoreError, + >, + > + Send + Sync + 'static, + L::Service: tower::Service< + StageMetricsSplitsRequest, + Response = EmptyResponse, + Error = crate::metastore::MetastoreError, + > + Clone + Send + Sync + 'static, + >::Future: Send + 'static, + { + self.stage_metrics_splits_layers + .push(quickwit_common::tower::BoxLayer::new(layer)); + self + } + pub fn stack_publish_metrics_splits_layer(mut self, layer: L) -> Self + where + L: tower::Layer< + quickwit_common::tower::BoxService< + PublishMetricsSplitsRequest, + EmptyResponse, + crate::metastore::MetastoreError, + >, + > + Send + Sync + 'static, + L::Service: tower::Service< + PublishMetricsSplitsRequest, + Response = EmptyResponse, + Error = crate::metastore::MetastoreError, + > + Clone + Send + Sync + 'static, + >::Future: Send + 'static, + { + self.publish_metrics_splits_layers + .push(quickwit_common::tower::BoxLayer::new(layer)); + self + } + pub fn stack_list_metrics_splits_layer(mut self, layer: L) -> Self + where + L: tower::Layer< + quickwit_common::tower::BoxService< + ListMetricsSplitsRequest, + ListMetricsSplitsResponse, + crate::metastore::MetastoreError, + >, + > + Send + Sync + 'static, + L::Service: tower::Service< + ListMetricsSplitsRequest, + Response = ListMetricsSplitsResponse, + Error = crate::metastore::MetastoreError, + > + Clone + Send + Sync + 'static, + >::Future: Send + 'static, + { + self.list_metrics_splits_layers + .push(quickwit_common::tower::BoxLayer::new(layer)); + self + } + pub fn stack_mark_metrics_splits_for_deletion_layer(mut self, layer: L) -> Self + where + L: tower::Layer< + quickwit_common::tower::BoxService< + MarkMetricsSplitsForDeletionRequest, + EmptyResponse, + crate::metastore::MetastoreError, + >, + > + Send + Sync + 'static, + L::Service: tower::Service< + MarkMetricsSplitsForDeletionRequest, + Response = EmptyResponse, + Error = crate::metastore::MetastoreError, + > + Clone + Send + Sync + 'static, + >::Future: Send + 'static, + { + self.mark_metrics_splits_for_deletion_layers + .push(quickwit_common::tower::BoxLayer::new(layer)); + self + } + pub fn stack_delete_metrics_splits_layer(mut self, layer: L) -> Self + where + L: tower::Layer< + quickwit_common::tower::BoxService< + DeleteMetricsSplitsRequest, + EmptyResponse, + crate::metastore::MetastoreError, + >, + > + Send + Sync + 'static, + L::Service: tower::Service< + DeleteMetricsSplitsRequest, + Response = EmptyResponse, + Error = crate::metastore::MetastoreError, + > + Clone + Send + Sync + 'static, + >::Future: Send + 'static, + { + self.delete_metrics_splits_layers + .push(quickwit_common::tower::BoxLayer::new(layer)); + self + } pub fn build(self, instance: T) -> MetastoreServiceClient where T: MetastoreService, @@ -4212,6 +5094,14 @@ impl MetastoreServiceTowerLayerStack { quickwit_common::tower::BoxService::new(inner_client.clone()), |svc, layer| layer.layer(svc), ); + let list_index_stats_svc = self + .list_index_stats_layers + .into_iter() + .rev() + .fold( + quickwit_common::tower::BoxService::new(inner_client.clone()), + |svc, layer| layer.layer(svc), + ); let list_splits_svc = self .list_splits_layers .into_iter() @@ -4412,6 +5302,54 @@ impl MetastoreServiceTowerLayerStack { quickwit_common::tower::BoxService::new(inner_client.clone()), |svc, layer| layer.layer(svc), ); + let get_cluster_identity_svc = self + .get_cluster_identity_layers + .into_iter() + .rev() + .fold( + quickwit_common::tower::BoxService::new(inner_client.clone()), + |svc, layer| layer.layer(svc), + ); + let stage_metrics_splits_svc = self + .stage_metrics_splits_layers + .into_iter() + .rev() + .fold( + quickwit_common::tower::BoxService::new(inner_client.clone()), + |svc, layer| layer.layer(svc), + ); + let publish_metrics_splits_svc = self + .publish_metrics_splits_layers + .into_iter() + .rev() + .fold( + quickwit_common::tower::BoxService::new(inner_client.clone()), + |svc, layer| layer.layer(svc), + ); + let list_metrics_splits_svc = self + .list_metrics_splits_layers + .into_iter() + .rev() + .fold( + quickwit_common::tower::BoxService::new(inner_client.clone()), + |svc, layer| layer.layer(svc), + ); + let mark_metrics_splits_for_deletion_svc = self + .mark_metrics_splits_for_deletion_layers + .into_iter() + .rev() + .fold( + quickwit_common::tower::BoxService::new(inner_client.clone()), + |svc, layer| layer.layer(svc), + ); + let delete_metrics_splits_svc = self + .delete_metrics_splits_layers + .into_iter() + .rev() + .fold( + quickwit_common::tower::BoxService::new(inner_client.clone()), + |svc, layer| layer.layer(svc), + ); let tower_svc_stack = MetastoreServiceTowerServiceStack { inner: inner_client, create_index_svc, @@ -4420,6 +5358,7 @@ impl MetastoreServiceTowerLayerStack { indexes_metadata_svc, list_indexes_metadata_svc, delete_index_svc, + list_index_stats_svc, list_splits_svc, stage_splits_svc, publish_splits_svc, @@ -4445,6 +5384,12 @@ impl MetastoreServiceTowerLayerStack { find_index_template_matches_svc, list_index_templates_svc, delete_index_templates_svc, + get_cluster_identity_svc, + stage_metrics_splits_svc, + publish_metrics_splits_svc, + list_metrics_splits_svc, + mark_metrics_splits_for_deletion_svc, + delete_metrics_splits_svc, }; MetastoreServiceClient::new(tower_svc_stack) } @@ -4560,6 +5505,12 @@ where Error = crate::metastore::MetastoreError, Future = BoxFuture, > + + tower::Service< + ListIndexStatsRequest, + Response = ListIndexStatsResponse, + Error = crate::metastore::MetastoreError, + Future = BoxFuture, + > + tower::Service< ListSplitsRequest, Response = MetastoreServiceStream, @@ -4727,6 +5678,48 @@ where Response = EmptyResponse, Error = crate::metastore::MetastoreError, Future = BoxFuture, + > + + tower::Service< + GetClusterIdentityRequest, + Response = GetClusterIdentityResponse, + Error = crate::metastore::MetastoreError, + Future = BoxFuture< + GetClusterIdentityResponse, + crate::metastore::MetastoreError, + >, + > + + tower::Service< + StageMetricsSplitsRequest, + Response = EmptyResponse, + Error = crate::metastore::MetastoreError, + Future = BoxFuture, + > + + tower::Service< + PublishMetricsSplitsRequest, + Response = EmptyResponse, + Error = crate::metastore::MetastoreError, + Future = BoxFuture, + > + + tower::Service< + ListMetricsSplitsRequest, + Response = ListMetricsSplitsResponse, + Error = crate::metastore::MetastoreError, + Future = BoxFuture< + ListMetricsSplitsResponse, + crate::metastore::MetastoreError, + >, + > + + tower::Service< + MarkMetricsSplitsForDeletionRequest, + Response = EmptyResponse, + Error = crate::metastore::MetastoreError, + Future = BoxFuture, + > + + tower::Service< + DeleteMetricsSplitsRequest, + Response = EmptyResponse, + Error = crate::metastore::MetastoreError, + Future = BoxFuture, >, { async fn create_index( @@ -4765,6 +5758,12 @@ where ) -> crate::metastore::MetastoreResult { self.clone().call(request).await } + async fn list_index_stats( + &self, + request: ListIndexStatsRequest, + ) -> crate::metastore::MetastoreResult { + self.clone().call(request).await + } async fn list_splits( &self, request: ListSplitsRequest, @@ -4915,6 +5914,42 @@ where ) -> crate::metastore::MetastoreResult { self.clone().call(request).await } + async fn get_cluster_identity( + &self, + request: GetClusterIdentityRequest, + ) -> crate::metastore::MetastoreResult { + self.clone().call(request).await + } + async fn stage_metrics_splits( + &self, + request: StageMetricsSplitsRequest, + ) -> crate::metastore::MetastoreResult { + self.clone().call(request).await + } + async fn publish_metrics_splits( + &self, + request: PublishMetricsSplitsRequest, + ) -> crate::metastore::MetastoreResult { + self.clone().call(request).await + } + async fn list_metrics_splits( + &self, + request: ListMetricsSplitsRequest, + ) -> crate::metastore::MetastoreResult { + self.clone().call(request).await + } + async fn mark_metrics_splits_for_deletion( + &self, + request: MarkMetricsSplitsForDeletionRequest, + ) -> crate::metastore::MetastoreResult { + self.clone().call(request).await + } + async fn delete_metrics_splits( + &self, + request: DeleteMetricsSplitsRequest, + ) -> crate::metastore::MetastoreResult { + self.clone().call(request).await + } async fn check_connectivity(&self) -> anyhow::Result<()> { if self.inner.is_disconnected() { anyhow::bail!("actor `{}` is disconnected", self.inner.actor_instance_id()) @@ -5046,6 +6081,20 @@ where DeleteIndexRequest::rpc_name(), )) } + async fn list_index_stats( + &self, + request: ListIndexStatsRequest, + ) -> crate::metastore::MetastoreResult { + self.inner + .clone() + .list_index_stats(request) + .await + .map(|response| response.into_inner()) + .map_err(|status| crate::error::grpc_status_to_service_error( + status, + ListIndexStatsRequest::rpc_name(), + )) + } async fn list_splits( &self, request: ListSplitsRequest, @@ -5404,6 +6453,90 @@ where DeleteIndexTemplatesRequest::rpc_name(), )) } + async fn get_cluster_identity( + &self, + request: GetClusterIdentityRequest, + ) -> crate::metastore::MetastoreResult { + self.inner + .clone() + .get_cluster_identity(request) + .await + .map(|response| response.into_inner()) + .map_err(|status| crate::error::grpc_status_to_service_error( + status, + GetClusterIdentityRequest::rpc_name(), + )) + } + async fn stage_metrics_splits( + &self, + request: StageMetricsSplitsRequest, + ) -> crate::metastore::MetastoreResult { + self.inner + .clone() + .stage_metrics_splits(request) + .await + .map(|response| response.into_inner()) + .map_err(|status| crate::error::grpc_status_to_service_error( + status, + StageMetricsSplitsRequest::rpc_name(), + )) + } + async fn publish_metrics_splits( + &self, + request: PublishMetricsSplitsRequest, + ) -> crate::metastore::MetastoreResult { + self.inner + .clone() + .publish_metrics_splits(request) + .await + .map(|response| response.into_inner()) + .map_err(|status| crate::error::grpc_status_to_service_error( + status, + PublishMetricsSplitsRequest::rpc_name(), + )) + } + async fn list_metrics_splits( + &self, + request: ListMetricsSplitsRequest, + ) -> crate::metastore::MetastoreResult { + self.inner + .clone() + .list_metrics_splits(request) + .await + .map(|response| response.into_inner()) + .map_err(|status| crate::error::grpc_status_to_service_error( + status, + ListMetricsSplitsRequest::rpc_name(), + )) + } + async fn mark_metrics_splits_for_deletion( + &self, + request: MarkMetricsSplitsForDeletionRequest, + ) -> crate::metastore::MetastoreResult { + self.inner + .clone() + .mark_metrics_splits_for_deletion(request) + .await + .map(|response| response.into_inner()) + .map_err(|status| crate::error::grpc_status_to_service_error( + status, + MarkMetricsSplitsForDeletionRequest::rpc_name(), + )) + } + async fn delete_metrics_splits( + &self, + request: DeleteMetricsSplitsRequest, + ) -> crate::metastore::MetastoreResult { + self.inner + .clone() + .delete_metrics_splits(request) + .await + .map(|response| response.into_inner()) + .map_err(|status| crate::error::grpc_status_to_service_error( + status, + DeleteMetricsSplitsRequest::rpc_name(), + )) + } async fn check_connectivity(&self) -> anyhow::Result<()> { if self.connection_addrs_rx.borrow().is_empty() { anyhow::bail!("no server currently available") @@ -5503,6 +6636,17 @@ for MetastoreServiceGrpcServerAdapter { .map(tonic::Response::new) .map_err(crate::error::grpc_error_to_grpc_status) } + async fn list_index_stats( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.inner + .0 + .list_index_stats(request.into_inner()) + .await + .map(tonic::Response::new) + .map_err(crate::error::grpc_error_to_grpc_status) + } type ListSplitsStream = quickwit_common::ServiceStream< tonic::Result, >; @@ -5708,77 +6852,143 @@ for MetastoreServiceGrpcServerAdapter { } async fn prune_shards( &self, - request: tonic::Request, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.inner + .0 + .prune_shards(request.into_inner()) + .await + .map(tonic::Response::new) + .map_err(crate::error::grpc_error_to_grpc_status) + } + async fn list_shards( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.inner + .0 + .list_shards(request.into_inner()) + .await + .map(tonic::Response::new) + .map_err(crate::error::grpc_error_to_grpc_status) + } + async fn create_index_template( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.inner + .0 + .create_index_template(request.into_inner()) + .await + .map(tonic::Response::new) + .map_err(crate::error::grpc_error_to_grpc_status) + } + async fn get_index_template( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.inner + .0 + .get_index_template(request.into_inner()) + .await + .map(tonic::Response::new) + .map_err(crate::error::grpc_error_to_grpc_status) + } + async fn find_index_template_matches( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.inner + .0 + .find_index_template_matches(request.into_inner()) + .await + .map(tonic::Response::new) + .map_err(crate::error::grpc_error_to_grpc_status) + } + async fn list_index_templates( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.inner + .0 + .list_index_templates(request.into_inner()) + .await + .map(tonic::Response::new) + .map_err(crate::error::grpc_error_to_grpc_status) + } + async fn delete_index_templates( + &self, + request: tonic::Request, ) -> Result, tonic::Status> { self.inner .0 - .prune_shards(request.into_inner()) + .delete_index_templates(request.into_inner()) .await .map(tonic::Response::new) .map_err(crate::error::grpc_error_to_grpc_status) } - async fn list_shards( + async fn get_cluster_identity( &self, - request: tonic::Request, - ) -> Result, tonic::Status> { + request: tonic::Request, + ) -> Result, tonic::Status> { self.inner .0 - .list_shards(request.into_inner()) + .get_cluster_identity(request.into_inner()) .await .map(tonic::Response::new) .map_err(crate::error::grpc_error_to_grpc_status) } - async fn create_index_template( + async fn stage_metrics_splits( &self, - request: tonic::Request, + request: tonic::Request, ) -> Result, tonic::Status> { self.inner .0 - .create_index_template(request.into_inner()) + .stage_metrics_splits(request.into_inner()) .await .map(tonic::Response::new) .map_err(crate::error::grpc_error_to_grpc_status) } - async fn get_index_template( + async fn publish_metrics_splits( &self, - request: tonic::Request, - ) -> Result, tonic::Status> { + request: tonic::Request, + ) -> Result, tonic::Status> { self.inner .0 - .get_index_template(request.into_inner()) + .publish_metrics_splits(request.into_inner()) .await .map(tonic::Response::new) .map_err(crate::error::grpc_error_to_grpc_status) } - async fn find_index_template_matches( + async fn list_metrics_splits( &self, - request: tonic::Request, - ) -> Result, tonic::Status> { + request: tonic::Request, + ) -> Result, tonic::Status> { self.inner .0 - .find_index_template_matches(request.into_inner()) + .list_metrics_splits(request.into_inner()) .await .map(tonic::Response::new) .map_err(crate::error::grpc_error_to_grpc_status) } - async fn list_index_templates( + async fn mark_metrics_splits_for_deletion( &self, - request: tonic::Request, - ) -> Result, tonic::Status> { + request: tonic::Request, + ) -> Result, tonic::Status> { self.inner .0 - .list_index_templates(request.into_inner()) + .mark_metrics_splits_for_deletion(request.into_inner()) .await .map(tonic::Response::new) .map_err(crate::error::grpc_error_to_grpc_status) } - async fn delete_index_templates( + async fn delete_metrics_splits( &self, - request: tonic::Request, + request: tonic::Request, ) -> Result, tonic::Status> { self.inner .0 - .delete_index_templates(request.into_inner()) + .delete_metrics_splits(request.into_inner()) .await .map(tonic::Response::new) .map_err(crate::error::grpc_error_to_grpc_status) @@ -5804,18 +7014,25 @@ pub mod metastore_service_grpc_client { /// We rely on atomically transitioning the status of splits. /// /// The split state goes through the following life cycle: + /// /// 1. `Staged` - /// - Start uploading the split files. + /// + /// * Start uploading the split files. + /// /// 2. `Published` - /// - Uploading the split files is complete and the split is searchable. + /// + /// * Uploading the split files is complete and the split is searchable. + /// /// 3. `MarkedForDeletion` - /// - Mark the split for deletion. + /// + /// * Mark the split for deletion. /// /// If a split has a file in the storage, it MUST be registered in the metastore, /// and its state can be as follows: - /// - `Staged`: The split is almost ready. Some of its files may have been uploaded in the storage. - /// - `Published`: The split is ready and published. - /// - `MarkedForDeletion`: The split is marked for deletion. + /// + /// * `Staged`: The split is almost ready. Some of its files may have been uploaded in the storage. + /// * `Published`: The split is ready and published. + /// * `MarkedForDeletion`: The split is marked for deletion. /// /// Before creating any file, we need to stage the split. If there is a failure, upon recovery, we /// schedule for deletion all the staged splits. A client may not necessarily remove files from @@ -5830,13 +7047,14 @@ pub mod metastore_service_grpc_client { /// /// Quickwit needs a way to track that a delete task has been applied to a split. This is ensured /// by two mechanisms: - /// - On creation of a delete task, we give to the task a monotically increasing opstamp (uniqueness - /// and monotonically increasing must be true at the index level). - /// - When a delete task is executed on a split, that is when the documents matched by the search - /// query are removed from the splits, we update the split's `delete_opstamp` to the value of the - /// task's opstamp. This marks the split as "up-to-date" regarding this delete task. If new delete - /// tasks are added, we will know that we need to run these delete tasks on the splits as its - /// `delete_optstamp` will be inferior to the `opstamp` of the new tasks. + /// + /// * On creation of a delete task, we give to the task a monotically increasing opstamp (uniqueness + /// and monotonically increasing must be true at the index level). + /// * When a delete task is executed on a split, that is when the documents matched by the search + /// query are removed from the splits, we update the split's `delete_opstamp` to the value of the + /// task's opstamp. This marks the split as "up-to-date" regarding this delete task. If new delete + /// tasks are added, we will know that we need to run these delete tasks on the splits as its + /// `delete_optstamp` will be inferior to the `opstamp` of the new tasks. /// /// For splits created after a given delete task, Quickwit's indexing ensures that these splits /// are created with a `delete_opstamp` equal the latest opstamp of the tasks of the @@ -5940,7 +7158,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/CreateIndex", ); @@ -5967,7 +7185,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/UpdateIndex", ); @@ -5994,7 +7212,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/IndexMetadata", ); @@ -6024,7 +7242,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/IndexesMetadata", ); @@ -6054,7 +7272,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/ListIndexesMetadata", ); @@ -6081,7 +7299,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/DeleteIndex", ); @@ -6092,6 +7310,36 @@ pub mod metastore_service_grpc_client { ); self.inner.unary(req, path, codec).await } + /// Returns a list of size info for each index. + pub async fn list_index_stats( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/quickwit.metastore.MetastoreService/ListIndexStats", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "quickwit.metastore.MetastoreService", + "ListIndexStats", + ), + ); + self.inner.unary(req, path, codec).await + } /// Streams splits from index. pub async fn list_splits( &mut self, @@ -6108,7 +7356,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/ListSplits", ); @@ -6132,7 +7380,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/StageSplits", ); @@ -6156,7 +7404,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/PublishSplits", ); @@ -6183,7 +7431,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/MarkSplitsForDeletion", ); @@ -6210,7 +7458,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/DeleteSplits", ); @@ -6237,7 +7485,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/AddSource", ); @@ -6261,7 +7509,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/UpdateSource", ); @@ -6288,7 +7536,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/ToggleSource", ); @@ -6315,7 +7563,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/DeleteSource", ); @@ -6342,7 +7590,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/ResetSourceCheckpoint", ); @@ -6372,7 +7620,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/LastDeleteOpstamp", ); @@ -6399,7 +7647,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/CreateDeleteTask", ); @@ -6429,7 +7677,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/UpdateSplitsDeleteOpstamp", ); @@ -6459,7 +7707,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/ListDeleteTasks", ); @@ -6473,7 +7721,7 @@ pub mod metastore_service_grpc_client { ); self.inner.unary(req, path, codec).await } - /// Lists splits with `split.delete_opstamp` < `delete_opstamp` for a given `index_id`. + /// Lists splits with `split.delete_opstamp` \< `delete_opstamp` for a given `index_id`. pub async fn list_stale_splits( &mut self, request: impl tonic::IntoRequest, @@ -6489,7 +7737,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/ListStaleSplits", ); @@ -6524,7 +7772,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/OpenShards", ); @@ -6559,7 +7807,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/AcquireShards", ); @@ -6590,7 +7838,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/DeleteShards", ); @@ -6617,7 +7865,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/PruneShards", ); @@ -6643,7 +7891,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/ListShards", ); @@ -6667,7 +7915,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/CreateIndexTemplate", ); @@ -6697,7 +7945,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/GetIndexTemplate", ); @@ -6727,7 +7975,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/FindIndexTemplateMatches", ); @@ -6757,7 +8005,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/ListIndexTemplates", ); @@ -6784,7 +8032,7 @@ pub mod metastore_service_grpc_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.metastore.MetastoreService/DeleteIndexTemplates", ); @@ -6798,6 +8046,174 @@ pub mod metastore_service_grpc_client { ); self.inner.unary(req, path, codec).await } + /// Get cluster identity + pub async fn get_cluster_identity( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/quickwit.metastore.MetastoreService/GetClusterIdentity", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "quickwit.metastore.MetastoreService", + "GetClusterIdentity", + ), + ); + self.inner.unary(req, path, codec).await + } + /// Stages metrics splits in the metastore. + pub async fn stage_metrics_splits( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/quickwit.metastore.MetastoreService/StageMetricsSplits", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "quickwit.metastore.MetastoreService", + "StageMetricsSplits", + ), + ); + self.inner.unary(req, path, codec).await + } + /// Publishes metrics splits. + pub async fn publish_metrics_splits( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/quickwit.metastore.MetastoreService/PublishMetricsSplits", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "quickwit.metastore.MetastoreService", + "PublishMetricsSplits", + ), + ); + self.inner.unary(req, path, codec).await + } + /// Lists metrics splits. + pub async fn list_metrics_splits( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/quickwit.metastore.MetastoreService/ListMetricsSplits", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "quickwit.metastore.MetastoreService", + "ListMetricsSplits", + ), + ); + self.inner.unary(req, path, codec).await + } + /// Marks metrics splits for deletion. + pub async fn mark_metrics_splits_for_deletion( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/quickwit.metastore.MetastoreService/MarkMetricsSplitsForDeletion", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "quickwit.metastore.MetastoreService", + "MarkMetricsSplitsForDeletion", + ), + ); + self.inner.unary(req, path, codec).await + } + /// Deletes metrics splits. + pub async fn delete_metrics_splits( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/quickwit.metastore.MetastoreService/DeleteMetricsSplits", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "quickwit.metastore.MetastoreService", + "DeleteMetricsSplits", + ), + ); + self.inner.unary(req, path, codec).await + } } } /// Generated server implementations. @@ -6861,6 +8277,14 @@ pub mod metastore_service_grpc_server { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + /// Returns a list of size info for each index. + async fn list_index_stats( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; /// Server streaming response type for the ListSplits method. type ListSplitsStream: tonic::codegen::tokio_stream::Stream< Item = std::result::Result, @@ -6946,7 +8370,7 @@ pub mod metastore_service_grpc_server { tonic::Response, tonic::Status, >; - /// Lists splits with `split.delete_opstamp` < `delete_opstamp` for a given `index_id`. + /// Lists splits with `split.delete_opstamp` \< `delete_opstamp` for a given `index_id`. async fn list_stale_splits( &self, request: tonic::Request, @@ -7038,6 +8462,42 @@ pub mod metastore_service_grpc_server { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + /// Get cluster identity + async fn get_cluster_identity( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + /// Stages metrics splits in the metastore. + async fn stage_metrics_splits( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + /// Publishes metrics splits. + async fn publish_metrics_splits( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + /// Lists metrics splits. + async fn list_metrics_splits( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + /// Marks metrics splits for deletion. + async fn mark_metrics_splits_for_deletion( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + /// Deletes metrics splits. + async fn delete_metrics_splits( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; } /// Metastore meant to manage Quickwit's indexes, their splits and delete tasks. /// @@ -7048,18 +8508,25 @@ pub mod metastore_service_grpc_server { /// We rely on atomically transitioning the status of splits. /// /// The split state goes through the following life cycle: + /// /// 1. `Staged` - /// - Start uploading the split files. + /// + /// * Start uploading the split files. + /// /// 2. `Published` - /// - Uploading the split files is complete and the split is searchable. + /// + /// * Uploading the split files is complete and the split is searchable. + /// /// 3. `MarkedForDeletion` - /// - Mark the split for deletion. + /// + /// * Mark the split for deletion. /// /// If a split has a file in the storage, it MUST be registered in the metastore, /// and its state can be as follows: - /// - `Staged`: The split is almost ready. Some of its files may have been uploaded in the storage. - /// - `Published`: The split is ready and published. - /// - `MarkedForDeletion`: The split is marked for deletion. + /// + /// * `Staged`: The split is almost ready. Some of its files may have been uploaded in the storage. + /// * `Published`: The split is ready and published. + /// * `MarkedForDeletion`: The split is marked for deletion. /// /// Before creating any file, we need to stage the split. If there is a failure, upon recovery, we /// schedule for deletion all the staged splits. A client may not necessarily remove files from @@ -7074,13 +8541,14 @@ pub mod metastore_service_grpc_server { /// /// Quickwit needs a way to track that a delete task has been applied to a split. This is ensured /// by two mechanisms: - /// - On creation of a delete task, we give to the task a monotically increasing opstamp (uniqueness - /// and monotonically increasing must be true at the index level). - /// - When a delete task is executed on a split, that is when the documents matched by the search - /// query are removed from the splits, we update the split's `delete_opstamp` to the value of the - /// task's opstamp. This marks the split as "up-to-date" regarding this delete task. If new delete - /// tasks are added, we will know that we need to run these delete tasks on the splits as its - /// `delete_optstamp` will be inferior to the `opstamp` of the new tasks. + /// + /// * On creation of a delete task, we give to the task a monotically increasing opstamp (uniqueness + /// and monotonically increasing must be true at the index level). + /// * When a delete task is executed on a split, that is when the documents matched by the search + /// query are removed from the splits, we update the split's `delete_opstamp` to the value of the + /// task's opstamp. This marks the split as "up-to-date" regarding this delete task. If new delete + /// tasks are added, we will know that we need to run these delete tasks on the splits as its + /// `delete_optstamp` will be inferior to the `opstamp` of the new tasks. /// /// For splits created after a given delete task, Quickwit's indexing ensures that these splits /// are created with a `delete_opstamp` equal the latest opstamp of the tasks of the @@ -7193,7 +8661,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = CreateIndexSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -7239,7 +8707,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = UpdateIndexSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -7285,7 +8753,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = IndexMetadataSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -7334,7 +8802,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = IndexesMetadataSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -7383,7 +8851,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = ListIndexesMetadataSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -7412,11 +8880,60 @@ pub mod metastore_service_grpc_server { >; fn call( &mut self, - request: tonic::Request, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::delete_index(&inner, request) + .await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = DeleteIndexSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/quickwit.metastore.MetastoreService/ListIndexStats" => { + #[allow(non_camel_case_types)] + struct ListIndexStatsSvc(pub Arc); + impl< + T: MetastoreServiceGrpc, + > tonic::server::UnaryService + for ListIndexStatsSvc { + type Response = super::ListIndexStatsResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, ) -> Self::Future { let inner = Arc::clone(&self.0); let fut = async move { - ::delete_index(&inner, request) + ::list_index_stats( + &inner, + request, + ) .await }; Box::pin(fut) @@ -7428,8 +8945,8 @@ pub mod metastore_service_grpc_server { let max_encoding_message_size = self.max_encoding_message_size; let inner = self.inner.clone(); let fut = async move { - let method = DeleteIndexSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let method = ListIndexStatsSvc(inner); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -7476,7 +8993,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = ListSplitsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -7522,7 +9039,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = StageSplitsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -7568,7 +9085,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = PublishSplitsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -7617,7 +9134,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = MarkSplitsForDeletionSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -7663,7 +9180,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = DeleteSplitsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -7709,7 +9226,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = AddSourceSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -7755,7 +9272,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = UpdateSourceSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -7801,7 +9318,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = ToggleSourceSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -7847,7 +9364,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = DeleteSourceSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -7896,7 +9413,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = ResetSourceCheckpointSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -7945,7 +9462,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = LastDeleteOpstampSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -7994,7 +9511,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = CreateDeleteTaskSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -8048,7 +9565,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = UpdateSplitsDeleteOpstampSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -8097,7 +9614,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = ListDeleteTasksSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -8146,7 +9663,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = ListStaleSplitsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -8192,7 +9709,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = OpenShardsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -8238,7 +9755,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = AcquireShardsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -8284,7 +9801,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = DeleteShardsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -8330,7 +9847,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = PruneShardsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -8376,7 +9893,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = ListShardsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -8425,7 +9942,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = CreateIndexTemplateSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -8474,7 +9991,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = GetIndexTemplateSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -8527,7 +10044,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = FindIndexTemplateMatchesSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -8576,7 +10093,7 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = ListIndexTemplatesSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -8625,7 +10142,306 @@ pub mod metastore_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let method = DeleteIndexTemplatesSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/quickwit.metastore.MetastoreService/GetClusterIdentity" => { + #[allow(non_camel_case_types)] + struct GetClusterIdentitySvc(pub Arc); + impl< + T: MetastoreServiceGrpc, + > tonic::server::UnaryService + for GetClusterIdentitySvc { + type Response = super::GetClusterIdentityResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::get_cluster_identity( + &inner, + request, + ) + .await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = GetClusterIdentitySvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/quickwit.metastore.MetastoreService/StageMetricsSplits" => { + #[allow(non_camel_case_types)] + struct StageMetricsSplitsSvc(pub Arc); + impl< + T: MetastoreServiceGrpc, + > tonic::server::UnaryService + for StageMetricsSplitsSvc { + type Response = super::EmptyResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::stage_metrics_splits( + &inner, + request, + ) + .await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = StageMetricsSplitsSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/quickwit.metastore.MetastoreService/PublishMetricsSplits" => { + #[allow(non_camel_case_types)] + struct PublishMetricsSplitsSvc(pub Arc); + impl< + T: MetastoreServiceGrpc, + > tonic::server::UnaryService + for PublishMetricsSplitsSvc { + type Response = super::EmptyResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::publish_metrics_splits( + &inner, + request, + ) + .await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = PublishMetricsSplitsSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/quickwit.metastore.MetastoreService/ListMetricsSplits" => { + #[allow(non_camel_case_types)] + struct ListMetricsSplitsSvc(pub Arc); + impl< + T: MetastoreServiceGrpc, + > tonic::server::UnaryService + for ListMetricsSplitsSvc { + type Response = super::ListMetricsSplitsResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::list_metrics_splits( + &inner, + request, + ) + .await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ListMetricsSplitsSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/quickwit.metastore.MetastoreService/MarkMetricsSplitsForDeletion" => { + #[allow(non_camel_case_types)] + struct MarkMetricsSplitsForDeletionSvc( + pub Arc, + ); + impl< + T: MetastoreServiceGrpc, + > tonic::server::UnaryService< + super::MarkMetricsSplitsForDeletionRequest, + > for MarkMetricsSplitsForDeletionSvc { + type Response = super::EmptyResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request< + super::MarkMetricsSplitsForDeletionRequest, + >, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::mark_metrics_splits_for_deletion( + &inner, + request, + ) + .await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = MarkMetricsSplitsForDeletionSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/quickwit.metastore.MetastoreService/DeleteMetricsSplits" => { + #[allow(non_camel_case_types)] + struct DeleteMetricsSplitsSvc(pub Arc); + impl< + T: MetastoreServiceGrpc, + > tonic::server::UnaryService + for DeleteMetricsSplitsSvc { + type Response = super::EmptyResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::delete_metrics_splits( + &inner, + request, + ) + .await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = DeleteMetricsSplitsSvc(inner); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, diff --git a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.search.rs b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.search.rs index 6d4b8ae137a..16c11358ab8 100644 --- a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.search.rs +++ b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.search.rs @@ -1,7 +1,7 @@ // This file is @generated by prost-build. /// / Scroll Request #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ScrollRequest { /// / The `scroll_id` is the given in the response of a search request including a scroll. #[prost(string, tag = "1")] @@ -10,7 +10,7 @@ pub struct ScrollRequest { pub scroll_ttl_secs: ::core::option::Option, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PutKvRequest { #[prost(bytes = "vec", tag = "1")] pub key: ::prost::alloc::vec::Vec, @@ -20,22 +20,22 @@ pub struct PutKvRequest { pub ttl_secs: u32, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct PutKvResponse {} #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetKvRequest { #[prost(bytes = "vec", tag = "1")] pub key: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetKvResponse { #[prost(bytes = "vec", optional, tag = "1")] pub payload: ::core::option::Option<::prost::alloc::vec::Vec>, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReportSplit { /// Split id (ULID format `01HAV29D4XY3D462FS3D8K5Q2H`) #[prost(string, tag = "2")] @@ -51,10 +51,10 @@ pub struct ReportSplitsRequest { pub report_splits: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReportSplitsResponse {} #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ListFieldsRequest { /// Index ID patterns #[prost(string, repeated, tag = "1")] @@ -65,11 +65,15 @@ pub struct ListFieldsRequest { pub fields: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, /// Time filter, expressed in seconds since epoch. /// That filter is to be interpreted as the semi-open interval: - /// [start_timestamp, end_timestamp). + /// \[start_timestamp, end_timestamp). #[prost(int64, optional, tag = "3")] pub start_timestamp: ::core::option::Option, #[prost(int64, optional, tag = "4")] pub end_timestamp: ::core::option::Option, + /// JSON-serialized QueryAst for index_filter support. + /// When provided, only fields from documents matching this query are returned. + #[prost(string, optional, tag = "5")] + pub query_ast: ::core::option::Option<::prost::alloc::string::String>, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -96,7 +100,7 @@ pub struct ListFieldsResponse { pub fields: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ListFieldsEntryResponse { #[prost(string, tag = "1")] pub field_name: ::prost::alloc::string::String, @@ -131,7 +135,7 @@ pub struct ListFields { pub fields: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Eq, Hash)] +#[derive(Hash, Eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct SearchRequest { /// Index ID patterns @@ -142,7 +146,10 @@ pub struct SearchRequest { pub query_ast: ::prost::alloc::string::String, /// Time filter, expressed in seconds since epoch. /// That filter is to be interpreted as the semi-open interval: - /// [start_timestamp, end_timestamp). + /// \[start_timestamp, end_timestamp). + /// If the query AST contains a range query over the timestamp field, + /// then the the bounds of the range query are used directly and + /// these two fields are ignored. #[prost(int64, optional, tag = "4")] pub start_timestamp: ::core::option::Option, #[prost(int64, optional, tag = "5")] @@ -154,7 +161,7 @@ pub struct SearchRequest { /// can be used for pagination. /// /// E.g. - /// The results with rank [start_offset..start_offset + max_hits) are returned. + /// The results with rank \[start_offset..start_offset + max_hits) are returned. #[prost(uint64, tag = "7")] pub start_offset: u64, /// json serialized aggregation_request @@ -173,15 +180,22 @@ pub struct SearchRequest { pub scroll_ttl_secs: ::core::option::Option, /// Document with sort tuple smaller or equal to this are discarded to /// enable pagination. - /// If split_id is empty, no comparison with _shard_doc should be done + /// If split_id is empty, no comparison with \_shard_doc should be done #[prost(message, optional, tag = "16")] pub search_after: ::core::option::Option, #[prost(enumeration = "CountHits", tag = "17")] pub count_hits: i32, + /// When an exact index ID is provided (not a pattern), the query fails only if + /// that index is not found and this parameter is set to `false`. + #[prost(bool, tag = "18")] + pub ignore_missing_indexes: bool, + /// When true, skip finalization of aggregation results and return + /// the raw IntermediateAggregationResults bytes instead. + #[prost(bool, tag = "19")] + pub skip_aggregation_finalization: bool, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Eq, Hash)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SortField { #[prost(string, tag = "1")] pub field_name: ::prost::alloc::string::String, @@ -228,13 +242,13 @@ pub struct SearchResponse { pub num_successful_splits: u64, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SearchPlanResponse { #[prost(string, tag = "1")] pub result: ::prost::alloc::string::String, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SplitSearchError { /// The searcherror that occurred formatted as string. #[prost(string, tag = "1")] @@ -246,8 +260,7 @@ pub struct SplitSearchError { #[prost(bool, tag = "3")] pub retryable_error: bool, } -/// / A LeafSearchRequest can span multiple indices. -/// / +/// A LeafSearchRequest can span multiple indices. #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct LeafSearchRequest { @@ -269,7 +282,7 @@ pub struct LeafSearchRequest { pub index_uris: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct ResourceStats { #[prost(uint64, tag = "1")] pub short_lived_cache_num_bytes: u64, @@ -282,7 +295,7 @@ pub struct ResourceStats { #[prost(uint64, tag = "5")] pub cpu_microsecs: u64, } -/// / LeafRequestRef references data in LeafSearchRequest to deduplicate data. +/// LeafRequestRef references data in LeafSearchRequest to deduplicate data. #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct LeafRequestRef { @@ -298,7 +311,7 @@ pub struct LeafRequestRef { pub split_offsets: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SplitIdAndFooterOffsets { /// Index split id to apply the query on. /// This id is resolved from the index_uri defined in the search_request. @@ -326,10 +339,11 @@ pub struct SplitIdAndFooterOffsets { /// It is very different from a quickwit json doc. /// /// For instance: -/// - it may contain a _source and a _dynamic field. -/// - since tantivy has no notion of cardinality, +/// +/// * it may contain a \_source and a \_dynamic field. +/// * since tantivy has no notion of cardinality, /// all fields are arrays. -/// - since tantivy has no notion of object, the object is +/// * since tantivy has no notion of object, the object is /// flattened by concatenating the path to the root. /// /// See `quickwit_search::convert_leaf_hit` @@ -374,11 +388,12 @@ pub struct Hit { /// emits an decreasing mapping of this fast field. /// /// In case of a tie, quickwit uses the increasing order of -/// - the split_id, -/// - the segment_ord, -/// - the doc id. +/// +/// * the split_id, +/// * the segment_ord, +/// * the doc id. #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Eq, Hash)] +#[derive(Hash, Eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct PartialHit { #[prost(message, optional, tag = "10")] @@ -448,8 +463,40 @@ pub struct LeafSearchResponse { #[prost(message, optional, tag = "8")] pub resource_stats: ::core::option::Option, } +/// The result of searching a single split in a Lambda invocation. +/// Each result is tagged with its split_id so that ordering is irrelevant. #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct LambdaSingleSplitResult { + /// The split that was searched. + #[prost(string, tag = "1")] + pub split_id: ::prost::alloc::string::String, + #[prost(oneof = "lambda_single_split_result::Outcome", tags = "2, 3")] + pub outcome: ::core::option::Option, +} +/// Nested message and enum types in `LambdaSingleSplitResult`. +pub mod lambda_single_split_result { + #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] + #[serde(rename_all = "snake_case")] + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Outcome { + /// On success, the leaf search response for this split. + #[prost(message, tag = "2")] + Response(super::LeafSearchResponse), + /// On failure, the error message. + #[prost(string, tag = "3")] + Error(::prost::alloc::string::String), + } +} +/// Wrapper for per-split results from a Lambda invocation. +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct LambdaSearchResponses { + #[prost(message, repeated, tag = "2")] + pub split_results: ::prost::alloc::vec::Vec, +} +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SnippetRequest { #[prost(string, repeated, tag = "1")] pub snippet_fields: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, @@ -485,7 +532,7 @@ pub struct FetchDocsResponse { pub hits: ::prost::alloc::vec::Vec, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ListTermsRequest { /// Index ID patterns #[prost(string, repeated, tag = "1")] @@ -508,7 +555,7 @@ pub struct ListTermsRequest { pub end_key: ::core::option::Option<::prost::alloc::vec::Vec>, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ListTermsResponse { /// Number of hits matching the query. #[prost(uint64, tag = "1")] @@ -556,62 +603,6 @@ pub struct LeafListTermsResponse { pub num_attempted_splits: u64, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SearchStreamRequest { - /// Index ID - #[prost(string, tag = "1")] - pub index_id: ::prost::alloc::string::String, - /// Quickwit Query AST encoded in Json - #[prost(string, tag = "11")] - pub query_ast: ::prost::alloc::string::String, - /// The time filter is interpreted as a semi-open interval. [start, end) - #[prost(int64, optional, tag = "4")] - pub start_timestamp: ::core::option::Option, - #[prost(int64, optional, tag = "5")] - pub end_timestamp: ::core::option::Option, - /// Name of the fast field to extract - #[prost(string, tag = "6")] - pub fast_field: ::prost::alloc::string::String, - /// The output format - #[prost(enumeration = "OutputFormat", tag = "7")] - pub output_format: i32, - /// The field by which we want to partition - #[prost(string, optional, tag = "9")] - pub partition_by_field: ::core::option::Option<::prost::alloc::string::String>, - /// Fields to extract snippet on. - #[prost(string, repeated, tag = "10")] - pub snippet_fields: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, -} -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct LeafSearchStreamRequest { - /// Stream request. This is a perfect copy of the original stream request, - /// that was sent to root. - #[prost(message, optional, tag = "1")] - pub request: ::core::option::Option, - /// Index split ids to apply the query on. - /// This ids are resolved from the index_uri defined in the stream request. - #[prost(message, repeated, tag = "2")] - pub split_offsets: ::prost::alloc::vec::Vec, - /// `DocMapper` as json serialized trait. - #[prost(string, tag = "5")] - pub doc_mapper: ::prost::alloc::string::String, - /// Index URI. The index URI defines the location of the storage that contains the - /// split files. - #[prost(string, tag = "6")] - pub index_uri: ::prost::alloc::string::String, -} -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct LeafSearchStreamResponse { - /// Row of data serialized in bytes. - #[prost(bytes = "vec", tag = "1")] - pub data: ::prost::alloc::vec::Vec, - /// Split id. - #[prost(string, tag = "2")] - pub split_id: ::prost::alloc::string::String, -} -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] @@ -703,7 +694,7 @@ pub enum SortOrder { Asc = 0, /// Descending order. /// - /// < This will be the default value; + /// \< This will be the default value; Desc = 1, } impl SortOrder { @@ -757,40 +748,6 @@ impl SortDatetimeFormat { } } } -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[serde(rename_all = "snake_case")] -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum OutputFormat { - /// Comma Separated Values format (). - /// The delimiter is `,`. - /// - /// < This will be the default value - Csv = 0, - /// Format data by row in ClickHouse binary format. - /// - ClickHouseRowBinary = 1, -} -impl OutputFormat { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - Self::Csv => "CSV", - Self::ClickHouseRowBinary => "CLICK_HOUSE_ROW_BINARY", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "CSV" => Some(Self::Csv), - "CLICK_HOUSE_ROW_BINARY" => Some(Self::ClickHouseRowBinary), - _ => None, - } - } -} /// Generated client implementations. pub mod search_service_client { #![allow( @@ -899,7 +856,7 @@ pub mod search_service_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.search.SearchService/RootSearch", ); @@ -911,10 +868,11 @@ pub mod search_service_client { /// Perform a leaf search on a given set of splits. /// /// It is like a regular search except that: - /// - the node should perform the search locally instead of dispatching - /// it to other nodes. - /// - it should be applied on the given subset of splits - /// - Hit content is not fetched, and we instead return so called `PartialHit`. + /// + /// * the node should perform the search locally instead of dispatching + /// it to other nodes. + /// * it should be applied on the given subset of splits + /// * Hit content is not fetched, and we instead return so called `PartialHit`. pub async fn leaf_search( &mut self, request: impl tonic::IntoRequest, @@ -930,7 +888,7 @@ pub mod search_service_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.search.SearchService/LeafSearch", ); @@ -956,7 +914,7 @@ pub mod search_service_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.search.SearchService/FetchDocs", ); @@ -965,33 +923,6 @@ pub mod search_service_client { .insert(GrpcMethod::new("quickwit.search.SearchService", "FetchDocs")); self.inner.unary(req, path, codec).await } - /// Perform a leaf stream on a given set of splits. - pub async fn leaf_search_stream( - &mut self, - request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response>, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/quickwit.search.SearchService/LeafSearchStream", - ); - let mut req = request.into_request(); - req.extensions_mut() - .insert( - GrpcMethod::new("quickwit.search.SearchService", "LeafSearchStream"), - ); - self.inner.server_streaming(req, path, codec).await - } /// Root list terms API. /// This RPC identifies the set of splits on which the query should run on, /// and dispatches the several calls to `LeafListTerms`. @@ -1012,7 +943,7 @@ pub mod search_service_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.search.SearchService/RootListTerms", ); @@ -1026,9 +957,10 @@ pub mod search_service_client { /// Performs a leaf list terms on a given set of splits. /// /// It is like a regular list term except that: - /// - the node should perform the listing locally instead of dispatching - /// it to other nodes. - /// - it should be applied on the given subset of splits + /// + /// * the node should perform the listing locally instead of dispatching + /// it to other nodes. + /// * it should be applied on the given subset of splits pub async fn leaf_list_terms( &mut self, request: impl tonic::IntoRequest, @@ -1044,7 +976,7 @@ pub mod search_service_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.search.SearchService/LeafListTerms", ); @@ -1068,7 +1000,7 @@ pub mod search_service_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.search.SearchService/Scroll", ); @@ -1091,7 +1023,7 @@ pub mod search_service_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.search.SearchService/PutKV", ); @@ -1114,7 +1046,7 @@ pub mod search_service_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.search.SearchService/GetKV", ); @@ -1138,7 +1070,7 @@ pub mod search_service_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.search.SearchService/ReportSplits", ); @@ -1164,7 +1096,7 @@ pub mod search_service_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.search.SearchService/ListFields", ); @@ -1188,7 +1120,7 @@ pub mod search_service_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.search.SearchService/LeafListFields", ); @@ -1215,7 +1147,7 @@ pub mod search_service_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/quickwit.search.SearchService/SearchPlan", ); @@ -1251,10 +1183,11 @@ pub mod search_service_server { /// Perform a leaf search on a given set of splits. /// /// It is like a regular search except that: - /// - the node should perform the search locally instead of dispatching - /// it to other nodes. - /// - it should be applied on the given subset of splits - /// - Hit content is not fetched, and we instead return so called `PartialHit`. + /// + /// * the node should perform the search locally instead of dispatching + /// it to other nodes. + /// * it should be applied on the given subset of splits + /// * Hit content is not fetched, and we instead return so called `PartialHit`. async fn leaf_search( &self, request: tonic::Request, @@ -1271,23 +1204,6 @@ pub mod search_service_server { tonic::Response, tonic::Status, >; - /// Server streaming response type for the LeafSearchStream method. - type LeafSearchStreamStream: tonic::codegen::tokio_stream::Stream< - Item = std::result::Result< - super::LeafSearchStreamResponse, - tonic::Status, - >, - > - + std::marker::Send - + 'static; - /// Perform a leaf stream on a given set of splits. - async fn leaf_search_stream( - &self, - request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; /// Root list terms API. /// This RPC identifies the set of splits on which the query should run on, /// and dispatches the several calls to `LeafListTerms`. @@ -1303,9 +1219,10 @@ pub mod search_service_server { /// Performs a leaf list terms on a given set of splits. /// /// It is like a regular list term except that: - /// - the node should perform the listing locally instead of dispatching - /// it to other nodes. - /// - it should be applied on the given subset of splits + /// + /// * the node should perform the listing locally instead of dispatching + /// it to other nodes. + /// * it should be applied on the given subset of splits async fn leaf_list_terms( &self, request: tonic::Request, @@ -1466,7 +1383,7 @@ pub mod search_service_server { let inner = self.inner.clone(); let fut = async move { let method = RootSearchSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -1511,7 +1428,7 @@ pub mod search_service_server { let inner = self.inner.clone(); let fut = async move { let method = LeafSearchSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -1556,7 +1473,7 @@ pub mod search_service_server { let inner = self.inner.clone(); let fut = async move { let method = FetchDocsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -1571,54 +1488,6 @@ pub mod search_service_server { }; Box::pin(fut) } - "/quickwit.search.SearchService/LeafSearchStream" => { - #[allow(non_camel_case_types)] - struct LeafSearchStreamSvc(pub Arc); - impl< - T: SearchService, - > tonic::server::ServerStreamingService< - super::LeafSearchStreamRequest, - > for LeafSearchStreamSvc { - type Response = super::LeafSearchStreamResponse; - type ResponseStream = T::LeafSearchStreamStream; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { - let inner = Arc::clone(&self.0); - let fut = async move { - ::leaf_search_stream(&inner, request) - .await - }; - Box::pin(fut) - } - } - let accept_compression_encodings = self.accept_compression_encodings; - let send_compression_encodings = self.send_compression_encodings; - let max_decoding_message_size = self.max_decoding_message_size; - let max_encoding_message_size = self.max_encoding_message_size; - let inner = self.inner.clone(); - let fut = async move { - let method = LeafSearchStreamSvc(inner); - let codec = tonic::codec::ProstCodec::default(); - let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); - let res = grpc.server_streaming(method, req).await; - Ok(res) - }; - Box::pin(fut) - } "/quickwit.search.SearchService/RootListTerms" => { #[allow(non_camel_case_types)] struct RootListTermsSvc(pub Arc); @@ -1649,7 +1518,7 @@ pub mod search_service_server { let inner = self.inner.clone(); let fut = async move { let method = RootListTermsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -1694,7 +1563,7 @@ pub mod search_service_server { let inner = self.inner.clone(); let fut = async move { let method = LeafListTermsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -1739,7 +1608,7 @@ pub mod search_service_server { let inner = self.inner.clone(); let fut = async move { let method = ScrollSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -1783,7 +1652,7 @@ pub mod search_service_server { let inner = self.inner.clone(); let fut = async move { let method = PutKVSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -1827,7 +1696,7 @@ pub mod search_service_server { let inner = self.inner.clone(); let fut = async move { let method = GetKVSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -1872,7 +1741,7 @@ pub mod search_service_server { let inner = self.inner.clone(); let fut = async move { let method = ReportSplitsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -1917,7 +1786,7 @@ pub mod search_service_server { let inner = self.inner.clone(); let fut = async move { let method = ListFieldsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -1963,7 +1832,7 @@ pub mod search_service_server { let inner = self.inner.clone(); let fut = async move { let method = LeafListFieldsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -2008,7 +1877,7 @@ pub mod search_service_server { let inner = self.inner.clone(); let fut = async move { let method = SearchPlanSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, diff --git a/quickwit/quickwit-proto/src/error.rs b/quickwit/quickwit-proto/src/error.rs index b03536d0283..d9904cb7611 100644 --- a/quickwit/quickwit-proto/src/error.rs +++ b/quickwit/quickwit-proto/src/error.rs @@ -85,7 +85,8 @@ impl ServiceError for Infallible { } impl ServiceError for AskError -where E: ServiceError +where + E: ServiceError, { fn error_code(&self) -> ServiceErrorCode { match self { @@ -97,7 +98,7 @@ where E: ServiceError } /// A trait for encoding/decoding service errors to/from gRPC statuses. Errors are stored in JSON -/// in the gRPC header [`QW_ERROR_HEADER_NAME`]. This allows for propagating them transparently +/// in the gRPC header `qw-error-bin`. This allows for propagating them transparently /// between clients and servers over the network without being semantically limited to a status code /// and a message. However, it also means that modifying the serialization format of existing errors /// or introducing new ones is not backward compatible. @@ -117,7 +118,9 @@ pub trait GrpcServiceError: ServiceError + Serialize + DeserializeOwned + Send + /// Converts a service error into a gRPC status. pub fn grpc_error_to_grpc_status(service_error: E) -> tonic::Status -where E: GrpcServiceError { +where + E: GrpcServiceError, +{ let code = service_error.error_code().grpc_status_code(); let message = service_error.to_string(); let mut status = tonic::Status::new(code, message); @@ -137,7 +140,9 @@ where E: GrpcServiceError { /// Converts a gRPC status into a service error. pub fn grpc_status_to_service_error(status: tonic::Status, rpc_name: &'static str) -> E -where E: GrpcServiceError { +where + E: GrpcServiceError, +{ if let Some(header_value) = status.metadata().get_bin(QW_ERROR_HEADER_NAME) { let service_error = match decode_error(header_value) { Ok(service_error) => service_error, diff --git a/quickwit/quickwit-proto/src/getters.rs b/quickwit/quickwit-proto/src/getters.rs index a327c1717a7..9cf5bd77a26 100644 --- a/quickwit/quickwit-proto/src/getters.rs +++ b/quickwit/quickwit-proto/src/getters.rs @@ -111,6 +111,7 @@ generate_getters! { ShardIds, ShardPKey, TruncateShardsSubrequest, + SourceShardUpdate, // Metastore API AcquireShardsRequest, @@ -136,7 +137,14 @@ generate_getters! { ToggleSourceRequest, UpdateIndexRequest, UpdateSourceRequest, - UpdateSplitsDeleteOpstampRequest + UpdateSplitsDeleteOpstampRequest, + + // Metrics Split API + StageMetricsSplitsRequest, + PublishMetricsSplitsRequest, + ListMetricsSplitsRequest, + MarkMetricsSplitsForDeletionRequest, + DeleteMetricsSplitsRequest } // [`PipelineUid`] getters @@ -209,8 +217,6 @@ generate_getters! { InitShardFailure, OpenFetchStreamRequest, OpenShardSubrequest, - PersistFailure, - PersistSubrequest, PersistSuccess, ReplicateFailure, ReplicateSubrequest, diff --git a/quickwit/quickwit-proto/src/ingest/ingester.rs b/quickwit/quickwit-proto/src/ingest/ingester.rs index d2da3f8d9bd..b7308d05d05 100644 --- a/quickwit/quickwit-proto/src/ingest/ingester.rs +++ b/quickwit/quickwit-proto/src/ingest/ingester.rs @@ -72,20 +72,42 @@ impl IngesterStatus { Self::Unspecified => "unspecified", Self::Initializing => "initializing", Self::Ready => "ready", + Self::Retiring => "retiring", Self::Decommissioning => "decommissioning", Self::Decommissioned => "decommissioned", Self::Failed => "failed", } } + + pub fn from_json_str_name(value: &str) -> Option { + match value { + "unspecified" => Some(Self::Unspecified), + "initializing" => Some(Self::Initializing), + "ready" => Some(Self::Ready), + "retiring" => Some(Self::Retiring), + "decommissioning" => Some(Self::Decommissioning), + "decommissioned" => Some(Self::Decommissioned), + "failed" => Some(Self::Failed), + _ => None, + } + } + + pub fn is_ready(&self) -> bool { + matches!(self, Self::Ready) + } + + pub fn accepts_write_requests(&self) -> bool { + matches!(self, Self::Ready | Self::Retiring) + } } -impl OpenFetchStreamRequest { - pub fn queue_id(&self) -> QueueId { - queue_id(self.index_uid(), &self.source_id, self.shard_id()) +impl std::fmt::Display for IngesterStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.as_json_str_name()) } } -impl PersistSubrequest { +impl OpenFetchStreamRequest { pub fn queue_id(&self) -> QueueId { queue_id(self.index_uid(), &self.source_id, self.shard_id()) } diff --git a/quickwit/quickwit-proto/src/ingest/mod.rs b/quickwit/quickwit-proto/src/ingest/mod.rs index fda347d7931..55110775e68 100644 --- a/quickwit/quickwit-proto/src/ingest/mod.rs +++ b/quickwit/quickwit-proto/src/ingest/mod.rs @@ -189,6 +189,7 @@ impl DocBatchV2 { doc_uids, doc_buffer: Bytes::from(doc_buffer), doc_lengths, + doc_format: DocFormat::Json as i32, } } } @@ -313,11 +314,10 @@ impl From for IngestFailureReason { fn from(reason: PersistFailureReason) -> Self { match reason { PersistFailureReason::Unspecified => IngestFailureReason::Unspecified, - PersistFailureReason::ShardNotFound => IngestFailureReason::NoShardsAvailable, - PersistFailureReason::ShardClosed => IngestFailureReason::NoShardsAvailable, + PersistFailureReason::NoShardsAvailable => IngestFailureReason::NoShardsAvailable, PersistFailureReason::WalFull => IngestFailureReason::WalFull, - PersistFailureReason::ShardRateLimited => IngestFailureReason::ShardRateLimited, PersistFailureReason::Timeout => IngestFailureReason::Timeout, + PersistFailureReason::NodeUnavailable => IngestFailureReason::NoShardsAvailable, } } } @@ -326,8 +326,8 @@ impl From for PersistFailureReason { fn from(reason: ReplicateFailureReason) -> Self { match reason { ReplicateFailureReason::Unspecified => PersistFailureReason::Unspecified, - ReplicateFailureReason::ShardNotFound => PersistFailureReason::ShardNotFound, - ReplicateFailureReason::ShardClosed => PersistFailureReason::ShardClosed, + ReplicateFailureReason::ShardNotFound => PersistFailureReason::NoShardsAvailable, + ReplicateFailureReason::ShardClosed => PersistFailureReason::NoShardsAvailable, ReplicateFailureReason::WalFull => PersistFailureReason::WalFull, } } diff --git a/quickwit/quickwit-proto/src/lib.rs b/quickwit/quickwit-proto/src/lib.rs index c61fa799491..dbe850b55b7 100644 --- a/quickwit/quickwit-proto/src/lib.rs +++ b/quickwit/quickwit-proto/src/lib.rs @@ -14,6 +14,7 @@ #![allow(clippy::derive_partial_eq_without_eq)] #![allow(clippy::disallowed_methods)] +#![allow(clippy::doc_lazy_continuation)] #![allow(rustdoc::invalid_html_tags)] use std::cmp::Ordering; @@ -27,7 +28,8 @@ use tracing_opentelemetry::OpenTelemetrySpanExt; pub mod cluster; pub mod control_plane; -pub use {bytes, tonic}; +pub use bytes; +pub use tonic; pub mod developer; pub mod error; mod getters; @@ -48,6 +50,9 @@ pub mod jaeger { pub mod v1 { include!("codegen/jaeger/jaeger.storage.v1.rs"); } + pub mod v2 { + include!("codegen/jaeger/jaeger.storage.v2.rs"); + } } } @@ -61,12 +66,11 @@ pub mod opentelemetry { include!("codegen/opentelemetry/opentelemetry.proto.collector.logs.v1.rs"); } } - // One can dream. - // pub mod metrics { - // pub mod v1 { - // include!("codegen/opentelemetry/opentelemetry.proto.collector.metrics.v1.rs" - // ); } - // } + pub mod metrics { + pub mod v1 { + include!("codegen/opentelemetry/opentelemetry.proto.collector.metrics.v1.rs"); + } + } pub mod trace { pub mod v1 { include!("codegen/opentelemetry/opentelemetry.proto.collector.trace.v1.rs"); @@ -83,14 +87,11 @@ pub mod opentelemetry { include!("codegen/opentelemetry/opentelemetry.proto.logs.v1.rs"); } } - // pub mod metrics { - // pub mod experimental { - // include!("codegen/opentelemetry/opentelemetry.proto.metrics.experimental.rs"); - // } - // pub mod v1 { - // tonic::include_proto!("codegen/opentelemetry/opentelemetry.proto.metrics.v1"); - // } - // } + pub mod metrics { + pub mod v1 { + include!("codegen/opentelemetry/opentelemetry.proto.metrics.v1.rs"); + } + } pub mod resource { pub mod v1 { include!("codegen/opentelemetry/opentelemetry.proto.resource.v1.rs"); @@ -104,21 +105,6 @@ pub mod opentelemetry { } } -impl TryFrom for search::SearchRequest { - type Error = anyhow::Error; - - fn try_from(search_stream_req: search::SearchStreamRequest) -> Result { - Ok(Self { - index_id_patterns: vec![search_stream_req.index_id], - query_ast: search_stream_req.query_ast, - snippet_fields: search_stream_req.snippet_fields, - start_timestamp: search_stream_req.start_timestamp, - end_timestamp: search_stream_req.end_timestamp, - ..Default::default() - }) - } -} - impl TryFrom for search::SearchRequest { type Error = anyhow::Error; @@ -139,10 +125,10 @@ pub struct MutMetadataMap<'a>(&'a mut tonic::metadata::MetadataMap); impl Injector for MutMetadataMap<'_> { /// Sets a key-value pair in the [`MetadataMap`]. No-op if the key or value is invalid. fn set(&mut self, key: &str, value: String) { - if let Ok(metadata_key) = tonic::metadata::MetadataKey::from_bytes(key.as_bytes()) { - if let Ok(metadata_value) = tonic::metadata::MetadataValue::try_from(&value) { - self.0.insert(metadata_key, metadata_value); - } + if let Ok(metadata_key) = tonic::metadata::MetadataKey::from_bytes(key.as_bytes()) + && let Ok(metadata_value) = tonic::metadata::MetadataValue::try_from(&value) + { + self.0.insert(metadata_key, metadata_value); } } } @@ -210,7 +196,7 @@ impl Extractor for MetadataMap<'_> { pub fn set_parent_span_from_request_metadata(request_metadata: &tonic::metadata::MetadataMap) { let parent_cx = global::get_text_map_propagator(|prop| prop.extract(&MetadataMap(request_metadata))); - Span::current().set_parent(parent_cx); + let _ = Span::current().set_parent(parent_cx); } impl search::SortOrder { diff --git a/quickwit/quickwit-proto/src/metastore/mod.rs b/quickwit/quickwit-proto/src/metastore/mod.rs index a855ba5a665..ba371c13d4a 100644 --- a/quickwit/quickwit-proto/src/metastore/mod.rs +++ b/quickwit/quickwit-proto/src/metastore/mod.rs @@ -387,6 +387,13 @@ impl ListDeleteTasksRequest { } } +impl SplitStats { + pub fn add_split(&mut self, size_bytes: u64) { + self.num_splits += 1; + self.total_size_bytes += size_bytes; + } +} + pub mod serde_utils { use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; diff --git a/quickwit/quickwit-proto/src/search/mod.rs b/quickwit/quickwit-proto/src/search/mod.rs index 307de262a70..86298db0f32 100644 --- a/quickwit/quickwit-proto/src/search/mod.rs +++ b/quickwit/quickwit-proto/src/search/mod.rs @@ -12,12 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +mod span_id; +mod trace_id; + use std::cmp::Ordering; use std::fmt; use std::io::{self, Read}; use prost::Message; pub use sort_by_value::SortValue; +pub use span_id::{SpanId, TryFromSpanIdError}; +pub use trace_id::{TraceId, TryFromTraceIdError}; include!("../codegen/quickwit/quickwit.search.rs"); diff --git a/quickwit/quickwit-opentelemetry/src/otlp/span_id.rs b/quickwit/quickwit-proto/src/search/span_id.rs similarity index 98% rename from quickwit/quickwit-opentelemetry/src/otlp/span_id.rs rename to quickwit/quickwit-proto/src/search/span_id.rs index 558a76d64ff..a255a1d70f8 100644 --- a/quickwit/quickwit-opentelemetry/src/otlp/span_id.rs +++ b/quickwit/quickwit-proto/src/search/span_id.rs @@ -42,7 +42,9 @@ impl Serialize for SpanId { impl<'de> Deserialize<'de> for SpanId { fn deserialize(deserializer: D) -> Result - where D: Deserializer<'de> { + where + D: Deserializer<'de>, + { let hexspan_id = String::deserialize(deserializer)?; if hexspan_id.len() != SpanId::HEX_LENGTH { diff --git a/quickwit/quickwit-opentelemetry/src/otlp/trace_id.rs b/quickwit/quickwit-proto/src/search/trace_id.rs similarity index 94% rename from quickwit/quickwit-opentelemetry/src/otlp/trace_id.rs rename to quickwit/quickwit-proto/src/search/trace_id.rs index b5399a95ec6..1f499a47294 100644 --- a/quickwit/quickwit-opentelemetry/src/otlp/trace_id.rs +++ b/quickwit/quickwit-proto/src/search/trace_id.rs @@ -50,7 +50,9 @@ impl Serialize for TraceId { impl<'de> Deserialize<'de> for TraceId { fn deserialize(deserializer: D) -> Result - where D: Deserializer<'de> { + where + D: Deserializer<'de>, + { if deserializer.is_human_readable() { let hextrace_id = String::deserialize(deserializer)?; if hextrace_id.len() != TraceId::HEX_LENGTH { @@ -97,6 +99,12 @@ impl TryFrom> for TraceId { } } +impl From for tonic::Status { + fn from(error: TryFromTraceIdError) -> Self { + tonic::Status::invalid_argument(error.to_string()) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/quickwit/quickwit-proto/src/types/doc_mapping_uid.rs b/quickwit/quickwit-proto/src/types/doc_mapping_uid.rs index a18db4235ca..e74da587db2 100644 --- a/quickwit/quickwit-proto/src/types/doc_mapping_uid.rs +++ b/quickwit/quickwit-proto/src/types/doc_mapping_uid.rs @@ -59,7 +59,9 @@ impl DocMappingUid { impl<'de> Deserialize<'de> for DocMappingUid { fn deserialize(deserializer: D) -> Result - where D: Deserializer<'de> { + where + D: Deserializer<'de>, + { let doc_mapping_uid_str: Cow<'de, str> = Cow::deserialize(deserializer)?; doc_mapping_uid_str.parse().map_err(D::Error::custom) } @@ -67,7 +69,9 @@ impl<'de> Deserialize<'de> for DocMappingUid { impl Serialize for DocMappingUid { fn serialize(&self, serializer: S) -> Result - where S: Serializer { + where + S: Serializer, + { serializer.collect_str(&self.0) } } @@ -151,7 +155,10 @@ impl sqlx::Type for DocMappingUid { #[cfg(feature = "postgres")] impl sqlx::Encode<'_, sqlx::Postgres> for DocMappingUid { - fn encode_by_ref(&self, buf: &mut sqlx::postgres::PgArgumentBuffer) -> sqlx::encode::IsNull { + fn encode_by_ref( + &self, + buf: &mut sqlx::postgres::PgArgumentBuffer, + ) -> Result { sqlx::Encode::::encode(self.0.to_string(), buf) } } diff --git a/quickwit/quickwit-proto/src/types/doc_uid.rs b/quickwit/quickwit-proto/src/types/doc_uid.rs index b0bd1b1e788..3cbcb653e53 100644 --- a/quickwit/quickwit-proto/src/types/doc_uid.rs +++ b/quickwit/quickwit-proto/src/types/doc_uid.rs @@ -57,7 +57,9 @@ impl DocUid { impl<'de> Deserialize<'de> for DocUid { fn deserialize(deserializer: D) -> Result - where D: Deserializer<'de> { + where + D: Deserializer<'de>, + { let doc_uid_str: Cow<'de, str> = Cow::deserialize(deserializer)?; let doc_uid = Ulid::from_string(&doc_uid_str).map_err(D::Error::custom)?; Ok(Self(doc_uid)) @@ -66,7 +68,9 @@ impl<'de> Deserialize<'de> for DocUid { impl Serialize for DocUid { fn serialize(&self, serializer: S) -> Result - where S: Serializer { + where + S: Serializer, + { serializer.collect_str(&self.0) } } diff --git a/quickwit/quickwit-proto/src/types/index_uid.rs b/quickwit/quickwit-proto/src/types/index_uid.rs index ab3950772d2..062fec57210 100644 --- a/quickwit/quickwit-proto/src/types/index_uid.rs +++ b/quickwit/quickwit-proto/src/types/index_uid.rs @@ -86,7 +86,9 @@ impl FromStr for IndexUid { impl<'de> Deserialize<'de> for IndexUid { fn deserialize(deserializer: D) -> Result - where D: Deserializer<'de> { + where + D: Deserializer<'de>, + { let index_uid_str: Cow<'de, str> = Cow::deserialize(deserializer)?; let index_uid = IndexUid::from_str(&index_uid_str).map_err(D::Error::custom)?; Ok(index_uid) @@ -95,7 +97,9 @@ impl<'de> Deserialize<'de> for IndexUid { impl Serialize for IndexUid { fn serialize(&self, serializer: S) -> Result - where S: Serializer { + where + S: Serializer, + { serializer.collect_str(&self) } } @@ -187,9 +191,12 @@ impl sqlx::Type for IndexUid { #[cfg(feature = "postgres")] impl sqlx::Encode<'_, sqlx::Postgres> for IndexUid { - fn encode_by_ref(&self, buf: &mut sqlx::postgres::PgArgumentBuffer) -> sqlx::encode::IsNull { - let _ = sqlx::Encode::::encode(&self.index_id, buf); - let _ = sqlx::Encode::::encode(":", buf); + fn encode_by_ref( + &self, + buf: &mut sqlx::postgres::PgArgumentBuffer, + ) -> Result { + let _ = sqlx::Encode::::encode(&self.index_id, buf)?; + let _ = sqlx::Encode::::encode(":", buf)?; sqlx::Encode::::encode(self.incarnation_id.to_string(), buf) } } diff --git a/quickwit/quickwit-proto/src/types/mod.rs b/quickwit/quickwit-proto/src/types/mod.rs index f103e5c60f2..d7e5d1ebf5f 100644 --- a/quickwit/quickwit-proto/src/types/mod.rs +++ b/quickwit/quickwit-proto/src/types/mod.rs @@ -20,7 +20,7 @@ use std::ops::Deref; use std::str::FromStr; use serde::{Deserialize, Serialize}; -use tracing::warn; +use tracing::error; pub use ulid::Ulid; mod doc_mapping_uid; @@ -62,7 +62,7 @@ pub fn split_queue_id(queue_id: &str) -> Option<(IndexUid, SourceId, ShardId)> { let parts_opt = split_queue_id_inner(queue_id); if parts_opt.is_none() { - warn!("failed to parse queue ID `{queue_id}`: this should never happen, please report"); + error!("failed to parse queue ID `{queue_id}`: this should never happen, please report"); } parts_opt } diff --git a/quickwit/quickwit-proto/src/types/position.rs b/quickwit/quickwit-proto/src/types/position.rs index ba2cafddcfc..983aeae2b65 100644 --- a/quickwit/quickwit-proto/src/types/position.rs +++ b/quickwit/quickwit-proto/src/types/position.rs @@ -18,6 +18,7 @@ use std::{fmt, mem}; use bytes::{Bytes, BytesMut}; use bytestring::ByteString; use prost::{self, DecodeError}; +use quickwit_common::pretty::PrettyDisplay; use serde::{Deserialize, Serialize}; const BEGINNING: &str = ""; @@ -109,6 +110,37 @@ impl Debug for Position { } } +// Caution: This is also the serialization format for chitchat and serde. Modify with care. +impl Display for Position { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::Beginning => write!(f, "{BEGINNING}"), + Self::Offset(offset) => write!(f, "{offset}"), + Self::Eof(Some(offset)) => write!(f, "{EOF_PREFIX}{offset}"), + Self::Eof(None) => write!(f, "{EOF_PREFIX}"), + } + } +} + +struct PositionPrettyDisplay<'a>(&'a Position); + +impl fmt::Display for PositionPrettyDisplay<'_> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self.0 { + Position::Beginning => write!(f, "beginning"), + Position::Offset(offset) => write!(f, "{offset}"), + Position::Eof(Some(offset)) => write!(f, "eof({offset})"), + Position::Eof(None) => write!(f, "eof"), + } + } +} + +impl PrettyDisplay for Position { + fn pretty_display(&self) -> impl fmt::Display { + PositionPrettyDisplay(self) + } +} + impl Position { pub fn offset(offset: impl Into) -> Self { Self::Offset(offset.into()) @@ -178,17 +210,6 @@ impl Position { } } -impl Display for Position { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - Self::Beginning => write!(f, "{BEGINNING}"), - Self::Offset(offset) => write!(f, "{offset}"), - Self::Eof(Some(offset)) => write!(f, "{EOF_PREFIX}{offset}"), - Self::Eof(None) => write!(f, "{EOF_PREFIX}"), - } - } -} - impl From for Position { fn from(position: ByteString) -> Self { match &position[..] { diff --git a/quickwit/quickwit-proto/src/types/shard_id.rs b/quickwit/quickwit-proto/src/types/shard_id.rs index de5c292e897..7a73feb6baa 100644 --- a/quickwit/quickwit-proto/src/types/shard_id.rs +++ b/quickwit/quickwit-proto/src/types/shard_id.rs @@ -139,7 +139,10 @@ impl sqlx::Type for ShardId { #[cfg(feature = "postgres")] impl sqlx::Encode<'_, sqlx::Postgres> for ShardId { - fn encode_by_ref(&self, buf: &mut sqlx::postgres::PgArgumentBuffer) -> sqlx::encode::IsNull { + fn encode_by_ref( + &self, + buf: &mut sqlx::postgres::PgArgumentBuffer, + ) -> Result { sqlx::Encode::::encode(self.as_str(), buf) } } diff --git a/quickwit/quickwit-query/Cargo.toml b/quickwit/quickwit-query/Cargo.toml index b6f5f173a26..f24d8662715 100644 --- a/quickwit/quickwit-query/Cargo.toml +++ b/quickwit/quickwit-query/Cargo.toml @@ -13,10 +13,8 @@ license.workspace = true [dependencies] anyhow = { workspace = true } base64 = { workspace = true } +bitpacking = { workspace = true } hex = { workspace = true } -lindera-core = { workspace = true, optional = true } -lindera-dictionary = { workspace = true, optional = true } -lindera-tokenizer = { workspace = true, optional = true } once_cell = { workspace = true } regex = { workspace = true } serde = { workspace = true } @@ -24,13 +22,14 @@ serde_json = { workspace = true } serde_with = { workspace = true } tantivy = { workspace = true } tantivy-fst = { workspace = true } +tracing = { workspace = true } time = { workspace = true } thiserror = { workspace = true } rustc-hash = { workspace = true } -whichlang = { workspace = true, optional = true } quickwit-common = { workspace = true } quickwit-datetime = { workspace = true } +quickwit-proto = { workspace = true } [dev-dependencies] criterion = { workspace = true } @@ -39,18 +38,6 @@ time = { workspace = true } quickwit-common = { workspace = true, features = ["testsuite"] } -[features] -multilang = [ - "lindera-core", - "lindera-dictionary", - "lindera-tokenizer", - "whichlang", -] - [[bench]] name = "tokenizers_bench" harness = false - -[[bench]] -name = "multilang_tokenizers_bench" -harness = false diff --git a/quickwit/quickwit-query/benches/multilang_tokenizers_bench.rs b/quickwit/quickwit-query/benches/multilang_tokenizers_bench.rs deleted file mode 100644 index 61755dea556..00000000000 --- a/quickwit/quickwit-query/benches/multilang_tokenizers_bench.rs +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use criterion::{Criterion, Throughput, black_box, criterion_group, criterion_main}; -use quickwit_query::create_default_quickwit_tokenizer_manager; -use tantivy::tokenizer::{TextAnalyzer, Token, TokenStream}; - -// A random ascii string of length 100 chars. -const ASCII_SHORT: &str = "It is a long established fact"; -static ASCII_LONG: &str = r#"It is a long established fact that a reader will be distracted by the readable content of a - page when looking at its layout. The point of using Lorem Ipsum is that it has a - more-or-less normal distribution of letters, as opposed to using 'Content here, content - here', making it look like readable English. Many desktop publishing packages and web page - editors now use Lorem Ipsum as their default model text, and a search for 'lorem ipsum' will - uncover many web sites still in their infancy. Various versions have evolved over the years, - sometimes by accident, sometimes on purpose (injected humour and the like)."#; -const JPN_SHORT: &str = "日本ごです。 とても素敵な言葉ですね"; -const JPN_LONG: &str = r#"日本ごです。 和名の由来は、 - 太陽の動きにつれてその方向を追うように花が回るといわれたことから。 - ただしこの動きは生長に伴うものであるため、 - 実際に太陽を追って動くのは生長が盛んな若い時期だけである。 - 若いヒマワリの茎の上部の葉は太陽に正対になるように動き、 - 朝には東を向いていたのが夕方には西を向く。日没後はまもなく起きあがり、 - 夜明け前にはふたたび東に向く。この運動はつぼみを付ける頃まで続くが、 - つぼみが大きくなり花が開く素敵な言葉ですね."#; -const CMN_SHORT: &str = "滚滚长江东逝水,浪花淘尽英雄。"; -const CMN_LONG: &str = r#"滚滚长江东逝水,浪花淘尽英雄。是非成败转头空,青山依旧在,几度夕阳红。 - 白发渔樵江渚上,惯看秋月春风。一壶浊酒喜相逢,古今多少事,都付笑谈中。 - 是非成败转头空,青山依旧在,惯看秋月春风。一壶浊酒喜相逢,古今多少事, - 滚滚长江东逝水,浪花淘尽英雄。 几度夕阳红。白发渔樵江渚上,都付笑谈中。"#; -const KOR_SHORT: &str = "안녕하세요. 반갑습니다."; -const KOR_LONG: &str = r#" -포근히 내려오는 눈밭속에서는 -낯이 붉은 處女아이들도 깃들이어 오는 소리… -울고 -웃고 -수구리고 -새파라니 얼어서 -運命들이 모두다 안끼어 드는 소리… -큰놈에겐 큰 눈물자국, 작은놈에겐 작은 웃음 흔적 -큰이얘기 작은이얘기들이 오부록이 도란 그리며 안끼어 오는 소리 -끊임없이 내리는 눈발 속에서는 -山도 山도 靑山도 안끼어 드는 소리 -"#; - -fn process_tokens(analyzer: &mut TextAnalyzer, text: &str) -> Vec { - let mut token_stream = analyzer.token_stream(text); - let mut tokens: Vec = Vec::new(); - token_stream.process(&mut |token: &Token| tokens.push(token.clone())); - tokens -} - -pub fn tokenizers_throughput_benchmark(c: &mut Criterion) { - let mut group = c.benchmark_group("multilang"); - let tokenizer_manager = create_default_quickwit_tokenizer_manager(); - let mut default_tokenizer = tokenizer_manager.get_tokenizer("default").unwrap(); - let mut multilang_tokenizer = tokenizer_manager.get_tokenizer("multilang").unwrap(); - let mut chinese_tokenizer = tokenizer_manager - .get_tokenizer("chinese_compatible") - .unwrap(); - - group - .throughput(Throughput::Bytes(ASCII_SHORT.len() as u64)) - .bench_with_input("default-tokenize-short", ASCII_SHORT, |b, text| { - b.iter(|| process_tokens(&mut default_tokenizer, black_box(text))); - }); - group - .throughput(Throughput::Bytes(ASCII_LONG.len() as u64)) - .bench_with_input("default-tokenize-long", ASCII_LONG, |b, text| { - b.iter(|| process_tokens(&mut default_tokenizer, black_box(text))); - }); - group - .throughput(Throughput::Bytes(ASCII_SHORT.len() as u64)) - .bench_with_input("multilang-eng-tokenize-short", ASCII_SHORT, |b, text| { - b.iter(|| process_tokens(&mut multilang_tokenizer, black_box(text))); - }); - group - .throughput(Throughput::Bytes(ASCII_LONG.len() as u64)) - .bench_with_input("multilang-eng-tokenize-long", ASCII_LONG, |b, text| { - b.iter(|| process_tokens(&mut multilang_tokenizer, black_box(text))); - }); - let short_with_prefix = "ENG:".to_string() + ASCII_SHORT; - group - .throughput(Throughput::Bytes(ASCII_SHORT.len() as u64)) - .bench_with_input( - "multilang-tokenize-short-with-prefix", - &short_with_prefix, - |b, text| { - b.iter(|| process_tokens(&mut multilang_tokenizer, black_box(text))); - }, - ); - let long_with_prefix = "ENG:".to_string() + ASCII_LONG; - group - .throughput(Throughput::Bytes(ASCII_LONG.len() as u64)) - .bench_with_input( - "multilang-tokenize-long-with-prefix", - &long_with_prefix, - |b, text| { - b.iter(|| process_tokens(&mut multilang_tokenizer, black_box(text))); - }, - ); - group - .throughput(Throughput::Bytes(JPN_SHORT.len() as u64)) - .bench_with_input("multilang-tokenize-jpn-short", JPN_SHORT, |b, text| { - b.iter(|| process_tokens(&mut multilang_tokenizer, black_box(text))); - }); - group - .throughput(Throughput::Bytes(JPN_LONG.len() as u64)) - .bench_with_input("multilang-tokenize-jpn-long", JPN_LONG, |b, text| { - b.iter(|| process_tokens(&mut multilang_tokenizer, black_box(text))); - }); - group - .throughput(Throughput::Bytes(CMN_SHORT.len() as u64)) - .bench_with_input("multilang-tokenize-cmn-short", CMN_SHORT, |b, text| { - b.iter(|| process_tokens(&mut multilang_tokenizer, black_box(text))); - }); - group - .throughput(Throughput::Bytes(CMN_LONG.len() as u64)) - .bench_with_input("multilang-tokenize-cmn-long", CMN_LONG, |b, text| { - b.iter(|| process_tokens(&mut multilang_tokenizer, black_box(text))); - }); - group - .throughput(Throughput::Bytes(KOR_SHORT.len() as u64)) - .bench_with_input("multilang-tokenize-kor-short", KOR_SHORT, |b, text| { - b.iter(|| process_tokens(&mut multilang_tokenizer, black_box(text))); - }); - group - .throughput(Throughput::Bytes(KOR_LONG.len() as u64)) - .bench_with_input("multilang-tokenize-kor-long", KOR_LONG, |b, text| { - b.iter(|| process_tokens(&mut multilang_tokenizer, black_box(text))); - }); - group - .throughput(Throughput::Bytes(CMN_SHORT.len() as u64)) - .bench_with_input( - "chinese-compatible-tokenize-cmn-short", - CMN_SHORT, - |b, text| { - b.iter(|| process_tokens(&mut chinese_tokenizer, black_box(text))); - }, - ); - group - .throughput(Throughput::Bytes(CMN_LONG.len() as u64)) - .bench_with_input( - "chinese-compatible-tokenize-cmn-long", - CMN_LONG, - |b, text| { - b.iter(|| process_tokens(&mut chinese_tokenizer, black_box(text))); - }, - ); -} - -criterion_group!( - tokenizers_throughput_benches, - tokenizers_throughput_benchmark -); -criterion_main!(tokenizers_throughput_benches); diff --git a/quickwit/quickwit-query/src/aggregations.rs b/quickwit/quickwit-query/src/aggregations.rs index eca275ad265..5a70e5d4105 100644 --- a/quickwit/quickwit-query/src/aggregations.rs +++ b/quickwit/quickwit-query/src/aggregations.rs @@ -18,9 +18,11 @@ use tantivy::aggregation::Key as TantivyKey; use tantivy::aggregation::agg_result::{ AggregationResult as TantivyAggregationResult, AggregationResults as TantivyAggregationResults, BucketEntries as TantivyBucketEntries, BucketEntry as TantivyBucketEntry, - BucketResult as TantivyBucketResult, MetricResult as TantivyMetricResult, + BucketResult as TantivyBucketResult, CompositeBucketEntry as TantivyCompositeBucketEntry, + CompositeKey as TantivyCompositeKey, MetricResult as TantivyMetricResult, RangeBucketEntry as TantivyRangeBucketEntry, }; +use tantivy::aggregation::bucket::AfterKey as TantivyAfterKey; use tantivy::aggregation::metric::{ ExtendedStats, PercentileValues as TantivyPercentileValues, PercentileValuesVecEntry, PercentilesMetricResult as TantivyPercentilesMetricResult, SingleMetricResult, Stats, @@ -169,6 +171,15 @@ pub enum BucketResult { /// The upper bound error for the doc count of each term. doc_count_error_upper_bound: Option, }, + /// This is the composite aggregation result + Composite { + /// The buckets + buckets: Vec, + /// The key to start after when paginating. + /// Uses tantivy's AfterKey directly since it carries type-tagged + /// serialization needed for correct pagination round-tripping. + after_key: FxHashMap, + }, } impl From for BucketResult { @@ -189,6 +200,13 @@ impl From for BucketResult { sum_other_doc_count, doc_count_error_upper_bound, }, + TantivyBucketResult::Filter(_filter_bucket_result) => { + unimplemented!("filter aggregation is not yet supported in quickwit") + } + TantivyBucketResult::Composite { buckets, after_key } => BucketResult::Composite { + buckets: buckets.into_iter().map(Into::into).collect(), + after_key, + }, } } } @@ -211,6 +229,10 @@ impl From for TantivyBucketResult { sum_other_doc_count, doc_count_error_upper_bound, }, + BucketResult::Composite { buckets, after_key } => TantivyBucketResult::Composite { + buckets: buckets.into_iter().map(Into::into).collect(), + after_key, + }, } } } @@ -226,7 +248,8 @@ pub enum BucketEntries { } impl From> for BucketEntries -where U: From +where + U: From, { fn from(value: TantivyBucketEntries) -> BucketEntries { match value { @@ -241,7 +264,8 @@ where U: From } impl From> for TantivyBucketEntries -where U: From +where + U: From, { fn from(value: BucketEntries) -> TantivyBucketEntries { match value { @@ -410,3 +434,75 @@ impl From for TantivyPercentilesMetricResult { TantivyPercentilesMetricResult { values } } } + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum CompositeKey { + /// Boolean key + Bool(bool), + /// String key + Str(String), + /// `i64` key + I64(i64), + /// `u64` key + U64(u64), + /// `f64` key + F64(f64), + /// Null key + Null, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct CompositeBucketEntry { + /// The identifier of the bucket. + pub key: FxHashMap, + /// Number of documents in the bucket. + pub doc_count: u64, + /// Sub-aggregations in this bucket. + pub sub_aggregation: AggregationResults, +} + +impl From for CompositeKey { + fn from(value: TantivyCompositeKey) -> CompositeKey { + match value { + TantivyCompositeKey::Bool(b) => CompositeKey::Bool(b), + TantivyCompositeKey::Str(s) => CompositeKey::Str(s), + TantivyCompositeKey::I64(i) => CompositeKey::I64(i), + TantivyCompositeKey::U64(u) => CompositeKey::U64(u), + TantivyCompositeKey::F64(f) => CompositeKey::F64(f), + TantivyCompositeKey::Null => CompositeKey::Null, + } + } +} + +impl From for TantivyCompositeKey { + fn from(value: CompositeKey) -> TantivyCompositeKey { + match value { + CompositeKey::Bool(b) => TantivyCompositeKey::Bool(b), + CompositeKey::Str(s) => TantivyCompositeKey::Str(s), + CompositeKey::I64(i) => TantivyCompositeKey::I64(i), + CompositeKey::U64(u) => TantivyCompositeKey::U64(u), + CompositeKey::F64(f) => TantivyCompositeKey::F64(f), + CompositeKey::Null => TantivyCompositeKey::Null, + } + } +} + +impl From for CompositeBucketEntry { + fn from(value: TantivyCompositeBucketEntry) -> CompositeBucketEntry { + CompositeBucketEntry { + key: value.key.into_iter().map(|(k, v)| (k, v.into())).collect(), + doc_count: value.doc_count, + sub_aggregation: value.sub_aggregation.into(), + } + } +} + +impl From for TantivyCompositeBucketEntry { + fn from(value: CompositeBucketEntry) -> TantivyCompositeBucketEntry { + TantivyCompositeBucketEntry { + key: value.key.into_iter().map(|(k, v)| (k, v.into())).collect(), + doc_count: value.doc_count, + sub_aggregation: value.sub_aggregation.into(), + } + } +} diff --git a/quickwit/quickwit-query/src/elastic_query_dsl/bool_query.rs b/quickwit/quickwit-query/src/elastic_query_dsl/bool_query.rs index da87d498139..a744f0b3801 100644 --- a/quickwit/quickwit-query/src/elastic_query_dsl/bool_query.rs +++ b/quickwit/quickwit-query/src/elastic_query_dsl/bool_query.rs @@ -21,10 +21,9 @@ use crate::not_nan_f32::NotNaNf32; use crate::query_ast::{self, QueryAst}; /// # Unsupported features -/// - minimum_should_match /// - named queries #[serde_as] -#[derive(Deserialize, Debug, PartialEq, Eq, Clone)] +#[derive(Deserialize, Debug, PartialEq, Clone)] #[serde(deny_unknown_fields)] pub struct BoolQuery { #[serde_as(deserialize_as = "DefaultOnNull>")] @@ -43,8 +42,14 @@ pub struct BoolQuery { pub boost: Option, #[serde(default)] pub minimum_should_match: Option, + #[serde(alias = "adjust_pure_negative", default, skip_serializing)] + _adjust_pure_negative: Option, } +// `IgnoredAny` implements `PartialEq` but not `Eq`, so we derive `PartialEq` +// and manually assert `Eq` (safe because `IgnoredAny` is a unit struct). +impl Eq for BoolQuery {} + #[derive(Deserialize, Debug, Eq, PartialEq, Clone)] #[serde(untagged)] pub enum MinimumShouldMatch { @@ -126,6 +131,7 @@ impl BoolQuery { filter: Vec::new(), boost: None, minimum_should_match: None, + _adjust_pure_negative: None, } } } @@ -199,7 +205,8 @@ mod tests { should: Vec::new(), filter: Vec::new(), boost: None, - minimum_should_match: None + minimum_should_match: None, + _adjust_pure_negative: None, } ); } @@ -220,6 +227,7 @@ mod tests { filter: vec![term_query_from_field_value("product_id", "2").into(),], boost: None, minimum_should_match: None, + _adjust_pure_negative: None, } ); } @@ -243,10 +251,25 @@ mod tests { filter: Vec::new(), boost: None, minimum_should_match: None, + _adjust_pure_negative: None, } ); } + #[test] + fn test_dsl_bool_query_deserialize_adjust_pure_negative() { + let bool_query_json = r#"{ + "must": [ + { "term": {"product_id": {"value": "1" }} } + ], + "adjust_pure_negative": true + }"#; + let bool_query: BoolQuery = serde_json::from_str(bool_query_json).unwrap(); + assert!(bool_query._adjust_pure_negative.is_some()); + assert_eq!(bool_query.must.len(), 1); + bool_query.convert_to_query_ast().unwrap(); + } + #[test] fn test_dsl_bool_query_deserialize_minimum_should_match() { let bool_query: super::BoolQuery = serde_json::from_str( diff --git a/quickwit/quickwit-query/src/elastic_query_dsl/mod.rs b/quickwit/quickwit-query/src/elastic_query_dsl/mod.rs index 96fac10c32c..871032951e2 100644 --- a/quickwit/quickwit-query/src/elastic_query_dsl/mod.rs +++ b/quickwit/quickwit-query/src/elastic_query_dsl/mod.rs @@ -22,16 +22,19 @@ mod match_query; mod multi_match; mod one_field_map; mod phrase_prefix_query; +mod prefix_query; mod query_string_query; mod range_query; mod regex_query; mod string_or_struct; mod term_query; mod terms_query; +mod wildcard_query; use bool_query::BoolQuery; pub use one_field_map::OneFieldMap; use phrase_prefix_query::MatchPhrasePrefixQuery; +use prefix_query::PrefixQuery; pub(crate) use query_string_query::QueryStringQuery; use range_query::RangeQuery; pub(crate) use string_or_struct::StringOrStructForSerialization; @@ -44,6 +47,7 @@ use crate::elastic_query_dsl::match_query::MatchQuery; use crate::elastic_query_dsl::multi_match::MultiMatchQuery; use crate::elastic_query_dsl::regex_query::RegexQuery; use crate::elastic_query_dsl::terms_query::TermsQuery; +use crate::elastic_query_dsl::wildcard_query::WildcardQuery; use crate::not_nan_f32::NotNaNf32; use crate::query_ast::QueryAst; @@ -85,6 +89,8 @@ pub(crate) enum ElasticQueryDslInner { Range(RangeQuery), Exists(ExistsQuery), Regexp(RegexQuery), + Wildcard(WildcardQuery), + Prefix(PrefixQuery), } #[derive(Deserialize, Debug, Eq, PartialEq, Clone)] @@ -133,6 +139,8 @@ impl ConvertibleToQueryAst for ElasticQueryDslInner { Self::Exists(exists_query) => exists_query.convert_to_query_ast(), Self::MultiMatch(multi_match_query) => multi_match_query.convert_to_query_ast(), Self::Regexp(regex_query) => regex_query.convert_to_query_ast(), + Self::Wildcard(wildcard_query) => wildcard_query.convert_to_query_ast(), + Self::Prefix(prefix_query) => prefix_query.convert_to_query_ast(), } } } diff --git a/quickwit/quickwit-query/src/elastic_query_dsl/one_field_map.rs b/quickwit/quickwit-query/src/elastic_query_dsl/one_field_map.rs index e18158e6e70..de67190019f 100644 --- a/quickwit/quickwit-query/src/elastic_query_dsl/one_field_map.rs +++ b/quickwit/quickwit-query/src/elastic_query_dsl/one_field_map.rs @@ -47,13 +47,15 @@ impl<'de, V: Deserialize<'de>> Visitor<'de> for OneFieldMapVisitor { } fn visit_map(self, mut map: A) -> Result - where A: serde::de::MapAccess<'de> { - if let Some(num_keys) = map.size_hint() { - if num_keys != 1 { - return Err(serde::de::Error::custom(format!( - "expected a single field. got {num_keys}" - ))); - } + where + A: serde::de::MapAccess<'de>, + { + if let Some(num_keys) = map.size_hint() + && num_keys != 1 + { + return Err(serde::de::Error::custom(format!( + "expected a single field. got {num_keys}" + ))); } let Some((key, val)) = map.next_entry()? else { return Err(serde::de::Error::custom( @@ -74,7 +76,9 @@ impl<'de, V: Deserialize<'de>> Visitor<'de> for OneFieldMapVisitor { impl<'de, V: Deserialize<'de>> Deserialize<'de> for OneFieldMap { fn deserialize(deserializer: D) -> Result - where D: serde::Deserializer<'de> { + where + D: serde::Deserializer<'de>, + { deserializer.deserialize_map(OneFieldMapVisitor { _data: Default::default(), }) diff --git a/quickwit/quickwit-query/src/elastic_query_dsl/prefix_query.rs b/quickwit/quickwit-query/src/elastic_query_dsl/prefix_query.rs new file mode 100644 index 00000000000..f19fad61037 --- /dev/null +++ b/quickwit/quickwit-query/src/elastic_query_dsl/prefix_query.rs @@ -0,0 +1,118 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use serde::Deserialize; + +use crate::elastic_query_dsl::one_field_map::OneFieldMap; +use crate::elastic_query_dsl::{ConvertibleToQueryAst, StringOrStructForSerialization}; +use crate::query_ast::{QueryAst, WildcardQuery as AstWildcardQuery}; + +#[derive(Deserialize, Clone, Eq, PartialEq, Debug)] +#[serde(from = "OneFieldMap>")] +pub(crate) struct PrefixQuery { + pub(crate) field: String, + pub(crate) params: PrefixQueryParams, +} + +#[derive(Deserialize, Debug, Default, Eq, PartialEq, Clone)] +#[serde(deny_unknown_fields)] +pub struct PrefixQueryParams { + value: String, + #[serde(default)] + case_insensitive: bool, +} + +impl ConvertibleToQueryAst for PrefixQuery { + fn convert_to_query_ast(self) -> anyhow::Result { + let wildcard = format!( + "{}*", + self.params + .value + .replace(r"\", r"\\") + .replace("*", r"\*") + .replace("?", r"\?") + ); + Ok(AstWildcardQuery { + field: self.field, + value: wildcard, + lenient: true, + case_insensitive: self.params.case_insensitive, + } + .into()) + } +} + +impl From>> for PrefixQuery { + fn from( + match_query_params: OneFieldMap>, + ) -> Self { + let OneFieldMap { field, value } = match_query_params; + PrefixQuery { + field, + params: value.inner, + } + } +} + +impl From for PrefixQueryParams { + fn from(value: String) -> PrefixQueryParams { + PrefixQueryParams { + value, + case_insensitive: false, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_prefix_query_convert_to_query_ast() { + let prefix_query_json = r#"{ + "user_name": { + "value": "john" + } + }"#; + let prefix_query: PrefixQuery = serde_json::from_str(prefix_query_json).unwrap(); + let query_ast = prefix_query.convert_to_query_ast().unwrap(); + + if let QueryAst::Wildcard(prefix) = query_ast { + assert_eq!(prefix.field, "user_name"); + assert_eq!(prefix.value, "john*"); + assert!(prefix.lenient); + } else { + panic!("Expected QueryAst::Prefix, got {:?}", query_ast); + } + } + + #[test] + fn test_prefix_query_convert_to_query_ast_special_chars() { + let prefix_query_json = r#"{ + "user_name": { + "value": "a\\dm?n*" + } + }"#; + let prefix_query: PrefixQuery = serde_json::from_str(prefix_query_json).unwrap(); + let query_ast = prefix_query.convert_to_query_ast().unwrap(); + + if let QueryAst::Wildcard(prefix) = query_ast { + assert_eq!(prefix.field, "user_name"); + assert_eq!(prefix.value, r"a\\dm\?n\**"); + assert!(prefix.lenient); + } else { + panic!("Expected QueryAst::Prefix, got {:?}", query_ast); + } + } +} diff --git a/quickwit/quickwit-query/src/elastic_query_dsl/range_query.rs b/quickwit/quickwit-query/src/elastic_query_dsl/range_query.rs index c28fde94ee0..4a062d08fcf 100644 --- a/quickwit/quickwit-query/src/elastic_query_dsl/range_query.rs +++ b/quickwit/quickwit-query/src/elastic_query_dsl/range_query.rs @@ -39,6 +39,14 @@ pub struct RangeQueryParams { boost: Option, #[serde(default)] format: Option, + #[serde(default)] + from: Option, + #[serde(default)] + to: Option, + #[serde(default)] + include_lower: Option, + #[serde(default)] + include_upper: Option, } pub type RangeQuery = OneFieldMap; @@ -53,7 +61,34 @@ impl ConvertibleToQueryAst for RangeQuery { lte, boost, format, + from, + to, + include_lower, + include_upper, } = self.value; + + let (mut gt, mut gte, mut lt, mut lte) = (gt, gte, lt, lte); + if let Some(from_val) = from + && gt.is_none() + && gte.is_none() + { + if include_lower.unwrap_or(true) { + gte = Some(from_val); + } else { + gt = Some(from_val); + } + } + if let Some(to_val) = to + && lt.is_none() + && lte.is_none() + { + if include_upper.unwrap_or(true) { + lte = Some(to_val); + } else { + lt = Some(to_val); + } + } + let (gt, gte, lt, lte) = if let Some(JsonLiteral::String(java_date_format)) = format { let parser = StrptimeParser::from_java_datetime_format(&java_date_format) .map_err(|err| anyhow::anyhow!("failed to parse range query date format. {err}"))?; @@ -121,6 +156,7 @@ mod tests { lte: None, boost: None, format: JsonLiteral::String("yyyy-MM-dd['T'HH:mm:ss]".to_string()).into(), + ..Default::default() }; let range_query: ElasticRangeQuery = ElasticRangeQuery { field: "date".to_string(), @@ -138,6 +174,51 @@ mod tests { )); } + fn into_json_number(n: u64) -> JsonLiteral { + JsonLiteral::Number(serde_json::Number::from(n)) + } + + #[test] + fn test_range_query_with_from_to_inclusive() { + let range_json = + r#"{"score": {"from": 50, "to": 100, "include_lower": true, "include_upper": true}}"#; + let range_query: ElasticRangeQuery = serde_json::from_str(range_json).unwrap(); + let ast = range_query.convert_to_query_ast().unwrap(); + let QueryAst::Range(rq) = ast else { + panic!("expected Range, got {ast:?}"); + }; + assert_eq!(rq.field, "score"); + assert_eq!(rq.lower_bound, Bound::Included(into_json_number(50))); + assert_eq!(rq.upper_bound, Bound::Included(into_json_number(100))); + } + + #[test] + fn test_range_query_with_from_to_exclusive() { + let range_json = + r#"{"score": {"from": 50, "to": 100, "include_lower": false, "include_upper": false}}"#; + let range_query: ElasticRangeQuery = serde_json::from_str(range_json).unwrap(); + let ast = range_query.convert_to_query_ast().unwrap(); + let QueryAst::Range(rq) = ast else { + panic!("expected Range, got {ast:?}"); + }; + assert_eq!(rq.field, "score"); + assert_eq!(rq.lower_bound, Bound::Excluded(into_json_number(50))); + assert_eq!(rq.upper_bound, Bound::Excluded(into_json_number(100))); + } + + #[test] + fn test_range_query_with_from_to_defaults() { + let range_json = r#"{"score": {"from": 50, "to": 100}}"#; + let range_query: ElasticRangeQuery = serde_json::from_str(range_json).unwrap(); + let ast = range_query.convert_to_query_ast().unwrap(); + let QueryAst::Range(rq) = ast else { + panic!("expected Range, got {ast:?}"); + }; + assert_eq!(rq.field, "score"); + assert_eq!(rq.lower_bound, Bound::Included(into_json_number(50))); + assert_eq!(rq.upper_bound, Bound::Included(into_json_number(100))); + } + #[test] fn test_date_range_query_with_strict_date_optional_time_format() { let range_query_params = ElasticRangeQueryParams { @@ -147,6 +228,7 @@ mod tests { lte: Some(JsonLiteral::String("2024-09-28T10:22:55.797Z".to_string())), boost: None, format: JsonLiteral::String("strict_date_optional_time".to_string()).into(), + ..Default::default() }; let range_query: ElasticRangeQuery = ElasticRangeQuery { field: "timestamp".to_string(), diff --git a/quickwit/quickwit-query/src/elastic_query_dsl/regex_query.rs b/quickwit/quickwit-query/src/elastic_query_dsl/regex_query.rs index 747e1321811..80eb899077a 100644 --- a/quickwit/quickwit-query/src/elastic_query_dsl/regex_query.rs +++ b/quickwit/quickwit-query/src/elastic_query_dsl/regex_query.rs @@ -18,21 +18,83 @@ use crate::elastic_query_dsl::ConvertibleToQueryAst; use crate::elastic_query_dsl::one_field_map::OneFieldMap; use crate::query_ast::{QueryAst, RegexQuery as AstRegexQuery}; -#[derive(Deserialize, Debug, Default, Eq, PartialEq, Clone)] -#[serde(deny_unknown_fields)] -pub struct RegexQueryParams { - value: String, - // we could probably add case_insensitive +/// Elasticsearch supports two formats for regexp queries: +/// - Shorthand: `{"regexp": {"field": "pattern"}}` +/// - Full: `{"regexp": {"field": {"value": "pattern", "case_insensitive": true}}}` +#[derive(Deserialize, Debug, Eq, PartialEq, Clone)] +#[serde(untagged)] +pub enum RegexQueryParams { + Full { + #[serde(rename = "value")] + pattern: String, + #[serde(default)] + case_insensitive: bool, + }, + Shorthand(String), +} + +impl RegexQueryParams { + fn into_tuple(self) -> (String, bool) { + match self { + RegexQueryParams::Full { + pattern, + case_insensitive, + } => (pattern, case_insensitive), + RegexQueryParams::Shorthand(pattern) => (pattern, false), + } + } } pub type RegexQuery = OneFieldMap; impl ConvertibleToQueryAst for RegexQuery { fn convert_to_query_ast(self) -> anyhow::Result { + let (pattern, case_insensitive) = self.value.into_tuple(); + + let regex = if case_insensitive { + format!("(?i){pattern}") + } else { + pattern + }; Ok(AstRegexQuery { field: self.field, - regex: self.value.value, + regex, } .into()) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_regex_query_shorthand_format() { + let json = serde_json::json!({"service": ".*logs.*"}); + let query: RegexQuery = serde_json::from_value(json).unwrap(); + assert_eq!(query.field, "service"); + let (pattern, case_insensitive) = query.value.into_tuple(); + assert_eq!(pattern, ".*logs.*"); + assert!(!case_insensitive); + } + + #[test] + fn test_regex_query_full_format() { + let json = serde_json::json!({"service": {"value": ".*logs.*", "case_insensitive": true}}); + let query: RegexQuery = serde_json::from_value(json).unwrap(); + assert_eq!(query.field, "service"); + let (pattern, case_insensitive) = query.value.into_tuple(); + assert_eq!(pattern, ".*logs.*"); + assert!(case_insensitive); + } + + #[test] + fn test_regex_query_full_format_default_case() { + let json = serde_json::json!({"service": {"value": ".*logs.*"}}); + let query: RegexQuery = serde_json::from_value(json).unwrap(); + assert_eq!(query.field, "service"); + let (pattern, case_insensitive) = query.value.into_tuple(); + assert_eq!(pattern, ".*logs.*"); + assert!(!case_insensitive); + } +} diff --git a/quickwit/quickwit-query/src/elastic_query_dsl/string_or_struct.rs b/quickwit/quickwit-query/src/elastic_query_dsl/string_or_struct.rs index c2be03c8367..0048a0119fd 100644 --- a/quickwit/quickwit-query/src/elastic_query_dsl/string_or_struct.rs +++ b/quickwit/quickwit-query/src/elastic_query_dsl/string_or_struct.rs @@ -69,22 +69,30 @@ where } fn visit_i64(self, v: i64) -> Result - where E: de::Error { + where + E: de::Error, + { self.visit_str(&v.to_string()) } fn visit_u64(self, v: u64) -> Result - where E: de::Error { + where + E: de::Error, + { self.visit_str(&v.to_string()) } fn visit_str(self, query: &str) -> Result - where E: serde::de::Error { + where + E: serde::de::Error, + { Ok(T::from(query.to_string())) } fn visit_map(self, map: M) -> Result - where M: MapAccess<'de> { + where + M: MapAccess<'de>, + { Deserialize::deserialize(de::value::MapAccessDeserializer::new(map)) } } diff --git a/quickwit/quickwit-query/src/elastic_query_dsl/term_query.rs b/quickwit/quickwit-query/src/elastic_query_dsl/term_query.rs index 9f9d32ba685..9eb2c57c178 100644 --- a/quickwit/quickwit-query/src/elastic_query_dsl/term_query.rs +++ b/quickwit/quickwit-query/src/elastic_query_dsl/term_query.rs @@ -41,6 +41,7 @@ impl From for TermQueryParams { TermQueryParams { value: query, boost: None, + case_insensitive: false, } } } @@ -54,7 +55,9 @@ enum TermValue { } fn deserialize_term_value<'de, D>(deserializer: D) -> Result -where D: Deserializer<'de> { +where + D: Deserializer<'de>, +{ let term_value = TermValue::deserialize(deserializer)?; match term_value { TermValue::I64(i64) => Ok(i64.to_string()), @@ -70,14 +73,18 @@ pub struct TermQueryParams { pub value: String, #[serde(default)] pub boost: Option, + #[serde(default)] + case_insensitive: bool, } +#[cfg(test)] pub fn term_query_from_field_value(field: impl ToString, value: impl ToString) -> TermQuery { TermQuery { field: field.to_string(), value: TermQueryParams { value: value.to_string(), boost: None, + case_insensitive: false, }, } } @@ -90,7 +97,20 @@ impl From for ElasticQueryDslInner { impl ConvertibleToQueryAst for TermQuery { fn convert_to_query_ast(self) -> anyhow::Result { - let TermQueryParams { value, boost } = self.value; + let TermQueryParams { + value, + boost, + case_insensitive, + } = self.value; + if case_insensitive { + let ci_value = format!("(?i){}", regex::escape(&value)); + let term_ast: QueryAst = query_ast::RegexQuery { + field: self.field, + regex: ci_value, + } + .into(); + return Ok(term_ast.boost(boost)); + } let term_ast: QueryAst = query_ast::TermQuery { field: self.field, value, diff --git a/quickwit/quickwit-query/src/elastic_query_dsl/terms_query.rs b/quickwit/quickwit-query/src/elastic_query_dsl/terms_query.rs index f6c0545a33a..5b3078e7073 100644 --- a/quickwit/quickwit-query/src/elastic_query_dsl/terms_query.rs +++ b/quickwit/quickwit-query/src/elastic_query_dsl/terms_query.rs @@ -12,14 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::{BTreeSet, HashMap}; + use serde::Deserialize; -use crate::elastic_query_dsl::bool_query::BoolQuery; use crate::elastic_query_dsl::one_field_map::OneFieldMap; -use crate::elastic_query_dsl::term_query::term_query_from_field_value; use crate::elastic_query_dsl::{ConvertibleToQueryAst, ElasticQueryDslInner}; use crate::not_nan_f32::NotNaNf32; -use crate::query_ast::QueryAst; +use crate::query_ast::{QueryAst, TermSetQuery}; #[derive(PartialEq, Eq, Debug, Deserialize, Clone)] #[serde(try_from = "TermsQueryForSerialization")] @@ -37,17 +37,36 @@ struct TermsQueryForSerialization { capture_other: serde_json::Value, } +#[derive(Deserialize)] +#[serde(untagged)] +enum TermValue { + I64(i64), + U64(u64), + Str(String), +} + +impl From for String { + fn from(term_value: TermValue) -> String { + match term_value { + TermValue::I64(val) => val.to_string(), + TermValue::U64(val) => val.to_string(), + TermValue::Str(val) => val, + } + } +} + #[derive(Deserialize)] #[serde(untagged)] enum OneOrMany { - One(String), - Many(Vec), + One(TermValue), + Many(Vec), } + impl From for Vec { fn from(one_or_many: OneOrMany) -> Vec { match one_or_many { - OneOrMany::One(one_value) => vec![one_value], - OneOrMany::Many(values) => values, + OneOrMany::One(one_value) => vec![String::from(one_value)], + OneOrMany::Many(values) => values.into_iter().map(String::from).collect(), } } } @@ -68,15 +87,14 @@ impl TryFrom for TermsQuery { impl ConvertibleToQueryAst for TermsQuery { fn convert_to_query_ast(self) -> anyhow::Result { - let term_queries: Vec = self - .values - .into_iter() - .map(|value| term_query_from_field_value(self.field.clone(), value)) - .map(ElasticQueryDslInner::from) - .collect(); - let mut union = BoolQuery::union(term_queries); - union.boost = self.boost; - union.convert_to_query_ast() + let mut terms_per_field = HashMap::new(); + let values_set: BTreeSet = self.values.into_iter().collect(); + terms_per_field.insert(self.field, values_set); + + let term_set_query = TermSetQuery { terms_per_field }; + let query_ast: QueryAst = term_set_query.into(); + + Ok(query_ast.boost(self.boost)) } } @@ -109,6 +127,14 @@ mod tests { assert_eq!(&terms_query.values[..], &["hello".to_string()]); } + #[test] + fn test_terms_query_not_string() { + let terms_query_json = r#"{ "user.id": [1, 2] }"#; + let terms_query: TermsQuery = serde_json::from_str(terms_query_json).unwrap(); + assert_eq!(&terms_query.field, "user.id"); + assert_eq!(&terms_query.values[..], &["1".to_string(), "2".to_string()]); + } + #[test] fn test_terms_query_single_term_boost() { let terms_query_json = r#"{ "user.id": ["hello", "happy"], "boost": 2 }"#; diff --git a/quickwit/quickwit-query/src/elastic_query_dsl/wildcard_query.rs b/quickwit/quickwit-query/src/elastic_query_dsl/wildcard_query.rs new file mode 100644 index 00000000000..3b975e896e9 --- /dev/null +++ b/quickwit/quickwit-query/src/elastic_query_dsl/wildcard_query.rs @@ -0,0 +1,121 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use serde::Deserialize; + +use crate::NotNaNf32; +use crate::elastic_query_dsl::one_field_map::OneFieldMap; +use crate::elastic_query_dsl::{ConvertibleToQueryAst, StringOrStructForSerialization}; +use crate::query_ast::{QueryAst, WildcardQuery as AstWildcardQuery}; + +#[derive(Deserialize, Clone, Eq, PartialEq, Debug)] +#[serde(from = "OneFieldMap>")] +pub(crate) struct WildcardQuery { + pub(crate) field: String, + pub(crate) params: WildcardQueryParams, +} + +#[derive(Deserialize, Debug, Default, Eq, PartialEq, Clone)] +#[serde(deny_unknown_fields)] +pub struct WildcardQueryParams { + value: String, + #[serde(default)] + pub boost: Option, + #[serde(default)] + case_insensitive: bool, +} + +impl ConvertibleToQueryAst for WildcardQuery { + fn convert_to_query_ast(self) -> anyhow::Result { + let wildcard_ast: QueryAst = AstWildcardQuery { + field: self.field, + value: self.params.value, + lenient: true, + case_insensitive: self.params.case_insensitive, + } + .into(); + Ok(wildcard_ast.boost(self.params.boost)) + } +} + +impl From>> for WildcardQuery { + fn from( + match_query_params: OneFieldMap>, + ) -> Self { + let OneFieldMap { field, value } = match_query_params; + WildcardQuery { + field, + params: value.inner, + } + } +} + +impl From for WildcardQueryParams { + fn from(value: String) -> WildcardQueryParams { + WildcardQueryParams { + value, + boost: None, + case_insensitive: false, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_wildcard_query_convert_to_query_ast() { + let wildcard_query_json = r#"{ + "user_name": { + "value": "john*" + } + }"#; + let wildcard_query: WildcardQuery = serde_json::from_str(wildcard_query_json).unwrap(); + let query_ast = wildcard_query.convert_to_query_ast().unwrap(); + + if let QueryAst::Wildcard(wildcard) = query_ast { + assert_eq!(wildcard.field, "user_name"); + assert_eq!(wildcard.value, "john*"); + assert!(wildcard.lenient); + } else { + panic!("Expected QueryAst::Wildcard"); + } + } + + #[test] + fn test_boosted_wildcard_query_convert_to_query_ast() { + let wildcard_query_json = r#"{ + "user_name": { + "value": "john*", + "boost": 2.0 + } + }"#; + let wildcard_query: WildcardQuery = serde_json::from_str(wildcard_query_json).unwrap(); + let query_ast = wildcard_query.convert_to_query_ast().unwrap(); + + if let QueryAst::Boost { underlying, boost } = query_ast { + if let QueryAst::Wildcard(wildcard) = *underlying { + assert_eq!(wildcard.field, "user_name"); + assert_eq!(wildcard.value, "john*"); + assert!(wildcard.lenient); + } else { + panic!("Expected underlying QueryAst::Wildcard"); + } + assert_eq!(boost, NotNaNf32::try_from(2.0).unwrap()); + } else { + panic!("Expected QueryAst::Wildcard"); + } + } +} diff --git a/quickwit/quickwit-query/src/lib.rs b/quickwit/quickwit-query/src/lib.rs index b2040f73daa..8f70e155933 100644 --- a/quickwit/quickwit-query/src/lib.rs +++ b/quickwit/quickwit-query/src/lib.rs @@ -38,8 +38,6 @@ pub(crate) use not_nan_f32::NotNaNf32; pub use query_ast::utils::find_field_or_hit_dynamic; use serde::{Deserialize, Serialize}; pub use tantivy::query::Query as TantivyQuery; -#[cfg(feature = "multilang")] -pub use tokenizers::MultiLangTokenizer; pub use tokenizers::{ CodeTokenizer, DEFAULT_REMOVE_TOKEN_LENGTH, create_default_quickwit_tokenizer_manager, get_quickwit_fastfield_normalizer_manager, diff --git a/quickwit/quickwit-query/src/query_ast/bool_query.rs b/quickwit/quickwit-query/src/query_ast/bool_query.rs index ee9a46ea97f..b85fbacb4d4 100644 --- a/quickwit/quickwit-query/src/query_ast/bool_query.rs +++ b/quickwit/quickwit-query/src/query_ast/bool_query.rs @@ -13,15 +13,12 @@ // limitations under the License. use serde::{Deserialize, Serialize}; -use tantivy::schema::Schema as TantivySchema; -use super::{BuildTantivyAst, TantivyQueryAst}; +use super::{BuildTantivyAst, BuildTantivyAstContext, TantivyQueryAst}; use crate::InvalidQuery; use crate::query_ast::QueryAst; -use crate::tokenizers::TokenizerManager; /// # Unsupported features -/// - minimum_should_match /// - named queries /// /// Edge cases of BooleanQuery are not obvious, @@ -56,49 +53,26 @@ impl From for QueryAst { impl BuildTantivyAst for BoolQuery { fn build_tantivy_ast_impl( &self, - schema: &TantivySchema, - tokenizer_manager: &TokenizerManager, - search_fields: &[String], - with_validation: bool, + context: &BuildTantivyAstContext, ) -> Result { let mut boolean_query = super::tantivy_query_ast::TantivyBoolQuery { minimum_should_match: self.minimum_should_match, ..Default::default() }; for must in &self.must { - let must_leaf = must.build_tantivy_ast_call( - schema, - tokenizer_manager, - search_fields, - with_validation, - )?; + let must_leaf = must.build_tantivy_ast_call(context)?; boolean_query.must.push(must_leaf); } for must_not in &self.must_not { - let must_not_leaf = must_not.build_tantivy_ast_call( - schema, - tokenizer_manager, - search_fields, - with_validation, - )?; + let must_not_leaf = must_not.build_tantivy_ast_call(context)?; boolean_query.must_not.push(must_not_leaf); } for should in &self.should { - let should_leaf = should.build_tantivy_ast_call( - schema, - tokenizer_manager, - search_fields, - with_validation, - )?; + let should_leaf = should.build_tantivy_ast_call(context)?; boolean_query.should.push(should_leaf); } for filter in &self.filter { - let filter_leaf = filter.build_tantivy_ast_call( - schema, - tokenizer_manager, - search_fields, - with_validation, - )?; + let filter_leaf = filter.build_tantivy_ast_call(context)?; boolean_query.filter.push(filter_leaf); } Ok(TantivyQueryAst::Bool(boolean_query)) diff --git a/quickwit/quickwit-query/src/query_ast/cache_node.rs b/quickwit/quickwit-query/src/query_ast/cache_node.rs new file mode 100644 index 00000000000..9c449aef4fb --- /dev/null +++ b/quickwit/quickwit-query/src/query_ast/cache_node.rs @@ -0,0 +1,870 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use bitpacking::{BitPacker, BitPacker1x}; +use quickwit_proto::types::SplitId; +use serde::{Deserialize, Serialize}; + +use super::{BuildTantivyAst, BuildTantivyAstContext, TantivyQueryAst}; +use crate::InvalidQuery; +use crate::query_ast::QueryAst; + +/// A node caching the result of an inner query. +/// +/// This can be used when it's known that some sub-ast might appear in many queries, +/// or that the same query might be run, with various aggregations. +/// +/// /!\ Sprinkling this everywhere can lead to performance degradations: the whole posting +/// list of the underlying query will need to be evaluated to build the cache, whereas it could +/// have been largely skipped if some other part of the query is very selective. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct CacheNode { + pub inner: Box, + #[serde(skip)] + pub state: CacheState, +} + +#[derive(Default, Clone)] +pub enum CacheState { + // This is the state a CacheNode should be before + #[default] + Uninitialized, + CacheHit(CacheEntry), + CacheMiss(CacheFiller), +} + +impl std::fmt::Debug for CacheState { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + CacheState::Uninitialized => fmt.debug_tuple("Uninitialized").finish(), + CacheState::CacheHit(_) => fmt.debug_tuple("CacheHit").finish_non_exhaustive(), + CacheState::CacheMiss(_) => fmt.debug_tuple("CacheMiss").finish_non_exhaustive(), + } + } +} + +// cache state shouldn't impact a CacheNode equality +impl Eq for CacheNode {} +impl PartialEq for CacheNode { + fn eq(&self, other: &Self) -> bool { + self.inner == other.inner + } +} + +impl From for QueryAst { + fn from(cache_node: CacheNode) -> Self { + QueryAst::Cache(cache_node) + } +} + +impl CacheNode { + pub fn new(ast: QueryAst) -> Self { + CacheNode { + inner: Box::new(ast), + state: CacheState::Uninitialized, + } + } + + pub fn fill_cache_state(&mut self, cache: &Arc, split_id: &str) { + let Ok(query) = serde_json::to_string(&self.inner) else { + return; + }; + if let Some((segment_id, hits)) = cache.get(split_id.to_string(), query.clone()) { + self.state = CacheState::CacheHit(CacheEntry { segment_id, hits }); + } else { + self.state = CacheState::CacheMiss(CacheFiller { + cache: cache.clone(), + split_id: split_id.to_string(), + query, + }); + } + } +} + +impl BuildTantivyAst for CacheNode { + fn build_tantivy_ast_impl( + &self, + context: &BuildTantivyAstContext, + ) -> Result { + match &self.state { + CacheState::Uninitialized => self.inner.build_tantivy_ast_call(context), + CacheState::CacheHit(cache_entry) => Ok(CacheHitQuery { + cache_entry: cache_entry.clone(), + } + .into()), + CacheState::CacheMiss(cache_filler) => { + let tantivy_query: Box = self + .inner + .build_tantivy_ast_call(context)? + .simplify() + .into(); + Ok(CacheFillerQuery { + inner_query: Box::new(tantivy_query), + cache_filler: cache_filler.clone(), + } + .into()) + } + } + } +} + +use tantivy::directory::OwnedBytes; +use tantivy::index::SegmentId; +use tantivy::query::{EnableScoring, Explanation, Query, Scorer, Weight}; +use tantivy::{DocId, DocSet, Score, SegmentReader, TantivyError}; + +#[derive(Clone, Debug)] +pub struct CacheHitQuery { + cache_entry: CacheEntry, +} + +impl Query for CacheHitQuery { + fn weight(&self, enable_scoring: EnableScoring<'_>) -> tantivy::Result> { + if enable_scoring.is_scoring_enabled() { + Err(tantivy::TantivyError::InternalError( + "Predicate cache doesn't support scoring yet".to_string(), + )) + } else { + Ok(Box::new(CacheHitWeight { + cache_entry: self.cache_entry.clone(), + })) + } + } +} + +/// Weight associated with the `AllQuery` query. +pub struct CacheHitWeight { + cache_entry: CacheEntry, +} + +impl Weight for CacheHitWeight { + fn scorer(&self, reader: &SegmentReader, boost: Score) -> tantivy::Result> { + // we could try to run the query if for some reason we don't actually find an entry in + // cache, but that would have required loading stuff during warmup which we skipped. + // An error is the best we can do + let mut hit_set = self + .cache_entry + .for_segment(reader.segment_id()) + .ok_or_else(|| TantivyError::InternalError("Segment not found in cache".to_string()))?; + hit_set.boost = boost; + Ok(Box::new(hit_set)) + } + + fn explain(&self, reader: &SegmentReader, doc: DocId) -> tantivy::Result { + let mut scorer = self.scorer(reader, 1.0)?; + if scorer.seek(doc) == doc { + Ok(Explanation::new("HitSet", 1.0)) + } else { + Err(TantivyError::InvalidArgument( + "Document does not exist".to_string(), + )) + } + } +} + +#[derive(Clone)] +pub struct CacheEntry { + segment_id: SegmentId, + hits: HitSet, +} + +impl std::fmt::Debug for CacheEntry { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fmt.debug_struct("CacheEntry") + .field("segment_id", &self.segment_id) + .finish_non_exhaustive() + } +} + +impl CacheEntry { + fn for_segment(&self, segment_id: SegmentId) -> Option { + if segment_id == self.segment_id { + Some(self.hits.clone()) + } else { + None + } + } +} + +#[derive(Debug, Clone)] +pub struct HitSet { + buffer: OwnedBytes, + buffer_pos: usize, + previous_last_val: Option, + current_block: [u32; BitPacker1x::BLOCK_LEN], + block_pos: usize, + boost: Score, +} + +const INCOMPLETE_BLOCK_MARKER: u8 = 0x80; + +impl HitSet { + #[cfg(test)] + fn empty() -> Self { + Self::from_buffer(OwnedBytes::new(vec![0, 0, 0, 0])) + } + + /// Build a HitSet from its serialized form. + /// + /// The provided buffer must come from `HitSet::into_buffer` + pub fn from_buffer(buffer: OwnedBytes) -> Self { + let mut this = Self { + buffer, + // skip count + buffer_pos: 4, + previous_last_val: None, + current_block: [0; BitPacker1x::BLOCK_LEN], + // we set this to block_len minus 1 so we can call advance() once to initialize + // everything + block_pos: BitPacker1x::BLOCK_LEN - 1, + boost: 1.0, + }; + this.advance(); + this + } + + /// Return a buffer representing the underlying data. + /// + /// This does not preserve where in the DocSet you are. + pub fn into_buffer(self) -> OwnedBytes { + self.buffer + } + + fn load_new_block(&mut self) { + let Some(num_bits) = self.buffer.get(self.buffer_pos) else { + // we ended iteration: simply fill the current_block full of TERMINATED + self.current_block = [tantivy::TERMINATED; 32]; + return; + }; + self.buffer_pos += 1; + if *num_bits == INCOMPLETE_BLOCK_MARKER { + // final block, decode as many ids as possible + let mut i = 0; + for chunk in self.buffer[self.buffer_pos..].as_chunks().0 { + self.current_block[i] = u32::from_ne_bytes(*chunk); + i += 1; + } + // pad with TERMINATED + while i < BitPacker1x::BLOCK_LEN { + self.current_block[i] = tantivy::TERMINATED; + i += 1; + } + self.buffer_pos = self.buffer.len(); + } else { + self.buffer_pos += BitPacker1x.decompress_strictly_sorted( + self.previous_last_val, + &self.buffer[self.buffer_pos..], + &mut self.current_block, + *num_bits, + ); + self.previous_last_val = self.current_block.last().copied(); + } + } +} + +impl DocSet for HitSet { + fn advance(&mut self) -> DocId { + self.block_pos += 1; + if let Some(doc_id) = self.current_block.get(self.block_pos) { + return *doc_id; + } + self.load_new_block(); + self.block_pos = 0; + self.current_block[0] + } + + // fn seek(&mut self, target: DocId) -> DocId { + // } + + #[inline(always)] + fn doc(&self) -> DocId { + self.current_block[self.block_pos] + } + + fn size_hint(&self) -> u32 { + u32::from_ne_bytes(self.buffer[0..4].try_into().unwrap()) + } +} + +impl Scorer for HitSet { + fn score(&mut self) -> f32 { + self.boost + } +} + +pub struct HitSetBuilder { + count: u32, + current_block: [u32; BitPacker1x::BLOCK_LEN], + previous_last_val: Option, + buffer: Vec, +} + +impl HitSetBuilder { + pub fn new() -> Self { + HitSetBuilder { + count: 0, + current_block: [0; BitPacker1x::BLOCK_LEN], + previous_last_val: None, + buffer: vec![0; 4], + } + } + + fn in_block_pos(&self) -> usize { + (self.count % BitPacker1x::BLOCK_LEN as u32) as usize + } + + fn end_of_block(&self) -> bool { + self.in_block_pos() == (BitPacker1x::BLOCK_LEN - 1) + } + + fn flush_block(&mut self) { + let num_bits = + BitPacker1x.num_bits_strictly_sorted(self.previous_last_val, &self.current_block); + self.buffer.push(num_bits); + let current_buffer_pos = self.buffer.len(); + let new_end = current_buffer_pos + (BitPacker1x::BLOCK_LEN * num_bits as usize) / 8; + self.buffer.resize(new_end, 0); + BitPacker1x.compress_strictly_sorted( + self.previous_last_val, + &self.current_block, + &mut self.buffer[current_buffer_pos..], + num_bits, + ); + self.previous_last_val = self.current_block.last().copied(); + } + + pub fn insert(&mut self, value: u32) { + self.current_block[self.in_block_pos()] = value; + if self.end_of_block() { + self.flush_block(); + } + self.count += 1; + } + + pub fn build(mut self) -> HitSet { + if self.in_block_pos() != 0 { + self.buffer.push(INCOMPLETE_BLOCK_MARKER); + for elem in &self.current_block[..self.in_block_pos()] { + self.buffer.extend_from_slice(&elem.to_ne_bytes()); + } + } + // write back the count of items + self.buffer[0..4].copy_from_slice(&self.count.to_ne_bytes()); + HitSet::from_buffer(OwnedBytes::new(self.buffer)) + } +} + +#[derive(Clone)] +pub struct CacheFiller { + cache: Arc, + split_id: String, + query: String, +} + +impl std::fmt::Debug for CacheFiller { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fmt.debug_struct("CacheFiller") + .field("split_id", &self.split_id) + .finish_non_exhaustive() + } +} + +impl CacheFiller { + fn fill_segment(&self, segment_id: SegmentId, value: HitSet) { + self.cache + .put(self.split_id.clone(), self.query.clone(), segment_id, value); + } +} + +#[derive(Debug)] +pub struct CacheFillerQuery { + inner_query: Box, + cache_filler: CacheFiller, +} + +impl Clone for CacheFillerQuery { + fn clone(&self) -> Self { + Self { + inner_query: self.inner_query.box_clone(), + cache_filler: self.cache_filler.clone(), + } + } +} + +impl Query for CacheFillerQuery { + fn weight(&self, enable_scoring: EnableScoring<'_>) -> tantivy::Result> { + if enable_scoring.is_scoring_enabled() { + Err(tantivy::TantivyError::InternalError( + "Predicate cache doesn't support scoring yet".to_string(), + )) + } else { + Ok(Box::new(CacheFillerWeight { + inner_weight: self.inner_query.weight(enable_scoring)?, + cache_filler: self.cache_filler.clone(), + })) + } + } + fn query_terms<'a>(&'a self, visitor: &mut dyn FnMut(&'a tantivy::Term, bool)) { + self.inner_query.query_terms(visitor) + } +} + +/// Weight associated with the `AllQuery` query. +pub struct CacheFillerWeight { + inner_weight: Box, + cache_filler: CacheFiller, +} + +impl Weight for CacheFillerWeight { + fn scorer(&self, reader: &SegmentReader, boost: Score) -> tantivy::Result> { + let mut hit_set_builder = HitSetBuilder::new(); + let mut scorer = self.inner_weight.scorer(reader, 1.0)?; + let mut doc_id = scorer.doc(); + while doc_id < tantivy::TERMINATED { + hit_set_builder.insert(doc_id); + doc_id = scorer.advance(); + } + let mut hit_set = hit_set_builder.build(); + self.cache_filler + .fill_segment(reader.segment_id(), hit_set.clone()); + hit_set.boost = boost; + Ok(Box::new(hit_set)) + } + + fn explain(&self, reader: &SegmentReader, doc: DocId) -> tantivy::Result { + self.inner_weight.explain(reader, doc) + } +} + +/// A transformer that goes through a QueryAst, and change the state of all CacheNodes +/// to Hit/Miss based on the provided cache. +/// +/// This must be called for any CacheNode inside a QueryAst to do anything (though not calling +/// it isn't an error, it just means no cache will be used). +pub struct PredicateCacheInjector { + pub cache: Arc, + pub split_id: String, +} + +impl crate::query_ast::QueryAstTransformer for PredicateCacheInjector { + type Err = std::convert::Infallible; + + fn transform_cache_node( + &mut self, + mut cache_node: CacheNode, + ) -> Result, Self::Err> { + cache_node.fill_cache_state(&self.cache, &self.split_id); + self.transform(*cache_node.inner).map(|maybe_ast| { + maybe_ast.map(|inner| { + QueryAst::Cache(CacheNode { + inner: Box::new(inner), + state: cache_node.state, + }) + }) + }) + } +} + +// we use a trait to dodge circular dependancies with quickwit-storage +pub trait PredicateCache: Send + Sync + 'static { + fn get(&self, split_id: SplitId, query_ast_json: String) -> Option<(SegmentId, HitSet)>; + + fn put(&self, split_id: SplitId, query_ast_json: String, segment: SegmentId, results: HitSet); +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Mutex; + + use tantivy::DocSet; + use tantivy::query::Query as TantivyQuery; + use tantivy::schema::{Schema, TEXT}; + + use super::*; + use crate::query_ast::{ + BuildTantivyAstContext, QueryAstTransformer, QueryAstVisitor, TermQuery, + }; + + impl PredicateCache for Mutex> { + fn get(&self, split_id: SplitId, query_ast_json: String) -> Option<(SegmentId, HitSet)> { + self.lock() + .unwrap() + .get(&(split_id, query_ast_json)) + .cloned() + } + + fn put( + &self, + split_id: SplitId, + query_ast_json: String, + segment: SegmentId, + results: HitSet, + ) { + self.lock() + .unwrap() + .insert((split_id, query_ast_json), (segment, results)); + } + } + + #[track_caller] + fn test_hit_set_roundtrip_helper + Clone>(iter: I) { + let mut hitset_builder = HitSetBuilder::new(); + for i in iter.clone() { + hitset_builder.insert(i); + } + let mut hitset = hitset_builder.build(); + + for val in iter { + assert_eq!(hitset.doc(), val); + hitset.advance(); + } + for _ in 0..96 { + assert_eq!(hitset.doc(), tantivy::TERMINATED); + hitset.advance(); + } + } + + #[test] + fn test_hit_set_roundtrip() { + // this generate a pseurorandom strictrly increasing sequence + let generator = std::iter::successors(Some(0u32), |x| Some(x + x.trailing_ones() + 1)); + + // empty + test_hit_set_roundtrip_helper(generator.clone().take(0)); + // one item + test_hit_set_roundtrip_helper(generator.clone().take(1)); + test_hit_set_roundtrip_helper(generator.clone().skip(10).take(1)); + // partial block + test_hit_set_roundtrip_helper(generator.clone().take(24)); + test_hit_set_roundtrip_helper(generator.clone().skip(10).take(24)); + + // one block + test_hit_set_roundtrip_helper(generator.clone().take(32)); + test_hit_set_roundtrip_helper(generator.clone().skip(10).take(32)); + // two blocks + test_hit_set_roundtrip_helper(generator.clone().take(64)); + test_hit_set_roundtrip_helper(generator.clone().skip(10).take(64)); + + // many blocks, partial last block + test_hit_set_roundtrip_helper(generator.clone().take(1024 + 6)); + test_hit_set_roundtrip_helper(generator.clone().skip(10).take(1024 + 6)); + } + + #[test] + fn test_built_tantivy_ast() { + let mut schema_builder = Schema::builder(); + schema_builder.add_text_field("body", TEXT); + let schema = schema_builder.build(); + let term_query: QueryAst = TermQuery { + field: "body".to_string(), + value: "val".to_string(), + } + .into(); + let tantivy_term_query: Box = term_query + .build_tantivy_ast_impl(&BuildTantivyAstContext::for_test(&schema)) + .unwrap() + .into(); + + { + let ast = CacheNode { + inner: Box::new(term_query.clone()), + state: CacheState::Uninitialized, + }; + let uninit_cache_query: Box = ast + .build_tantivy_ast_impl(&BuildTantivyAstContext::for_test(&schema)) + .unwrap() + .into(); + assert_eq!( + format!("{uninit_cache_query:?}"), + format!("{tantivy_term_query:?}") + ); + } + + { + let cache_entry = CacheEntry { + segment_id: SegmentId::from_uuid_string("1686a000d4f7a91939d0e71df1646d7a") + .unwrap(), + hits: HitSet::empty(), + }; + let ast = CacheNode { + inner: Box::new(term_query.clone()), + state: CacheState::CacheHit(cache_entry), + }; + let cache_hit_query: Box = ast + .build_tantivy_ast_impl(&BuildTantivyAstContext::for_test(&schema)) + .unwrap() + .into(); + + let debug_query = format!("{cache_hit_query:?}"); + assert!(debug_query.contains("CacheHitQuery")); + assert!(!debug_query.contains("TermQuery")); + } + { + let cache_filler = CacheFiller { + cache: Arc::new(Mutex::new(HashMap::new())), + split_id: "split_id".to_string(), + query: "{}".to_string(), + }; + let ast = CacheNode { + inner: Box::new(term_query.clone()), + state: CacheState::CacheMiss(cache_filler), + }; + let cache_miss_query: Box = ast + .build_tantivy_ast_impl(&BuildTantivyAstContext::for_test(&schema)) + .unwrap() + .into(); + + let debug_query = format!("{cache_miss_query:?}"); + assert!(debug_query.contains("CacheFillerQuery")); + assert!(debug_query.contains(&format!("{tantivy_term_query:?}"))); + } + } + + struct FoundATermVisitor(bool); + impl QueryAstVisitor<'_> for FoundATermVisitor { + type Err = std::convert::Infallible; + fn visit_term(&mut self, _term: &TermQuery) -> Result<(), Self::Err> { + self.0 = true; + Ok(()) + } + } + + impl QueryAstTransformer for FoundATermVisitor { + type Err = std::convert::Infallible; + fn transform_term(&mut self, term: TermQuery) -> Result, Self::Err> { + self.0 = true; + Ok(Some(term.into())) + } + } + + #[test] + fn test_default_visitor_ignore_cached_node() { + let term_query: QueryAst = TermQuery { + field: "body".to_string(), + value: "val".to_string(), + } + .into(); + { + let ast = CacheNode { + inner: Box::new(term_query.clone()), + state: CacheState::Uninitialized, + } + .into(); + + let mut visitor = FoundATermVisitor(false); + visitor.visit(&ast).unwrap(); + assert!(visitor.0); + let mut visitor = FoundATermVisitor(false); + visitor.transform(ast).unwrap(); + assert!(visitor.0); + } + { + let cache_entry = CacheEntry { + segment_id: SegmentId::from_uuid_string("1686a000d4f7a91939d0e71df1646d7a") + .unwrap(), + hits: HitSet::empty(), + }; + let ast = CacheNode { + inner: Box::new(term_query.clone()), + state: CacheState::CacheHit(cache_entry), + } + .into(); + + let mut visitor = FoundATermVisitor(false); + visitor.visit(&ast).unwrap(); + assert!(!visitor.0); + let mut visitor = FoundATermVisitor(false); + visitor.transform(ast).unwrap(); + assert!(!visitor.0); + } + { + let cache_filler = CacheFiller { + cache: Arc::new(Mutex::new(HashMap::new())), + split_id: "split_id".to_string(), + query: "{}".to_string(), + }; + let ast = CacheNode { + inner: Box::new(term_query.clone()), + state: CacheState::CacheMiss(cache_filler), + } + .into(); + + let mut visitor = FoundATermVisitor(false); + visitor.visit(&ast).unwrap(); + assert!(visitor.0); + let mut visitor = FoundATermVisitor(false); + visitor.transform(ast).unwrap(); + assert!(visitor.0); + } + } + + #[test] + fn test_cache_preigniter_fills_cache() { + let term_query: QueryAst = TermQuery { + field: "body".to_string(), + value: "val".to_string(), + } + .into(); + let cache_node = CacheNode { + inner: Box::new(term_query.clone()), + state: CacheState::Uninitialized, + }; + let query_json = serde_json::to_string(&cache_node.inner).unwrap(); + let ast: QueryAst = cache_node.into(); + let cache = Arc::new(Mutex::new(HashMap::new())); + cache.put( + "split_2".to_string(), + query_json, + SegmentId::from_uuid_string("1686a000d4f7a91939d0e71df1646d7a").unwrap(), + HitSet::empty(), + ); + + { + let mut pre_igniter = PredicateCacheInjector { + cache: cache.clone(), + split_id: "split_1".to_string(), + }; + let filled = pre_igniter.transform(ast.clone()).unwrap().unwrap(); + assert!(matches!( + filled, + QueryAst::Cache(CacheNode { + state: CacheState::CacheMiss(_), + .. + }) + )); + } + + { + let mut pre_igniter = PredicateCacheInjector { + cache: cache.clone(), + split_id: "split_2".to_string(), + }; + let filled = pre_igniter.transform(ast.clone()).unwrap().unwrap(); + assert!(matches!( + filled, + QueryAst::Cache(CacheNode { + state: CacheState::CacheHit(_), + .. + }) + )); + } + } + + #[test] + fn test_cache_hit_returns_correct_docs() { + let mut schema_builder = Schema::builder(); + let host_field = schema_builder.add_text_field("host", TEXT); + let schema = schema_builder.build(); + let index = tantivy::IndexBuilder::new() + .schema(schema.clone()) + .create_in_ram() + .unwrap(); + let mut index_writer = index.writer_with_num_threads(1, 20_000_000).unwrap(); + for count in 1..13 { + let mut doc = tantivy::TantivyDocument::default(); + doc.add_text(host_field, format!("host_{count}")); + for _ in 0..count { + index_writer.add_document(doc.clone()).unwrap(); + } + } + index_writer.commit().unwrap(); + let searcher = index.reader().unwrap().searcher(); + let segment_id = searcher.segment_readers()[0].segment_id(); + + let generator = + std::iter::successors(Some(0u32), |x| Some(x + x.trailing_ones() + 1)).take(500); + let mut hitset_builder = HitSetBuilder::new(); + for i in generator { + hitset_builder.insert(i); + } + let hitset = hitset_builder.build(); + + // this query isn't even valid for that split, but that's not relevant as it won't get run + let term_query: QueryAst = TermQuery { + field: "body".to_string(), + value: "val".to_string(), + } + .into(); + let cache_entry = CacheEntry { + segment_id, + hits: hitset, + }; + let ast = CacheNode { + inner: Box::new(term_query.clone()), + state: CacheState::CacheHit(cache_entry), + }; + let cache_hit_query: Box = ast + .build_tantivy_ast_impl(&BuildTantivyAstContext::for_test(&schema)) + .unwrap() + .into(); + + assert_eq!(cache_hit_query.count(&searcher).unwrap(), 500); + } + + #[test] + fn test_cache_miss_returns_correct_docs_and_fill_cache() { + let mut schema_builder = Schema::builder(); + let host_field = schema_builder.add_text_field("host", TEXT); + let schema = schema_builder.build(); + let index = tantivy::IndexBuilder::new() + .schema(schema.clone()) + .create_in_ram() + .unwrap(); + let mut index_writer = index.writer_with_num_threads(1, 20_000_000).unwrap(); + for count in 1..13 { + let mut doc = tantivy::TantivyDocument::default(); + doc.add_text(host_field, format!("host_{count}")); + for _ in 0..count { + index_writer.add_document(doc.clone()).unwrap(); + } + } + index_writer.commit().unwrap(); + let searcher = index.reader().unwrap().searcher(); + let segment_id = searcher.segment_readers()[0].segment_id(); + + let term_query: QueryAst = TermQuery { + field: "host".to_string(), + value: "11".to_string(), + } + .into(); + let cache = Arc::new(Mutex::new(HashMap::new())); + let cache_filler = CacheFiller { + cache: cache.clone(), + split_id: "split_id".to_string(), + query: "{some_query}".to_string(), + }; + let ast = CacheNode { + inner: Box::new(term_query.clone()), + state: CacheState::CacheMiss(cache_filler), + }; + let cache_hit_query: Box = ast + .build_tantivy_ast_impl(&BuildTantivyAstContext::for_test(&schema)) + .unwrap() + .into(); + + assert_eq!(cache_hit_query.count(&searcher).unwrap(), 11); + let mut cache_entry = cache + .get("split_id".to_string(), "{some_query}".to_string()) + .unwrap(); + assert_eq!(cache_entry.0, segment_id); + let expected = (10 * 11 / 2)..(11 * 12 / 2); + for doc_id in expected { + assert_eq!(cache_entry.1.doc(), doc_id); + cache_entry.1.advance(); + } + } +} diff --git a/quickwit/quickwit-query/src/query_ast/field_presence.rs b/quickwit/quickwit-query/src/query_ast/field_presence.rs index 708cb1e16bf..56fd05d3398 100644 --- a/quickwit/quickwit-query/src/query_ast/field_presence.rs +++ b/quickwit/quickwit-query/src/query_ast/field_presence.rs @@ -21,8 +21,7 @@ use tantivy::schema::{Field, FieldEntry, IndexRecordOption, Schema as TantivySch use super::tantivy_query_ast::TantivyBoolQuery; use super::utils::{DYNAMIC_FIELD_NAME, find_subfields}; use crate::query_ast::tantivy_query_ast::TantivyQueryAst; -use crate::query_ast::{BuildTantivyAst, QueryAst}; -use crate::tokenizers::TokenizerManager; +use crate::query_ast::{BuildTantivyAst, BuildTantivyAstContext, QueryAst}; use crate::{BooleanOperand, InvalidQuery, find_field_or_hit_dynamic}; #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] @@ -121,12 +120,9 @@ impl FieldPresenceQuery { impl BuildTantivyAst for FieldPresenceQuery { fn build_tantivy_ast_impl( &self, - schema: &TantivySchema, - _tokenizer_manager: &TokenizerManager, - _search_fields: &[String], - _with_validation: bool, + context: &BuildTantivyAstContext, ) -> Result { - let fields = self.find_field_and_subfields(schema); + let fields = self.find_field_and_subfields(context.schema); if fields.is_empty() { // the schema is not dynamic and no subfields are defined return Err(InvalidQuery::FieldDoesNotExist { @@ -159,9 +155,14 @@ impl BuildTantivyAst for FieldPresenceQuery { } // Some fields are not fast, need _field_presence field for fallback - let field_presence_field = schema.get_field(FIELD_PRESENCE_FIELD_NAME).map_err(|_| { - InvalidQuery::SchemaError("field presence is not available for this split".to_string()) - })?; + let field_presence_field = context + .schema + .get_field(FIELD_PRESENCE_FIELD_NAME) + .map_err(|_| { + InvalidQuery::SchemaError( + "field presence is not available for this split".to_string(), + ) + })?; let queries = fields .into_iter() .map(|(field, entry, path)| { diff --git a/quickwit/quickwit-query/src/query_ast/full_text_query.rs b/quickwit/quickwit-query/src/query_ast/full_text_query.rs index 1df3e9fd6a4..83c4ce5d7b3 100644 --- a/quickwit/quickwit-query/src/query_ast/full_text_query.rs +++ b/quickwit/quickwit-query/src/query_ast/full_text_query.rs @@ -27,7 +27,7 @@ use tantivy::tokenizer::{TextAnalyzer, TokenStream}; use crate::query_ast::tantivy_query_ast::{TantivyBoolQuery, TantivyQueryAst}; use crate::query_ast::utils::full_text_query; -use crate::query_ast::{BuildTantivyAst, QueryAst}; +use crate::query_ast::{BuildTantivyAst, BuildTantivyAstContext, QueryAst}; use crate::tokenizers::TokenizerManager; use crate::{BooleanOperand, InvalidQuery, MatchAllOrNone, find_field_or_hit_dynamic}; @@ -235,17 +235,14 @@ impl From for QueryAst { impl BuildTantivyAst for FullTextQuery { fn build_tantivy_ast_impl( &self, - schema: &TantivySchema, - tokenizer_manager: &TokenizerManager, - _search_fields: &[String], - _with_validation: bool, + context: &BuildTantivyAstContext, ) -> Result { full_text_query( &self.field, &self.text, &self.params, - schema, - tokenizer_manager, + context.schema, + context.tokenizer_manager, self.lenient, ) } @@ -304,11 +301,11 @@ impl FullTextQuery { #[cfg(test)] mod tests { - use tantivy::schema::{Schema, TEXT}; + use tantivy::schema::{DateOptions, DateTimePrecision, Schema, TEXT}; + use crate::BooleanOperand; use crate::query_ast::tantivy_query_ast::TantivyQueryAst; - use crate::query_ast::{BuildTantivyAst, FullTextMode, FullTextQuery}; - use crate::{BooleanOperand, create_default_quickwit_tokenizer_manager}; + use crate::query_ast::{BuildTantivyAst, BuildTantivyAstContext, FullTextMode, FullTextQuery}; #[test] fn test_zero_terms() { @@ -326,12 +323,7 @@ mod tests { schema_builder.add_text_field("body", TEXT); let schema = schema_builder.build(); let ast: TantivyQueryAst = full_text_query - .build_tantivy_ast_call( - &schema, - &create_default_quickwit_tokenizer_manager(), - &[], - true, - ) + .build_tantivy_ast_call(&BuildTantivyAstContext::for_test(&schema)) .unwrap(); assert_eq!(ast.const_predicate(), Some(crate::MatchAllOrNone::MatchAll)); } @@ -352,12 +344,7 @@ mod tests { schema_builder.add_text_field("body", TEXT); let schema = schema_builder.build(); let ast: TantivyQueryAst = full_text_query - .build_tantivy_ast_call( - &schema, - &create_default_quickwit_tokenizer_manager(), - &[], - true, - ) + .build_tantivy_ast_call(&BuildTantivyAstContext::for_test(&schema)) .unwrap(); let leaf = ast.as_leaf().unwrap(); assert_eq!( @@ -383,12 +370,7 @@ mod tests { schema_builder.add_text_field("body", TEXT); let schema = schema_builder.build(); let ast: TantivyQueryAst = full_text_query - .build_tantivy_ast_call( - &schema, - &create_default_quickwit_tokenizer_manager(), - &[], - true, - ) + .build_tantivy_ast_call(&BuildTantivyAstContext::for_test(&schema)) .unwrap(); let leaf = ast.as_leaf().unwrap(); assert_eq!( @@ -397,6 +379,59 @@ mod tests { ); } + #[test] + fn test_full_text_datetime() { + let full_text_query = FullTextQuery { + field: "ts".to_string(), + text: "2025-12-13T16:13:12.666777Z".to_string(), + params: super::FullTextParams { + tokenizer: Some("raw".to_string()), + mode: FullTextMode::Phrase { slop: 1 }, + zero_terms_query: crate::MatchAllOrNone::MatchAll, + }, + lenient: false, + }; + { + // indexed, we truncate to the second + let mut schema_builder = Schema::builder(); + schema_builder.add_date_field( + "ts", + DateOptions::default() + .set_precision(DateTimePrecision::Milliseconds) + .set_fast() + .set_indexed(), + ); + let schema = schema_builder.build(); + let ast: TantivyQueryAst = full_text_query + .build_tantivy_ast_call(&BuildTantivyAstContext::for_test(&schema)) + .unwrap(); + let leaf = ast.as_leaf().unwrap(); + assert_eq!( + &format!("{leaf:?}"), + r#"TermQuery(Term(field=0, type=Date, 2025-12-13T16:13:12Z))"# + ); + } + { + // not indexed, we truncate to fastfield precision + let mut schema_builder = Schema::builder(); + schema_builder.add_date_field( + "ts", + DateOptions::default() + .set_precision(DateTimePrecision::Milliseconds) + .set_fast(), + ); + let schema = schema_builder.build(); + let ast: TantivyQueryAst = full_text_query + .build_tantivy_ast_call(&BuildTantivyAstContext::for_test(&schema)) + .unwrap(); + let leaf = ast.as_leaf().unwrap(); + assert_eq!( + &format!("{leaf:?}"), + r#"TermQuery(Term(field=0, type=Date, 2025-12-13T16:13:12.666Z))"# + ); + } + } + #[test] fn test_full_text_bool_mode() { let full_text_query = FullTextQuery { @@ -413,12 +448,7 @@ mod tests { schema_builder.add_text_field("body", TEXT); let schema = schema_builder.build(); let ast: TantivyQueryAst = full_text_query - .build_tantivy_ast_call( - &schema, - &create_default_quickwit_tokenizer_manager(), - &[], - true, - ) + .build_tantivy_ast_call(&BuildTantivyAstContext::for_test(&schema)) .unwrap(); let bool_query = ast.as_bool_query().unwrap(); assert_eq!(bool_query.must.len(), 2); diff --git a/quickwit/quickwit-query/src/query_ast/mod.rs b/quickwit/quickwit-query/src/query_ast/mod.rs index c7487f1ed31..d51e36eb93a 100644 --- a/quickwit/quickwit-query/src/query_ast/mod.rs +++ b/quickwit/quickwit-query/src/query_ast/mod.rs @@ -19,6 +19,7 @@ use tantivy::schema::Schema as TantivySchema; use crate::tokenizers::TokenizerManager; mod bool_query; +mod cache_node; mod field_presence; mod full_text_query; mod phrase_prefix_query; @@ -33,6 +34,7 @@ mod visitor; mod wildcard_query; pub use bool_query::BoolQuery; +pub use cache_node::{CacheNode, HitSet, PredicateCache, PredicateCacheInjector}; pub use field_presence::FieldPresenceQuery; pub use full_text_query::{FullTextMode, FullTextParams, FullTextQuery}; pub use phrase_prefix_query::PhrasePrefixQuery; @@ -67,6 +69,7 @@ pub enum QueryAst { underlying: Box, boost: NotNaNf32, }, + Cache(CacheNode), } impl QueryAst { @@ -115,6 +118,30 @@ impl QueryAst { boost, }) } + QueryAst::Cache(cache_node) => { + let inner = cache_node.inner.parse_user_query(default_search_fields)?; + let uninitialized = + matches!(cache_node.state, cache_node::CacheState::Uninitialized); + debug_assert!( + uninitialized, + "QueryAst::parse_user_query called on initialized CacheNode, this is probably \ + a misstake" + ); + if !uninitialized { + tracing::warn!( + "QueryAst::parse_user_query called on initialized CacheNode, cache \ + discarded" + ); + } + Ok(CacheNode { + inner: Box::new(inner), + // inner got modified, the result is supposed to be equivalent, but to be safe, + // lets reinitialize the cache in practice this function + // shouldn't ever be called after cache was resolved + state: cache_node::CacheState::Uninitialized, + } + .into()) + } } } @@ -144,6 +171,36 @@ impl QueryAst { } } +/// Context used when building a tantivy ast. +pub struct BuildTantivyAstContext<'a> { + pub schema: &'a TantivySchema, + pub tokenizer_manager: &'a TokenizerManager, + pub search_fields: &'a [String], + pub with_validation: bool, +} + +impl<'a> BuildTantivyAstContext<'a> { + pub fn for_test(schema: &'a TantivySchema) -> Self { + use once_cell::sync::Lazy; + + // we do that to have a TokenizerManager with a long enough lifetime + static DEFAULT_TOKENIZER_MANAGER: Lazy = + Lazy::new(crate::create_default_quickwit_tokenizer_manager); + + BuildTantivyAstContext { + schema, + tokenizer_manager: &DEFAULT_TOKENIZER_MANAGER, + search_fields: &[], + with_validation: true, + } + } + + pub fn without_validation(mut self) -> Self { + self.with_validation = false; + self + } +} + trait BuildTantivyAst { /// Transforms a query Ast node into a TantivyQueryAst. /// @@ -151,23 +208,16 @@ trait BuildTantivyAst { /// It can call `into_tantivy_ast_call_me` but should never call `into_tantivy_ast_impl`. fn build_tantivy_ast_impl( &self, - schema: &TantivySchema, - tokenizer_manager: &TokenizerManager, - search_fields: &[String], - with_validation: bool, + context: &BuildTantivyAstContext, ) -> Result; /// This method is meant to be called, but should never be overloaded. fn build_tantivy_ast_call( &self, - schema: &TantivySchema, - tokenizer_manager: &TokenizerManager, - search_fields: &[String], - with_validation: bool, + context: &BuildTantivyAstContext, ) -> Result { - let tantivy_ast_res = - self.build_tantivy_ast_impl(schema, tokenizer_manager, search_fields, with_validation); - if !with_validation && tantivy_ast_res.is_err() { + let tantivy_ast_res = self.build_tantivy_ast_impl(context); + if !context.with_validation && tantivy_ast_res.is_err() { return match tantivy_ast_res { res @ Ok(_) | res @ Err(InvalidQuery::UserQueryNotParsed) => res, Err(_) => Ok(TantivyQueryAst::match_none()), @@ -180,80 +230,31 @@ trait BuildTantivyAst { impl BuildTantivyAst for QueryAst { fn build_tantivy_ast_impl( &self, - schema: &TantivySchema, - tokenizer_manager: &TokenizerManager, - search_fields: &[String], - with_validation: bool, + context: &BuildTantivyAstContext, ) -> Result { match self { - QueryAst::Bool(bool_query) => bool_query.build_tantivy_ast_call( - schema, - tokenizer_manager, - search_fields, - with_validation, - ), - QueryAst::Term(term_query) => term_query.build_tantivy_ast_call( - schema, - tokenizer_manager, - search_fields, - with_validation, - ), - QueryAst::Range(range_query) => range_query.build_tantivy_ast_call( - schema, - tokenizer_manager, - search_fields, - with_validation, - ), + QueryAst::Bool(bool_query) => bool_query.build_tantivy_ast_call(context), + QueryAst::Term(term_query) => term_query.build_tantivy_ast_call(context), + QueryAst::Range(range_query) => range_query.build_tantivy_ast_call(context), QueryAst::MatchAll => Ok(TantivyQueryAst::match_all()), QueryAst::MatchNone => Ok(TantivyQueryAst::match_none()), QueryAst::Boost { boost, underlying } => { - let underlying = underlying.build_tantivy_ast_call( - schema, - tokenizer_manager, - search_fields, - with_validation, - )?; + let underlying = underlying.build_tantivy_ast_call(context)?.simplify(); let boost_query = TantivyBoostQuery::new(underlying.into(), (*boost).into()); Ok(boost_query.into()) } - QueryAst::TermSet(term_set) => term_set.build_tantivy_ast_call( - schema, - tokenizer_manager, - search_fields, - with_validation, - ), - QueryAst::FullText(full_text_query) => full_text_query.build_tantivy_ast_call( - schema, - tokenizer_manager, - search_fields, - with_validation, - ), - QueryAst::PhrasePrefix(phrase_prefix_query) => phrase_prefix_query - .build_tantivy_ast_call(schema, tokenizer_manager, search_fields, with_validation), - QueryAst::UserInput(user_text_query) => user_text_query.build_tantivy_ast_call( - schema, - tokenizer_manager, - search_fields, - with_validation, - ), - QueryAst::FieldPresence(field_presence) => field_presence.build_tantivy_ast_call( - schema, - tokenizer_manager, - search_fields, - with_validation, - ), - QueryAst::Wildcard(wildcard) => wildcard.build_tantivy_ast_call( - schema, - tokenizer_manager, - search_fields, - with_validation, - ), - QueryAst::Regex(regex) => regex.build_tantivy_ast_call( - schema, - tokenizer_manager, - search_fields, - with_validation, - ), + QueryAst::TermSet(term_set) => term_set.build_tantivy_ast_call(context), + QueryAst::FullText(full_text_query) => full_text_query.build_tantivy_ast_call(context), + QueryAst::PhrasePrefix(phrase_prefix_query) => { + phrase_prefix_query.build_tantivy_ast_call(context) + } + QueryAst::UserInput(user_text_query) => user_text_query.build_tantivy_ast_call(context), + QueryAst::FieldPresence(field_presence) => { + field_presence.build_tantivy_ast_call(context) + } + QueryAst::Wildcard(wildcard) => wildcard.build_tantivy_ast_call(context), + QueryAst::Regex(regex) => regex.build_tantivy_ast_call(context), + QueryAst::Cache(cache_node) => cache_node.build_tantivy_ast_call(context), } } } @@ -261,13 +262,9 @@ impl BuildTantivyAst for QueryAst { impl QueryAst { pub fn build_tantivy_query( &self, - schema: &TantivySchema, - tokenizer_manager: &TokenizerManager, - search_fields: &[String], - with_validation: bool, + context: &BuildTantivyAstContext, ) -> Result, InvalidQuery> { - let tantivy_query_ast = - self.build_tantivy_ast_call(schema, tokenizer_manager, search_fields, with_validation)?; + let tantivy_query_ast = self.build_tantivy_ast_call(context)?; Ok(tantivy_query_ast.simplify().into()) } } @@ -332,9 +329,10 @@ pub fn query_ast_from_user_text(user_text: &str, default_fields: Option for QueryAst { impl BuildTantivyAst for PhrasePrefixQuery { fn build_tantivy_ast_impl( &self, - schema: &TantivySchema, - tokenizer_manager: &TokenizerManager, - _search_fields: &[String], - _with_validation: bool, + context: &BuildTantivyAstContext, ) -> Result { - let (_, terms) = match self.get_terms(schema, tokenizer_manager) { + let (_, terms) = match self.get_terms(context.schema, context.tokenizer_manager) { Ok(res) => res, Err(InvalidQuery::FieldDoesNotExist { .. }) if self.lenient => { return Ok(TantivyQueryAst::match_none()); diff --git a/quickwit/quickwit-query/src/query_ast/range_query.rs b/quickwit/quickwit-query/src/query_ast/range_query.rs index 3cacf697cf9..fbe03d62d73 100644 --- a/quickwit/quickwit-query/src/query_ast/range_query.rs +++ b/quickwit/quickwit-query/src/query_ast/range_query.rs @@ -17,16 +17,13 @@ use std::ops::Bound; use serde::{Deserialize, Serialize}; use tantivy::fastfield::FastValue; use tantivy::query::FastFieldRangeQuery; -use tantivy::schema::Schema as TantivySchema; use tantivy::tokenizer::TextAnalyzer; use tantivy::{DateTime, Term}; use super::QueryAst; use super::tantivy_query_ast::TantivyBoolQuery; use crate::json_literal::InterpretUserInput; -use crate::query_ast::BuildTantivyAst; -use crate::query_ast::tantivy_query_ast::TantivyQueryAst; -use crate::tokenizers::TokenizerManager; +use crate::query_ast::{BuildTantivyAst, BuildTantivyAstContext, TantivyQueryAst}; use crate::{InvalidQuery, JsonLiteral}; #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] @@ -38,7 +35,9 @@ pub struct RangeQuery { /// Converts a given bound JsonLiteral bound into a bound of type T. fn convert_bound<'a, T>(bound: &'a Bound) -> Option> -where T: InterpretUserInput<'a> { +where + T: InterpretUserInput<'a>, +{ match bound { Bound::Included(val) => { let val = T::interpret_json(val)?; @@ -110,17 +109,14 @@ fn get_normalized_text(normalizer: &mut Option, text: &str) -> Str impl BuildTantivyAst for RangeQuery { fn build_tantivy_ast_impl( &self, - schema: &TantivySchema, - tokenizer_manager: &TokenizerManager, - _search_fields: &[String], - _with_validation: bool, + context: &BuildTantivyAstContext, ) -> Result { let (field, field_entry, json_path) = - super::utils::find_field_or_hit_dynamic(&self.field, schema).ok_or_else(|| { - InvalidQuery::FieldDoesNotExist { + super::utils::find_field_or_hit_dynamic(&self.field, context.schema).ok_or_else( + || InvalidQuery::FieldDoesNotExist { full_path: self.field.clone(), - } - })?; + }, + )?; if !field_entry.is_fast() { return Err(InvalidQuery::SchemaError(format!( "range queries are only supported for fast fields. (`{}` is not a fast field)", @@ -129,9 +125,12 @@ impl BuildTantivyAst for RangeQuery { } Ok(match field_entry.field_type() { tantivy::schema::FieldType::Str(options) => { - let mut normalizer = options - .get_fast_field_tokenizer_name() - .and_then(|tokenizer_name| tokenizer_manager.get_normalizer(tokenizer_name)); + let mut normalizer = + options + .get_fast_field_tokenizer_name() + .and_then(|tokenizer_name| { + context.tokenizer_manager.get_normalizer(tokenizer_name) + }); let (lower_bound, upper_bound) = convert_bounds(&self.lower_bound, &self.upper_bound, field_entry.name())?; @@ -184,8 +183,8 @@ impl BuildTantivyAst for RangeQuery { convert_bounds(&self.lower_bound, &self.upper_bound, field_entry.name())?; let truncate_datetime = |date: &DateTime| date.truncate(date_options.get_precision()); - let lower_bound = map_bound(&lower_bound, truncate_datetime); - let upper_bound = map_bound(&upper_bound, truncate_datetime); + let lower_bound = lower_bound.as_ref().map(truncate_datetime); + let upper_bound = upper_bound.as_ref().map(truncate_datetime); FastFieldRangeQuery::new( lower_bound.map(|val| Term::from_field_date(field, val)), upper_bound.map(|val| Term::from_field_date(field, val)), @@ -219,10 +218,17 @@ impl BuildTantivyAst for RangeQuery { } else if let Some(range) = bounds_range_f64 { sub_queries.push(query_from_fast_val_range(&empty_term, range).into()); } - - let mut normalizer = options - .get_fast_field_tokenizer_name() - .and_then(|tokenizer_name| tokenizer_manager.get_normalizer(tokenizer_name)); + let bounds_range_date: Option<(Bound, Bound)> = + convert_bound(&self.lower_bound).zip(convert_bound(&self.upper_bound)); + if let Some(range) = bounds_range_date { + sub_queries.push(query_from_fast_val_range(&empty_term, range).into()); + } + let mut normalizer = + options + .get_fast_field_tokenizer_name() + .and_then(|tokenizer_name| { + context.tokenizer_manager.get_normalizer(tokenizer_name) + }); let bounds_range_str: Option<(Bound<&str>, Bound<&str>)> = convert_bound(&self.lower_bound).zip(convert_bound(&self.upper_bound)); @@ -273,14 +279,6 @@ impl BuildTantivyAst for RangeQuery { } } -fn map_bound(bound: &Bound, transform: impl Fn(&TFrom) -> TTo) -> Bound { - match bound { - Bound::Excluded(from_val) => Bound::Excluded(transform(from_val)), - Bound::Included(from_val) => Bound::Included(transform(from_val)), - Bound::Unbounded => Bound::Unbounded, - } -} - #[cfg(test)] mod tests { use std::ops::Bound; @@ -288,10 +286,8 @@ mod tests { use tantivy::schema::{DateOptions, DateTimePrecision, FAST, STORED, Schema, TEXT}; use super::RangeQuery; - use crate::query_ast::BuildTantivyAst; - use crate::{ - InvalidQuery, JsonLiteral, MatchAllOrNone, create_default_quickwit_tokenizer_manager, - }; + use crate::query_ast::{BuildTantivyAst, BuildTantivyAstContext}; + use crate::{InvalidQuery, JsonLiteral, MatchAllOrNone}; fn make_schema(dynamic_mode: bool) -> Schema { let mut schema_builder = Schema::builder(); @@ -323,12 +319,7 @@ mod tests { upper_bound: Bound::Included(upper_value), }; let tantivy_ast = range_query - .build_tantivy_ast_call( - &schema, - &create_default_quickwit_tokenizer_manager(), - &[], - true, - ) + .build_tantivy_ast_call(&BuildTantivyAstContext::for_test(&schema)) .unwrap() .simplify(); let leaf = tantivy_ast.as_leaf().unwrap(); @@ -371,12 +362,7 @@ mod tests { }; // with validation let invalid_query: InvalidQuery = range_query - .build_tantivy_ast_call( - &schema, - &create_default_quickwit_tokenizer_manager(), - &[], - true, - ) + .build_tantivy_ast_call(&BuildTantivyAstContext::for_test(&schema)) .unwrap_err(); assert!( matches!(invalid_query, InvalidQuery::FieldDoesNotExist { full_path } if full_path == "missing_field.toto") @@ -385,10 +371,7 @@ mod tests { assert_eq!( range_query .build_tantivy_ast_call( - &schema, - &create_default_quickwit_tokenizer_manager(), - &[], - false + &BuildTantivyAstContext::for_test(&schema).without_validation() ) .unwrap() .const_predicate(), @@ -405,12 +388,7 @@ mod tests { }; let schema = make_schema(true); let tantivy_ast = range_query - .build_tantivy_ast_call( - &schema, - &create_default_quickwit_tokenizer_manager(), - &[], - true, - ) + .build_tantivy_ast_call(&BuildTantivyAstContext::for_test(&schema)) .unwrap(); assert_eq!( format!("{tantivy_ast:?}"), @@ -424,6 +402,34 @@ mod tests { ); } + #[test] + fn test_range_dynamic_datetime() { + let range_query = RangeQuery { + field: "hello".to_string(), + lower_bound: Bound::Included(JsonLiteral::String( + "2020-12-09T16:09:53+00:00".to_string(), + )), + upper_bound: Bound::Included(JsonLiteral::String( + "2020-12-09T16:09:53+00:00".to_string(), + )), + }; + let schema = make_schema(true); + let tantivy_ast = range_query + .build_tantivy_ast_call(&BuildTantivyAstContext::for_test(&schema)) + .unwrap(); + assert_eq!( + format!("{tantivy_ast:?}"), + "Bool(TantivyBoolQuery { must: [], must_not: [], should: [Leaf(FastFieldRangeQuery { \ + bounds: BoundsRange { lower_bound: Included(Term(field=6, type=Json, path=hello, \ + type=Date, 2020-12-09T16:09:53Z)), upper_bound: Included(Term(field=6, type=Json, \ + path=hello, type=Date, 2020-12-09T16:09:53Z)) } }), Leaf(FastFieldRangeQuery { \ + bounds: BoundsRange { lower_bound: Included(Term(field=6, type=Json, path=hello, \ + type=Str, \"2020-12-09T16:09:53+00:00\")), upper_bound: Included(Term(field=6, \ + type=Json, path=hello, type=Str, \"2020-12-09T16:09:53+00:00\")) } })], filter: [], \ + minimum_should_match: None })" + ); + } + #[test] fn test_range_query_not_fast_field() { let range_query = RangeQuery { @@ -433,12 +439,7 @@ mod tests { }; let schema = make_schema(false); let err = range_query - .build_tantivy_ast_call( - &schema, - &create_default_quickwit_tokenizer_manager(), - &[], - true, - ) + .build_tantivy_ast_call(&BuildTantivyAstContext::for_test(&schema)) .unwrap_err(); assert!(matches!(err, InvalidQuery::SchemaError { .. })); } diff --git a/quickwit/quickwit-query/src/query_ast/regex_query.rs b/quickwit/quickwit-query/src/query_ast/regex_query.rs index 17a6419e3c6..0429c387600 100644 --- a/quickwit/quickwit-query/src/query_ast/regex_query.rs +++ b/quickwit/quickwit-query/src/query_ast/regex_query.rs @@ -20,9 +20,8 @@ use serde::{Deserialize, Serialize}; use tantivy::Term; use tantivy::schema::{Field, FieldType, Schema as TantivySchema}; -use super::{BuildTantivyAst, QueryAst}; +use super::{BuildTantivyAst, BuildTantivyAstContext, QueryAst}; use crate::query_ast::TantivyQueryAst; -use crate::tokenizers::TokenizerManager; use crate::{InvalidQuery, find_field_or_hit_dynamic}; /// A Regex query @@ -70,7 +69,7 @@ impl RegexQuery { )) })?; - Ok((field, None, self.regex.clone())) + Ok((field, None, self.regex.to_string())) } FieldType::JsonObject(json_options) => { json_options.get_text_indexing_options().ok_or_else(|| { @@ -91,7 +90,7 @@ impl RegexQuery { // We skip the 1st byte which is a marker to tell this is json. This isn't present // in the dictionary let byte_path_prefix = value.as_serialized()[1..].to_owned(); - Ok((field, Some(byte_path_prefix), self.regex.clone())) + Ok((field, Some(byte_path_prefix), self.regex.to_string())) } _ => Err(InvalidQuery::SchemaError( "trying to run a regex query on a non-text field".to_string(), @@ -103,12 +102,9 @@ impl RegexQuery { impl BuildTantivyAst for RegexQuery { fn build_tantivy_ast_impl( &self, - schema: &TantivySchema, - _tokenizer_manager: &TokenizerManager, - _search_fields: &[String], - _with_validation: bool, + context: &BuildTantivyAstContext, ) -> Result { - let (field, path, regex) = self.to_field_and_regex(schema)?; + let (field, path, regex) = self.to_field_and_regex(context.schema)?; let regex = tantivy_fst::Regex::new(®ex).context("failed to parse regex")?; let regex_automaton_with_path = JsonPathPrefix { prefix: path.unwrap_or_default(), @@ -237,7 +233,8 @@ mod prefix { } impl Query for AutomatonQuery - where A::State: Clone + where + A::State: Clone, { fn weight(&self, _enabled_scoring: EnableScoring<'_>) -> tantivy::Result> { Ok(Box::new(AutomatonWeight::::new( diff --git a/quickwit/quickwit-query/src/query_ast/tantivy_query_ast.rs b/quickwit/quickwit-query/src/query_ast/tantivy_query_ast.rs index 6f26ad28532..3d168e3faaa 100644 --- a/quickwit/quickwit-query/src/query_ast/tantivy_query_ast.rs +++ b/quickwit/quickwit-query/src/query_ast/tantivy_query_ast.rs @@ -322,13 +322,14 @@ impl TantivyBoolQuery { } else { let num_children = self.must.len() + self.should.len() + self.must_not.len() + self.filter.len(); - if num_children == 1 && self.minimum_should_match.is_none() { - if let Some(ast) = self.must.pop().or(self.should.pop()) { - return ast; - } - // We do not optimize a single filter clause for the moment. - // We do need a mechanism to make sure we keep the boost of 0. + if num_children == 1 + && self.minimum_should_match.is_none() + && let Some(ast) = self.must.pop().or(self.should.pop()) + { + return ast; } + // We do not optimize a single filter clause for the moment. + // We do need a mechanism to make sure we keep the boost of 0. } TantivyQueryAst::Bool(self) @@ -895,10 +896,10 @@ mod tests { } } - if let Some(minimum_should_match) = self.minimum_should_match { - if minimum_should_match > matching_should_count { - return None; - } + if let Some(minimum_should_match) = self.minimum_should_match + && minimum_should_match > matching_should_count + { + return None; } if self.must.len() + self.filter.len() > 0 { diff --git a/quickwit/quickwit-query/src/query_ast/term_query.rs b/quickwit/quickwit-query/src/query_ast/term_query.rs index 6a4c9511cf7..f0a732b6f8c 100644 --- a/quickwit/quickwit-query/src/query_ast/term_query.rs +++ b/quickwit/quickwit-query/src/query_ast/term_query.rs @@ -15,11 +15,9 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; -use tantivy::schema::Schema as TantivySchema; use super::{BuildTantivyAst, QueryAst}; -use crate::query_ast::{FullTextParams, TantivyQueryAst}; -use crate::tokenizers::TokenizerManager; +use crate::query_ast::{BuildTantivyAstContext, FullTextParams, TantivyQueryAst}; use crate::{BooleanOperand, InvalidQuery}; /// The TermQuery acts exactly like a FullTextQuery with @@ -49,10 +47,7 @@ impl TermQuery { impl BuildTantivyAst for TermQuery { fn build_tantivy_ast_impl( &self, - schema: &TantivySchema, - tokenizer_manager: &TokenizerManager, - _search_fields: &[String], - _with_validation: bool, + context: &BuildTantivyAstContext, ) -> Result { let full_text_params = FullTextParams { tokenizer: Some("raw".to_string()), @@ -64,8 +59,8 @@ impl BuildTantivyAst for TermQuery { &self.field, &self.value, &full_text_params, - schema, - tokenizer_manager, + context.schema, + context.tokenizer_manager, false, ) } @@ -123,8 +118,7 @@ impl From for HashMap { mod tests { use tantivy::schema::{INDEXED, Schema}; - use crate::create_default_quickwit_tokenizer_manager; - use crate::query_ast::{BuildTantivyAst, TermQuery}; + use crate::query_ast::{BuildTantivyAst, BuildTantivyAstContext, TermQuery}; #[test] fn test_term_query_with_ipaddr_ipv4() { @@ -136,12 +130,7 @@ mod tests { schema_builder.add_ip_addr_field("ip", INDEXED); let schema = schema_builder.build(); let tantivy_query_ast = term_query - .build_tantivy_ast_call( - &schema, - &create_default_quickwit_tokenizer_manager(), - &[], - true, - ) + .build_tantivy_ast_call(&BuildTantivyAstContext::for_test(&schema)) .unwrap(); let leaf = tantivy_query_ast.as_leaf().unwrap(); assert_eq!( @@ -160,12 +149,7 @@ mod tests { schema_builder.add_ip_addr_field("ip", INDEXED); let schema = schema_builder.build(); let tantivy_query_ast = term_query - .build_tantivy_ast_call( - &schema, - &create_default_quickwit_tokenizer_manager(), - &[], - true, - ) + .build_tantivy_ast_call(&BuildTantivyAstContext::for_test(&schema)) .unwrap(); let leaf = tantivy_query_ast.as_leaf().unwrap(); assert_eq!( @@ -184,12 +168,7 @@ mod tests { schema_builder.add_bytes_field("bytes", INDEXED); let schema = schema_builder.build(); let tantivy_query_ast = term_query - .build_tantivy_ast_call( - &schema, - &create_default_quickwit_tokenizer_manager(), - &[], - true, - ) + .build_tantivy_ast_call(&BuildTantivyAstContext::for_test(&schema)) .unwrap(); let leaf = tantivy_query_ast.as_leaf().unwrap(); assert_eq!( @@ -208,12 +187,7 @@ mod tests { schema_builder.add_bytes_field("bytes", INDEXED); let schema = schema_builder.build(); let tantivy_query_ast = term_query - .build_tantivy_ast_call( - &schema, - &create_default_quickwit_tokenizer_manager(), - &[], - true, - ) + .build_tantivy_ast_call(&BuildTantivyAstContext::for_test(&schema)) .unwrap(); let leaf = tantivy_query_ast.as_leaf().unwrap(); assert_eq!( @@ -232,12 +206,7 @@ mod tests { schema_builder.add_date_field("timestamp", INDEXED); let schema = schema_builder.build(); let tantivy_query_ast = term_query - .build_tantivy_ast_call( - &schema, - &create_default_quickwit_tokenizer_manager(), - &[], - true, - ) + .build_tantivy_ast_call(&BuildTantivyAstContext::for_test(&schema)) .unwrap(); let leaf = tantivy_query_ast.as_leaf().unwrap(); // The date should have been truncated to seconds precision. diff --git a/quickwit/quickwit-query/src/query_ast/term_set_query.rs b/quickwit/quickwit-query/src/query_ast/term_set_query.rs index c70d71cd94c..0605d854863 100644 --- a/quickwit/quickwit-query/src/query_ast/term_set_query.rs +++ b/quickwit/quickwit-query/src/query_ast/term_set_query.rs @@ -16,11 +16,11 @@ use std::collections::{BTreeSet, HashMap, HashSet}; use serde::{Deserialize, Serialize}; use tantivy::Term; -use tantivy::schema::Schema as TantivySchema; use crate::InvalidQuery; -use crate::query_ast::{BuildTantivyAst, QueryAst, TantivyQueryAst, TermQuery}; -use crate::tokenizers::TokenizerManager; +use crate::query_ast::{ + BoolQuery, BuildTantivyAst, BuildTantivyAstContext, QueryAst, TantivyQueryAst, TermQuery, +}; /// TermSetQuery matches the same document set as if it was a union of /// the equivalent set of TermQueries. @@ -32,12 +32,59 @@ pub struct TermSetQuery { } impl TermSetQuery { + fn has_fast_only_field(&self, context: &BuildTantivyAstContext) -> bool { + for full_path in self.terms_per_field.keys() { + if let Some((_, field_entry, _)) = + super::utils::find_field_or_hit_dynamic(full_path, context.schema) + && field_entry.is_fast() + && !field_entry.is_indexed() + { + return true; + } + } + false + } + + fn build_bool_query( + &self, + context: &BuildTantivyAstContext, + ) -> Result { + let should_clauses = self + .terms_per_field + .iter() + .flat_map(|(full_path, values)| { + values.iter().map(|value| { + QueryAst::Term(TermQuery { + field: full_path.to_string(), + value: value.to_string(), + }) + }) + }) + .collect(); + + let bool_query = BoolQuery { + should: should_clauses, + ..Default::default() + }; + + bool_query.build_tantivy_ast_impl(context) + } + + fn build_term_set_query( + &self, + context: &BuildTantivyAstContext, + ) -> Result { + let terms_it = self.make_term_iterator(context)?; + let term_set_query = tantivy::query::TermSetQuery::new(terms_it); + Ok(term_set_query.into()) + } + fn make_term_iterator( &self, - schema: &TantivySchema, - tokenizer_manager: &TokenizerManager, + context: &BuildTantivyAstContext, ) -> Result, InvalidQuery> { let mut terms: HashSet = HashSet::default(); + for (full_path, values) in &self.terms_per_field { for value in values { // Mapping a text (field, value) is non-trivial: @@ -51,8 +98,7 @@ impl TermSetQuery { field: full_path.to_string(), value: value.to_string(), }; - let ast = - term_query.build_tantivy_ast_call(schema, tokenizer_manager, &[], false)?; + let ast = term_query.build_tantivy_ast_call(context)?; let tantivy_query: Box = ast.simplify().into(); tantivy_query.query_terms(&mut |term, _| { terms.insert(term.clone()); @@ -66,14 +112,13 @@ impl TermSetQuery { impl BuildTantivyAst for TermSetQuery { fn build_tantivy_ast_impl( &self, - schema: &TantivySchema, - tokenizer_manager: &TokenizerManager, - _search_fields: &[String], - _with_validation: bool, + context: &BuildTantivyAstContext, ) -> Result { - let terms_it = self.make_term_iterator(schema, tokenizer_manager)?; - let term_set_query = tantivy::query::TermSetQuery::new(terms_it); - Ok(term_set_query.into()) + if self.has_fast_only_field(context) { + self.build_bool_query(context) + } else { + self.build_term_set_query(context) + } } } @@ -82,3 +127,67 @@ impl From for QueryAst { QueryAst::TermSet(term_set_query) } } + +#[cfg(test)] +mod tests { + use std::collections::{BTreeSet, HashMap}; + + use tantivy::schema::{FAST, INDEXED, Schema}; + + use super::TermSetQuery; + use crate::query_ast::{BuildTantivyAst, BuildTantivyAstContext}; + + #[test] + fn test_term_set_query_with_fast_only_field_returns_bool_query() { + let mut schema_builder = Schema::builder(); + schema_builder.add_u64_field("fast_field", FAST); + let schema = schema_builder.build(); + + let terms_per_field = HashMap::from([( + "fast_field".to_string(), + BTreeSet::from(["1".to_string(), "2".to_string()]), + )]); + let term_set_query = TermSetQuery { terms_per_field }; + + let tantivy_query_ast = term_set_query + .build_tantivy_ast_call(&BuildTantivyAstContext::for_test(&schema)) + .unwrap(); + + let bool_query = tantivy_query_ast + .as_bool_query() + .expect("Expected BoolQuery for fast-only field, but got a different query type"); + assert_eq!(bool_query.should.len(), 2); + assert_eq!(bool_query.must.len(), 0); + assert_eq!(bool_query.must_not.len(), 0); + assert_eq!(bool_query.filter.len(), 0); + } + + #[test] + fn test_term_set_query_with_indexed_field_uses_term_set() { + let mut schema_builder = Schema::builder(); + schema_builder.add_u64_field("indexed_field", FAST | INDEXED); + let schema = schema_builder.build(); + + let terms_per_field = HashMap::from([( + "indexed_field".to_string(), + BTreeSet::from(["1".to_string(), "2".to_string()]), + )]); + let term_set_query = TermSetQuery { terms_per_field }; + + let tantivy_query_ast = term_set_query + .build_tantivy_ast_call(&BuildTantivyAstContext::for_test(&schema)) + .unwrap(); + + // Should return a leaf query (TermSetQuery wrapped in TantivyQueryAst) + let leaf = tantivy_query_ast + .as_leaf() + .expect("Expected a leaf query (TermSetQuery), but got a complex query"); + + // Verify it's a TermSetQuery by checking the debug representation + let debug_str = format!("{leaf:?}"); + assert!( + debug_str.contains("TermSetQuery"), + "Expected TermSetQuery, got: {debug_str}" + ); + } +} diff --git a/quickwit/quickwit-query/src/query_ast/user_input_query.rs b/quickwit/quickwit-query/src/query_ast/user_input_query.rs index 542a3f45d9d..8f83461932e 100644 --- a/quickwit/quickwit-query/src/query_ast/user_input_query.rs +++ b/quickwit/quickwit-query/src/query_ast/user_input_query.rs @@ -20,14 +20,12 @@ use serde::{Deserialize, Serialize}; use tantivy::query_grammar::{ Delimiter, Occur, UserInputAst, UserInputBound, UserInputLeaf, UserInputLiteral, }; -use tantivy::schema::Schema as TantivySchema; use crate::not_nan_f32::NotNaNf32; -use crate::query_ast::tantivy_query_ast::TantivyQueryAst; use crate::query_ast::{ - self, BuildTantivyAst, FieldPresenceQuery, FullTextMode, FullTextParams, QueryAst, + self, BuildTantivyAst, BuildTantivyAstContext, FieldPresenceQuery, FullTextMode, + FullTextParams, QueryAst, TantivyQueryAst, }; -use crate::tokenizers::TokenizerManager; use crate::{BooleanOperand, InvalidQuery, JsonLiteral}; const DEFAULT_PHRASE_QUERY_MAX_EXPANSION: u32 = 50; @@ -88,10 +86,7 @@ impl From for QueryAst { impl BuildTantivyAst for UserInputQuery { fn build_tantivy_ast_impl( &self, - _schema: &TantivySchema, - _tokenizer_manager: &TokenizerManager, - _default_search_fields: &[String], - _with_validation: bool, + _context: &BuildTantivyAstContext, ) -> Result { Err(InvalidQuery::UserQueryNotParsed) } @@ -178,6 +173,22 @@ fn convert_user_input_ast_to_query_ast( Ok(term_set_query.into()) } UserInputLeaf::Exists { field } => Ok(FieldPresenceQuery { field }.into()), + UserInputLeaf::Regex { field, pattern } => { + let field = if let Some(field) = field { + field + } else if default_search_fields.len() == 1 { + default_search_fields[0].clone() + } else if default_search_fields.is_empty() { + bail!("regex query without field is not supported"); + } else { + bail!("regex query with multiple fields is not supported"); + }; + let regex_query = query_ast::RegexQuery { + field, + regex: pattern, + }; + Ok(regex_query.into()) + } }, UserInputAst::Boost(underlying, boost) => { let query_ast = convert_user_input_ast_to_query_ast( @@ -186,7 +197,7 @@ fn convert_user_input_ast_to_query_ast( default_search_fields, lenient, )?; - let boost: NotNaNf32 = (boost as f32) + let boost: NotNaNf32 = (boost.into_inner() as f32) .try_into() .map_err(|err_msg: &str| anyhow::anyhow!(err_msg))?; Ok(QueryAst::Boost { @@ -277,6 +288,7 @@ fn convert_user_input_literal( field: field_name, value: phrase.clone(), lenient, + case_insensitive: false, } .into() } else { @@ -306,9 +318,10 @@ fn convert_user_input_literal( #[cfg(test)] mod tests { use crate::query_ast::{ - BoolQuery, BuildTantivyAst, FullTextMode, FullTextQuery, QueryAst, UserInputQuery, + BoolQuery, BuildTantivyAst, BuildTantivyAstContext, FullTextMode, FullTextQuery, QueryAst, + UserInputQuery, }; - use crate::{BooleanOperand, InvalidQuery, create_default_quickwit_tokenizer_manager}; + use crate::{BooleanOperand, InvalidQuery}; #[test] fn test_user_input_query_not_parsed_error() { @@ -321,23 +334,13 @@ mod tests { let schema = tantivy::schema::Schema::builder().build(); { let invalid_query = user_input_query - .build_tantivy_ast_call( - &schema, - &create_default_quickwit_tokenizer_manager(), - &[], - true, - ) + .build_tantivy_ast_call(&BuildTantivyAstContext::for_test(&schema)) .unwrap_err(); assert!(matches!(invalid_query, InvalidQuery::UserQueryNotParsed)); } { let invalid_query = user_input_query - .build_tantivy_ast_call( - &schema, - &create_default_quickwit_tokenizer_manager(), - &[], - false, - ) + .build_tantivy_ast_call(&BuildTantivyAstContext::for_test(&schema)) .unwrap_err(); assert!(matches!(invalid_query, InvalidQuery::UserQueryNotParsed)); } @@ -535,4 +538,21 @@ mod tests { ); } } + + #[test] + fn test_user_input_query_regex() { + let ast = UserInputQuery { + user_text: "field: /.*/".to_string(), + default_fields: None, + default_operator: BooleanOperand::And, + lenient: false, + } + .parse_user_query(&[]) + .unwrap(); + let QueryAst::Regex(regex_query) = ast else { + panic!() + }; + assert_eq!(®ex_query.field, "field"); + assert_eq!(®ex_query.regex, ".*"); + } } diff --git a/quickwit/quickwit-query/src/query_ast/utils.rs b/quickwit/quickwit-query/src/query_ast/utils.rs index b322e51c520..71f3d4e5303 100644 --- a/quickwit/quickwit-query/src/query_ast/utils.rs +++ b/quickwit/quickwit-query/src/query_ast/utils.rs @@ -17,7 +17,7 @@ use tantivy::json_utils::convert_to_fast_value_and_append_to_json_term; use tantivy::query::TermQuery as TantivyTermQuery; use tantivy::schema::{ Field, FieldEntry, FieldType, IndexRecordOption, JsonObjectOptions, Schema as TantivySchema, - Type, + TextFieldIndexing, Type, }; use crate::InvalidQuery; @@ -25,7 +25,7 @@ use crate::MatchAllOrNone::MatchNone as TantivyEmptyQuery; use crate::json_literal::InterpretUserInput; use crate::query_ast::full_text_query::FullTextParams; use crate::query_ast::tantivy_query_ast::{TantivyBoolQuery, TantivyQueryAst}; -use crate::tokenizers::TokenizerManager; +use crate::tokenizers::{RAW_TOKENIZER_NAME, TokenizerManager}; pub(crate) const DYNAMIC_FIELD_NAME: &str = "_dynamic"; @@ -141,18 +141,28 @@ fn compute_query_with_field( let term = Term::from_field_bool(field, bool_val); Ok(make_term_query(term)) } - FieldType::Date(_) => { + FieldType::Date(date_options) => { let dt = parse_value_from_user_text(value, field_entry.name())?; - let term = Term::from_field_date_for_search(field, dt); + let term = if date_options.is_indexed() { + Term::from_field_date_for_search(field, dt) + } else { + Term::from_field_date(field, dt.truncate(date_options.get_precision())) + }; Ok(make_term_query(term)) } FieldType::Str(text_options) => { - let text_field_indexing = text_options.get_indexing_options().ok_or_else(|| { - InvalidQuery::SchemaError(format!( - "field {} is not full-text searchable", - field_entry.name() - )) - })?; + let columnar_opt = TextFieldIndexing::default() + .set_fieldnorms(false) + .set_tokenizer(RAW_TOKENIZER_NAME); + let text_field_indexing = text_options + .get_indexing_options() + .or_else(|| text_options.is_fast().then_some(&columnar_opt)) + .ok_or_else(|| { + InvalidQuery::SchemaError(format!( + "field {} is not full-text searchable", + field_entry.name() + )) + })?; let terms = full_text_params.tokenize_text_into_terms( field, value, @@ -195,7 +205,7 @@ fn compute_tantivy_ast_query_for_json( ) -> Result { let mut bool_query = TantivyBoolQuery::default(); let term = Term::from_field_json_path(field, json_path, json_options.is_expand_dots_enabled()); - if let Some(term) = convert_to_fast_value_and_append_to_json_term(term, text, true) { + if let Some(term) = convert_to_fast_value_and_append_to_json_term(&term, text, true) { bool_query .should .push(TantivyTermQuery::new(term, IndexRecordOption::Basic).into()); diff --git a/quickwit/quickwit-query/src/query_ast/visitor.rs b/quickwit/quickwit-query/src/query_ast/visitor.rs index b94893fb54d..a6d2a5c07d4 100644 --- a/quickwit/quickwit-query/src/query_ast/visitor.rs +++ b/quickwit/quickwit-query/src/query_ast/visitor.rs @@ -13,11 +13,12 @@ // limitations under the License. use crate::not_nan_f32::NotNaNf32; +use crate::query_ast::cache_node::CacheState; use crate::query_ast::field_presence::FieldPresenceQuery; use crate::query_ast::user_input_query::UserInputQuery; use crate::query_ast::{ - BoolQuery, FullTextQuery, PhrasePrefixQuery, QueryAst, RangeQuery, RegexQuery, TermQuery, - TermSetQuery, WildcardQuery, + BoolQuery, CacheNode, FullTextQuery, PhrasePrefixQuery, QueryAst, RangeQuery, RegexQuery, + TermQuery, TermSetQuery, WildcardQuery, }; /// Simple trait to implement a Visitor over the QueryAst. @@ -41,6 +42,7 @@ pub trait QueryAstVisitor<'a> { QueryAst::FieldPresence(exists) => self.visit_exists(exists), QueryAst::Wildcard(wildcard) => self.visit_wildcard(wildcard), QueryAst::Regex(regex) => self.visit_regex(regex), + QueryAst::Cache(cache_node) => self.visit_cache_node(cache_node), } } @@ -111,6 +113,19 @@ pub trait QueryAstVisitor<'a> { fn visit_regex(&mut self, _regex_query: &'a RegexQuery) -> Result<(), Self::Err> { Ok(()) } + + fn visit_cache_node(&mut self, cache_node: &'a CacheNode) -> Result<(), Self::Err> { + // this goes a bit again how the rest of the default Visitor behave. The rational is that in + // practice, on a cache hit, we don't want to do anything with that node. + // On unitialized cache, any kind of data extract could make sense (extracing tags or + // timestamp bounds) On cache miss, we still want to know what we need for warmup. + // But on cache hit, it's too late to do optimisation based on tags and timestamps, and we + // don't want to warmup anything. + if !matches!(cache_node.state, CacheState::CacheHit(_)) { + self.visit(&cache_node.inner)? + } + Ok(()) + } } /// Simple trait to implement a Visitor over the QueryAst. @@ -134,6 +149,7 @@ pub trait QueryAstTransformer { QueryAst::FieldPresence(exists) => self.transform_exists(exists), QueryAst::Wildcard(wildcard) => self.transform_wildcard(wildcard), QueryAst::Regex(regex) => self.transform_regex(regex), + QueryAst::Cache(cache_node) => self.transform_cache_node(cache_node), } } @@ -236,4 +252,21 @@ pub trait QueryAstTransformer { fn transform_regex(&mut self, regex_query: RegexQuery) -> Result, Self::Err> { Ok(Some(QueryAst::Regex(regex_query))) } + + fn transform_cache_node( + &mut self, + cache_node: CacheNode, + ) -> Result, Self::Err> { + if matches!(cache_node.state, CacheState::CacheHit(_)) { + return Ok(Some(cache_node.into())); + } + self.transform(*cache_node.inner).map(|maybe_ast| { + maybe_ast.map(|inner| { + QueryAst::Cache(CacheNode { + inner: Box::new(inner), + state: Default::default(), + }) + }) + }) + } } diff --git a/quickwit/quickwit-query/src/query_ast/wildcard_query.rs b/quickwit/quickwit-query/src/query_ast/wildcard_query.rs index 74e3acf0bc5..7b24a66163d 100644 --- a/quickwit/quickwit-query/src/query_ast/wildcard_query.rs +++ b/quickwit/quickwit-query/src/query_ast/wildcard_query.rs @@ -21,7 +21,7 @@ use tantivy::Term; use tantivy::schema::{Field, FieldType, Schema as TantivySchema}; use super::{BuildTantivyAst, QueryAst}; -use crate::query_ast::{AutomatonQuery, JsonPathPrefix, TantivyQueryAst}; +use crate::query_ast::{AutomatonQuery, BuildTantivyAstContext, JsonPathPrefix, TantivyQueryAst}; use crate::tokenizers::TokenizerManager; use crate::{InvalidQuery, find_field_or_hit_dynamic}; @@ -32,6 +32,7 @@ pub struct WildcardQuery { pub value: String, /// Support missing fields pub lenient: bool, + pub case_insensitive: bool, } impl From for QueryAst { @@ -133,6 +134,11 @@ impl WildcardQuery { let tokenizer_name = text_field_indexing.tokenizer(); let regex = sub_query_parts_to_regex(sub_query_parts, tokenizer_name, tokenizer_manager)?; + let regex = if self.case_insensitive { + format!("(?i){}", regex) + } else { + regex + }; Ok((field, None, regex)) } @@ -147,6 +153,11 @@ impl WildcardQuery { let tokenizer_name = text_field_indexing.tokenizer(); let regex = sub_query_parts_to_regex(sub_query_parts, tokenizer_name, tokenizer_manager)?; + let regex = if self.case_insensitive { + format!("(?i){}", regex) + } else { + regex + }; let mut term_for_path = Term::from_field_json_path( field, @@ -172,12 +183,9 @@ impl WildcardQuery { impl BuildTantivyAst for WildcardQuery { fn build_tantivy_ast_impl( &self, - schema: &TantivySchema, - tokenizer_manager: &TokenizerManager, - _search_fields: &[String], - _with_validation: bool, + context: &BuildTantivyAstContext, ) -> Result { - let (field, path, regex) = match self.to_regex(schema, tokenizer_manager) { + let (field, path, regex) = match self.to_regex(context.schema, context.tokenizer_manager) { Ok(res) => res, Err(InvalidQuery::FieldDoesNotExist { .. }) if self.lenient => { return Ok(TantivyQueryAst::match_none()); @@ -219,6 +227,7 @@ mod tests { field: "text_field".to_string(), value: "MyString Wh1ch?a.nOrMal Tokenizer would*cut".to_string(), lenient: false, + case_insensitive: false, }; let tokenizer_manager = create_default_quickwit_tokenizer_manager(); @@ -238,7 +247,6 @@ mod tests { "raw_lowercase", "lowercase", "default", - "en_stem", "chinese_compatible", "source_code_default", "source_code_with_hex", @@ -255,6 +263,48 @@ mod tests { } } + #[test] + fn test_wildcard_query_to_regex_on_escaped_text() { + let query = WildcardQuery { + field: "text_field".to_string(), + value: "MyString Wh1ch\\?a.nOrMal Tokenizer would\\*cut".to_string(), + lenient: false, + case_insensitive: false, + }; + + let tokenizer_manager = create_default_quickwit_tokenizer_manager(); + for tokenizer in ["raw", "whitespace"] { + let mut schema_builder = TantivySchema::builder(); + let text_options = TextOptions::default() + .set_indexing_options(TextFieldIndexing::default().set_tokenizer(tokenizer)); + schema_builder.add_text_field("text_field", text_options); + let schema = schema_builder.build(); + + let (_field, path, regex) = query.to_regex(&schema, &tokenizer_manager).unwrap(); + assert_eq!(regex, "MyString Wh1ch\\?a\\.nOrMal Tokenizer would\\*cut"); + assert!(path.is_none()); + } + + for tokenizer in [ + "raw_lowercase", + "lowercase", + "default", + "chinese_compatible", + "source_code_default", + "source_code_with_hex", + ] { + let mut schema_builder = TantivySchema::builder(); + let text_options = TextOptions::default() + .set_indexing_options(TextFieldIndexing::default().set_tokenizer(tokenizer)); + schema_builder.add_text_field("text_field", text_options); + let schema = schema_builder.build(); + + let (_field, path, regex) = query.to_regex(&schema, &tokenizer_manager).unwrap(); + assert_eq!(regex, "mystring wh1ch\\?a\\.normal tokenizer would\\*cut"); + assert!(path.is_none()); + } + } + #[test] fn test_wildcard_query_to_regex_on_json() { let query = WildcardQuery { @@ -263,6 +313,7 @@ mod tests { field: "json_field.Inner.Fie*ld".to_string(), value: "MyString Wh1ch?a.nOrMal Tokenizer would*cut".to_string(), lenient: false, + case_insensitive: false, }; let tokenizer_manager = create_default_quickwit_tokenizer_manager(); @@ -282,7 +333,6 @@ mod tests { "raw_lowercase", "lowercase", "default", - "en_stem", "chinese_compatible", "source_code_default", "source_code_with_hex", @@ -305,6 +355,7 @@ mod tests { field: "my_missing_field".to_string(), value: "My query value*".to_string(), lenient: false, + case_insensitive: false, }; let tokenizer_manager = create_default_quickwit_tokenizer_manager(); let schema = single_text_field_schema("my_field", "whitespace"); @@ -317,4 +368,46 @@ mod tests { }; assert_eq!(missing_field_full_path, "my_missing_field"); } + + #[test] + fn test_wildcard_query_to_regex_on_text_case_insensitive() { + let query = WildcardQuery { + field: "text_field".to_string(), + value: "MyString Wh1ch?a.nOrMal Tokenizer would*cut".to_string(), + lenient: false, + case_insensitive: true, + }; + + let tokenizer_manager = create_default_quickwit_tokenizer_manager(); + for tokenizer in ["raw", "whitespace"] { + let mut schema_builder = TantivySchema::builder(); + let text_options = TextOptions::default() + .set_indexing_options(TextFieldIndexing::default().set_tokenizer(tokenizer)); + schema_builder.add_text_field("text_field", text_options); + let schema = schema_builder.build(); + + let (_field, path, regex) = query.to_regex(&schema, &tokenizer_manager).unwrap(); + assert_eq!(regex, "(?i)MyString Wh1ch.a\\.nOrMal Tokenizer would.*cut"); + assert!(path.is_none()); + } + + for tokenizer in [ + "raw_lowercase", + "lowercase", + "default", + "chinese_compatible", + "source_code_default", + "source_code_with_hex", + ] { + let mut schema_builder = TantivySchema::builder(); + let text_options = TextOptions::default() + .set_indexing_options(TextFieldIndexing::default().set_tokenizer(tokenizer)); + schema_builder.add_text_field("text_field", text_options); + let schema = schema_builder.build(); + + let (_field, path, regex) = query.to_regex(&schema, &tokenizer_manager).unwrap(); + assert_eq!(regex, "(?i)mystring wh1ch.a\\.normal tokenizer would.*cut"); + assert!(path.is_none()); + } + } } diff --git a/quickwit/quickwit-query/src/tokenizers/code_tokenizer.rs b/quickwit/quickwit-query/src/tokenizers/code_tokenizer.rs index 0e33a926d5e..57dfbc57037 100644 --- a/quickwit/quickwit-query/src/tokenizers/code_tokenizer.rs +++ b/quickwit/quickwit-query/src/tokenizers/code_tokenizer.rs @@ -313,9 +313,9 @@ fn get_char_type(c: char) -> CharType { CharType::LowerCase } } else if c.is_numeric() { - return CharType::Numeric; + CharType::Numeric } else { - return CharType::Delimiter; + CharType::Delimiter } } @@ -377,7 +377,7 @@ impl ProcessingHexState { // end of sequence, check if size is multiple of 2, or try to generate code // state. We use next_char_offset as it already takes into account the size of // the last character - if (next_char_offset - self.start_offset) % 2 == 0 { + if (next_char_offset - self.start_offset).is_multiple_of(2) { return HexResult::Emit(self.start_offset..next_char_offset); } } @@ -411,7 +411,7 @@ impl ProcessingHexState { fn finalize(&self) -> HexResult { let next_char_offset = self.current_char_offset + self.current_char.len_utf8(); - if (next_char_offset - self.start_offset) % 2 == 0 { + if (next_char_offset - self.start_offset).is_multiple_of(2) { return HexResult::Emit(self.start_offset..next_char_offset); } self.to_processing_chars_state() diff --git a/quickwit/quickwit-query/src/tokenizers/mod.rs b/quickwit/quickwit-query/src/tokenizers/mod.rs index d3763238d9d..5a90715075e 100644 --- a/quickwit/quickwit-query/src/tokenizers/mod.rs +++ b/quickwit/quickwit-query/src/tokenizers/mod.rs @@ -14,21 +14,17 @@ mod chinese_compatible; mod code_tokenizer; -#[cfg(feature = "multilang")] -mod multilang; mod tokenizer_manager; use once_cell::sync::Lazy; use tantivy::tokenizer::{ - AsciiFoldingFilter, Language, LowerCaser, RawTokenizer, RemoveLongFilter, SimpleTokenizer, - Stemmer, TextAnalyzer, WhitespaceTokenizer, + AsciiFoldingFilter, LowerCaser, RawTokenizer, RemoveLongFilter, SimpleTokenizer, TextAnalyzer, + WhitespaceTokenizer, }; use self::chinese_compatible::ChineseTokenizer; pub use self::code_tokenizer::CodeTokenizer; -#[cfg(feature = "multilang")] -pub use self::multilang::MultiLangTokenizer; -pub use self::tokenizer_manager::TokenizerManager; +pub use self::tokenizer_manager::{RAW_TOKENIZER_NAME, TokenizerManager}; pub const DEFAULT_REMOVE_TOKEN_LENGTH: usize = 255; @@ -58,14 +54,6 @@ pub fn create_default_quickwit_tokenizer_manager() -> TokenizerManager { .filter(LowerCaser) .build(); tokenizer_manager.register("default", default_tokenizer, true); - - let en_stem_tokenizer = TextAnalyzer::builder(SimpleTokenizer::default()) - .filter(RemoveLongFilter::limit(DEFAULT_REMOVE_TOKEN_LENGTH)) - .filter(LowerCaser) - .filter(Stemmer::new(Language::English)) - .build(); - tokenizer_manager.register("en_stem", en_stem_tokenizer, true); - tokenizer_manager.register("whitespace", WhitespaceTokenizer::default(), false); let chinese_tokenizer = TextAnalyzer::builder(ChineseTokenizer) @@ -91,15 +79,6 @@ pub fn create_default_quickwit_tokenizer_manager() -> TokenizerManager { .build(), true, ); - #[cfg(feature = "multilang")] - tokenizer_manager.register( - "multilang_default", - TextAnalyzer::builder(MultiLangTokenizer::default()) - .filter(RemoveLongFilter::limit(DEFAULT_REMOVE_TOKEN_LENGTH)) - .filter(LowerCaser) - .build(), - true, - ); tokenizer_manager } diff --git a/quickwit/quickwit-query/src/tokenizers/multilang.rs b/quickwit/quickwit-query/src/tokenizers/multilang.rs deleted file mode 100644 index 4c1d8446eb8..00000000000 --- a/quickwit/quickwit-query/src/tokenizers/multilang.rs +++ /dev/null @@ -1,334 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use lindera_core::mode::Mode; -use lindera_dictionary::{DictionaryConfig, DictionaryKind, load_dictionary_from_config}; -use lindera_tokenizer::token::Token as LinderaToken; -use lindera_tokenizer::tokenizer::Tokenizer as LinderaTokenizer; -use once_cell::sync::Lazy; -use tantivy::tokenizer::{SimpleTokenStream, SimpleTokenizer, Token, TokenStream, Tokenizer}; -use whichlang::{Lang, detect_language}; - -// Note(fmassot): we use `lindera_tokenizer::tokenizer::Tokenizer` and not -// `use lindera_tantivy::tokenizer::LinderaTokenizer` to avoid -// costly copy of lindera dictionaries each time we clone the `MultiLangTokenizer`. - -/// Mandarin chinese tokenizer. -static CMN_TOKENIZER: Lazy = Lazy::new(|| { - let cmn_dictionary_config = DictionaryConfig { - kind: Some(DictionaryKind::CcCedict), - path: None, - }; - let cmn_dictionary = load_dictionary_from_config(cmn_dictionary_config) - .expect("Lindera `CcCedict` dictionary must be present"); - LinderaTokenizer::new(cmn_dictionary, None, Mode::Normal) -}); - -/// Japanese tokenizer. -static JPN_TOKENIZER: Lazy = Lazy::new(|| { - let jpn_dictionary_config = DictionaryConfig { - kind: Some(DictionaryKind::IPADIC), - path: None, - }; - let jpn_dictionary = load_dictionary_from_config(jpn_dictionary_config) - .expect("Lindera `IPAD` dictionary must be present"); - LinderaTokenizer::new(jpn_dictionary, None, Mode::Normal) -}); - -/// Korean tokenizer. -static KOR_TOKENIZER: Lazy = Lazy::new(|| { - let kor_dictionary_config = DictionaryConfig { - kind: Some(DictionaryKind::KoDic), - path: None, - }; - let kor_dictionary = load_dictionary_from_config(kor_dictionary_config) - .expect("Lindera `KoDic` dictionary must be present"); - LinderaTokenizer::new(kor_dictionary, None, Mode::Normal) -}); - -/// Multilanguage tokenizer that uses the `whichlang` to detect the language of the text -/// and uses the appropriate tokenizer for the detected language: -/// - lindera for Chinese, Japanese, and Korean. -/// - Quickwit's default tokenizer for other languages. -/// -/// It is possible to bypass the language detection by prefixing the text with the language code -/// followed by a colon. For example, `KOR:일본입니다` will be tokenized by the korean tokenizer. -/// Current supported prefix are: -/// - `KOR:` for Korean tokenizer -/// - `JPN:` for Japanese tokenizer -/// - `CMN:` for Chinese tokenizer -/// - `ENG:` for Quickwit's default tokenizer -#[derive(Clone, Default)] -pub struct MultiLangTokenizer { - default_tokenizer: SimpleTokenizer, - token: Token, -} - -impl Tokenizer for MultiLangTokenizer { - type TokenStream<'a> = MultiLanguageTokenStream<'a>; - fn token_stream<'a>(&'a mut self, text: &'a str) -> MultiLanguageTokenStream<'a> { - self.token.reset(); - let (language_prefix, text_to_tokenize) = get_language_from_prefix(text); - // If the text is empty, we return an empty token stream. - // `whichlang::detect_language` panicks if the text is empty. - if text.trim().is_empty() { - return MultiLanguageTokenStream::Empty; - } - let language = language_prefix.unwrap_or_else(|| detect_language(text_to_tokenize)); - match language { - Lang::Cmn => { - let lindera_token_stream = LinderaTokenStream { - tokens: CMN_TOKENIZER - .tokenize(text_to_tokenize) - .expect("tokenize method should never fail"), - token: &mut self.token, - }; - MultiLanguageTokenStream::Lindera(lindera_token_stream) - } - Lang::Jpn => { - let lindera_token_stream = LinderaTokenStream { - tokens: JPN_TOKENIZER - .tokenize(text_to_tokenize) - .expect("tokenize method should never fail"), - token: &mut self.token, - }; - MultiLanguageTokenStream::Lindera(lindera_token_stream) - } - Lang::Kor => { - let lindera_token_stream = LinderaTokenStream { - tokens: KOR_TOKENIZER - .tokenize(text_to_tokenize) - .expect("tokenize method should never fail"), - token: &mut self.token, - }; - MultiLanguageTokenStream::Lindera(lindera_token_stream) - } - _ => MultiLanguageTokenStream::Simple( - self.default_tokenizer.token_stream(text_to_tokenize), - ), - } - } -} - -/// Gets the language defined by a prefix `{ID}:text` where ID being the 3-letter language used by -/// whichlang) and returns the language and the text without the prefix. If the prefix is not -/// recognized, the language is `None` and the text is the original. -fn get_language_from_prefix(text: &str) -> (Option, &str) { - let prefix_bytes = &text.as_bytes()[0..std::cmp::min(4, text.len())]; - // TODO: refactor. - let prefix_language = match prefix_bytes { - b"CMN:" => Some(Lang::Cmn), - b"ENG:" => Some(Lang::Eng), - b"JPN:" => Some(Lang::Jpn), - b"KOR:" => Some(Lang::Kor), - _ => None, - }; - let text_without_prefix = if prefix_language.is_some() { - // This is safe as we know that the prefix is made of 4 ascii characters. - &text[4..] - } else { - text - }; - (prefix_language, text_without_prefix) -} -pub enum MultiLanguageTokenStream<'a> { - Empty, - Lindera(LinderaTokenStream<'a>), - Simple(SimpleTokenStream<'a>), -} - -impl TokenStream for MultiLanguageTokenStream<'_> { - fn advance(&mut self) -> bool { - match self { - MultiLanguageTokenStream::Empty => false, - MultiLanguageTokenStream::Lindera(tokenizer) => tokenizer.advance(), - MultiLanguageTokenStream::Simple(tokenizer) => tokenizer.advance(), - } - } - - fn token(&self) -> &Token { - match self { - MultiLanguageTokenStream::Empty => { - panic!("Cannot call token() on an empty token stream.") - } - MultiLanguageTokenStream::Lindera(tokenizer) => tokenizer.token(), - MultiLanguageTokenStream::Simple(tokenizer) => tokenizer.token(), - } - } - - fn token_mut(&mut self) -> &mut Token { - match self { - MultiLanguageTokenStream::Empty => { - panic!("Cannot call token_mut() on an empty token stream.") - } - MultiLanguageTokenStream::Lindera(tokenizer) => tokenizer.token_mut(), - MultiLanguageTokenStream::Simple(tokenizer) => tokenizer.token_mut(), - } - } -} - -pub struct LinderaTokenStream<'a> { - pub tokens: Vec>, - pub token: &'a mut Token, -} - -impl TokenStream for LinderaTokenStream<'_> { - fn advance(&mut self) -> bool { - if self.tokens.is_empty() { - return false; - } - let token = self.tokens.remove(0); - self.token.text = token.text.to_string(); - self.token.offset_from = token.byte_start; - self.token.offset_to = token.byte_end; - self.token.position = token.position; - self.token.position_length = token.position_length; - - true - } - - fn token(&self) -> &Token { - self.token - } - - fn token_mut(&mut self) -> &mut Token { - self.token - } -} - -#[cfg(test)] -mod tests { - use tantivy::tokenizer::{Token, TokenStream, Tokenizer}; - - use super::{MultiLangTokenizer, MultiLanguageTokenStream, get_language_from_prefix}; - - fn test_helper(mut tokenizer: MultiLanguageTokenStream) -> Vec { - let mut tokens: Vec = Vec::new(); - tokenizer.process(&mut |token: &Token| tokens.push(token.clone())); - tokens - } - - #[test] - fn test_multilanguage_tokenizer_cmn() { - let mut tokenizer = MultiLangTokenizer::default(); - let tokens = test_helper( - tokenizer.token_stream("地址1,包含無效的字元 (包括符號與不標準的asci阿爾發字元"), - ); - assert_eq!(tokens.len(), 19); - { - let token = &tokens[0]; - assert_eq!(token.text, "地址"); - assert_eq!(token.offset_from, 0); - assert_eq!(token.offset_to, 6); - assert_eq!(token.position, 0); - assert_eq!(token.position_length, 1); - } - } - - #[test] - fn test_multilanguage_tokenizer_jpn() { - let mut tokenizer = MultiLangTokenizer::default(); - { - let tokens = test_helper(tokenizer.token_stream("すもももももももものうち")); - assert_eq!(tokens.len(), 7); - { - let token = &tokens[0]; - assert_eq!(token.text, "すもも"); - assert_eq!(token.offset_from, 0); - assert_eq!(token.offset_to, 9); - assert_eq!(token.position, 0); - assert_eq!(token.position_length, 1); - } - } - { - // Force usage of JPN tokenizer. - let tokens = test_helper(tokenizer.token_stream("JPN:すもももももももものうち")); - assert_eq!(tokens.len(), 7); - } - { - // Force usage of ENG tokenizer. - // This tokenizer will return only one token. - let tokens = test_helper(tokenizer.token_stream("ENG:すもももももももものうち")); - assert_eq!(tokens.len(), 1); - } - } - - #[test] - fn test_multilanguage_tokenizer_kor() { - let mut tokenizer = MultiLangTokenizer::default(); - { - let tokens = test_helper(tokenizer.token_stream("일본입니다. 매우 멋진 단어입니다.")); - assert_eq!(tokens.len(), 11); - { - let token = &tokens[0]; - assert_eq!(token.text, "일본"); - assert_eq!(token.offset_from, 0); - assert_eq!(token.offset_to, 6); - assert_eq!(token.position, 0); - assert_eq!(token.position_length, 1); - } - } - { - let tokens = - test_helper(tokenizer.token_stream("KOR:일본입니다. 매우 멋진 단어입니다.")); - assert_eq!(tokens.len(), 11); - } - { - let tokens = test_helper(tokenizer.token_stream("ENG:일본입니다")); - assert_eq!(tokens.len(), 1); - } - } - - #[test] - fn test_multilanguage_tokenizer_with_empty_string() { - let mut tokenizer = MultiLangTokenizer::default(); - { - let tokens = test_helper(tokenizer.token_stream("")); - assert_eq!(tokens.len(), 0); - } - { - let tokens = test_helper(tokenizer.token_stream(" ")); - assert_eq!(tokens.len(), 0); - } - } - - #[test] - fn test_multilanguage_process_language_prefix() { - { - let (lang, text) = get_language_from_prefix("JPN:すもももももももものうち"); - assert_eq!(lang, Some(whichlang::Lang::Jpn)); - assert_eq!(text, "すもももももももものうち"); - } - { - let (lang, text) = get_language_from_prefix("CMN:地址1,包含無效的字元"); - assert_eq!(lang, Some(whichlang::Lang::Cmn)); - assert_eq!(text, "地址1,包含無效的字元"); - } - { - let (lang, text) = get_language_from_prefix("ENG:my address"); - assert_eq!(lang, Some(whichlang::Lang::Eng)); - assert_eq!(text, "my address"); - } - { - let (lang, text) = get_language_from_prefix("UNK:my address"); - assert!(lang.is_none()); - assert_eq!(text, "UNK:my address"); - } - { - let (lang, text) = get_language_from_prefix(""); - assert!(lang.is_none()); - assert_eq!(text, ""); - } - } -} diff --git a/quickwit/quickwit-query/src/tokenizers/tokenizer_manager.rs b/quickwit/quickwit-query/src/tokenizers/tokenizer_manager.rs index 74a56e84827..55fc4840b94 100644 --- a/quickwit/quickwit-query/src/tokenizers/tokenizer_manager.rs +++ b/quickwit/quickwit-query/src/tokenizers/tokenizer_manager.rs @@ -22,7 +22,7 @@ use tantivy::tokenizer::{ use crate::DEFAULT_REMOVE_TOKEN_LENGTH; -const RAW_TOKENIZER_NAME: &str = "raw"; +pub const RAW_TOKENIZER_NAME: &str = "raw"; const LOWERCASE_TOKENIZER_NAME: &str = "lowercase"; const RAW_LOWERCASE_TOKENIZER_NAME: &str = "raw_lowercase"; @@ -62,7 +62,9 @@ impl TokenizerManager { /// Registers a new tokenizer associated with a given name. pub fn register(&self, tokenizer_name: &str, tokenizer: T, does_lowercasing: bool) - where TextAnalyzer: From { + where + TextAnalyzer: From, + { self.inner.register(tokenizer_name, tokenizer); self.is_lowercaser .write() diff --git a/quickwit/quickwit-rest-client/Cargo.toml b/quickwit/quickwit-rest-client/Cargo.toml index c13c3f08e8a..18ae1e05625 100644 --- a/quickwit/quickwit-rest-client/Cargo.toml +++ b/quickwit/quickwit-rest-client/Cargo.toml @@ -29,7 +29,6 @@ quickwit-indexing = { workspace = true } quickwit-ingest = { workspace = true } quickwit-metastore = { workspace = true } quickwit-proto = { workspace = true } -quickwit-search = { workspace = true } quickwit-serve = { workspace = true } [dev-dependencies] diff --git a/quickwit/quickwit-rest-client/src/models.rs b/quickwit/quickwit-rest-client/src/models.rs index 5420cbd3b1c..2857495803f 100644 --- a/quickwit/quickwit-rest-client/src/models.rs +++ b/quickwit/quickwit-rest-client/src/models.rs @@ -73,7 +73,7 @@ impl ApiResponse { } } -/// A cousin of [`quickwit_search::SearchResponseRest`] that implements [`Deserialize`] +/// A cousin of `quickwit_search::SearchResponseRest` that implements [`Deserialize`] /// /// This version of the response is necessary because /// `serde_json_borrow::OwnedValue` is not deserializeable. diff --git a/quickwit/quickwit-rest-client/src/rest_client.rs b/quickwit/quickwit-rest-client/src/rest_client.rs index 32ca1e6ac4e..1fb2b5c9812 100644 --- a/quickwit/quickwit-rest-client/src/rest_client.rs +++ b/quickwit/quickwit-rest-client/src/rest_client.rs @@ -276,7 +276,7 @@ impl QuickwitClient { Ok(search_response) } - pub fn indexes(&self) -> IndexClient { + pub fn indexes(&self) -> IndexClient<'_> { IndexClient::new(&self.transport, self.timeout) } @@ -288,15 +288,15 @@ impl QuickwitClient { SourceClient::new(&self.transport, self.timeout, index_id) } - pub fn cluster(&self) -> ClusterClient { + pub fn cluster(&self) -> ClusterClient<'_> { ClusterClient::new(&self.transport, self.timeout) } - pub fn node_stats(&self) -> NodeStatsClient { + pub fn node_stats(&self) -> NodeStatsClient<'_> { NodeStatsClient::new(&self.transport, self.timeout) } - pub fn node_health(&self) -> NodeHealthClient { + pub fn node_health(&self) -> NodeHealthClient<'_> { NodeHealthClient::new(&self.transport, self.timeout) } diff --git a/quickwit/quickwit-search/Cargo.toml b/quickwit/quickwit-search/Cargo.toml index bb092fec0ce..7d0744d9e48 100644 --- a/quickwit/quickwit-search/Cargo.toml +++ b/quickwit/quickwit-search/Cargo.toml @@ -26,15 +26,12 @@ once_cell = { workspace = true } pin-project = { workspace = true } postcard = { workspace = true } prost = { workspace = true } -rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -serde_json_borrow = { workspace = true } tantivy = { workspace = true } tantivy-fst = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } -tokio-stream = { workspace = true } tower = { workspace = true, features = ["timeout"] } tracing = { workspace = true } ttl_cache = { workspace = true } @@ -46,7 +43,6 @@ quickwit-config = { workspace = true } quickwit-directories = { workspace = true } quickwit-doc-mapper = { workspace = true } quickwit-metastore = { workspace = true } -quickwit-opentelemetry = { workspace = true } quickwit-proto = { workspace = true } quickwit-query = { workspace = true } quickwit-storage = { workspace = true } diff --git a/quickwit/quickwit-search/src/client.rs b/quickwit/quickwit-search/src/client.rs index 0715e74cf1f..194bf0b2bd0 100644 --- a/quickwit/quickwit-search/src/client.rs +++ b/quickwit/quickwit-search/src/client.rs @@ -18,18 +18,14 @@ use std::sync::Arc; use std::time::Duration; use bytesize::ByteSize; -use futures::{StreamExt, TryStreamExt}; use http::Uri; -use quickwit_proto::search::{ - GetKvRequest, LeafSearchStreamResponse, PutKvRequest, ReportSplitsRequest, -}; +use quickwit_proto::search::{GetKvRequest, PutKvRequest, ReportSplitsRequest}; use quickwit_proto::tonic::Request; use quickwit_proto::tonic::codegen::InterceptedService; use quickwit_proto::tonic::transport::{Channel, Endpoint}; use quickwit_proto::{SpanContextInterceptor, tonic}; -use tokio_stream::wrappers::UnboundedReceiverStream; use tower::timeout::Timeout; -use tracing::{Instrument, info_span, warn}; +use tracing::warn; use crate::SearchService; use crate::error::parse_grpc_error; @@ -148,60 +144,6 @@ impl SearchServiceClient { } } - /// Perform leaf stream. - pub async fn leaf_search_stream( - &mut self, - request: quickwit_proto::search::LeafSearchStreamRequest, - ) -> UnboundedReceiverStream> { - match &mut self.client_impl { - SearchServiceClientImpl::Grpc(grpc_client) => { - let mut grpc_client_clone = grpc_client.clone(); - let span = info_span!( - "client:leaf_search_stream", - grpc_addr=?self.grpc_addr() - ); - let tonic_request = Request::new(request); - let (result_sender, result_receiver) = tokio::sync::mpsc::unbounded_channel(); - tokio::spawn( - async move { - let tonic_result = grpc_client_clone - .leaf_search_stream(tonic_request) - .await - .map_err(|tonic_error| parse_grpc_error(&tonic_error)); - // If the grpc client fails, send the error in the channel and stop. - if let Err(error) = tonic_result { - // It is ok to ignore error sending error. - let _ = result_sender.send(Err(error)); - return; - } - let mut results_stream = tonic_result - .unwrap() - .into_inner() - .map_err(|tonic_error| parse_grpc_error(&tonic_error)); - while let Some(search_result) = results_stream.next().await { - let send_result = result_sender.send(search_result); - // If we get a sending error, stop consuming the stream. - if send_result.is_err() { - break; - } - } - } - .instrument(span), - ); - UnboundedReceiverStream::new(result_receiver) - } - SearchServiceClientImpl::Local(service) => { - let stream_result = service.leaf_search_stream(request).await; - stream_result.unwrap_or_else(|error| { - let (result_sender, result_receiver) = tokio::sync::mpsc::unbounded_channel(); - // Receiver cannot be closed here, ignore error. - let _ = result_sender.send(Err(error)); - UnboundedReceiverStream::new(result_receiver) - }) - } - } - } - /// Perform fetch docs. pub async fn fetch_docs( &mut self, diff --git a/quickwit/quickwit-search/src/cluster_client.rs b/quickwit/quickwit-search/src/cluster_client.rs index c4e78179b73..79f6ba81702 100644 --- a/quickwit/quickwit-search/src/cluster_client.rs +++ b/quickwit/quickwit-search/src/cluster_client.rs @@ -19,19 +19,14 @@ use futures::future::ready; use futures::{Future, StreamExt}; use quickwit_proto::search::{ FetchDocsRequest, FetchDocsResponse, GetKvRequest, LeafListFieldsRequest, LeafListTermsRequest, - LeafListTermsResponse, LeafSearchRequest, LeafSearchResponse, LeafSearchStreamRequest, - LeafSearchStreamResponse, ListFieldsResponse, PutKvRequest, + LeafListTermsResponse, LeafSearchRequest, LeafSearchResponse, ListFieldsResponse, PutKvRequest, }; use tantivy::aggregation::intermediate_agg_result::IntermediateAggregationResults; -use tokio::sync::mpsc::error::SendError; -use tokio::sync::mpsc::{UnboundedSender, unbounded_channel}; -use tokio_stream::wrappers::UnboundedReceiverStream; use tracing::{debug, error, info, warn}; use crate::retry::search::LeafSearchRetryPolicy; -use crate::retry::search_stream::{LeafSearchStreamRetryPolicy, SuccessfulSplitIds}; use crate::retry::{DefaultRetryPolicy, RetryPolicy, retry_client}; -use crate::{SearchError, SearchJobPlacer, SearchServiceClient, merge_resource_stats_it}; +use crate::{SearchJobPlacer, SearchServiceClient, merge_resource_stats_it}; /// Maximum number of put requests emitted to perform a replicated given PUT KV. const MAX_PUT_KV_ATTEMPTS: usize = 6; @@ -42,9 +37,9 @@ const MAX_GET_KV_ATTEMPTS: usize = 6; /// We attempt to store our KVs on two nodes. const TARGET_NUM_REPLICATION: usize = 2; -/// Client that executes placed requests (Request, `SearchServiceClient`) and provides -/// retry policies for `FetchDocsRequest`, `LeafSearchRequest` and `LeafSearchStreamRequest` -/// to retry on other `SearchServiceClient`. +/// Client that executes placed requests (Request, `SearchServiceClient`) and +/// provides retry policies for `FetchDocsRequest` and `LeafSearchRequest` to +/// retry on other `SearchServiceClient`. #[derive(Clone)] pub struct ClusterClient { pub(crate) search_job_placer: SearchJobPlacer, @@ -131,56 +126,6 @@ impl ClusterClient { client.leaf_list_fields(request.clone()).await } - /// Leaf search stream with retry on another node client. - pub async fn leaf_search_stream( - &self, - request: LeafSearchStreamRequest, - mut client: SearchServiceClient, - ) -> UnboundedReceiverStream> { - // We need a dedicated channel to send results with retry. First we send only the successful - // responses and ignore errors. If there are some errors, we make one retry and - // in this case we send all results. - let (result_sender, result_receiver) = unbounded_channel(); - let client_pool = self.search_job_placer.clone(); - let retry_policy = LeafSearchStreamRetryPolicy {}; - tokio::spawn(async move { - let result_stream = client.leaf_search_stream(request.clone()).await; - // Forward only responses and not errors to the sender as we will make one retry on - // errors. - let forward_result = - forward_leaf_search_stream(result_stream, result_sender.clone(), false).await; - if let Some(retry_request) = retry_policy.retry_request(request, &forward_result) { - assert!(!retry_request.split_offsets.is_empty()); - let retry_client_res = retry_client( - &client_pool, - client.grpc_addr(), - &retry_request.split_offsets[0].split_id, - ) - .await; - let mut retry_client = match retry_client_res { - Ok(retry_client) => retry_client, - Err(error) => { - // Propagates the error if we cannot get a new client and stops the task. - let _ = result_sender.send(Err(SearchError::from(error))); - return; - } - }; - debug!( - "Leaf search stream response error. Retry once to execute {:?} with {:?}", - retry_request, client - ); - let retry_results_stream = retry_client.leaf_search_stream(retry_request).await; - // Forward all results to the result_sender as we won't do another retry. - // It is ok to ignore send errors, there is nothing else to do. - let _ = - forward_leaf_search_stream(retry_results_stream, result_sender.clone(), true) - .await; - } - }); - - UnboundedReceiverStream::new(result_receiver) - } - /// Leaf search with retry on another node client. pub async fn leaf_list_terms( &self, @@ -347,46 +292,20 @@ fn merge_original_with_retry_leaf_search_results( } } -// Forward leaf search stream results into a sender and -// returns the split ids of Ok(response). -// If `send_error` is false, errors are ignored and not forwarded. This is -// useful if you want to make a retry before propagating errors. -async fn forward_leaf_search_stream( - mut stream: UnboundedReceiverStream>, - sender: UnboundedSender>, - send_error: bool, -) -> Result>> { - let mut successful_split_ids: Vec = Vec::new(); - while let Some(result) = stream.next().await { - match result { - Ok(response) => { - successful_split_ids.push(response.split_id.clone()); - sender.send(Ok(response))?; - } - Err(error) => { - if send_error { - sender.send(Err(error))?; - } - } - } - } - Ok(SuccessfulSplitIds(successful_split_ids)) -} - #[cfg(test)] mod tests { use std::collections::HashSet; use std::net::SocketAddr; use quickwit_proto::search::{ - LeafRequestRef, PartialHit, SearchRequest, SearchStreamRequest, SortValue, - SplitIdAndFooterOffsets, SplitSearchError, + LeafRequestRef, PartialHit, SearchRequest, SortValue, SplitIdAndFooterOffsets, + SplitSearchError, }; use quickwit_query::query_ast::qast_json_helper; use super::*; use crate::root::SearchJob; - use crate::{MockSearchService, searcher_pool_for_test}; + use crate::{MockSearchService, SearchError, searcher_pool_for_test}; fn mock_partial_hit(split_id: &str, sort_value: u64, doc_id: u32) -> PartialHit { PartialHit { @@ -450,42 +369,6 @@ mod tests { } } - fn mock_leaf_search_stream_request() -> LeafSearchStreamRequest { - let search_request = SearchStreamRequest { - index_id: "test-idx".to_string(), - query_ast: qast_json_helper("text", &["body"]), - snippet_fields: Vec::new(), - start_timestamp: None, - end_timestamp: None, - fast_field: "fast".to_string(), - output_format: 0, - partition_by_field: None, - }; - LeafSearchStreamRequest { - request: Some(search_request), - doc_mapper: "doc_mapper".to_string(), - index_uri: "uri".to_string(), - split_offsets: vec![ - SplitIdAndFooterOffsets { - split_id: "split_1".to_string(), - split_footer_start: 0, - split_footer_end: 100, - timestamp_start: None, - timestamp_end: None, - num_docs: 0, - }, - SplitIdAndFooterOffsets { - split_id: "split_2".to_string(), - split_footer_start: 0, - split_footer_end: 100, - timestamp_start: None, - timestamp_end: None, - num_docs: 0, - }, - ], - } - } - #[tokio::test] async fn test_cluster_client_fetch_docs_no_retry() { let request = mock_doc_request("split_1"); @@ -703,49 +586,6 @@ mod tests { Ok(()) } - #[tokio::test] - async fn test_cluster_client_leaf_stream_retry_on_error() { - let request = mock_leaf_search_stream_request(); - - let mut mock_search_service_1 = MockSearchService::new(); - mock_search_service_1 - .expect_leaf_search_stream() - .return_once(|_| Err(SearchError::Internal("error".to_string()))); - - let mut mock_search_service_2 = MockSearchService::new(); - let (result_sender, result_receiver) = tokio::sync::mpsc::unbounded_channel(); - mock_search_service_2 - .expect_leaf_search_stream() - .return_once(|_| Ok(UnboundedReceiverStream::new(result_receiver))); - - let searcher_pool = searcher_pool_for_test([ - ("127.0.0.1:1001", mock_search_service_1), - ("127.0.0.1:1002", mock_search_service_2), - ]); - let search_job_placer = SearchJobPlacer::new(searcher_pool.clone()); - let cluster_client = ClusterClient::new(search_job_placer); - - result_sender - .send(Ok(LeafSearchStreamResponse { - data: Vec::new(), - split_id: "split_1".to_string(), - })) - .unwrap(); - result_sender - .send(Err(SearchError::Internal("last split error".to_string()))) - .unwrap(); - drop(result_sender); - - let first_client_addr: SocketAddr = "127.0.0.1:1001".parse().unwrap(); - let first_client = searcher_pool.get(&first_client_addr).unwrap(); - let result = cluster_client - .leaf_search_stream(request, first_client) - .await; - let results: Vec<_> = result.collect().await; - assert_eq!(results.len(), 2); - assert!(results[0].is_ok()); - } - #[tokio::test] async fn test_put_kv_happy_path() { // 3 servers 1, 2, 3 diff --git a/quickwit/quickwit-search/src/collector.rs b/quickwit/quickwit-search/src/collector.rs index 65afcaf303b..7cbc9ca7b55 100644 --- a/quickwit/quickwit-search/src/collector.rs +++ b/quickwit/quickwit-search/src/collector.rs @@ -27,10 +27,11 @@ use quickwit_proto::types::SplitId; use serde::Deserialize; use tantivy::aggregation::agg_req::{Aggregations, get_fast_field_names}; use tantivy::aggregation::intermediate_agg_result::IntermediateAggregationResults; -use tantivy::aggregation::{AggregationLimitsGuard, AggregationSegmentCollector}; +use tantivy::aggregation::{AggContextParams, AggregationLimitsGuard, AggregationSegmentCollector}; use tantivy::collector::{Collector, SegmentCollector}; use tantivy::columnar::{ColumnType, MonotonicallyMappableToU64}; use tantivy::fastfield::Column; +use tantivy::tokenizer::TokenizerManager; use tantivy::{DateTime, DocId, Score, SegmentOrdinal, SegmentReader, TantivyError}; use crate::find_trace_ids_collector::{FindTraceIdsCollector, FindTraceIdsSegmentCollector, Span}; @@ -660,21 +661,20 @@ impl QuickwitIncrementalAggregations { fn virtual_worst_hit(&self) -> Option { match self { QuickwitIncrementalAggregations::FindTraceIdsAggregation(collector, state) => { - if let Some(first) = state.first() { - if first.len() >= collector.num_traces { - if let Some(last_elem) = first.last() { - let timestamp = last_elem.span_timestamp.into_timestamp_nanos(); - return Some(PartialHit { - sort_value: Some(SortByValue { - sort_value: Some(SortValue::I64(timestamp)), - }), - sort_value2: None, - split_id: SplitId::new(), - segment_ord: 0, - doc_id: 0, - }); - } - } + if let Some(first) = state.first() + && first.len() >= collector.num_traces + && let Some(last_elem) = first.last() + { + let timestamp = last_elem.span_timestamp.into_timestamp_nanos(); + return Some(PartialHit { + sort_value: Some(SortByValue { + sort_value: Some(SortValue::I64(timestamp)), + }), + sort_value2: None, + split_id: SplitId::new(), + segment_ord: 0, + doc_id: 0, + }); } None } @@ -716,7 +716,7 @@ pub struct QuickwitCollector { pub max_hits: usize, pub sort_by: SortByPair, pub aggregation: Option, - pub aggregation_limits: AggregationLimitsGuard, + pub agg_context_params: AggContextParams, search_after: Option, } @@ -786,7 +786,7 @@ impl Collector for QuickwitCollector { aggs, segment_reader, segment_ord, - &self.aggregation_limits, + &self.agg_context_params, )?, ), ), @@ -861,8 +861,10 @@ impl Collector for QuickwitCollector { } } -fn map_error(err: postcard::Error) -> TantivyError { - TantivyError::InternalError(format!("merge result Postcard error: {err}")) +fn map_error(error: postcard::Error) -> TantivyError { + TantivyError::InternalError(format!( + "failed to merge intermediate aggregation results: Postcard error: {error}" + )) } /// Merges a set of Leaf Results. @@ -884,24 +886,24 @@ fn merge_intermediate_aggregation_result<'a>( Some(serialized) } Some(QuickwitAggregations::TantivyAggregations(_)) => { - let fruits: Vec = intermediate_aggregation_results - .map(|intermediate_aggregation_result| { - postcard::from_bytes(intermediate_aggregation_result).map_err(map_error) - }) - .collect::>()?; - - let mut fruit_iter = fruits.into_iter(); - if let Some(first_fruit) = fruit_iter.next() { - let mut merged_fruit = first_fruit; - for fruit in fruit_iter { - merged_fruit.merge_fruits(fruit)?; - } - let serialized = postcard::to_allocvec(&merged_fruit).map_err(map_error)?; - - Some(serialized) - } else { - None - } + let merged_opt = intermediate_aggregation_results + .map(|bytes| postcard::from_bytes(bytes).map_err(map_error)) + .try_fold::<_, _, Result<_, TantivyError>>( + None, + |acc: Option, fruits_res| { + let fruits = fruits_res?; + match acc { + Some(mut merged_fruits) => { + merged_fruits.merge_fruits(fruits)?; + Ok(Some(merged_fruits)) + } + None => Ok(Some(fruits)), + } + }, + )?; + let serialized = + postcard::to_allocvec(&merged_opt.unwrap_or_default()).map_err(map_error)?; + Some(serialized) } None => None, }; @@ -1032,7 +1034,7 @@ pub(crate) fn sort_by_from_request(search_request: &SearchRequest) -> SortByPair pub fn make_collector_for_split( split_id: SplitId, search_request: &SearchRequest, - aggregation_limits: AggregationLimitsGuard, + agg_context_params: AggContextParams, ) -> crate::Result { let aggregation = match &search_request.aggregation_request { Some(aggregation) => Some(serde_json::from_str(aggregation)?), @@ -1045,7 +1047,7 @@ pub fn make_collector_for_split( max_hits: search_request.max_hits as usize, sort_by, aggregation, - aggregation_limits, + agg_context_params, search_after: search_request.search_after.clone(), }) } @@ -1053,8 +1055,15 @@ pub fn make_collector_for_split( /// Builds a QuickwitCollector that's only useful for merging fruits. pub(crate) fn make_merge_collector( search_request: &SearchRequest, - aggregation_limits: &AggregationLimitsGuard, + agg_limits: AggregationLimitsGuard, ) -> crate::Result { + // Note: at this point the tokenizer manager is not used anymore by aggregations (filter query), + // so we can create an empty one. So if it will ever be used, it would panic. + let agg_context_params = AggContextParams { + limits: agg_limits, + tokenizers: TokenizerManager::new(), + }; + let aggregation = match &search_request.aggregation_request { Some(aggregation) => Some(serde_json::from_str(aggregation)?), None => None, @@ -1066,7 +1075,7 @@ pub(crate) fn make_merge_collector( max_hits: search_request.max_hits as usize, sort_by, aggregation, - aggregation_limits: aggregation_limits.clone(), + agg_context_params, search_after: search_request.search_after.clone(), }) } @@ -1251,7 +1260,7 @@ impl IncrementalCollector { /// Get the worst top-hit. Can be used to skip splits if they can't possibly do better. /// /// Only returns a result if enough hits were recorded already. - pub(crate) fn peek_worst_hit(&self) -> Option> { + pub(crate) fn peek_worst_hit(&self) -> Option> { if self.top_k_hits.max_len() == 0 { return self .incremental_aggregation @@ -1294,10 +1303,13 @@ mod tests { SortOrder, SortValue, SplitSearchError, }; use tantivy::TantivyDocument; + use tantivy::aggregation::agg_req::Aggregations; + use tantivy::aggregation::intermediate_agg_result::IntermediateAggregationResults; use tantivy::collector::Collector; use super::{IncrementalCollector, make_merge_collector}; - use crate::collector::top_k_partial_hits; + use crate::QuickwitAggregations; + use crate::collector::{merge_intermediate_aggregation_result, top_k_partial_hits}; #[test] fn test_merge_partial_hits_no_tie() { @@ -1744,7 +1756,7 @@ mod tests { request: &SearchRequest, results: Vec, ) -> LeafSearchResponse { - let collector = make_merge_collector(request, &Default::default()).unwrap(); + let collector = make_merge_collector(request, Default::default()).unwrap(); let mut incremental_collector = IncrementalCollector::new(collector.clone()); let result = collector @@ -2003,4 +2015,22 @@ mod tests { ); // TODO would be nice to test aggregation too. } + + #[test] + fn test_merge_empty_intermediate_aggregation_result() { + let merged = merge_intermediate_aggregation_result(&None, std::iter::empty()).unwrap(); + assert!(merged.is_none()); + + let aggregations_json = r#"{ + "avg_price": { "avg": { "field": "price" } } + }"#; + let ttv_aggregations: Aggregations = serde_json::from_str(aggregations_json).unwrap(); + let qw_aggregations = QuickwitAggregations::TantivyAggregations(ttv_aggregations); + let serialized = + merge_intermediate_aggregation_result(&Some(qw_aggregations), std::iter::empty()) + .unwrap() + .unwrap(); + let _merged: IntermediateAggregationResults = postcard::from_bytes(&serialized).unwrap(); + // Hopefully `_merged` is empty but the API does not allow us to assert that. + } } diff --git a/quickwit/quickwit-search/src/error.rs b/quickwit/quickwit-search/src/error.rs index 21141a3035a..073a1ddedc3 100644 --- a/quickwit/quickwit-search/src/error.rs +++ b/quickwit/quickwit-search/src/error.rs @@ -14,6 +14,7 @@ use itertools::Itertools; use quickwit_common::rate_limited_error; +use quickwit_common::retry::Retryable; use quickwit_doc_mapper::QueryParserError; use quickwit_proto::error::grpc_error_to_grpc_status; use quickwit_proto::metastore::{EntityKind, MetastoreError}; @@ -175,6 +176,12 @@ impl From for SearchError { } } +impl Retryable for SearchError { + fn is_retryable(&self) -> bool { + matches!(self, SearchError::TooManyRequests | SearchError::Timeout(_)) + } +} + impl From for SearchError { fn from(join_error: JoinError) -> SearchError { SearchError::Internal(format!("spawned task in root join failed: {join_error}")) diff --git a/quickwit/quickwit-search/src/fetch_docs.rs b/quickwit/quickwit-search/src/fetch_docs.rs index c1c1d3a73b7..9d225113249 100644 --- a/quickwit/quickwit-search/src/fetch_docs.rs +++ b/quickwit/quickwit-search/src/fetch_docs.rs @@ -129,14 +129,14 @@ pub async fn fetch_docs( .await?; let hits: Vec = partial_hits - .iter() + .into_iter() .flat_map(|partial_hit| { - let global_doc_addr = GlobalDocAddress::from_partial_hit(partial_hit); + let global_doc_addr = GlobalDocAddress::from_partial_hit(&partial_hit); if let Some((_, document)) = global_doc_addr_to_doc_json.remove_entry(&global_doc_addr) { Some(quickwit_proto::search::LeafHit { leaf_json: document.content_json, - partial_hit: Some(partial_hit.clone()), + partial_hit: Some(partial_hit), leaf_snippet_json: document.snippet_json, }) } else { @@ -304,7 +304,7 @@ async fn create_fields_snippet_generator( let schema = searcher.schema(); let query_ast_resolved = serde_json::from_str(&snippet_request.query_ast_resolved) .context("failed to deserialize QueryAst")?; - let (query, _) = doc_mapper.query(schema.clone(), &query_ast_resolved, false)?; + let (query, _) = doc_mapper.query(schema.clone(), query_ast_resolved, false, None)?; let mut snippet_generators = HashMap::new(); for field_name in &snippet_request.snippet_fields { let field = schema.get_field(field_name)?; diff --git a/quickwit/quickwit-search/src/filters.rs b/quickwit/quickwit-search/src/filters.rs deleted file mode 100644 index 72acfab61e4..00000000000 --- a/quickwit/quickwit-search/src/filters.rs +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::ops::{Bound, RangeBounds, RangeInclusive}; - -use tantivy::columnar::Cardinality; -use tantivy::fastfield::Column; -use tantivy::{DateTime, DocId, SegmentReader}; - -/// A filter that only retains docs within a time range. -#[derive(Clone)] -pub struct TimestampFilter { - /// The time range represented as (lower_bound, upper_bound). - // TODO replace this with a RangeInclusive if it improves perf? - time_range: (Bound, Bound), - timestamp_column: Column, -} - -impl TimestampFilter { - #[inline] - pub fn contains_timestamp(&self, ts: &DateTime) -> bool { - self.time_range.contains(ts) - } - - #[inline] - /// Fetches the timestamp of a given doc from the column storage and checks if it is within the - /// time range. - pub fn contains_doc_timestamp(&self, doc_id: DocId) -> bool { - if let Some(ts) = self.timestamp_column.first(doc_id) { - self.contains_timestamp(&ts) - } else { - false - } - } -} - -/// Creates a timestamp field depending on the user request. -/// -/// The start/end timestamp are in seconds and are interpreted as -/// a semi-open interval [start, end). -pub fn create_timestamp_filter_builder( - timestamp_field_opt: Option<&str>, - start_timestamp_secs: Option, - end_timestamp_secs: Option, -) -> Option { - let timestamp_field = timestamp_field_opt?; - if start_timestamp_secs.is_none() && end_timestamp_secs.is_none() { - return None; - } - let start_timestamp_bound: Bound = start_timestamp_secs - .map(|timestamp_secs| Bound::Included(DateTime::from_timestamp_secs(timestamp_secs))) - .unwrap_or(Bound::Unbounded); - let end_timestamp_bound: Bound = end_timestamp_secs - .map(|timestamp_secs| Bound::Excluded(DateTime::from_timestamp_secs(timestamp_secs))) - .unwrap_or(Bound::Unbounded); - Some(TimestampFilterBuilder::new( - timestamp_field.to_string(), - start_timestamp_bound, - end_timestamp_bound, - )) -} - -#[derive(Clone, Debug)] -pub struct TimestampFilterBuilder { - pub timestamp_field_name: String, - start_timestamp: Bound, - end_timestamp: Bound, -} - -impl TimestampFilterBuilder { - pub fn new( - timestamp_field_name: String, - start_timestamp: Bound, - end_timestamp: Bound, - ) -> TimestampFilterBuilder { - TimestampFilterBuilder { - timestamp_field_name, - start_timestamp, - end_timestamp, - } - } - - /// None means that all documents are matching the timestamp range. - pub fn build( - &self, - segment_reader: &SegmentReader, - ) -> tantivy::Result> { - let timestamp_column_opt: Option> = - segment_reader - .fast_fields() - .column_opt::(&self.timestamp_field_name)?; - let timestamp_column = timestamp_column_opt - .unwrap_or_else(|| Column::build_empty_column(segment_reader.max_doc())); - let time_range = (self.start_timestamp, self.end_timestamp); - if time_range == (Bound::Unbounded, Bound::Unbounded) { - return Ok(None); - } - if timestamp_column.index.get_cardinality() == Cardinality::Full { - let segment_range: RangeInclusive = - timestamp_column.min_value()..=timestamp_column.max_value(); - if is_segment_always_within_timestamp_range(segment_range, time_range) { - return Ok(None); - } - } - Ok(Some(TimestampFilter { - time_range, - timestamp_column, - })) - } -} - -/// Determine if all docs of a segment always satisfy the requested timestamp range. -/// -/// Note: -/// - segment_range: is an inclusive range on both ends `[min, max]`. -/// - timestamp_range: is a half open range `[min, max[`. -fn is_segment_always_within_timestamp_range( - segment_range: RangeInclusive, - timestamp_range: impl RangeBounds, -) -> bool { - timestamp_range.contains(segment_range.start()) && timestamp_range.contains(segment_range.end()) -} - -#[cfg(test)] -mod tests { - use tantivy::DateTime; - - use super::is_segment_always_within_timestamp_range; - - const TEST_START: DateTime = DateTime::from_timestamp_secs(1_662_529_435); - const TEST_MIDDLE: DateTime = DateTime::from_timestamp_secs(1_662_629_435); - const TEST_END: DateTime = DateTime::from_timestamp_secs(1_662_639_435); - - #[test] - fn test_is_segment_always_within_timestamp_range() { - assert_eq!( - is_segment_always_within_timestamp_range(TEST_START..=TEST_END, ..), - true - ); - - assert_eq!( - is_segment_always_within_timestamp_range( - TEST_START..=TEST_MIDDLE, - TEST_START..TEST_END - ), - true - ); - - assert_eq!( - is_segment_always_within_timestamp_range( - TEST_START..=TEST_END, - TEST_START..TEST_MIDDLE - ), - false - ); - - assert_eq!( - is_segment_always_within_timestamp_range(TEST_START..=TEST_END, TEST_START..TEST_END), - false - ); - } -} diff --git a/quickwit/quickwit-search/src/find_trace_ids_collector.rs b/quickwit/quickwit-search/src/find_trace_ids_collector.rs index 510cb8f81d4..2c1dc93e85a 100644 --- a/quickwit/quickwit-search/src/find_trace_ids_collector.rs +++ b/quickwit/quickwit-search/src/find_trace_ids_collector.rs @@ -17,7 +17,7 @@ use std::collections::HashSet; use fnv::{FnvHashMap, FnvHashSet}; use itertools::Itertools; -use quickwit_opentelemetry::otlp::TraceId; +use quickwit_proto::search::TraceId; use serde::{Deserialize, Serialize}; use tantivy::collector::{Collector, SegmentCollector}; use tantivy::columnar::BytesColumn; @@ -344,12 +344,16 @@ mod serde_datetime { use tantivy::DateTime; pub(crate) fn serialize(datetime: &DateTime, serializer: S) -> Result - where S: Serializer { + where + S: Serializer, + { serializer.serialize_i64(datetime.into_timestamp_nanos()) } pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result - where D: Deserializer<'de> { + where + D: Deserializer<'de>, + { let datetime_i64: i64 = Deserialize::deserialize(deserializer)?; Ok(DateTime::from_timestamp_nanos(datetime_i64)) } diff --git a/quickwit/quickwit-search/src/invoker.rs b/quickwit/quickwit-search/src/invoker.rs new file mode 100644 index 00000000000..90203052eac --- /dev/null +++ b/quickwit/quickwit-search/src/invoker.rs @@ -0,0 +1,64 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Trait for invoking remote serverless functions for leaf search. + +use async_trait::async_trait; +use quickwit_proto::search::{LambdaSingleSplitResult, LeafSearchRequest}; + +use crate::SearchError; + +/// Trait for invoking remote serverless functions (e.g., AWS Lambda) for leaf search. +/// +/// This abstraction allows different cloud providers to be supported. +/// Implementations are provided by the `quickwit-lambda` crate. +#[async_trait] +pub trait LambdaLeafSearchInvoker: Send + Sync + 'static { + /// Invoke the remote function with a LeafSearchRequest. + /// + /// Returns one `LambdaSingleSplitResult` per split in the request. + /// Each result is tagged with its split_id so ordering is irrelevant. + /// Individual split failures are reported per-split; the outer `Result` + /// only represents transport-level errors. + async fn invoke_leaf_search( + &self, + request: LeafSearchRequest, + ) -> Result, SearchError>; +} + +#[async_trait] +impl LambdaLeafSearchInvoker for Box +where + T: LambdaLeafSearchInvoker + ?Sized, +{ + async fn invoke_leaf_search( + &self, + request: LeafSearchRequest, + ) -> Result, SearchError> { + (**self).invoke_leaf_search(request).await + } +} + +#[async_trait] +impl LambdaLeafSearchInvoker for std::sync::Arc +where + T: LambdaLeafSearchInvoker + ?Sized, +{ + async fn invoke_leaf_search( + &self, + request: LeafSearchRequest, + ) -> Result, SearchError> { + (**self).invoke_leaf_search(request).await + } +} diff --git a/quickwit/quickwit-search/src/leaf.rs b/quickwit/quickwit-search/src/leaf.rs index 0742a5f55ff..7eef0e08575 100644 --- a/quickwit/quickwit-search/src/leaf.rs +++ b/quickwit/quickwit-search/src/leaf.rs @@ -12,7 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::{HashMap, HashSet}; +use std::cmp::Reverse; +use std::collections::binary_heap::PeekMut; +use std::collections::{BinaryHeap, HashMap, HashSet}; +use std::num::NonZeroUsize; use std::ops::Bound; use std::path::PathBuf; use std::str::FromStr; @@ -23,26 +26,31 @@ use anyhow::Context; use bytesize::ByteSize; use futures::future::try_join_all; use quickwit_common::pretty::PrettySample; -use tantivy::HasLen; +use quickwit_common::uri::Uri; use quickwit_directories::{CachingDirectory, HotDirectory, StorageDirectory}; use quickwit_doc_mapper::{Automaton, DocMapper, FastFieldWarmupInfo, TermRange, WarmupInfo}; +use quickwit_proto::search::lambda_single_split_result::Outcome; use quickwit_proto::search::{ CountHits, LeafSearchRequest, LeafSearchResponse, PartialHit, ResourceStats, SearchRequest, SortOrder, SortValue, SplitIdAndFooterOffsets, SplitSearchError, }; -use quickwit_query::query_ast::{BoolQuery, QueryAst, QueryAstTransformer, RangeQuery, TermQuery}; +use quickwit_query::query_ast::{ + BoolQuery, CacheNode, QueryAst, QueryAstTransformer, RangeQuery, TermQuery, +}; use quickwit_query::tokenizers::TokenizerManager; use quickwit_storage::{ BundleStorage, ByteRangeCache, MemorySizedCache, OwnedBytes, SplitCache, Storage, StorageResolver, TimeoutAndRetryStorage, wrap_storage_with_cache, }; -use tantivy::aggregation::AggregationLimitsGuard; +use tantivy::HasLen; +use tantivy::aggregation::AggContextParams; use tantivy::aggregation::agg_req::{AggregationVariants, Aggregations}; +use tantivy::collector::Collector; use tantivy::directory::FileSlice; use tantivy::fastfield::FastFieldReaders; use tantivy::schema::Field; use tantivy::{DateTime, Index, ReloadPolicy, Searcher, TantivyError, Term}; -use tokio::task::JoinError; +use tokio::task::{JoinError, JoinSet}; use tracing::*; // Debug macro that checks TANTIVY4JAVA_DEBUG environment variable @@ -55,12 +63,68 @@ macro_rules! debug_println { } use crate::collector::{IncrementalCollector, make_collector_for_split, make_merge_collector}; +use crate::leaf_cache::LeafSearchCache; +use crate::metrics::SplitSearchOutcomeCounters; use crate::root::is_metadata_count_request_with_ast; -use crate::search_permit_provider::{SearchPermit, compute_initial_memory_allocation}; +use crate::search_permit_provider::{ + SearchPermit, SearchPermitFuture, compute_initial_memory_allocation, +}; use crate::service::{SearcherContext, deserialize_doc_mapper}; use crate::{QuickwitAggregations, SearchError}; -#[instrument(skip_all)] +/// Distributes items across batches using a greedy LPT (Longest Processing Time) +/// algorithm to balance total weight across batches. +/// +/// Items are sorted by weight descending, then each item is assigned to the +/// batch with the smallest current total weight. This produces a good +/// approximation of balanced batches. +fn greedy_batch_split( + items: Vec, + weight_fn: impl Fn(&T) -> u64, + max_items_per_batch: NonZeroUsize, +) -> Vec> { + if items.is_empty() { + return Vec::new(); + } + + let num_items = items.len(); + let max_items_per_batch: usize = max_items_per_batch.get(); + let num_batches = num_items.div_ceil(max_items_per_batch); + + // Compute weights, then sort descending by weight + let mut weighted_items: Vec<(u64, T)> = Vec::with_capacity(num_items); + for item in items { + let weight = weight_fn(&item); + weighted_items.push((weight, item)); + } + weighted_items.sort_unstable_by_key(|(weight, _)| std::cmp::Reverse(*weight)); + + let mut batches: Vec> = std::iter::repeat_with(Vec::new).take(num_batches).collect(); + + // Min-heap of (weight, item_count, batch_index). + // Reverse turns BinaryHeap into a min-heap. + // Ties break naturally: lighter weight → fewer items → lower index. + let mut heap: BinaryHeap> = BinaryHeap::with_capacity(num_batches); + for batch_idx in 0..num_batches { + heap.push(Reverse((0, 0, batch_idx))); + } + + // Greedily assign each item to the lightest batch. + // Full batches are removed via PeekMut::pop(). + for (weight, item) in weighted_items { + let mut top = heap.peek_mut().unwrap(); + let Reverse((ref mut batch_weight, ref mut batch_count, batch_idx)) = *top; + batches[batch_idx].push(item); + *batch_weight += weight; + *batch_count += 1; + if *batch_count >= max_items_per_batch { + PeekMut::pop(top); + } + } + + batches +} + async fn get_split_footer_from_cache_or_fetch( index_storage: Arc, split_and_footer_offsets: &SplitIdAndFooterOffsets, @@ -90,7 +154,6 @@ async fn get_split_footer_from_cache_or_fetch( ) })?; - footer_cache.put( split_and_footer_offsets.split_id.to_owned(), footer_data_opt.clone(), @@ -107,7 +170,6 @@ pub async fn open_split_bundle( index_storage: Arc, split_and_footer_offsets: &SplitIdAndFooterOffsets, ) -> anyhow::Result<(FileSlice, BundleStorage)> { - let split_file = PathBuf::from(format!("{}.split", split_and_footer_offsets.split_id)); let footer_data = get_split_footer_from_cache_or_fetch( @@ -156,8 +218,8 @@ fn configure_storage_retries( /// Opens a `tantivy::Index` for the given split with several cache layers: /// - A split footer cache given by `SearcherContext.split_footer_cache`. /// - A fast fields cache given by `SearcherContext.storage_long_term_cache`. -/// - An ephemeral unbounded cache directory (whose lifetime is tied to the -/// returned `Index` if no `ByteRangeCache` is provided). +/// - An ephemeral unbounded cache directory (whose lifetime is tied to the returned `Index` if no +/// `ByteRangeCache` is provided). #[instrument(skip_all, fields(split_footer_start=split_and_footer_offsets.split_footer_start, split_footer_end=split_and_footer_offsets.split_footer_end))] pub async fn open_index_with_caches( searcher_context: &SearcherContext, @@ -201,36 +263,50 @@ impl tantivy::directory::Directory for OverlayDirectory { fn get_file_handle( &self, path: &std::path::Path, - ) -> Result, tantivy::directory::error::OpenReadError> { + ) -> Result, tantivy::directory::error::OpenReadError> + { if let Some(data) = self.file_overrides.get(path) { debug_println!("📊 OVERLAY: HIT for {:?} ({} bytes)", path, data.len()); Ok(Arc::new(data.clone())) } else { - Err(tantivy::directory::error::OpenReadError::FileDoesNotExist(path.to_path_buf())) + Err(tantivy::directory::error::OpenReadError::FileDoesNotExist( + path.to_path_buf(), + )) } } - fn exists(&self, path: &std::path::Path) -> Result { + fn exists( + &self, + path: &std::path::Path, + ) -> Result { if path == std::path::Path::new("meta.json") && self.meta_json.is_some() { return Ok(true); } Ok(self.file_overrides.contains_key(path)) } - fn atomic_read(&self, path: &std::path::Path) -> Result, tantivy::directory::error::OpenReadError> { + fn atomic_read( + &self, + path: &std::path::Path, + ) -> Result, tantivy::directory::error::OpenReadError> { if path == std::path::Path::new("meta.json") { if let Some(meta) = &self.meta_json { return Ok(meta.clone()); } } - Err(tantivy::directory::error::OpenReadError::FileDoesNotExist(path.to_path_buf())) + Err(tantivy::directory::error::OpenReadError::FileDoesNotExist( + path.to_path_buf(), + )) } fn atomic_write(&self, _path: &std::path::Path, _data: &[u8]) -> std::io::Result<()> { Ok(()) // no-op } - fn delete(&self, _path: &std::path::Path) -> Result<(), tantivy::directory::error::DeleteError> { + fn delete( + &self, + _path: &std::path::Path, + ) -> Result<(), tantivy::directory::error::DeleteError> { Ok(()) // no-op } @@ -241,7 +317,10 @@ impl tantivy::directory::Directory for OverlayDirectory { Err(tantivy::directory::error::OpenWriteError::FileAlreadyExists(path.to_path_buf())) } - fn watch(&self, _watch_callback: tantivy::directory::WatchCallback) -> tantivy::Result { + fn watch( + &self, + _watch_callback: tantivy::directory::WatchCallback, + ) -> tantivy::Result { Ok(tantivy::directory::WatchHandle::empty()) } @@ -288,7 +367,6 @@ pub async fn open_index_with_caches_and_schema_override( ephemeral_unbounded_cache: Option, overrides: Option, ) -> anyhow::Result<(Index, HotDirectory)> { - let index_storage_with_retry_on_timeout = configure_storage_retries(searcher_context, index_storage); @@ -587,26 +665,27 @@ fn compute_index_size(hot_directory: &HotDirectory) -> ByteSize { /// Apply a leaf search on a single split. #[allow(clippy::too_many_arguments)] pub async fn leaf_search_single_split( - searcher_context: &SearcherContext, - mut search_request: SearchRequest, + search_request: SearchRequest, + searcher_context: Arc, storage: Arc, split: SplitIdAndFooterOffsets, - doc_mapper: Arc, - split_filter: Arc>, - aggregations_limits: AggregationLimitsGuard, + doc_mapper: Arc, + split_filter: Arc>, search_permit: &mut SearchPermit, overrides: Option, -) -> crate::Result { - rewrite_request( - &mut search_request, - &split, - doc_mapper.timestamp_field_name(), - ); +) -> crate::Result> { + let split_outcome_counters = + Arc::new(crate::metrics::SplitSearchOutcomeCounters::new_unregistered()); + let mut leaf_search_state_guard = SplitSearchStateGuard::new(split_outcome_counters); + + // We already checked if the result was already in the partial result cache, + // but it's not a bad idea to check again. if let Some(cached_answer) = searcher_context .leaf_search_cache .get(split.clone(), search_request.clone()) { - return Ok(cached_answer); + leaf_search_state_guard.set_state(SplitSearchState::CacheHit); + return Ok(Some(cached_answer)); } let query_ast: QueryAst = serde_json::from_str(search_request.query_ast.as_str()) @@ -615,16 +694,16 @@ pub async fn leaf_search_single_split( // CanSplitDoBetter or rewrite_request may have changed the request to be a count only request // This may be the case for AllQuery with a sort by date and time filter, where the current // split can't have better results. - // if is_metadata_count_request_with_ast(&query_ast, &search_request) { - return Ok(get_leaf_resp_from_count(split.num_docs)); + leaf_search_state_guard.set_state(SplitSearchState::PrunedBeforeWarmup); + return Ok(Some(get_leaf_resp_from_count(split.num_docs))); } let split_id = split.split_id.to_string(); let byte_range_cache = ByteRangeCache::with_infinite_capacity(&quickwit_storage::STORAGE_METRICS.shortlived_cache); let (index, hot_directory) = open_index_with_caches_and_schema_override( - searcher_context, + &searcher_context, storage, &split, Some(doc_mapper.tokenizer_manager()), @@ -638,23 +717,42 @@ pub async fn leaf_search_single_split( search_permit.update_memory_usage(index_size); } - let reader = index + let searcher = index .reader_builder() .reload_policy(ReloadPolicy::Manual) - .try_into()?; - let searcher = reader.searcher(); + .try_into()? + .searcher(); + let agg_context_params = AggContextParams { + limits: searcher_context.get_aggregation_limits(), + tokenizers: doc_mapper.tokenizer_manager().tantivy_manager().clone(), + }; let mut collector = - make_collector_for_split(split_id.clone(), &search_request, aggregations_limits)?; + make_collector_for_split(split_id.clone(), &search_request, agg_context_params)?; + let predicate_cache = if collector.requires_scoring() { + // at the moment the predicate cache doesn't support scoring + None + } else { + Some(( + searcher_context.predicate_cache.clone() as _, + split.split_id.clone(), + )) + }; let split_schema = index.schema(); - let (query, mut warmup_info) = doc_mapper.query(split_schema.clone(), &query_ast, false)?; + let (query, mut warmup_info) = doc_mapper.query( + split_schema.clone(), + query_ast.clone(), + false, + predicate_cache, + )?; let collector_warmup_info = collector.warmup_info(); warmup_info.merge(collector_warmup_info); warmup_info.simplify(); let warmup_start = Instant::now(); + leaf_search_state_guard.set_state(SplitSearchState::WarmUp); warmup(&searcher, &warmup_info).await?; let warmup_end = Instant::now(); let warmup_duration: Duration = warmup_end.duration_since(warmup_start); @@ -676,20 +774,28 @@ pub async fn leaf_search_single_split( let span = info_span!("tantivy_search"); - let (search_request, leaf_search_response) = { - let split = split.clone(); + let split_clone = split.clone(); + let split_filter_clone = split_filter.clone(); + leaf_search_state_guard.set_state(SplitSearchState::CpuQueue); + let search_request_and_result: Option<(SearchRequest, LeafSearchResponse)> = crate::search_thread_pool() .run_cpu_intensive(move || { + leaf_search_state_guard.set_state(SplitSearchState::Cpu); let cpu_start = Instant::now(); let cpu_thread_pool_wait_microsecs = cpu_start.duration_since(warmup_end); let _span_guard = span.enter(); // Our search execution has been scheduled, let's check if we can improve the // request based on the results of the preceding searches - check_optimize_search_request(&mut search_request, &split, &split_filter); - collector.update_search_param(&search_request); + let Some(simplified_search_request) = + simplify_search_request(search_request, &split_clone, &split_filter_clone) + else { + leaf_search_state_guard.set_state(SplitSearchState::PrunedAfterWarmup); + return Ok(None); + }; + collector.update_search_param(&simplified_search_request); let mut leaf_search_response: LeafSearchResponse = - if is_metadata_count_request_with_ast(&query_ast, &search_request) { + if is_metadata_count_request_with_ast(&query_ast, &simplified_search_request) { get_leaf_resp_from_count(searcher.num_docs()) } else if collector.is_count_only() { let count = query.count(&searcher)? as u64; @@ -705,18 +811,26 @@ pub async fn leaf_search_single_split( cpu_thread_pool_wait_microsecs: cpu_thread_pool_wait_microsecs.as_micros() as u64, }); - Result::<_, TantivyError>::Ok((search_request, leaf_search_response)) + leaf_search_state_guard.set_state(SplitSearchState::Success); + Result::<_, TantivyError>::Ok(Some(( + simplified_search_request, + leaf_search_response, + ))) }) .await .map_err(|_| { crate::SearchError::Internal(format!("leaf search panicked. split={split_id}")) - })?? - }; + })??; + // Let's cache this result in the partial result cache. + let Some((leaf_search_req, leaf_search_resp)) = search_request_and_result else { + return Ok(None); + }; + // We save our result in the cache. searcher_context .leaf_search_cache - .put(split, search_request, leaf_search_response.clone()); - Ok(leaf_search_response) + .put(split, leaf_search_req, leaf_search_resp.clone()); + Ok(Some(leaf_search_resp)) } /// Rewrite a request removing parts which incur additional download or computation with no @@ -736,6 +850,20 @@ fn rewrite_request( remove_redundant_timestamp_range(search_request, split, timestamp_field); } rewrite_aggregation(search_request); + // we add a top level cache node when search_after is set, this won't help for this query (which + // is the 2nd in its series), but should speedup every other request that comes after + if search_request.search_after.is_some() { + add_top_cache_node(search_request) + } +} + +fn add_top_cache_node(search_request: &mut SearchRequest) { + let Ok(query_ast) = serde_json::from_str(search_request.query_ast.as_str()) else { + // an error will get raised a bit after anyway + return; + }; + let new_ast: QueryAst = CacheNode::new(query_ast).into(); + search_request.query_ast = serde_json::to_string(&new_ast).unwrap(); } /// Rewrite aggregation to make them easier to cache @@ -783,16 +911,6 @@ fn visit_aggregation_mut( modified_something } -// equivalent to Bound::map, which is unstable -pub fn map_bound(bound: Bound, f: impl FnOnce(T) -> U) -> Bound { - use Bound::*; - match bound { - Unbounded => Unbounded, - Included(x) => Included(f(x)), - Excluded(x) => Excluded(f(x)), - } -} - // returns the max of left and right, that isn't unbounded. Useful for making // the intersection of lower bound of ranges fn max_bound(left: Bound, right: Bound) -> Bound { @@ -925,12 +1043,8 @@ fn remove_redundant_timestamp_range( if final_start_timestamp != Bound::Unbounded || final_end_timestamp != Bound::Unbounded { let range = RangeQuery { field: timestamp_field.to_string(), - lower_bound: map_bound(final_start_timestamp, |bound| { - bound.into_timestamp_nanos().into() - }), - upper_bound: map_bound(final_end_timestamp, |bound| { - bound.into_timestamp_nanos().into() - }), + lower_bound: final_start_timestamp.map(|bound| bound.into_timestamp_nanos().into()), + upper_bound: final_end_timestamp.map(|bound| bound.into_timestamp_nanos().into()), }; new_ast = if let QueryAst::Bool(mut bool_query) = new_ast { if bool_query.must.is_empty() @@ -1064,27 +1178,6 @@ impl QueryAstTransformer for RemoveTimestampRange<'_> { } } -pub(crate) fn rewrite_start_end_time_bounds( - start_timestamp_opt: &mut Option, - end_timestamp_opt: &mut Option, - split: &SplitIdAndFooterOffsets, -) { - if let (Some(split_start), Some(split_end)) = (split.timestamp_start, split.timestamp_end) { - if let Some(start_timestamp) = start_timestamp_opt { - // both starts are inclusive - if *start_timestamp <= split_start { - *start_timestamp_opt = None; - } - } - if let Some(end_timestamp) = end_timestamp_opt { - // search end is exclusive, split end is inclusive - if *end_timestamp > split_end { - *end_timestamp_opt = None; - } - } - } -} - /// Checks if request is a simple all query. /// Simple in this case would still including sorting fn is_simple_all_query(search_request: &SearchRequest) -> bool { @@ -1120,19 +1213,14 @@ pub enum CanSplitDoBetter { impl CanSplitDoBetter { /// Create a CanSplitDoBetter from a SearchRequest fn from_request(request: &SearchRequest, timestamp_field_name: Option<&str>) -> Self { - if request.max_hits == 0 { - if let Some(aggregation) = &request.aggregation_request { - if let Ok(crate::QuickwitAggregations::FindTraceIdsAggregation( - find_trace_aggregation, - )) = serde_json::from_str(aggregation) - { - if Some(find_trace_aggregation.span_timestamp_field_name.as_str()) - == timestamp_field_name - { - return CanSplitDoBetter::FindTraceIdsAggregation(None); - } - } - } + if request.max_hits == 0 + && let Some(aggregation) = &request.aggregation_request + && let Ok(crate::QuickwitAggregations::FindTraceIdsAggregation(find_trace_aggregation)) = + serde_json::from_str(aggregation) + && Some(find_trace_aggregation.span_timestamp_field_name.as_str()) + == timestamp_field_name + { + return CanSplitDoBetter::FindTraceIdsAggregation(None); } if request.sort_fields.is_empty() { @@ -1185,12 +1273,12 @@ impl CanSplitDoBetter { /// Returns the search_requests with their split. fn optimize( &self, - request: Arc, + request: &SearchRequest, mut splits: Vec, ) -> Result, SearchError> { self.optimize_split_order(&mut splits); - if !is_simple_all_query(&request) { + if !is_simple_all_query(request) { // no optimization opportunity here. return Ok(splits .into_iter() @@ -1335,7 +1423,7 @@ impl CanSplitDoBetter { pub async fn multi_index_leaf_search( searcher_context: Arc, leaf_search_request: LeafSearchRequest, - storage_resolver: &StorageResolver, + storage_resolver: StorageResolver, ) -> Result { let search_request: Arc = leaf_search_request .search_request @@ -1347,8 +1435,7 @@ pub async fn multi_index_leaf_search( .iter() .map(|doc_mapper| deserialize_doc_mapper(doc_mapper)) .collect::>()?; - // Creates a collector which merges responses into one - let aggregation_limits = searcher_context.get_aggregation_limits(); + // TODO: to avoid lockstep, we should pull up the future creation over the list of split ids // and have the semaphore on this level. // This will lower resource consumption due to less in-flight futures and avoid contention. @@ -1356,8 +1443,7 @@ pub async fn multi_index_leaf_search( // // It is a little bit tricky how to handle which is now the incremental_merge_collector, one // per index, e.g. when to merge results and how to avoid lock contention. - let mut leaf_request_tasks = Vec::new(); - + let mut leaf_request_futures = JoinSet::new(); for leaf_search_request_ref in leaf_search_request.leaf_requests.into_iter() { let index_uri = quickwit_common::uri::Uri::from_str( leaf_search_request @@ -1380,11 +1466,11 @@ pub async fn multi_index_leaf_search( })? .clone(); - let leaf_request_future = tokio::spawn({ - let storage_resolver = storage_resolver.clone(); - let searcher_context = searcher_context.clone(); - let search_request = search_request.clone(); - let aggregation_limits = aggregation_limits.clone(); + let storage_resolver = storage_resolver.clone(); + let searcher_context = searcher_context.clone(); + let search_request = search_request.clone(); + + leaf_request_futures.spawn({ async move { let storage = storage_resolver.resolve(&index_uri).await?; single_doc_mapping_leaf_search( @@ -1393,26 +1479,24 @@ pub async fn multi_index_leaf_search( storage, leaf_search_request_ref.split_offsets, doc_mapper, - aggregation_limits, ) + .in_current_span() .await } - .in_current_span() }); - leaf_request_tasks.push(leaf_request_future); } - let leaf_responses: Vec> = tokio::time::timeout( - searcher_context.searcher_config.request_timeout(), - try_join_all(leaf_request_tasks), - ) - .await??; - let merge_collector = make_merge_collector(&search_request, &aggregation_limits)?; + // Creates a collector which merges responses into one + let merge_collector = + make_merge_collector(&search_request, searcher_context.get_aggregation_limits())?; let mut incremental_merge_collector = IncrementalCollector::new(merge_collector); - for result in leaf_responses { - match result { - Ok(result) => { - incremental_merge_collector.add_result(result)?; + + while let Some(leaf_response_join_result) = leaf_request_futures.join_next().await { + // abort the search on join errors + let leaf_response_result = leaf_response_join_result?; + match leaf_response_result { + Ok(leaf_response) => { + incremental_merge_collector.add_result(leaf_response)?; } Err(err) => { incremental_merge_collector.add_failed_split(SplitSearchError { @@ -1432,17 +1516,42 @@ pub async fn multi_index_leaf_search( } /// Optimizes the search_request based on CanSplitDoBetter -/// Returns true if the split can return better results -fn check_optimize_search_request( - search_request: &mut SearchRequest, +/// Returns None if the search request does nothing can be skipped. +#[must_use] +fn simplify_search_request( + mut search_request: SearchRequest, split: &SplitIdAndFooterOffsets, - split_filter: &Arc>, -) -> bool { - let can_be_better = split_filter.read().unwrap().can_be_better(split); + split_filter_lock: &Arc>, +) -> Option { + let can_be_better: bool; + let is_trace_req: bool; + { + let split_filter_guard = split_filter_lock.read().unwrap(); + can_be_better = split_filter_guard.can_be_better(split); + // The info is originally from the search_request.aggregation as a string (yes we need to + // clean this eventually). We don't want to parse it again, so we use the + // split_filter variant to get that info. + is_trace_req = matches!( + &*split_filter_guard, + &CanSplitDoBetter::FindTraceIdsAggregation(_) + ); + } if !can_be_better { - disable_search_request_hits(search_request); + disable_search_request_hits(&mut search_request); + } + if is_trace_req { + return Some(search_request); } - can_be_better + if search_request.max_hits > 0 { + return Some(search_request); + } + if search_request.aggregation_request.is_some() { + return Some(search_request); + } + if search_request.count_hits() == CountHits::CountAll { + return Some(search_request); + } + None } /// Alter the search request so it does not return any docs. @@ -1452,189 +1561,511 @@ fn disable_search_request_hits(search_request: &mut SearchRequest) { search_request.max_hits = 0; search_request.start_offset = 0; search_request.sort_fields.clear(); + search_request.search_after = None; } /// Searches multiple splits for a specific index and a single doc mapping -/// +/// Offloads splits to Lambda invocations, distributing them across batches +/// balanced by document count. Each batch is invoked independently; a failure +/// in one batch does not affect others. +async fn run_offloaded_search_tasks( + searcher_context: &SearcherContext, + search_request: &SearchRequest, + doc_mapper: &DocMapper, + index_uri: Uri, + splits_with_requests: Vec<(SplitIdAndFooterOffsets, SearchRequest)>, + incremental_merge_collector: &Mutex, +) -> Result<(), SearchError> { + if splits_with_requests.is_empty() { + return Ok(()); + } + + info!( + num_offloaded_splits = splits_with_requests.len(), + "offloading to lambda" + ); + + let lambda_invoker = searcher_context.lambda_invoker.as_ref().expect( + "did not receive enough permit futures despite not having any lambda invoker to offload to", + ); + let lambda_config = searcher_context.searcher_config.lambda.as_ref().unwrap(); + + let doc_mapper_str = serde_json::to_string(doc_mapper) + .map_err(|err| SearchError::Internal(format!("failed to serialize doc mapper: {err}")))?; + + // Build a lookup so we can match lambda results (tagged by split_id) back to the + // split metadata and per-split SearchRequest needed for caching. + let mut split_lookup: HashMap = + HashMap::with_capacity(splits_with_requests.len()); + let splits: Vec = splits_with_requests + .into_iter() + .map(|(split, search_req)| { + split_lookup.insert(split.split_id.clone(), (split.clone(), search_req)); + split + }) + .collect(); + + let batches: Vec> = greedy_batch_split( + splits, + |split| split.num_docs, + lambda_config.max_splits_per_invocation, + ); + + let mut lambda_tasks_joinset = JoinSet::new(); + for batch in batches { + let batch_split_ids: Vec = + batch.iter().map(|split| split.split_id.clone()).collect(); + let leaf_request = LeafSearchRequest { + // Note this is not the split-specific rewritten request, we ship the main request, + // and the leaf will apply the split specific rewrite on its own. + search_request: Some(search_request.clone()), + doc_mappers: vec![doc_mapper_str.clone()], + index_uris: vec![index_uri.as_str().to_string()], //< careful here. Calling to_string() directly would return a redacted uri. + leaf_requests: vec![quickwit_proto::search::LeafRequestRef { + index_uri_ord: 0, + doc_mapper_ord: 0, + split_offsets: batch, + }], + }; + let invoker = lambda_invoker.clone(); + lambda_tasks_joinset.spawn(async move { + ( + batch_split_ids, + invoker.invoke_leaf_search(leaf_request).await, + ) + }); + } + + while let Some(join_res) = lambda_tasks_joinset.join_next().await { + let Ok((batch_split_ids, result)) = join_res else { + error!("lambda join error"); + return Err(SearchError::Internal("lambda join error".to_string())); + }; + match result { + Ok(split_results) => { + let mut locked = incremental_merge_collector.lock().unwrap(); + for split_result in split_results { + match split_result.outcome { + Some(Outcome::Response(response)) => { + if let Some((split_info, single_split_search_req)) = + split_lookup.remove(&split_result.split_id) + { + // We use the single_split_search_req to perform the search + searcher_context.leaf_search_cache.put( + split_info, + single_split_search_req, + response.clone(), + ); + } + if let Err(err) = locked.add_result(response) { + error!(error = %err, "failed to add lambda result to collector"); + } + } + Some(Outcome::Error(error_msg)) => { + locked.add_failed_split(SplitSearchError { + split_id: split_result.split_id, + error: format!("lambda split error: {error_msg}"), + retryable_error: true, + }); + } + None => { + locked.add_failed_split(SplitSearchError { + split_id: split_result.split_id, + error: "lambda returned empty outcome".to_string(), + retryable_error: true, + }); + } + } + } + } + Err(err) => { + // Transport-level failure: the Lambda invocation itself failed. + // Mark all splits in this batch as failed. + error!( + error = %err, + num_splits = batch_split_ids.len(), + "lambda invocation failed for batch" + ); + let mut locked = incremental_merge_collector.lock().unwrap(); + for split_id in batch_split_ids { + locked.add_failed_split(SplitSearchError { + split_id, + error: format!("lambda invocation error: {err}"), + retryable_error: true, + }); + } + } + } + } + + Ok(()) +} + +struct LocalSearchTask { + split: SplitIdAndFooterOffsets, + search_request: SearchRequest, + search_permit_future: SearchPermitFuture, +} + +struct ScheduleSearchTaskResult { + // The search permit futures associated to each local_search_task are + // guaranteed to resolve in order. + local_search_tasks: Vec, + // The per-split SearchRequest (already rewritten by `rewrite_request()`) is preserved + // so that lambda results can be cached with the correct cache key in `leaf_search_cache`. + offloaded_search_tasks: Vec<(SplitIdAndFooterOffsets, SearchRequest)>, +} + +/// Schedule search tasks, either: +/// - locally +/// - remotely on lambdas, if lambda are configured, and the number of tasks scheduled exceed the +/// offload threshold. +async fn schedule_search_tasks( + mut splits: Vec<(SplitIdAndFooterOffsets, SearchRequest)>, + searcher_context: &SearcherContext, +) -> ScheduleSearchTaskResult { + let permit_sizes: Vec = splits + .iter() + .map(|(split, _)| { + compute_initial_memory_allocation( + split, + searcher_context + .searcher_config + .warmup_single_split_initial_allocation, + ) + }) + .collect(); + + let offload_threshold: usize = if searcher_context.lambda_invoker.is_some() + && let Some(lambda_config) = &searcher_context.searcher_config.lambda + { + lambda_config.offload_threshold + } else { + usize::MAX + }; + + let search_permit_futures = searcher_context + .search_permit_provider + .get_permits_with_offload(permit_sizes, offload_threshold) + .await; + + let splits_to_run_on_lambda: Vec<(SplitIdAndFooterOffsets, SearchRequest)> = + splits.drain(search_permit_futures.len()..).collect(); + + let splits_to_run_locally: Vec = splits + .into_iter() + .zip(search_permit_futures) + .map( + |((split, search_request), search_permit_future)| LocalSearchTask { + split, + search_request, + search_permit_future, + }, + ) + .collect(); + + ScheduleSearchTaskResult { + local_search_tasks: splits_to_run_locally, + offloaded_search_tasks: splits_to_run_on_lambda, + } +} + /// The leaf search collects all kind of information, and returns a set of -/// [PartialHit](quickwit_proto::search::PartialHit) candidates. The root will be in +/// [PartialHit] candidates. The root will be in /// charge to consolidate, identify the actual final top hits to display, and /// fetch the actual documents to convert the partial hits into actual Hits. -#[instrument(skip_all, fields(index = ?request.index_id_patterns))] pub async fn single_doc_mapping_leaf_search( searcher_context: Arc, request: Arc, index_storage: Arc, splits: Vec, doc_mapper: Arc, - aggregations_limits: AggregationLimitsGuard, ) -> Result { let num_docs: u64 = splits.iter().map(|split| split.num_docs).sum(); let num_splits = splits.len(); - let current_span = tracing::Span::current(); - current_span.record("num_docs", num_docs); - current_span.record("num_splits", num_splits); - info!(num_docs, num_splits, split_offsets = ?PrettySample::new(&splits, 5)); - let split_filter = CanSplitDoBetter::from_request(&request, doc_mapper.timestamp_field_name()); - let split_with_req = split_filter.optimize(request.clone(), splits)?; + // We simplify the request as much as possible. + let split_filter: CanSplitDoBetter = + CanSplitDoBetter::from_request(&request, doc_mapper.timestamp_field_name()); + let mut split_with_req: Vec<(SplitIdAndFooterOffsets, SearchRequest)> = + split_filter.optimize(&request, splits)?; + for (split, single_split_search_request) in &mut split_with_req { + rewrite_request( + single_split_search_request, + split, + doc_mapper.timestamp_field_name(), + ); + } + let split_filter_arc: Arc> = Arc::new(RwLock::new(split_filter)); + + let merge_collector = + make_merge_collector(&request, searcher_context.get_aggregation_limits())?; + let mut incremental_merge_collector = IncrementalCollector::new(merge_collector); - // if client wants full count, or we are doing an aggregation, we want to run every splits. - // However if the aggregation is the tracing aggregation, we don't actually need all splits. - let run_all_splits = request.count_hits() == CountHits::CountAll - || (request.aggregation_request.is_some() - && !matches!(split_filter, CanSplitDoBetter::FindTraceIdsAggregation(_))); + let split_outcome_counters = Arc::new(SplitSearchOutcomeCounters::new_unregistered()); - let split_filter = Arc::new(RwLock::new(split_filter)); + // Sort out the splits that are already in the partial result cache. + let uncached_splits: Vec<(SplitIdAndFooterOffsets, SearchRequest)> = + process_partial_result_cache( + &searcher_context.leaf_search_cache, + split_with_req, + split_outcome_counters.clone(), + &mut incremental_merge_collector, + )?; + let incremental_merge_collector_arc: Arc> = + Arc::new(Mutex::new(incremental_merge_collector)); - let mut leaf_search_single_split_join_handles: Vec<(String, tokio::task::JoinHandle<()>)> = - Vec::with_capacity(split_with_req.len()); + // Determine which uncached splits to process locally vs offload. + let ScheduleSearchTaskResult { + local_search_tasks, + offloaded_search_tasks, + } = schedule_search_tasks(uncached_splits, &searcher_context).await; - let merge_collector = make_merge_collector(&request, &aggregations_limits)?; - let incremental_merge_collector = IncrementalCollector::new(merge_collector); - let incremental_merge_collector = Arc::new(Mutex::new(incremental_merge_collector)); + // Offload splits to Lambda. + let run_offloaded_search_tasks_fut = run_offloaded_search_tasks( + &searcher_context, + &request, + &doc_mapper, + index_storage.uri().clone(), + offloaded_search_tasks, + &incremental_merge_collector_arc, + ); - // We acquire all of the leaf search permits to make sure our single split search tasks - // do no interleave with other leaf search requests. - let permit_sizes = split_with_req.iter().map(|(split, _)| { - compute_initial_memory_allocation( - split, - searcher_context - .searcher_config - .warmup_single_split_initial_allocation, - ) + // Spawn local split search tasks. + let leaf_search_context = Arc::new(LeafSearchContext { + searcher_context: searcher_context.clone(), + split_outcome_counters, + incremental_merge_collector: incremental_merge_collector_arc.clone(), + doc_mapper: doc_mapper.clone(), + split_filter: split_filter_arc.clone(), }); - let permit_futures = searcher_context - .search_permit_provider - .get_permits(permit_sizes) - .await; + let run_local_search_tasks_fut = run_local_search_tasks( + local_search_tasks, + index_storage, + split_filter_arc, + leaf_search_context, + ); + + let (offloaded_res, _) = + tokio::join!(run_offloaded_search_tasks_fut, run_local_search_tasks_fut); + offloaded_res?; + + // we can't use unwrap_or_clone because mutexes aren't Clone + let incremental_merge_collector = match Arc::try_unwrap(incremental_merge_collector_arc) { + Ok(filter_merger) => filter_merger.into_inner().unwrap(), + Err(filter_merger) => filter_merger.lock().unwrap().clone(), + }; - for ((split, mut request), permit_fut) in - split_with_req.into_iter().zip(permit_futures.into_iter()) + let leaf_search_response_result: tantivy::Result = + crate::search_thread_pool() + .run_cpu_intensive(|| incremental_merge_collector.finalize()) + .instrument(info_span!("incremental_merge_intermediate")) + .await + .context("failed to merge split search responses: thread panicked")?; + + Ok(leaf_search_response_result?) +} + +async fn run_local_search_tasks( + local_search_tasks: Vec, + index_storage: Arc, + split_filter_arc: Arc>, + leaf_search_context: Arc, +) { + let mut split_search_joinset = JoinSet::new(); + let mut task_id_to_split_id_map = HashMap::with_capacity(local_search_tasks.len()); + + for LocalSearchTask { + split, + search_request, + search_permit_future, + } in local_search_tasks { - let leaf_split_search_permit = permit_fut + let leaf_split_search_permit = search_permit_future .instrument(info_span!("waiting_for_leaf_search_split_semaphore")) .await; - let can_be_better = check_optimize_search_request(&mut request, &split, &split_filter); - if !can_be_better && !run_all_splits { + // We run simplify search request again: as we push split into the merge collector, + // we may have discovered that we won't find any better candidates for top hits in this + // split, in which case we can remove top hits collection. + let Some(simplified_search_request) = + simplify_search_request(search_request, &split, &split_filter_arc) + else { + let mut leaf_search_state_guard = + SplitSearchStateGuard::new(leaf_search_context.split_outcome_counters.clone()); + leaf_search_state_guard.set_state(SplitSearchState::PrunedBeforeWarmup); continue; - } - - leaf_search_single_split_join_handles.push(( - split.split_id.clone(), - tokio::spawn( - leaf_search_single_split_wrapper( - request, - searcher_context.clone(), - index_storage.clone(), - doc_mapper.clone(), - split, - split_filter.clone(), - incremental_merge_collector.clone(), - leaf_split_search_permit, - aggregations_limits.clone(), - ) - .in_current_span(), - ), - )); + }; + let split_id = split.split_id.clone(); + let handle = split_search_joinset.spawn( + leaf_search_single_split_wrapper( + simplified_search_request, + leaf_search_context.clone(), + index_storage.clone(), + split.clone(), + leaf_split_search_permit, + ) + .in_current_span(), + ); + task_id_to_split_id_map.insert(handle.id(), split_id); } - // TODO we could cancel running splits when !run_all_splits and the running split can no - // longer give better results after some other split answered. + // Await all local tasks. let mut split_search_join_errors: Vec<(String, JoinError)> = Vec::new(); - // There is no need to use `join_all`, as these are spawned tasks. - for (split, leaf_search_join_handle) in leaf_search_single_split_join_handles { - // splits that did not panic were already added to the collector - if let Err(join_error) = leaf_search_join_handle.await { + while let Some(leaf_search_join_result) = split_search_joinset.join_next().await { + if let Err(join_error) = leaf_search_join_result { if join_error.is_cancelled() { - // An explicit task cancellation is not an error. continue; } + let split_id = task_id_to_split_id_map.get(&join_error.id()).unwrap(); if join_error.is_panic() { - error!(split=%split, "leaf search task panicked"); + error!(split=%split_id, "leaf search task panicked"); } else { - error!(split=%split, "please report: leaf search was not cancelled, and could not extract panic. this should never happen"); + error!(split=%split_id, "please report: leaf search was not cancelled, and could not extract panic. this should never happen"); } - split_search_join_errors.push((split, join_error)); + split_search_join_errors.push((split_id.clone(), join_error)); } } - // we can't use unwrap_or_clone because mutexes aren't Clone - let mut incremental_merge_collector = match Arc::try_unwrap(incremental_merge_collector) { - Ok(filter_merger) => filter_merger.into_inner().unwrap(), - Err(filter_merger) => filter_merger.lock().unwrap().clone(), - }; - + let mut incremental_merge_collector_lock = leaf_search_context + .incremental_merge_collector + .lock() + .unwrap(); for (split_id, split_search_join_error) in split_search_join_errors { - incremental_merge_collector.add_failed_split(SplitSearchError { + incremental_merge_collector_lock.add_failed_split(SplitSearchError { split_id, error: SearchError::from(split_search_join_error).to_string(), retryable_error: true, }); } - let leaf_search_response_reresult: Result, _> = - crate::search_thread_pool() - .run_cpu_intensive(|| incremental_merge_collector.finalize()) - .instrument(info_span!("incremental_merge_intermediate")) - .await - .context("failed to merge split search responses"); + info!(split_outcome_counters=%leaf_search_context.split_outcome_counters, "leaf split search finished"); +} + +/// We identify the splits that are in the cache and append them to the incremental merge collector. +/// The (split, request) that are yet to be processed are returned. +fn process_partial_result_cache( + leaf_search_cache: &LeafSearchCache, + split_with_req: Vec<(SplitIdAndFooterOffsets, SearchRequest)>, + split_outcome_counters: Arc, + incremental_merge_collector: &mut IncrementalCollector, +) -> Result, SearchError> { + let mut uncached_splits: Vec<(SplitIdAndFooterOffsets, SearchRequest)> = + Vec::with_capacity(split_with_req.len()); + for (split, search_request) in split_with_req { + if let Some(cached_response) = leaf_search_cache + // TODO remove the clone here. + .get(split.clone(), search_request.clone()) + { + let mut split_search_guard = SplitSearchStateGuard::new(split_outcome_counters.clone()); + split_search_guard.set_state(SplitSearchState::CacheHit); + incremental_merge_collector.add_result(cached_response)?; + } else { + uncached_splits.push((split, search_request)); + } + } + Ok(uncached_splits) +} + +#[derive(Copy, Clone)] +enum SplitSearchState { + Start, + CacheHit, + PrunedBeforeWarmup, + WarmUp, + PrunedAfterWarmup, + CpuQueue, + Cpu, + Success, +} + +impl SplitSearchState { + pub fn inc(self, counters: &SplitSearchOutcomeCounters) { + match self { + SplitSearchState::Start => counters.cancel_before_warmup.inc(), + SplitSearchState::CacheHit => counters.cache_hit.inc(), + SplitSearchState::PrunedBeforeWarmup => counters.pruned_before_warmup.inc(), + SplitSearchState::WarmUp => counters.cancel_warmup.inc(), + SplitSearchState::PrunedAfterWarmup => counters.pruned_after_warmup.inc(), + SplitSearchState::CpuQueue => counters.cancel_cpu_queue.inc(), + SplitSearchState::Cpu => counters.cancel_cpu.inc(), + SplitSearchState::Success => counters.success.inc(), + } + } +} + +impl Drop for SplitSearchStateGuard { + fn drop(&mut self) { + self.state + .inc(&crate::metrics::SEARCH_METRICS.split_search_outcome_total); + self.state.inc(&self.local_split_search_outcome_counters); + } +} + +struct SplitSearchStateGuard { + state: SplitSearchState, + local_split_search_outcome_counters: Arc, +} + +impl SplitSearchStateGuard { + pub fn new(local_split_search_outcome_counters: Arc) -> Self { + SplitSearchStateGuard { + state: SplitSearchState::Start, + local_split_search_outcome_counters: local_split_search_outcome_counters.clone(), + } + } + + pub fn set_state(&mut self, state: SplitSearchState) { + self.state = state; + } +} - Ok(leaf_search_response_reresult??) +struct LeafSearchContext { + searcher_context: Arc, + split_outcome_counters: Arc, + incremental_merge_collector: Arc>, + doc_mapper: Arc, + split_filter: Arc>, } #[allow(clippy::too_many_arguments)] #[instrument(skip_all, fields(split_id = split.split_id, num_docs = split.num_docs))] async fn leaf_search_single_split_wrapper( request: SearchRequest, - searcher_context: Arc, + ctx: Arc, index_storage: Arc, - doc_mapper: Arc, split: SplitIdAndFooterOffsets, - split_filter: Arc>, - incremental_merge_collector: Arc>, mut search_permit: SearchPermit, - aggregations_limits: AggregationLimitsGuard, ) { - crate::SEARCH_METRICS.leaf_searches_splits_total.inc(); let timer = crate::SEARCH_METRICS .leaf_search_split_duration_secs .start_timer(); - - // Debug: Dump the QueryAst before calling leaf_search_single_split - debug!( - split_id = split.split_id, - query_ast = %request.query_ast, - "QueryAst debug dump before leaf_search_single_split" - ); - - let leaf_search_single_split_res = leaf_search_single_split( - &searcher_context, - request, - index_storage, - split.clone(), - doc_mapper, - split_filter.clone(), - aggregations_limits, - &mut search_permit, - None, - ) - .await; + let leaf_search_single_split_opt_res: crate::Result> = + leaf_search_single_split( + request, + ctx.searcher_context.clone(), + index_storage, + split.clone(), + ctx.doc_mapper.clone(), + ctx.split_filter.clone(), + &mut search_permit, + None, + ) + .await; // Explicitly drop the permit for readability. // This should always happen after the ephemeral search cache is dropped. std::mem::drop(search_permit); - if leaf_search_single_split_res.is_ok() { + if leaf_search_single_split_opt_res.is_ok() { timer.observe_duration(); } - let mut locked_incremental_merge_collector = incremental_merge_collector.lock().unwrap(); - match leaf_search_single_split_res { - Ok(split_search_res) => { + let mut locked_incremental_merge_collector = ctx.incremental_merge_collector.lock().unwrap(); + match leaf_search_single_split_opt_res { + Ok(Some(split_search_res)) => { if let Err(err) = locked_incremental_merge_collector.add_result(split_search_res) { locked_incremental_merge_collector.add_failed_split(SplitSearchError { split_id: split.split_id.clone(), @@ -1643,6 +2074,7 @@ async fn leaf_search_single_split_wrapper( }); } } + Ok(None) => {} Err(err) => locked_incremental_merge_collector.add_failed_split(SplitSearchError { split_id: split.split_id.clone(), error: format!("{err}"), @@ -1652,7 +2084,7 @@ async fn leaf_search_single_split_wrapper( if let Some(last_hit) = locked_incremental_merge_collector.peek_worst_hit() { // TODO: we could use the RWLock instead and read the value instead of updating it // unconditionally. - split_filter + ctx.split_filter .write() .unwrap() .record_new_worst_hit(last_hit.as_ref()); @@ -1663,9 +2095,12 @@ async fn leaf_search_single_split_wrapper( mod tests { use std::ops::Bound; + use async_trait::async_trait; use bytes::BufMut; + use quickwit_config::{LambdaConfig, SearcherConfig}; use quickwit_directories::write_hotcache; - use rand::{Rng, thread_rng}; + use quickwit_proto::search::LambdaSingleSplitResult; + use rand::Rng; use tantivy::TantivyDocument; use tantivy::directory::RamDirectory; use tantivy::schema::{ @@ -1673,6 +2108,7 @@ mod tests { }; use super::*; + use crate::LambdaLeafSearchInvoker; fn bool_filter(ast: impl Into) -> QueryAst { BoolQuery { @@ -2147,7 +2583,7 @@ mod tests { // We use random bytes so that the store can't compress them let mut payload = vec![0u8; 1024]; - thread_rng().fill(&mut payload[..]); + rand::rng().fill(&mut payload[..]); let (hotcache_directory_stored_payload, directory_size_stored_payload) = create_tantivy_dir_with_hotcache( @@ -2202,4 +2638,241 @@ mod tests { assert!(directory_size_larger > directory_size_smaller + 100); assert!(larger_size > smaller_size + 100); } + + fn nz(n: usize) -> std::num::NonZeroUsize { + std::num::NonZeroUsize::new(n).unwrap() + } + + #[test] + fn test_greedy_batch_split_empty() { + let items: Vec = vec![]; + let batches = super::greedy_batch_split(items, |&x| x, nz(5)); + assert!(batches.is_empty()); + } + + #[test] + fn test_greedy_batch_split_single_batch() { + let items = vec![10u64, 20, 30]; + let batches = super::greedy_batch_split(items, |&x| x, nz(10)); + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].len(), 3); + } + + #[test] + fn test_greedy_batch_split_balances_weights() { + // 7 items with weights, max 3 per batch -> 3 batches + let items = vec![100u64, 80, 60, 50, 40, 30, 20]; + let batches = super::greedy_batch_split(items, |&x| x, nz(3)); + + assert_eq!(batches.len(), 3); + + // All items should be present + let mut all_items: Vec = batches.iter().flatten().copied().collect(); + all_items.sort_unstable(); + assert_eq!(all_items, vec![20, 30, 40, 50, 60, 80, 100]); + + // Check weights are reasonably balanced + let weights: Vec = batches.iter().map(|b| b.iter().sum()).collect(); + let max_weight = *weights.iter().max().unwrap(); + let min_weight = *weights.iter().min().unwrap(); + // With greedy LPT, the imbalance should be bounded + assert!( + max_weight <= min_weight * 2, + "weights should be reasonably balanced: {:?}", + weights + ); + } + + #[test] + fn test_greedy_batch_split_count_balance() { + // 10 items, max 3 per batch -> 4 batches + // counts should be either 2 or 3 per batch + let items: Vec = (0..10).collect(); + let batches = super::greedy_batch_split(items, |&x| x, nz(3)); + + assert_eq!(batches.len(), 4); + let counts: Vec = batches.iter().map(|b| b.len()).collect(); + for count in &counts { + assert!( + *count >= 2 && *count <= 3, + "count should be 2 or 3, got {}", + count + ); + } + assert_eq!(counts.iter().sum::(), 10); + } + + fn make_splits_with_requests( + num_splits: usize, + ) -> Vec<(SplitIdAndFooterOffsets, SearchRequest)> { + (0..num_splits) + .map(|idx| { + let split = SplitIdAndFooterOffsets { + split_id: format!("split_{idx}"), + num_docs: 100, + ..Default::default() + }; + (split, SearchRequest::default()) + }) + .collect() + } + + #[tokio::test] + async fn test_schedule_search_tasks_no_lambda_all_local() { + let searcher_context = SearcherContext::for_test(); + let splits = make_splits_with_requests(5); + let result = super::schedule_search_tasks(splits, &searcher_context).await; + assert_eq!(result.local_search_tasks.len(), 5); + assert!(result.offloaded_search_tasks.is_empty()); + for (idx, task) in result.local_search_tasks.iter().enumerate() { + assert_eq!(task.split.split_id, format!("split_{idx}")); + } + } + + struct DummyInvoker; + #[async_trait] + impl LambdaLeafSearchInvoker for DummyInvoker { + async fn invoke_leaf_search( + &self, + _req: LeafSearchRequest, + ) -> Result, SearchError> { + todo!() + } + } + + #[tokio::test] + async fn test_schedule_search_tasks_lambda_offloads_excess() { + let mut config = SearcherConfig::default(); + config.lambda = Some(LambdaConfig { + offload_threshold: 3, + ..LambdaConfig::for_test() + }); + let searcher_context = SearcherContext::new(config, None, Some(Arc::new(DummyInvoker))); + let splits = make_splits_with_requests(7); + let result = super::schedule_search_tasks(splits, &searcher_context).await; + assert_eq!(result.local_search_tasks.len(), 3); + assert_eq!(result.offloaded_search_tasks.len(), 4); + for (idx, task) in result.local_search_tasks.iter().enumerate() { + assert_eq!(task.split.split_id, format!("split_{idx}")); + } + for (idx, (split, _req)) in result.offloaded_search_tasks.iter().enumerate() { + assert_eq!(split.split_id, format!("split_{}", idx + 3)); + } + } + + #[tokio::test] + async fn test_schedule_search_tasks_lambda_threshold_zero_offloads_all() { + let mut config = SearcherConfig::default(); + config.lambda = Some(LambdaConfig { + offload_threshold: 0, + ..LambdaConfig::for_test() + }); + let searcher_context = SearcherContext::new(config, None, Some(Arc::new(DummyInvoker))); + let splits = make_splits_with_requests(5); + let result = super::schedule_search_tasks(splits, &searcher_context).await; + assert!(result.local_search_tasks.is_empty()); + assert_eq!(result.offloaded_search_tasks.len(), 5); + } + + #[tokio::test] + async fn test_schedule_search_tasks_lambda_threshold_above_split_count() { + let mut config = SearcherConfig::default(); + config.lambda = Some(LambdaConfig { + offload_threshold: 100, + ..LambdaConfig::for_test() + }); + let searcher_context = SearcherContext::new(config, None, Some(Arc::new(DummyInvoker))); + let splits = make_splits_with_requests(5); + let result = super::schedule_search_tasks(splits, &searcher_context).await; + assert_eq!(result.local_search_tasks.len(), 5); + assert!(result.offloaded_search_tasks.is_empty()); + } + + #[tokio::test] + async fn test_schedule_search_tasks_empty() { + let searcher_context = SearcherContext::for_test(); + let result = super::schedule_search_tasks(Vec::new(), &searcher_context).await; + assert!(result.local_search_tasks.is_empty()); + assert!(result.offloaded_search_tasks.is_empty()); + } + + mod proptest_greedy_batch { + use std::num::NonZeroUsize; + + use proptest::prelude::*; + + proptest! { + #[test] + fn all_items_preserved( + items in prop::collection::vec(0u64..1000, 0..100), + max_per_batch in 1usize..20 + ) { + let original: Vec = items.clone(); + let max_per_batch = NonZeroUsize::new(max_per_batch).unwrap(); + let batches = super::super::greedy_batch_split(items, |&x| x, max_per_batch); + + // All items should be present exactly once + let mut result: Vec = batches.into_iter().flatten().collect(); + result.sort_unstable(); + let mut expected = original; + expected.sort_unstable(); + prop_assert_eq!(result, expected); + } + + #[test] + fn batch_count_correct( + items in prop::collection::vec(0u64..1000, 1..100), + max_per_batch in 1usize..20 + ) { + let n = items.len(); + let max_per_batch_nz = NonZeroUsize::new(max_per_batch).unwrap(); + let batches = super::super::greedy_batch_split(items, |&x| x, max_per_batch_nz); + + let expected_batches = n.div_ceil(max_per_batch); + prop_assert_eq!(batches.len(), expected_batches); + } + + #[test] + fn total_items_matches( + items in prop::collection::vec(0u64..1000, 1..100), + max_per_batch in 1usize..20 + ) { + let n = items.len(); + let max_per_batch = NonZeroUsize::new(max_per_batch).unwrap(); + let batches = super::super::greedy_batch_split(items, |&x| x, max_per_batch); + + // Total items across all batches equals input + let total: usize = batches.iter().map(|b| b.len()).sum(); + prop_assert_eq!(total, n); + } + + #[test] + fn greedy_balances_by_weight_not_count( + // Use items with significant weights to test weight balancing + items in prop::collection::vec(100u64..1000, 4..30), + max_per_batch in 2usize..10 + ) { + let max_per_batch = NonZeroUsize::new(max_per_batch).unwrap(); + let batches = super::super::greedy_batch_split(items, |&x| x, max_per_batch); + + if batches.len() >= 2 { + let weights: Vec = batches.iter().map(|b| b.iter().sum()).collect(); + let total_weight: u64 = weights.iter().sum(); + let avg_weight = total_weight / batches.len() as u64; + + // LPT guarantees max makespan <= (4/3) * optimal + // With balanced input, max should be close to average + let max_weight = *weights.iter().max().unwrap(); + + // Max weight should be at most 2x average (generous bound) + prop_assert!( + max_weight <= avg_weight * 2 + 1000, // +1000 for rounding slack + "max weight {} too far from average {}", + max_weight, + avg_weight + ); + } + } + } + } } diff --git a/quickwit/quickwit-search/src/leaf_cache.rs b/quickwit/quickwit-search/src/leaf_cache.rs index e1f284214f7..cc4d27e464e 100644 --- a/quickwit/quickwit-search/src/leaf_cache.rs +++ b/quickwit/quickwit-search/src/leaf_cache.rs @@ -15,11 +15,13 @@ use std::ops::{Bound, RangeBounds}; use prost::Message; +use quickwit_config::CacheConfig; use quickwit_proto::search::{ CountHits, LeafSearchResponse, SearchRequest, SplitIdAndFooterOffsets, }; use quickwit_proto::types::SplitId; use quickwit_storage::{MemorySizedCache, OwnedBytes}; +use tantivy::index::SegmentId; /// A cache to memoize `leaf_search_single_split` results. pub struct LeafSearchCache { @@ -42,10 +44,10 @@ pub struct LeafSearchCache { // queries which vary only by search_after. impl LeafSearchCache { - pub fn new(capacity: usize) -> LeafSearchCache { + pub fn new(config: &CacheConfig) -> LeafSearchCache { LeafSearchCache { - content: MemorySizedCache::with_capacity_in_bytes( - capacity, + content: MemorySizedCache::from_config( + config, &quickwit_storage::STORAGE_METRICS.partial_request_cache, ), } @@ -68,14 +70,13 @@ impl LeafSearchCache { result: LeafSearchResponse, ) { let key = CacheKey::from_split_meta_and_request(split_info, search_request); - let encoded_result = result.encode_to_vec(); self.content.put(key, OwnedBytes::new(encoded_result)); } } /// A key inside a [`LeafSearchCache`]. -#[derive(Debug, Hash, PartialEq, Eq)] +#[derive(Debug, Hash, Clone, PartialEq, Eq)] struct CacheKey { /// The split this entry refers to split_id: SplitId, @@ -187,8 +188,54 @@ impl RangeBounds for HalfOpenRange { } } +pub struct PredicateCacheImpl { + content: MemorySizedCache<(SplitId, String)>, +} + +impl PredicateCacheImpl { + pub fn new(config: &CacheConfig) -> Self { + PredicateCacheImpl { + content: MemorySizedCache::from_config( + config, + &quickwit_storage::STORAGE_METRICS.predicate_cache, + ), + } + } +} + +impl quickwit_query::query_ast::PredicateCache for PredicateCacheImpl { + fn get( + &self, + split_id: SplitId, + query_ast_json: String, + ) -> Option<(SegmentId, quickwit_query::query_ast::HitSet)> { + let encoded_result = self.content.get(&(split_id, query_ast_json))?; + let (segment_id_bytes, hits_buffer) = encoded_result.split(32); + let segment_id = + SegmentId::from_uuid_string(str::from_utf8(&segment_id_bytes).ok()?).ok()?; + let hits = quickwit_query::query_ast::HitSet::from_buffer(hits_buffer); + Some((segment_id, hits)) + } + + fn put( + &self, + split_id: SplitId, + query_ast_json: String, + segment: SegmentId, + hits: quickwit_query::query_ast::HitSet, + ) { + let hits_buffer = hits.into_buffer(); + let mut buffer = Vec::with_capacity(32 + hits_buffer.len()); + buffer.extend_from_slice(segment.uuid_string().as_bytes()); + buffer.extend_from_slice(&hits_buffer); + self.content + .put((split_id, query_ast_json), OwnedBytes::new(buffer)); + } +} + #[cfg(test)] mod tests { + use bytesize::ByteSize; use quickwit_proto::search::{ LeafSearchResponse, PartialHit, ResourceStats, SearchRequest, SortValue, SplitIdAndFooterOffsets, @@ -198,7 +245,7 @@ mod tests { #[test] fn test_leaf_search_cache_no_timestamp() { - let cache = LeafSearchCache::new(64_000_000); + let cache = LeafSearchCache::new(&ByteSize::mb(64).into()); let split_1 = SplitIdAndFooterOffsets { split_id: "split_1".to_string(), @@ -264,7 +311,7 @@ mod tests { #[test] fn test_leaf_search_cache_timestamp() { - let cache = LeafSearchCache::new(64_000_000); + let cache = LeafSearchCache::new(&ByteSize::mb(64).into()); let split_1 = SplitIdAndFooterOffsets { split_id: "split_1".to_string(), diff --git a/quickwit/quickwit-search/src/lib.rs b/quickwit/quickwit-search/src/lib.rs index 57b1ba7a9be..19ac307fdfd 100644 --- a/quickwit/quickwit-search/src/lib.rs +++ b/quickwit/quickwit-search/src/lib.rs @@ -22,8 +22,10 @@ mod cluster_client; pub mod collector; mod error; mod fetch_docs; -mod filters; mod find_trace_ids_collector; + +mod invoker; +/// Leaf search operations. pub mod leaf; pub mod leaf_cache; mod list_fields; @@ -35,7 +37,6 @@ mod root; mod scroll_context; mod search_job_placer; mod search_response_rest; -mod search_stream; mod service; pub(crate) mod top_k_collector; @@ -82,17 +83,20 @@ pub use crate::client::{ pub use crate::cluster_client::ClusterClient; pub use crate::error::{SearchError, parse_grpc_error}; use crate::fetch_docs::fetch_docs; +pub use crate::invoker::LambdaLeafSearchInvoker; +pub use crate::leaf::{ + CanSplitDoBetter, SplitOverrides, leaf_search_single_split, open_index_with_caches, + open_split_bundle, warmup, +}; pub use crate::root::{ - IndexMetasForLeafSearch, SearchJob, check_all_index_metadata_found, jobs_to_leaf_request, + IndexMetasForLeafSearch, SearchJob, ensure_all_indexes_found, jobs_to_leaf_request, root_search, search_plan, }; pub use crate::search_job_placer::{Job, SearchJobPlacer}; pub use crate::search_response_rest::{ AggregationResults, SearchPlanResponseRest, SearchResponseRest, }; -pub use crate::search_stream::root_search_stream; pub use crate::service::{MockSearchService, SearchService, SearchServiceImpl}; -pub use crate::leaf::{leaf_search_single_split, warmup, CanSplitDoBetter, open_split_bundle, open_index_with_caches, SplitOverrides}; pub use quickwit_doc_mapper::WarmupInfo; /// A pool of searcher clients identified by their gRPC socket address. @@ -229,7 +233,7 @@ pub async fn resolve_index_patterns( ListIndexesMetadataRequest::all() } else { ListIndexesMetadataRequest { - index_id_patterns: index_id_patterns.to_owned(), + index_id_patterns: index_id_patterns.to_vec(), } }; @@ -239,7 +243,7 @@ pub async fn resolve_index_patterns( .await? .deserialize_indexes_metadata() .await?; - check_all_index_metadata_found(&indexes_metadata, index_id_patterns)?; + ensure_all_indexes_found(&indexes_metadata, index_id_patterns)?; Ok(indexes_metadata) } @@ -288,7 +292,7 @@ pub async fn single_node_search( let search_job_placer = SearchJobPlacer::new(searcher_pool.clone()); let cluster_client = ClusterClient::new(search_job_placer); let searcher_config = SearcherConfig::default(); - let searcher_context = Arc::new(SearcherContext::new(searcher_config, None)); + let searcher_context = Arc::new(SearcherContext::new_without_invoker(searcher_config, None)); let search_service = Arc::new(SearchServiceImpl::new( metastore.clone(), storage_resolver, @@ -311,14 +315,17 @@ pub async fn single_node_search( #[cfg(any(test, feature = "testsuite"))] #[macro_export] macro_rules! encode_term_for_test { - ($field:expr, $value:expr) => { - ::tantivy::schema::Term::from_field_text( - ::tantivy::schema::Field::from_field_id($field), - $value, - ) - .serialized_term() - .to_vec() - }; + ($field:expr, $value:expr) => {{ + #[allow(deprecated)] + { + ::tantivy::schema::Term::from_field_text( + ::tantivy::schema::Field::from_field_id($field), + $value, + ) + .serialized_term() + .to_vec() + } + }}; ($value:expr) => { encode_term_for_test!(0, $value) }; diff --git a/quickwit/quickwit-search/src/list_fields.rs b/quickwit/quickwit-search/src/list_fields.rs index aa8d473e8fb..b5974867cfd 100644 --- a/quickwit/quickwit-search/src/list_fields.rs +++ b/quickwit/quickwit-search/src/list_fields.rs @@ -12,17 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::cmp::Ordering; use std::collections::{HashMap, HashSet}; use std::path::Path; -use std::sync::Arc; +use std::str::FromStr; +use std::sync::{Arc, LazyLock}; use anyhow::Context; use futures::future; use futures::future::try_join_all; use itertools::Itertools; -use quickwit_common::shared_consts::SPLIT_FIELDS_FILE_NAME; +use quickwit_common::rate_limited_warn; +use quickwit_common::shared_consts::{FIELD_PRESENCE_FIELD_NAME, SPLIT_FIELDS_FILE_NAME}; use quickwit_common::uri::Uri; +use quickwit_config::build_doc_mapper; +use quickwit_doc_mapper::tag_pruning::extract_tags_from_query; use quickwit_metastore::SplitMetadata; use quickwit_proto::metastore::MetastoreServiceClient; use quickwit_proto::search::{ @@ -30,25 +33,42 @@ use quickwit_proto::search::{ ListFieldsResponse, SplitIdAndFooterOffsets, deserialize_split_fields, }; use quickwit_proto::types::{IndexId, IndexUid}; +use quickwit_query::query_ast::QueryAst; use quickwit_storage::Storage; use crate::leaf::open_split_bundle; use crate::search_job_placer::group_jobs_by_index_id; use crate::service::SearcherContext; -use crate::{ClusterClient, SearchError, SearchJob, list_relevant_splits, resolve_index_patterns}; +use crate::{ + ClusterClient, SearchError, SearchJob, list_relevant_splits, resolve_index_patterns, + search_thread_pool, +}; + +/// QW_FIELD_LIST_SIZE_LIMIT defines a hard limit on the number of fields that +/// can be returned (error otherwise). +/// +/// Having many fields can happen when a user is creating fields dynamically in +/// a JSON type with random field names. This leads to huge memory consumption +/// when building the response. This is a workaround until a way is found to +/// prune the long tail of rare fields. +static FIELD_LIST_SIZE_LIMIT: LazyLock = + LazyLock::new(|| quickwit_common::get_from_env("QW_FIELD_LIST_SIZE_LIMIT", 100_000, false)); -/// Get the list of splits for the request which we need to scan. -pub async fn get_fields_from_split( +const DYNAMIC_FIELD_PREFIX: &str = "_dynamic."; + +/// Get the list of fields in the given split. +/// The returned list is guaranteed to be strictly sorted by (field_name, field_type). +async fn get_fields_from_split( searcher_context: &SearcherContext, index_id: IndexId, split_and_footer_offsets: &SplitIdAndFooterOffsets, index_storage: Arc, -) -> anyhow::Result + Send>> { +) -> anyhow::Result> { if let Some(list_fields) = searcher_context .list_fields_cache .get(split_and_footer_offsets.clone()) { - return Ok(Box::new(list_fields.fields.into_iter())); + return Ok(list_fields.fields); } let (_, split_bundle) = open_split_bundle(searcher_context, index_storage, split_and_footer_offsets).await?; @@ -57,19 +77,31 @@ pub async fn get_fields_from_split( .get_all(Path::new(SPLIT_FIELDS_FILE_NAME)) .await?; let serialized_split_fields_len = serialized_split_fields.len(); - let mut list_fields = deserialize_split_fields(serialized_split_fields) - .with_context(|| { + let list_fields_proto = + deserialize_split_fields(serialized_split_fields).with_context(|| { format!("could not read split fields (serialized len: {serialized_split_fields_len})",) - })? - .fields; + })?; + + let mut list_fields = list_fields_proto.fields; + list_fields.retain(|list_field_entry| list_field_entry.field_name != FIELD_PRESENCE_FIELD_NAME); + for list_field_entry in list_fields.iter_mut() { list_field_entry.index_ids = vec![index_id.to_string()]; + + if list_field_entry + .field_name + .starts_with(DYNAMIC_FIELD_PREFIX) + { + list_field_entry + .field_name + .replace_range(..DYNAMIC_FIELD_PREFIX.len(), ""); + } } - // Prepare for grouping by field name and type - list_fields.sort_by(|left, right| match left.field_name.cmp(&right.field_name) { - Ordering::Equal => left.field_type.cmp(&right.field_type), - other => other, - }); + + // We sort our fields, as the removal of dynamic_field prefix could have caused them to be out + // of order. We also defensively make sure there are no duplicates here. + make_sorted_and_dedup(&mut list_fields); + // Put result into cache searcher_context.list_fields_cache.put( split_and_footer_offsets.clone(), @@ -78,7 +110,38 @@ pub async fn get_fields_from_split( }, ); - Ok(Box::new(list_fields.into_iter())) + Ok(list_fields) +} + +fn field_order( + left: &ListFieldsEntryResponse, + right: &ListFieldsEntryResponse, +) -> std::cmp::Ordering { + left.field_name + .cmp(&right.field_name) + .then_with(|| left.field_type.cmp(&right.field_type)) +} + +// Sorts and deduplicates the list of fields. +// +// If somehow we end up with duplicate fields, only the first one is kept, +// and we log a warning. +fn make_sorted_and_dedup(list_fields: &mut Vec) { + list_fields.sort_unstable_by(field_order); + + // We defensively make sure there are no duplicates here. + list_fields.dedup_by(|left, right| { + if left.field_name == right.field_name && left.field_type == right.field_type { + rate_limited_warn!( + limit_per_min = 1, + left.field_name, + "duplicate fields found, please report" + ); + true + } else { + false + } + }); } /// `current_group` needs to contain at least one element. @@ -96,9 +159,13 @@ fn merge_same_field_group( ); if current_group.len() == 1 { - return current_group.pop().unwrap(); + return current_group + .pop() + .expect("`current_group` should not be empty"); } - let metadata = ¤t_group.last().unwrap(); + let metadata = current_group + .last() + .expect("`current_group` should not be empty"); let searchable = current_group.iter().any(|entry| entry.searchable); let aggregatable = current_group.iter().any(|entry| entry.aggregatable); let field_name = metadata.field_name.to_string(); @@ -120,7 +187,7 @@ fn merge_same_field_group( // Not searchable => no need to list all the indices Vec::new() }; - non_searchable_index_ids.sort(); + non_searchable_index_ids.sort_unstable(); non_searchable_index_ids.dedup(); let mut non_aggregatable_index_ids = if aggregatable { @@ -140,14 +207,15 @@ fn merge_same_field_group( // Not aggregatable => no need to list all the indices Vec::new() }; - non_aggregatable_index_ids.sort(); + non_aggregatable_index_ids.sort_unstable(); non_aggregatable_index_ids.dedup(); let mut index_ids: Vec = current_group .drain(..) .flat_map(|entry| entry.index_ids.into_iter()) .collect(); - index_ids.sort(); + index_ids.sort_unstable(); index_ids.dedup(); + ListFieldsEntryResponse { field_name, field_type, @@ -179,10 +247,16 @@ fn merge_leaf_list_fields( }; for entry in merged { - if let Some(last) = current_group.last() { - if last.field_name != entry.field_name || last.field_type != entry.field_type { - flush_group(&mut responses, &mut current_group); - } + if let Some(last) = current_group.last() + && (last.field_name != entry.field_name || last.field_type != entry.field_type) + { + flush_group(&mut responses, &mut current_group); + } + if responses.len() >= *FIELD_LIST_SIZE_LIMIT { + return Err(SearchError::Internal(format!( + "list fields response exceeded {} fields", + *FIELD_LIST_SIZE_LIMIT + ))); } current_group.push(entry); } @@ -193,45 +267,73 @@ fn merge_leaf_list_fields( Ok(responses) } -fn matches_any_pattern(field_name: &str, field_patterns: &[String]) -> bool { - if field_patterns.is_empty() { - return true; - } +// Returns true if any of the patterns match the field name. +fn matches_any_pattern(field_name: &str, field_patterns: &[FieldPattern]) -> bool { field_patterns .iter() - .any(|pattern| matches_pattern(pattern, field_name)) + .any(|pattern| pattern.matches(field_name)) +} + +enum FieldPattern { + Match { field: String }, + Wildcard { prefix: String, suffix: String }, } -/// Supports up to 1 wildcard. -fn matches_pattern(field_pattern: &str, field_name: &str) -> bool { - match field_pattern.find('*') { - None => field_pattern == field_name, - Some(index) => { - if index == 0 { - // "*field" - field_name.ends_with(&field_pattern[1..]) - } else if index == field_pattern.len() - 1 { - // "field*" - field_name.starts_with(&field_pattern[..index]) - } else { - // "fi*eld" - field_name.starts_with(&field_pattern[..index]) - && field_name.ends_with(&field_pattern[index + 1..]) +impl FromStr for FieldPattern { + type Err = crate::SearchError; + + fn from_str(field_pattern: &str) -> crate::Result { + match field_pattern.find('*') { + None => Ok(FieldPattern::Match { + field: field_pattern.to_string(), + }), + Some(pos) => { + let prefix = field_pattern[..pos].to_string(); + let suffix = field_pattern[pos + 1..].to_string(); + if suffix.contains("*") { + return Err(crate::SearchError::InvalidArgument(format!( + "invalid field pattern `{field_pattern}`: we only support one wildcard" + ))); + } + Ok(FieldPattern::Wildcard { prefix, suffix }) + } + } + } +} + +impl FieldPattern { + pub fn matches(&self, field_name: &str) -> bool { + match self { + FieldPattern::Match { field } => field == field_name, + FieldPattern::Wildcard { prefix, suffix } => { + field_name.starts_with(prefix) && field_name.ends_with(suffix) } } } } /// `leaf` step of list fields. +/// +/// Returns field metadata from the assigned splits. pub async fn leaf_list_fields( index_id: IndexId, index_storage: Arc, searcher_context: &SearcherContext, split_ids: &[SplitIdAndFooterOffsets], - field_patterns: &[String], + field_patterns_str: &[String], ) -> crate::Result { - let mut iter_per_split = Vec::new(); - let get_field_futures: Vec<_> = split_ids + let field_patterns: Vec = field_patterns_str + .iter() + .map(|pattern_str| FieldPattern::from_str(pattern_str)) + .collect::>()?; + + // If no splits, return empty response + if split_ids.is_empty() { + return Ok(ListFieldsResponse { fields: Vec::new() }); + } + + // Get fields from all splits + let single_split_list_fields_futures: Vec<_> = split_ids .iter() .map(|split_id| { get_fields_from_split( @@ -242,31 +344,49 @@ pub async fn leaf_list_fields( ) }) .collect(); - let result = future::join_all(get_field_futures).await; - // This only works well, if the field data is in a local cache. - for fields in result { - let list_fields_iter = match fields { - Ok(fields) => fields, - Err(_err) => Box::new(std::iter::empty()), - }; - let list_fields_iter = list_fields_iter - .map(|mut entry| { - // We don't want to leak the _dynamic hack to the user API. - if entry.field_name.starts_with("_dynamic.") { - entry.field_name.replace_range(.."_dynamic.".len(), ""); + + let mut single_split_list_fields_vec: Vec> = + future::try_join_all(single_split_list_fields_futures).await?; + + let fields = search_thread_pool() + .run_cpu_intensive(move || { + for single_split_list_fields in &mut single_split_list_fields_vec { + // This contract is enforced on a different node, etc. so we defensively check that + // the fields are sorted and deduplicated. + if !single_split_list_fields.is_sorted_by(|left, right| { + // Checking on less ensure that this is both sorted AND that there are no + // duplicates + field_order(left, right) == std::cmp::Ordering::Less + }) { + rate_limited_warn!( + limit_per_min = 1, + "contract breach: fields returned by a leaf are not strictly sorted! \ + please report" + ); + make_sorted_and_dedup(single_split_list_fields); } - entry - }) - .filter(|field| matches_any_pattern(&field.field_name, field_patterns)) - // remove internal fields - .filter(|field| field.field_name != "_field_presence"); - iter_per_split.push(list_fields_iter); - } - let fields = merge_leaf_list_fields(iter_per_split)?; + } + + let filtered_list_fields_sorted_iters: Vec<_> = single_split_list_fields_vec + .into_iter() + .map(|list_fields_sorted| { + list_fields_sorted.into_iter().filter(|field| { + if field_patterns.is_empty() { + true + } else { + matches_any_pattern(&field.field_name, &field_patterns) + } + }) + }) + .collect(); + merge_leaf_list_fields(filtered_list_fields_sorted_iters) + }) + .await + .context("failed to merge single split list fields")??; Ok(ListFieldsResponse { fields }) } -/// Index metas needed for executing a leaf search request. +/// Index metas needed for executing a leaf list fields request. #[derive(Clone, Debug)] pub struct IndexMetasForLeafSearch { /// Index id. @@ -290,29 +410,63 @@ pub async fn root_list_fields( if indexes_metadata.is_empty() { return Ok(ListFieldsResponse { fields: Vec::new() }); } - let index_uid_to_index_meta: HashMap = indexes_metadata - .iter() - .map(|index_metadata| { - let index_metadata_for_leaf_search = IndexMetasForLeafSearch { - index_uri: index_metadata.index_uri().clone(), - index_id: index_metadata.index_config.index_id.to_string(), - }; - - ( - index_metadata.index_uid.clone(), - index_metadata_for_leaf_search, + + // Build index metadata map and extract timestamp field for time range refinement + let mut index_uid_to_index_meta: HashMap = HashMap::new(); + let mut index_uids: Vec = Vec::new(); + let mut timestamp_field_opt: Option = None; + + for index_metadata in indexes_metadata { + // Extract timestamp field for time range refinement (use first index's field) + if timestamp_field_opt.is_none() + && list_fields_req.query_ast.is_some() + && let Ok(doc_mapper) = build_doc_mapper( + &index_metadata.index_config.doc_mapping, + &index_metadata.index_config.search_settings, ) - }) - .collect(); - let index_uids: Vec = indexes_metadata - .into_iter() - .map(|index_metadata| index_metadata.index_uid) - .collect(); + { + timestamp_field_opt = doc_mapper.timestamp_field_name().map(|s| s.to_string()); + } + + let index_metadata_for_leaf_search = IndexMetasForLeafSearch { + index_uri: index_metadata.index_uri().clone(), + index_id: index_metadata.index_config.index_id.to_string(), + }; + + index_uids.push(index_metadata.index_uid.clone()); + index_uid_to_index_meta.insert( + index_metadata.index_uid.clone(), + index_metadata_for_leaf_search, + ); + } + + // Extract tags and refine time range from query_ast for split pruning + let mut start_timestamp = list_fields_req.start_timestamp; + let mut end_timestamp = list_fields_req.end_timestamp; + let tags_filter_opt = if let Some(ref query_ast_json) = list_fields_req.query_ast { + let query_ast: QueryAst = serde_json::from_str(query_ast_json) + .map_err(|err| SearchError::InvalidQuery(err.to_string()))?; + + // Refine time range from query AST if timestamp field is available + if let Some(ref timestamp_field) = timestamp_field_opt { + crate::root::refine_start_end_timestamp_from_ast( + &query_ast, + timestamp_field, + &mut start_timestamp, + &mut end_timestamp, + ); + } + + extract_tags_from_query(query_ast) + } else { + None + }; + let split_metadatas: Vec = list_relevant_splits( index_uids, - list_fields_req.start_timestamp, - list_fields_req.end_timestamp, - None, + start_timestamp, + end_timestamp, + tags_filter_opt, &mut metastore, ) .await?; @@ -332,13 +486,18 @@ pub async fn root_list_fields( leaf_request_tasks.push(cluster_client.leaf_list_fields(leaf_request, client.clone())); } } - let leaf_search_responses: Vec = try_join_all(leaf_request_tasks).await?; - let fields = merge_leaf_list_fields( - leaf_search_responses - .into_iter() - .map(|resp| resp.fields.into_iter()) - .collect_vec(), - )?; + let leaf_list_fields_protos: Vec = try_join_all(leaf_request_tasks).await?; + let fields = search_thread_pool() + .run_cpu_intensive(move || { + let leaf_list_fields = leaf_list_fields_protos + .into_iter() + .map(|leaf_list_fields_proto| leaf_list_fields_proto.fields.into_iter()) + .collect(); + merge_leaf_list_fields(leaf_list_fields) + }) + .await + .context("failed to merge leaf list fields responses")??; + Ok(ListFieldsResponse { fields }) } @@ -378,37 +537,6 @@ mod tests { use super::*; - #[test] - fn test_pattern() { - assert!(matches_any_pattern("field", &["field".to_string()])); - assert!(matches_any_pattern("field", &["fi*eld".to_string()])); - assert!(matches_any_pattern("field", &["*field".to_string()])); - assert!(matches_any_pattern("field", &["field*".to_string()])); - - assert!(matches_any_pattern("field1", &["field*".to_string()])); - assert!(!matches_any_pattern("field1", &["*field".to_string()])); - assert!(!matches_any_pattern("field1", &["fi*eld".to_string()])); - assert!(!matches_any_pattern("field1", &["field".to_string()])); - - // 2nd pattern matches - assert!(matches_any_pattern( - "field", - &["a".to_string(), "field".to_string()] - )); - assert!(matches_any_pattern( - "field", - &["a".to_string(), "fi*eld".to_string()] - )); - assert!(matches_any_pattern( - "field", - &["a".to_string(), "*field".to_string()] - )); - assert!(matches_any_pattern( - "field", - &["a".to_string(), "field*".to_string()] - )); - } - #[test] fn merge_leaf_list_fields_identical_test() { let entry1 = ListFieldsEntryResponse { @@ -688,4 +816,32 @@ mod tests { }; assert_eq!(resp, vec![expected]); } + + #[test] + fn test_field_pattern() { + let prefix_pattern = FieldPattern::from_str("toto*").unwrap(); + assert!(!prefix_pattern.matches("")); + assert!(!prefix_pattern.matches("tot3")); + assert!(!prefix_pattern.matches("atoto")); + assert!(prefix_pattern.matches("toto")); + assert!(prefix_pattern.matches("totowhatever")); + + let suffix_pattern = FieldPattern::from_str("*toto").unwrap(); + assert!(!suffix_pattern.matches("")); + assert!(!suffix_pattern.matches("3tot")); + assert!(!suffix_pattern.matches("totoa")); + assert!(suffix_pattern.matches("toto")); + assert!(suffix_pattern.matches("whatevertoto")); + + let inner_pattern = FieldPattern::from_str("to*ti").unwrap(); + assert!(!inner_pattern.matches("")); + assert!(!inner_pattern.matches("tot")); + assert!(!inner_pattern.matches("totia")); + assert!(!inner_pattern.matches("atoti")); + assert!(inner_pattern.matches("toti")); + assert!(!inner_pattern.matches("tito")); + assert!(inner_pattern.matches("towhateverti")); + + assert!(FieldPattern::from_str("to**").is_err()); + } } diff --git a/quickwit/quickwit-search/src/list_fields_cache.rs b/quickwit/quickwit-search/src/list_fields_cache.rs index 681ce7a2e77..531e1015e4e 100644 --- a/quickwit/quickwit-search/src/list_fields_cache.rs +++ b/quickwit/quickwit-search/src/list_fields_cache.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use quickwit_config::CacheConfig; use quickwit_proto::search::{ ListFields, SplitIdAndFooterOffsets, deserialize_split_fields, serialize_split_fields, }; @@ -26,10 +27,10 @@ pub struct ListFieldsCache { // TODO For now this simply caches the whole ListFieldsEntryResponse. We could // be more clever and cache aggregates instead. impl ListFieldsCache { - pub fn new(capacity: usize) -> ListFieldsCache { + pub fn new(config: &CacheConfig) -> ListFieldsCache { ListFieldsCache { - content: MemorySizedCache::with_capacity_in_bytes( - capacity, + content: MemorySizedCache::from_config( + config, &quickwit_storage::STORAGE_METRICS.partial_request_cache, ), } @@ -50,7 +51,7 @@ impl ListFieldsCache { } /// A key inside a [`ListFieldsCache`]. -#[derive(Debug, Hash, PartialEq, Eq)] +#[derive(Debug, Hash, Clone, PartialEq, Eq)] struct CacheKey { /// The split this entry refers to split_id: SplitId, @@ -66,6 +67,7 @@ impl CacheKey { #[cfg(test)] mod tests { + use bytesize::ByteSize; use quickwit_proto::search::{ ListFieldType, ListFields, ListFieldsEntryResponse, SplitIdAndFooterOffsets, }; @@ -74,7 +76,7 @@ mod tests { #[test] fn test_list_fields_cache() { - let cache = ListFieldsCache::new(64_000_000); + let cache = ListFieldsCache::new(&ByteSize::mb(64).into()); let split_1 = SplitIdAndFooterOffsets { split_id: "split_1".to_string(), diff --git a/quickwit/quickwit-search/src/list_terms.rs b/quickwit/quickwit-search/src/list_terms.rs index c69f232f9d8..5c69972728e 100644 --- a/quickwit/quickwit-search/src/list_terms.rs +++ b/quickwit/quickwit-search/src/list_terms.rs @@ -17,6 +17,7 @@ use std::ops::Bound; use std::sync::Arc; use anyhow::Context; +use bytesize::ByteSize; use futures::future::try_join_all; use itertools::{Either, Itertools}; use quickwit_common::pretty::PrettySample; @@ -206,6 +207,7 @@ pub fn jobs_to_leaf_requests( /// Apply a leaf list terms on a single split. #[instrument(skip_all, fields(split_id = split.split_id))] +#[allow(deprecated)] async fn leaf_list_terms_single_split( searcher_context: &SearcherContext, search_request: &ListTermsRequest, @@ -308,6 +310,7 @@ fn term_from_data(field: Field, field_type: &FieldType, data: &[u8]) -> Term { term } +#[allow(deprecated)] fn term_to_data(field: Field, field_type: &FieldType, field_value: &[u8]) -> Vec { let mut term = Term::from_field_bool(field, false); term.clear_with_type(field_type.value_type()); @@ -324,14 +327,20 @@ pub async fn leaf_list_terms( splits: &[SplitIdAndFooterOffsets], ) -> Result { info!(split_offsets = ?PrettySample::new(splits, 5)); - let permit_sizes = splits.iter().map(|split| { - compute_initial_memory_allocation( - split, - searcher_context - .searcher_config - .warmup_single_split_initial_allocation, - ) - }); + let permit_sizes: Vec = splits + .iter() + .map(|split| { + compute_initial_memory_allocation( + split, + searcher_context + .searcher_config + .warmup_single_split_initial_allocation, + ) + }) + .collect(); + // We have added offloading leaf search to lambdas, but not for list_terms yet. + // TODO (Add it) + // https://github.com/quickwit-oss/quickwit/issues/6150 let permits = searcher_context .search_permit_provider .get_permits(permit_sizes) @@ -345,7 +354,7 @@ pub async fn leaf_list_terms( async move { let leaf_split_search_permit = search_permit_recv.await; // TODO dedicated counter and timer? - crate::SEARCH_METRICS.leaf_searches_splits_total.inc(); + crate::SEARCH_METRICS.leaf_list_terms_splits_total.inc(); let timer = crate::SEARCH_METRICS .leaf_search_split_duration_secs .start_timer(); diff --git a/quickwit/quickwit-search/src/metrics.rs b/quickwit/quickwit-search/src/metrics.rs index 42e3d289d0c..9ed11d7ecbb 100644 --- a/quickwit/quickwit-search/src/metrics.rs +++ b/quickwit/quickwit-search/src/metrics.rs @@ -14,6 +14,8 @@ // See https://prometheus.io/docs/practices/naming/ +use std::fmt; + use bytesize::ByteSize; use once_cell::sync::Lazy; use quickwit_common::metrics::{ @@ -22,6 +24,86 @@ use quickwit_common::metrics::{ new_histogram_vec, }; +fn print_if_not_null( + field_name: &'static str, + counter: &IntCounter, + f: &mut fmt::Formatter, +) -> fmt::Result { + let val = counter.get(); + if val > 0 { + write!(f, "{}={} ", field_name, val)?; + } + Ok(()) +} + +pub struct SplitSearchOutcomeCounters { + pub cancel_before_warmup: IntCounter, + pub cache_hit: IntCounter, + pub pruned_before_warmup: IntCounter, + pub cancel_warmup: IntCounter, + pub pruned_after_warmup: IntCounter, + pub cancel_cpu_queue: IntCounter, + pub cancel_cpu: IntCounter, + pub success: IntCounter, +} + +impl fmt::Display for SplitSearchOutcomeCounters { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + print_if_not_null("cancel_before_warmup", &self.cancel_before_warmup, f)?; + print_if_not_null("cache_hit", &self.cache_hit, f)?; + print_if_not_null("pruned_before_warmup", &self.pruned_before_warmup, f)?; + print_if_not_null("cancel_warmup", &self.cancel_warmup, f)?; + print_if_not_null("pruned_after_warmup", &self.pruned_after_warmup, f)?; + print_if_not_null("cancel_cpu_queue", &self.cancel_cpu_queue, f)?; + print_if_not_null("cancel_cpu", &self.cancel_cpu, f)?; + print_if_not_null("success", &self.success, f)?; + Ok(()) + } +} + +impl SplitSearchOutcomeCounters { + /// Create a new SplitSearchOutcomeCounters instance, registered in prometheus. + pub fn new_registered() -> Self { + let search_split_outcome_vec = new_counter_vec( + "split_search_outcome", + "Count the state in which each leaf search split ended", + "search", + &[], + ["category"], + ); + Self::new_from_counter_vec(search_split_outcome_vec) + } + + /// Create a new SplitSearchOutcomeCounters instance, but this one won't be reported to + /// prometheus. + pub fn new_unregistered() -> Self { + let search_split_outcome_vec = IntCounterVec::new( + "split_search_outcome", + "Count the state in which each leaf search split ended", + "search", + &[], + ["category"], + ); + Self::new_from_counter_vec(search_split_outcome_vec) + } + + pub fn new_from_counter_vec(search_split_outcome_vec: IntCounterVec<1>) -> Self { + SplitSearchOutcomeCounters { + cancel_before_warmup: search_split_outcome_vec + .with_label_values(["cancel_before_warmup"]), + cache_hit: search_split_outcome_vec.with_label_values(["cache_hit"]), + pruned_before_warmup: search_split_outcome_vec + .with_label_values(["pruned_before_warmup"]), + cancel_warmup: search_split_outcome_vec.with_label_values(["cancel_warmup"]), + pruned_after_warmup: search_split_outcome_vec + .with_label_values(["pruned_after_warmup"]), + cancel_cpu_queue: search_split_outcome_vec.with_label_values(["cancel_cpu_queue"]), + cancel_cpu: search_split_outcome_vec.with_label_values(["cancel_cpu"]), + success: search_split_outcome_vec.with_label_values(["success"]), + } + } +} + pub struct SearchMetrics { pub root_search_requests_total: IntCounterVec<1>, pub root_search_request_duration_seconds: HistogramVec<1>, @@ -29,7 +111,8 @@ pub struct SearchMetrics { pub leaf_search_requests_total: IntCounterVec<1>, pub leaf_search_request_duration_seconds: HistogramVec<1>, pub leaf_search_targeted_splits: HistogramVec<1>, - pub leaf_searches_splits_total: IntCounter, + pub leaf_list_terms_splits_total: IntCounter, + pub split_search_outcome_total: SplitSearchOutcomeCounters, pub leaf_search_split_duration_secs: Histogram, pub job_assigned_total: IntCounterVec<1>, pub leaf_search_single_split_tasks_pending: IntGauge, @@ -123,12 +206,15 @@ impl Default for SearchMetrics { ["status"], targeted_splits_buckets, ), - leaf_searches_splits_total: new_counter( - "leaf_searches_splits_total", - "Number of leaf searches (count of splits) started.", + + leaf_list_terms_splits_total: new_counter( + "leaf_list_terms_splits_total", + "Number of list terms splits total", "search", &[], ), + split_search_outcome_total: SplitSearchOutcomeCounters::new_registered(), + leaf_search_split_duration_secs: new_histogram( "leaf_search_split_duration_secs", "Number of seconds required to run a leaf search over a single split. The timer \ diff --git a/quickwit/quickwit-search/src/metrics_trackers.rs b/quickwit/quickwit-search/src/metrics_trackers.rs index 7f2f9fbbfb3..2d8deee64f4 100644 --- a/quickwit/quickwit-search/src/metrics_trackers.rs +++ b/quickwit/quickwit-search/src/metrics_trackers.rs @@ -86,7 +86,8 @@ impl PinnedDrop for RootSearchMetricsFuture { } impl Future for RootSearchMetricsFuture -where F: Future> +where + F: Future>, { type Output = Result; @@ -103,7 +104,8 @@ where F: Future> /// Wrapper around the search future to track metrics. #[pin_project(PinnedDrop)] pub struct LeafSearchMetricsFuture -where F: Future> +where + F: Future>, { #[pin] pub tracked: F, @@ -114,7 +116,8 @@ where F: Future> #[pinned_drop] impl PinnedDrop for LeafSearchMetricsFuture -where F: Future> +where + F: Future>, { fn drop(self: Pin<&mut Self>) { let label_values = [self.status.unwrap_or("cancelled")]; @@ -134,7 +137,8 @@ where F: Future> } impl Future for LeafSearchMetricsFuture -where F: Future> +where + F: Future>, { type Output = Result; diff --git a/quickwit/quickwit-search/src/retry/mod.rs b/quickwit/quickwit-search/src/retry/mod.rs index 316569222ae..996665717cf 100644 --- a/quickwit/quickwit-search/src/retry/mod.rs +++ b/quickwit/quickwit-search/src/retry/mod.rs @@ -13,7 +13,6 @@ // limitations under the License. pub mod search; -pub mod search_stream; use std::collections::HashSet; use std::net::SocketAddr; diff --git a/quickwit/quickwit-search/src/retry/search_stream.rs b/quickwit/quickwit-search/src/retry/search_stream.rs deleted file mode 100644 index 0d94b896327..00000000000 --- a/quickwit/quickwit-search/src/retry/search_stream.rs +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use quickwit_proto::search::{LeafSearchStreamRequest, LeafSearchStreamResponse}; -use tokio::sync::mpsc::error::SendError; -use tracing::warn; - -use super::RetryPolicy; - -pub struct SuccessfulSplitIds(pub Vec); - -/// Retry policy for consuming the result stream of a LeafSearchStreamRequest. -/// A retry is only made if there are some missing splits. -/// As errors only come from a closed receiver, we ignore them. -pub struct LeafSearchStreamRetryPolicy {} - -impl - RetryPolicy< - LeafSearchStreamRequest, - SuccessfulSplitIds, - SendError>, - > for LeafSearchStreamRetryPolicy -{ - // Returns a retry request that is either: - // - a clone of the initial request on error - // - or a request on failing split ids only. - fn retry_request( - &self, - mut request: LeafSearchStreamRequest, - response_res: &Result< - SuccessfulSplitIds, - SendError>, - >, - ) -> Option { - match response_res { - Ok(SuccessfulSplitIds(successful_split_ids)) => { - if successful_split_ids.len() == request.split_offsets.len() { - // All splits were successful! - return None; - } - // We retry the failed splits. - request.split_offsets.retain(|split_metadata| { - !successful_split_ids.contains(&split_metadata.split_id) - }); - Some(request) - } - Err(SendError(_)) => { - // The receiver channel was dropped. - // There is no need to retry. - warn!( - "Receiver channel closed during stream search request. The client probably \ - closed the connection?" - ); - None - } - } - } -} - -#[cfg(test)] -mod tests { - use quickwit_proto::search::{ - LeafSearchStreamRequest, LeafSearchStreamResponse, SplitIdAndFooterOffsets, - }; - use tokio::sync::mpsc::error::SendError; - - use crate::retry::RetryPolicy; - use crate::retry::search_stream::{LeafSearchStreamRetryPolicy, SuccessfulSplitIds}; - - #[tokio::test] - async fn test_retry_policy_search_stream_should_not_retry_on_send_error() { - let retry_policy = LeafSearchStreamRetryPolicy {}; - let request = LeafSearchStreamRequest::default(); - let response = LeafSearchStreamResponse::default(); - let response_res = Err(SendError(Ok(response))); - let retry_req_opt = retry_policy.retry_request(request, &response_res); - assert!(retry_req_opt.is_none()); - } - - #[tokio::test] - async fn test_retry_policy_search_stream_should_not_retry_on_successful_response() { - let retry_policy = LeafSearchStreamRetryPolicy {}; - let request = LeafSearchStreamRequest::default(); - let response_res = Ok(SuccessfulSplitIds(Vec::new())); - let retry_req_opt = retry_policy.retry_request(request, &response_res); - assert!(retry_req_opt.is_none()); - } - - #[tokio::test] - async fn test_retry_policy_search_stream_should_retry_on_failed_splits() { - let split_1 = SplitIdAndFooterOffsets { - split_id: "split_1".to_string(), - split_footer_end: 100, - split_footer_start: 0, - timestamp_start: None, - timestamp_end: None, - num_docs: 0, - }; - let split_2 = SplitIdAndFooterOffsets { - split_id: "split_2".to_string(), - split_footer_end: 100, - split_footer_start: 0, - timestamp_start: None, - timestamp_end: None, - num_docs: 0, - }; - let retry_policy = LeafSearchStreamRetryPolicy {}; - let request = LeafSearchStreamRequest { - split_offsets: vec![split_1, split_2], - ..Default::default() - }; - let response_res = Ok(SuccessfulSplitIds(vec!["split_1".to_string()])); - let retry_req = retry_policy.retry_request(request, &response_res).unwrap(); - assert_eq!(retry_req.split_offsets.len(), 1); - assert_eq!(retry_req.split_offsets[0].split_id, "split_2"); - } -} diff --git a/quickwit/quickwit-search/src/root.rs b/quickwit/quickwit-search/src/root.rs index 0d1d30bcc3a..a8ca8a6cffc 100644 --- a/quickwit/quickwit-search/src/root.rs +++ b/quickwit/quickwit-search/src/root.rs @@ -45,7 +45,7 @@ use tantivy::aggregation::agg_result::AggregationResults; use tantivy::aggregation::intermediate_agg_result::IntermediateAggregationResults; use tantivy::collector::Collector; use tantivy::schema::{Field, FieldEntry, FieldType, Schema}; -use tracing::{debug, info_span, instrument}; +use tracing::{debug, error, info, info_span, instrument}; use crate::cluster_client::ClusterClient; use crate::collector::{QuickwitAggregations, make_merge_collector}; @@ -249,7 +249,12 @@ fn validate_request_and_build_metadata( )?; // Validates the query by effectively building it against the current schema. - doc_mapper.query(doc_mapper.schema(), &query_ast_resolved_for_index, true)?; + doc_mapper.query( + doc_mapper.schema(), + query_ast_resolved_for_index, + true, + None, + )?; let index_metadata_for_leaf_search = IndexMetasForLeafSearch { index_uri: index_metadata.index_uri().clone(), @@ -366,6 +371,8 @@ fn simplify_search_request_for_scroll_api(req: &SearchRequest) -> crate::Result< // request is simplified after initial query, and we cache the hit count, so we don't need // to recompute it afterward. count_hits: quickwit_proto::search::CountHits::Underestimate as i32, + ignore_missing_indexes: req.ignore_missing_indexes, + skip_aggregation_finalization: false, }) } @@ -647,6 +654,7 @@ pub fn is_metadata_count_request(request: &SearchRequest) -> bool { /// /// The passed query_ast should match the serialized on in request. pub fn is_metadata_count_request_with_ast(query_ast: &QueryAst, request: &SearchRequest) -> bool { + // TODO detect Cache(MatchAll), Boost(MatchAll) and Bool{must/should:MatchAll} if query_ast != &QueryAst::MatchAll { return false; } @@ -750,9 +758,8 @@ pub(crate) async fn search_partial_hits_phase( try_join_all(leaf_request_tasks).await? }; - // Creates a collector which merges responses into one let merge_collector = - make_merge_collector(search_request, &searcher_context.get_aggregation_limits())?; + make_merge_collector(search_request, searcher_context.get_aggregation_limits())?; // Merging is a cpu-bound task. // It should be executed by Tokio's blocking threads. @@ -777,14 +784,19 @@ pub(crate) async fn search_partial_hits_phase( "Merged leaf search response." ); - if let Some(resource_stats) = &leaf_search_response.resource_stats { - if is_top_5pct_memory_intensive( + if let Some(resource_stats) = &leaf_search_response.resource_stats + && is_top_5pct_memory_intensive( resource_stats.short_lived_cache_num_bytes, resource_stats.split_num_docs, - ) { - // We log at most 5 times per minute. - quickwit_common::rate_limited_info!(limit_per_min=5, split_num_docs=resource_stats.split_num_docs, %search_request.query_ast, short_lived_cached_num_bytes=resource_stats.short_lived_cache_num_bytes, query=%search_request.query_ast, "memory intensive query"); - } + ) + { + // We log at most 5 times per minute. + quickwit_common::rate_limited_info!( + limit_per_min = 5, + split_num_docs = resource_stats.split_num_docs, + short_lived_cached_num_bytes = resource_stats.short_lived_cache_num_bytes, + "memory intensive query" + ); } if !leaf_search_response.failed_splits.is_empty() { @@ -908,19 +920,19 @@ fn build_hit_with_position( .sort_value .as_mut() .and_then(|sort_field| sort_field.sort_value.as_mut()); - if let Some(sort_by_value) = sort_value_opt { - if let Some(output_datetime_format) = &sort_field_1_datetime_format_opt { - convert_sort_datetime_value(sort_by_value, *output_datetime_format)?; - } + if let Some(sort_by_value) = sort_value_opt + && let Some(output_datetime_format) = &sort_field_1_datetime_format_opt + { + convert_sort_datetime_value(sort_by_value, *output_datetime_format)?; } let sort_value_2_opt = partial_hit_ref .sort_value2 .as_mut() .and_then(|sort_field| sort_field.sort_value.as_mut()); - if let Some(sort_by_value) = sort_value_2_opt { - if let Some(output_datetime_format) = &sort_field_2_datetime_format_opt { - convert_sort_datetime_value(sort_by_value, *output_datetime_format)?; - } + if let Some(sort_by_value) = sort_value_2_opt + && let Some(output_datetime_format) = &sort_field_2_datetime_format_opt + { + convert_sort_datetime_value(sort_by_value, *output_datetime_format)?; } let position = *hit_order.get(&key).expect("hit order must be present"); let index_id = split_id_to_index_id_map @@ -942,13 +954,13 @@ fn build_hit_with_position( fn get_sort_field_datetime_format( sort_field: Option<&SortField>, ) -> crate::Result> { - if let Some(sort_field) = sort_field { - if let Some(sort_field_datetime_format_int) = &sort_field.sort_datetime_format { - let sort_field_datetime_format = - SortDatetimeFormat::try_from(*sort_field_datetime_format_int) - .context("invalid sort datetime format")?; - return Ok(Some(sort_field_datetime_format)); - } + if let Some(sort_field) = sort_field + && let Some(sort_field_datetime_format_int) = &sort_field.sort_datetime_format + { + let sort_field_datetime_format = + SortDatetimeFormat::try_from(*sort_field_datetime_format_int) + .context("invalid sort datetime format")?; + return Ok(Some(sort_field_datetime_format)); } Ok(None) } @@ -958,7 +970,6 @@ fn get_sort_field_datetime_format( /// 2. Merges the search results. /// 3. Sends fetch docs requests to multiple leaf nodes. /// 4. Builds the response with docs and returns. -#[instrument(skip_all, fields(num_splits=%split_metadatas.len()))] async fn root_search_aux( searcher_context: &SearcherContext, indexes_metas_for_leaf_search: &IndexesMetasForLeafSearch, @@ -1052,6 +1063,9 @@ fn finalize_aggregation_if_any( let Some(aggregations_json) = search_request.aggregation_request.as_ref() else { return Ok(None); }; + if search_request.skip_aggregation_finalization { + return Ok(intermediate_aggregation_result_bytes_opt); + } let aggregations: QuickwitAggregations = serde_json::from_str(aggregations_json)?; let aggregation_result_postcard = finalize_aggregation( intermediate_aggregation_result_bytes_opt, @@ -1069,36 +1083,34 @@ fn finalize_aggregation_if_any( /// We put this check here and not in the metastore to make sure the logic is independent /// of the metastore implementation, and some different use cases could require different /// behaviors. This specification was principally motivated by #4042. -pub fn check_all_index_metadata_found( - index_metadatas: &[IndexMetadata], +pub fn ensure_all_indexes_found( + indexes_metadata: &[IndexMetadata], index_id_patterns: &[String], ) -> crate::Result<()> { let mut index_ids: HashSet<&str> = index_id_patterns .iter() - .map(|index_ptn| index_ptn.as_str()) - .filter(|index_ptn| !index_ptn.contains('*') && !index_ptn.starts_with('-')) + .filter(|pattern| !pattern.contains('*') && !pattern.starts_with('-')) + .map(|pattern| pattern.as_str()) .collect(); if index_ids.is_empty() { - // All of the patterns are wildcard patterns. + // All the patterns are wildcard or negative patterns. return Ok(()); } - - for index_metadata in index_metadatas { - index_ids.remove(index_metadata.index_uid.index_id.as_str()); + for index_metadata in indexes_metadata { + index_ids.remove(index_metadata.index_id()); } - - if !index_ids.is_empty() { - let missing_index_ids = index_ids - .into_iter() - .map(|missing_index_id| missing_index_id.to_string()) - .collect(); - return Err(SearchError::IndexesNotFound { - index_ids: missing_index_ids, - }); + if index_ids.is_empty() { + return Ok(()); } + let not_found_index_ids = index_ids + .into_iter() + .map(|index_id| index_id.to_string()) + .collect(); - Ok(()) + Err(SearchError::IndexesNotFound { + index_ids: not_found_index_ids, + }) } async fn refine_and_list_matches( @@ -1156,7 +1168,9 @@ async fn plan_splits_for_root_search( .deserialize_indexes_metadata() .await?; - check_all_index_metadata_found(&indexes_metadata[..], &search_request.index_id_patterns[..])?; + if !search_request.ignore_missing_indexes { + ensure_all_indexes_found(&indexes_metadata[..], &search_request.index_id_patterns[..])?; + } if indexes_metadata.is_empty() { return Ok((Vec::new(), HashMap::default())); @@ -1202,9 +1216,33 @@ pub async fn root_search( let num_docs: usize = split_metadatas.iter().map(|split| split.num_docs).sum(); let num_splits = split_metadatas.len(); - let current_span = tracing::Span::current(); - current_span.record("num_docs", num_docs); - current_span.record("num_splits", num_splits); + + // It would have been nice to add those in the context of the trace span, + // but with our current logging setting, it makes logs too verbose. + info!( + query_ast = search_request.query_ast.as_str(), + agg = search_request.aggregation_request(), + start_ts = ?(search_request.start_timestamp()..search_request.end_timestamp()), + count_required = search_request.count_hits().as_str_name(), + num_docs = num_docs, + num_splits = num_splits, + "root_search" + ); + + if let Some(max_total_split_searches) = searcher_context.searcher_config.max_splits_per_search + && max_total_split_searches < num_splits + { + error!( + num_splits, + max_total_split_searches, + index=?search_request.index_id_patterns, + query=%search_request.query_ast, + "max total splits exceeded" + ); + return Err(SearchError::InvalidArgument(format!( + "Number of targeted splits {num_splits} exceeds the limit {max_total_split_searches}" + ))); + } let mut search_response_result = RootSearchMetricsFuture { start: start_instant, @@ -1243,7 +1281,9 @@ pub async fn search_plan( .deserialize_indexes_metadata() .await?; - check_all_index_metadata_found(&indexes_metadata[..], &search_request.index_id_patterns[..])?; + if !search_request.ignore_missing_indexes { + ensure_all_indexes_found(&indexes_metadata[..], &search_request.index_id_patterns[..])?; + } if indexes_metadata.is_empty() { return Ok(SearchPlanResponse { result: serde_json::to_string(&SearchPlanResponseRest { @@ -1273,10 +1313,11 @@ pub async fn search_plan( let (query, mut warmup_info) = doc_mapper.query( doc_mapper.schema(), - &request_metadata.query_ast_resolved, + request_metadata.query_ast_resolved.clone(), true, + None, )?; - let merge_collector = make_merge_collector(&search_request, &Default::default())?; + let merge_collector = make_merge_collector(&search_request, Default::default())?; warmup_info.merge(merge_collector.warmup_info()); warmup_info.simplify(); @@ -3240,6 +3281,102 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_root_search_missing_index() -> anyhow::Result<()> { + let mut mock_metastore = MockMetastoreService::new(); + let index_metadata = IndexMetadata::for_test("test-index1", "ram:///test-index"); + let index_uid = index_metadata.index_uid.clone(); + mock_metastore + .expect_list_indexes_metadata() + .returning(move |_index_ids_query| { + Ok(ListIndexesMetadataResponse::for_test(vec![ + index_metadata.clone(), + ])) + }); + mock_metastore + .expect_list_splits() + .returning(move |_list_splits_request| { + let splits = vec![ + MockSplitBuilder::new("split1") + .with_index_uid(&index_uid) + .build(), + ]; + let splits_response = ListSplitsResponse::try_from_splits(splits).unwrap(); + Ok(ServiceStream::from(vec![Ok(splits_response)])) + }); + let mock_metastore_client = MetastoreServiceClient::from_mock(mock_metastore); + let mut mock_search_service = MockSearchService::new(); + mock_search_service.expect_leaf_search().returning( + |_leaf_search_req: quickwit_proto::search::LeafSearchRequest| { + Ok(quickwit_proto::search::LeafSearchResponse { + num_hits: 3, + partial_hits: vec![ + mock_partial_hit("split1", 3, 1), + mock_partial_hit("split1", 2, 2), + mock_partial_hit("split1", 1, 3), + ], + failed_splits: Vec::new(), + num_attempted_splits: 1, + ..Default::default() + }) + }, + ); + mock_search_service.expect_fetch_docs().returning( + |fetch_docs_req: quickwit_proto::search::FetchDocsRequest| { + Ok(quickwit_proto::search::FetchDocsResponse { + hits: get_doc_for_fetch_req(fetch_docs_req), + }) + }, + ); + let searcher_pool = searcher_pool_for_test([("127.0.0.1:1001", mock_search_service)]); + let search_job_placer = SearchJobPlacer::new(searcher_pool); + let cluster_client = ClusterClient::new(search_job_placer.clone()); + + let searcher_context = SearcherContext::for_test(); + + // search with ignore_missing_indexes=true succeeds + let search_request = quickwit_proto::search::SearchRequest { + index_id_patterns: vec!["test-index1".to_string(), "test-index2".to_string()], + query_ast: qast_json_helper("test", &["body"]), + max_hits: 10, + ignore_missing_indexes: true, + ..Default::default() + }; + let search_response = root_search( + &searcher_context, + search_request, + mock_metastore_client.clone(), + &cluster_client, + ) + .await + .unwrap(); + assert_eq!(search_response.num_hits, 3); + assert_eq!(search_response.hits.len(), 3); + + // search with ignore_missing_indexes=false fails + let search_request = quickwit_proto::search::SearchRequest { + index_id_patterns: vec!["test-index1".to_string(), "test-index2".to_string()], + query_ast: qast_json_helper("test", &["body"]), + max_hits: 10, + ignore_missing_indexes: false, + ..Default::default() + }; + let search_error = root_search( + &searcher_context, + search_request, + mock_metastore_client, + &cluster_client, + ) + .await + .unwrap_err(); + if let SearchError::IndexesNotFound { index_ids } = search_error { + assert_eq!(index_ids, vec!["test-index2".to_string()]); + } else { + panic!("unexpected error type: {search_error}"); + } + Ok(()) + } + #[tokio::test] async fn test_root_search_multiple_splits_retry_on_other_node() -> anyhow::Result<()> { let search_request = quickwit_proto::search::SearchRequest { @@ -4112,6 +4249,69 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_search_plan_missing_index() -> anyhow::Result<()> { + let mut mock_metastore = MockMetastoreService::new(); + let index_metadata = IndexMetadata::for_test("test-index1", "ram:///test-index"); + let index_uid = index_metadata.index_uid.clone(); + mock_metastore + .expect_list_indexes_metadata() + .returning(move |_index_ids_query| { + Ok(ListIndexesMetadataResponse::for_test(vec![ + index_metadata.clone(), + ])) + }); + mock_metastore + .expect_list_splits() + .returning(move |_filter| { + let splits = vec![ + MockSplitBuilder::new("split1") + .with_index_uid(&index_uid) + .build(), + MockSplitBuilder::new("split2") + .with_index_uid(&index_uid) + .build(), + ]; + let splits_response = ListSplitsResponse::try_from_splits(splits).unwrap(); + Ok(ServiceStream::from(vec![Ok(splits_response)])) + }); + let mock_metastore_service = MetastoreServiceClient::from_mock(mock_metastore); + + // plan with ignore_missing_indexes=true succeeds + search_plan( + quickwit_proto::search::SearchRequest { + index_id_patterns: vec!["test-index1".to_string(), "test-index2".to_string()], + query_ast: qast_json_helper("test-query", &["body"]), + max_hits: 10, + ignore_missing_indexes: true, + ..Default::default() + }, + mock_metastore_service.clone(), + ) + .await + .unwrap(); + + // plan with ignore_missing_indexes=false fails + let search_error = search_plan( + quickwit_proto::search::SearchRequest { + index_id_patterns: vec!["test-index1".to_string(), "test-index2".to_string()], + query_ast: qast_json_helper("test-query", &["body"]), + max_hits: 10, + ignore_missing_indexes: false, + ..Default::default() + }, + mock_metastore_service.clone(), + ) + .await + .unwrap_err(); + if let SearchError::IndexesNotFound { index_ids } = search_error { + assert_eq!(index_ids, vec!["test-index2".to_string()]); + } else { + panic!("unexpected error type: {search_error}"); + } + Ok(()) + } + #[test] fn test_extract_timestamp_range_from_ast() { use std::ops::Bound; @@ -5046,4 +5246,144 @@ mod tests { assert_eq!(search_response.failed_splits.len(), 1); Ok(()) } + + #[tokio::test] + async fn test_root_search_too_many_splits() -> anyhow::Result<()> { + let search_request = quickwit_proto::search::SearchRequest { + index_id_patterns: vec!["test-index".to_string()], + query_ast: qast_json_helper("test", &["body"]), + max_hits: 10, + ..Default::default() + }; + let mut mock_metastore = MockMetastoreService::new(); + let index_metadata = IndexMetadata::for_test("test-index", "ram:///test-index"); + let index_uid = index_metadata.index_uid.clone(); + mock_metastore + .expect_list_indexes_metadata() + .returning(move |_index_ids_query| { + Ok(ListIndexesMetadataResponse::for_test(vec![ + index_metadata.clone(), + ])) + }); + mock_metastore + .expect_list_splits() + .returning(move |_filter| { + let splits = vec![ + MockSplitBuilder::new("split1") + .with_index_uid(&index_uid) + .build(), + MockSplitBuilder::new("split2") + .with_index_uid(&index_uid) + .build(), + ]; + let splits_response = ListSplitsResponse::try_from_splits(splits).unwrap(); + Ok(ServiceStream::from(vec![Ok(splits_response)])) + }); + let mock_search_service = MockSearchService::new(); + let searcher_pool = searcher_pool_for_test([("127.0.0.1:1001", mock_search_service)]); + let search_job_placer = SearchJobPlacer::new(searcher_pool); + let cluster_client = ClusterClient::new(search_job_placer.clone()); + + let mut searcher_context = SearcherContext::for_test(); + searcher_context.searcher_config.max_splits_per_search = Some(1); + let search_error = root_search( + &searcher_context, + search_request, + MetastoreServiceClient::from_mock(mock_metastore), + &cluster_client, + ) + .await + .unwrap_err(); + assert!(matches!(search_error, SearchError::InvalidArgument { .. })); + Ok(()) + } + + #[tokio::test] + async fn test_finalize_aggregation_if_any_no_aggregation_request() { + let search_request = SearchRequest { + aggregation_request: None, + skip_aggregation_finalization: false, + ..Default::default() + }; + let searcher_context = SearcherContext::for_test(); + let result = + finalize_aggregation_if_any(&search_request, Some(vec![1, 2, 3]), &searcher_context) + .unwrap(); + assert!(result.is_none()); + } + + #[tokio::test] + async fn test_finalize_aggregation_if_any_skip_finalization_returns_intermediate_bytes() { + let agg_req = r#"{"avg_price": {"avg": {"field": "price"}}}"#; + let intermediate_bytes = vec![42, 43, 44]; + let search_request = SearchRequest { + aggregation_request: Some(agg_req.to_string()), + skip_aggregation_finalization: true, + ..Default::default() + }; + let searcher_context = SearcherContext::for_test(); + let result = finalize_aggregation_if_any( + &search_request, + Some(intermediate_bytes.clone()), + &searcher_context, + ) + .unwrap(); + assert_eq!(result, Some(intermediate_bytes)); + } + + #[tokio::test] + async fn test_finalize_aggregation_if_any_skip_finalization_none_bytes() { + let agg_req = r#"{"avg_price": {"avg": {"field": "price"}}}"#; + let search_request = SearchRequest { + aggregation_request: Some(agg_req.to_string()), + skip_aggregation_finalization: true, + ..Default::default() + }; + let searcher_context = SearcherContext::for_test(); + let result = finalize_aggregation_if_any(&search_request, None, &searcher_context).unwrap(); + assert!(result.is_none()); + } + + #[tokio::test] + async fn test_finalize_aggregation_if_any_default_finalizes() { + let agg_req = r#"{"avg_price": {"avg": {"field": "price"}}}"#; + let intermediate_results = IntermediateAggregationResults::default(); + let intermediate_bytes = postcard::to_stdvec(&intermediate_results).unwrap(); + let search_request = SearchRequest { + aggregation_request: Some(agg_req.to_string()), + skip_aggregation_finalization: false, + ..Default::default() + }; + let searcher_context = SearcherContext::for_test(); + let result = finalize_aggregation_if_any( + &search_request, + Some(intermediate_bytes.clone()), + &searcher_context, + ) + .unwrap(); + // Result should be Some (finalized), but different from intermediate bytes + assert!(result.is_some()); + assert_ne!(result.unwrap(), intermediate_bytes); + } + + #[tokio::test] + async fn test_finalize_aggregation_if_any_false_flag_finalizes() { + let agg_req = r#"{"avg_price": {"avg": {"field": "price"}}}"#; + let intermediate_results = IntermediateAggregationResults::default(); + let intermediate_bytes = postcard::to_stdvec(&intermediate_results).unwrap(); + let search_request = SearchRequest { + aggregation_request: Some(agg_req.to_string()), + skip_aggregation_finalization: false, + ..Default::default() + }; + let searcher_context = SearcherContext::for_test(); + let result = finalize_aggregation_if_any( + &search_request, + Some(intermediate_bytes.clone()), + &searcher_context, + ) + .unwrap(); + assert!(result.is_some()); + assert_ne!(result.unwrap(), intermediate_bytes); + } } diff --git a/quickwit/quickwit-search/src/search_permit_provider.rs b/quickwit/quickwit-search/src/search_permit_provider.rs index b9b750cec3b..40995824215 100644 --- a/quickwit/quickwit-search/src/search_permit_provider.rs +++ b/quickwit/quickwit-search/src/search_permit_provider.rs @@ -12,7 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::VecDeque; +use std::collections::BinaryHeap; +use std::collections::binary_heap::PeekMut; use std::future::Future; use std::pin::Pin; use std::sync::{Arc, Mutex}; @@ -21,8 +22,6 @@ use std::task::{Context, Poll}; use bytesize::ByteSize; use quickwit_common::metrics::GaugeGuard; use quickwit_proto::search::SplitIdAndFooterOffsets; -#[cfg(test)] -use tokio::sync::watch; use tokio::sync::{mpsc, oneshot}; /// Distributor of permits to perform split search operation. @@ -41,6 +40,7 @@ pub struct SearchPermitProvider { enum SearchPermitMode { Async { message_sender: mpsc::UnboundedSender, + actor_join_handle: Arc>, #[cfg(test)] actor_stopped: watch::Receiver, }, @@ -58,11 +58,17 @@ struct SyncPermitState { current_memory: u64, } -#[derive(Debug)] pub enum SearchPermitMessage { - Request { - permit_sender: oneshot::Sender>, + RequestWithOffload { permit_sizes: Vec, + /// Maximum number of pending requests. If granting all + /// requested permits would cause the number of pending requests to exceed this threshold, + /// some permits will be offloaded to Lambda. + offload_threshold: usize, + /// Channel to return the result message from the actor. + /// When offloading permits, the number of futures can be < to the number of requested + /// permits. + permit_resp_tx: oneshot::Sender>, }, UpdateMemory { memory_delta: i64, @@ -101,22 +107,19 @@ pub fn compute_initial_memory_allocation( impl SearchPermitProvider { pub fn new(num_download_slots: usize, memory_budget: ByteSize) -> Self { let (message_sender, message_receiver) = mpsc::unbounded_channel(); - #[cfg(test)] - let (state_sender, state_receiver) = watch::channel(false); let actor = SearchPermitActor { msg_receiver: message_receiver, msg_sender: message_sender.downgrade(), num_warmup_slots_available: num_download_slots, total_memory_budget: memory_budget.as_u64(), - permits_requests: VecDeque::new(), + permits_requests: BinaryHeap::new(), total_memory_allocated: 0u64, - #[cfg(test)] - stopped: state_sender, }; - tokio::spawn(actor.run()); + let actor_join_handle = Arc::new(tokio::spawn(actor.run())); Self { mode: SearchPermitMode::Async { message_sender, + actor_join_handle, #[cfg(test)] actor_stopped: state_receiver, }, @@ -138,25 +141,54 @@ impl SearchPermitProvider { } } - /// Returns one permit future for each provided split metadata. + #[cfg(test)] + async fn stop_and_unwrap(self) -> SearchPermitActor { + let (message_sender, actor_join_handle) = match self.mode { + SearchPermitMode::Async { + message_sender, + actor_join_handle, + .. + } => (message_sender, actor_join_handle), + SearchPermitMode::Sync { .. } => panic!("Cannot stop_and_unwrap a sync provider"), + }; + drop(message_sender); + Arc::into_inner(actor_join_handle).unwrap().await.unwrap() + } + + /// Returns permits for local splits /// - /// The permits returned are guaranteed to be resolved in order. In - /// addition, the permits are guaranteed to be resolved before permits - /// returned by subsequent calls to this function. + /// The returned futures are guaranteed to resolve in order. + pub async fn get_permits(&self, splits: Vec) -> Vec { + self.get_permits_with_offload(splits, usize::MAX).await + } + + /// Returns permits for local splits and a list of split indices to offload. + /// + /// The actor checks the current pending queue depth. If adding all splits + /// would exceed `offload_threshold` pending requests, only enough splits + /// to fill up to the threshold are processed locally; the rest are offloaded. + /// + /// The returned futures are guaranteed to resolve in order. /// - /// The permit memory size is capped by per_permit_initial_memory_allocation. - pub async fn get_permits( + /// If `offload_threshold` is 0, all splits are offloaded. + /// If `offload_threshold` is usize::MAX, all splits are processed locally. + pub async fn get_permits_with_offload( &self, - splits: impl IntoIterator, + splits: Vec, + offload_threshold: usize, ) -> Vec { match &self.mode { SearchPermitMode::Async { message_sender, .. } => { + if splits.is_empty() { + return Vec::new(); + } let (permit_sender, permit_receiver) = oneshot::channel(); let permit_sizes = splits.into_iter().map(|size| size.as_u64()).collect(); message_sender - .send(SearchPermitMessage::Request { - permit_sender, + .send(SearchPermitMessage::RequestWithOffload { + permit_resp_tx: permit_sender, permit_sizes, + offload_threshold, }) .expect("Receiver lives longer than sender"); permit_receiver @@ -199,38 +231,123 @@ struct SearchPermitActor { /// When it happens, new permits will not be assigned until the memory is freed. total_memory_budget: u64, total_memory_allocated: u64, - permits_requests: VecDeque<(oneshot::Sender, u64)>, - #[cfg(test)] - stopped: watch::Sender, + permits_requests: BinaryHeap, +} + +struct SingleSplitPermitRequest { + permit_sender: oneshot::Sender, + permit_size: u64, +} + +struct LeafPermitRequest { + /// Single split permit requests for this leaf search. + single_split_permit_requests: std::vec::IntoIter, +} + +impl Ord for LeafPermitRequest { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + // we compare other with self and not the other way arround because we want a min-heap and + // Rust's is a max-heap + other + .single_split_permit_requests + .as_slice() + .len() + .cmp(&self.single_split_permit_requests.as_slice().len()) + } +} + +impl PartialOrd for LeafPermitRequest { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl PartialEq for LeafPermitRequest { + fn eq(&self, other: &Self) -> bool { + self.cmp(other).is_eq() + } +} + +impl Eq for LeafPermitRequest {} + +impl LeafPermitRequest { + // `permit_sizes` must not be empty. + fn from_estimated_costs(permit_sizes: Vec) -> (Self, Vec) { + assert!(!permit_sizes.is_empty(), "permit_sizes must not be empty"); + let mut permits = Vec::with_capacity(permit_sizes.len()); + let mut single_split_permit_requests = Vec::with_capacity(permit_sizes.len()); + for permit_size in permit_sizes { + let (tx, rx) = oneshot::channel(); + // we keep our internal list of permits and the returned wait handles in the + // same order to make sure we emit each permit in the right order. Doing otherwise + // may cause deadlocks + single_split_permit_requests.push(SingleSplitPermitRequest { + permit_sender: tx, + permit_size, + }); + permits.push(SearchPermitFuture::from_async_receiver(rx)); + } + ( + LeafPermitRequest { + single_split_permit_requests: single_split_permit_requests.into_iter(), + }, + permits, + ) + } + + fn pop_if_smaller_than(&mut self, max_size: u64) -> Option { + let peeked_single_split_req = self.single_split_permit_requests.as_slice().first()?; + if peeked_single_split_req.permit_size > max_size { + return None; + } + self.single_split_permit_requests.next() + } + + fn is_empty(&self) -> bool { + self.single_split_permit_requests.as_slice().is_empty() + } } impl SearchPermitActor { - async fn run(mut self) { + async fn run(mut self) -> Self { // Stops when the last clone of SearchPermitProvider is dropped. while let Some(msg) = self.msg_receiver.recv().await { self.handle_message(msg); } - #[cfg(test)] - self.stopped.send(true).ok(); + self } fn handle_message(&mut self, msg: SearchPermitMessage) { match msg { - SearchPermitMessage::Request { - permit_sizes, - permit_sender, + SearchPermitMessage::RequestWithOffload { + mut permit_sizes, + permit_resp_tx: permit_sender, + offload_threshold, } => { - let mut permits = Vec::with_capacity(permit_sizes.len()); - for permit_size in permit_sizes { - let (tx, rx) = oneshot::channel(); - self.permits_requests.push_back((tx, permit_size)); - permits.push(SearchPermitFuture::from_async_receiver(rx)); + let current_pending = self + .permits_requests + .iter() + .map(|req| req.single_split_permit_requests.as_slice().len()) + .sum(); + // How many new splits can we accept locally before hitting the threshold. + let local_capacity = offload_threshold.saturating_sub(current_pending); + + // If this indeed truncates the permit_sizes vector, other splits will be offloaded + // to lambdas. + permit_sizes.truncate(local_capacity); + + // We special case here in order to avoid pushing empty request in the queue. + // (they would never be removed) + if permit_sizes.is_empty() { + let _ = permit_sender.send(Vec::new()); + return; } + + let (leaf_permit_request, permit_futures) = + LeafPermitRequest::from_estimated_costs(permit_sizes); + self.permits_requests.push(leaf_permit_request); self.assign_available_permits(); - // The receiver could be dropped in the (unlikely) situation - // where the future requesting these permits is cancelled before - // this message is processed. - let _ = permit_sender.send(permits); + let _ = permit_sender.send(permit_futures); } SearchPermitMessage::UpdateMemory { memory_delta } => { if self.total_memory_allocated as i64 + memory_delta < 0 { @@ -260,34 +377,43 @@ impl SearchPermitActor { } } - fn pop_next_request_if_serviceable(&mut self) -> Option<(oneshot::Sender, u64)> { + fn pop_next_request_if_serviceable(&mut self) -> Option { if self.num_warmup_slots_available == 0 { return None; } - if let Some((_, next_permit_size)) = self.permits_requests.front() { - if self.total_memory_allocated + next_permit_size <= self.total_memory_budget { - return self.permits_requests.pop_front(); + let available_memory = self + .total_memory_budget + .checked_sub(self.total_memory_allocated)?; + let mut peeked = self.permits_requests.peek_mut()?; + + assert!( + !peeked.is_empty(), + "unexpected empty permits_requests present in the search permit provider queue" + ); + if let Some(permit_request) = peeked.pop_if_smaller_than(available_memory) { + if peeked.is_empty() { + PeekMut::pop(peeked); } + return Some(permit_request); } None } fn assign_available_permits(&mut self) { - while let Some((permit_requester_tx, next_permit_size)) = - self.pop_next_request_if_serviceable() - { + while let Some(permit_request) = self.pop_next_request_if_serviceable() { let mut ongoing_gauge_guard = GaugeGuard::from_gauge( &crate::SEARCH_METRICS.leaf_search_single_split_tasks_ongoing, ); ongoing_gauge_guard.add(1); - self.total_memory_allocated += next_permit_size; + self.total_memory_allocated += permit_request.permit_size; self.num_warmup_slots_available -= 1; - permit_requester_tx + permit_request + .permit_sender .send(SearchPermit { mode: SearchPermitMode2::Async { _ongoing_gauge_guard: ongoing_gauge_guard, msg_sender: self.msg_sender.clone(), - memory_allocation: next_permit_size, + memory_allocation: permit_request.permit_size, warmup_slot_freed: false, }, }) @@ -301,7 +427,6 @@ impl SearchPermitActor { } } -#[derive(Debug)] pub struct SearchPermit { mode: SearchPermitMode2, } @@ -333,11 +458,18 @@ impl SearchPermit { /// This will increase or decrease the available memory in the [`SearchPermitProvider`]. pub fn update_memory_usage(&mut self, new_memory_usage: ByteSize) { match &mut self.mode { - SearchPermitMode2::Async { memory_allocation, msg_sender, .. } => { + SearchPermitMode2::Async { + memory_allocation, + msg_sender, + .. + } => { let new_usage_bytes = new_memory_usage.as_u64(); let memory_delta = new_usage_bytes as i64 - *memory_allocation as i64; *memory_allocation = new_usage_bytes; - Self::send_if_still_running(msg_sender, SearchPermitMessage::UpdateMemory { memory_delta }); + Self::send_if_still_running( + msg_sender, + SearchPermitMessage::UpdateMemory { memory_delta }, + ); } SearchPermitMode2::Sync { sync_permit } => { let new_usage_bytes = new_memory_usage.as_u64(); @@ -354,7 +486,11 @@ impl SearchPermit { /// slot is attached to each permit so calling this again has no effect. pub fn free_warmup_slot(&mut self) { match &mut self.mode { - SearchPermitMode2::Async { warmup_slot_freed, msg_sender, .. } => { + SearchPermitMode2::Async { + warmup_slot_freed, + msg_sender, + .. + } => { if *warmup_slot_freed { return; } @@ -370,12 +506,17 @@ impl SearchPermit { pub fn memory_allocation(&self) -> ByteSize { match &self.mode { - SearchPermitMode2::Async { memory_allocation, .. } => ByteSize(*memory_allocation), + SearchPermitMode2::Async { + memory_allocation, .. + } => ByteSize(*memory_allocation), SearchPermitMode2::Sync { sync_permit } => ByteSize(sync_permit.memory_allocation), } } - fn send_if_still_running(msg_sender: &mpsc::WeakUnboundedSender, msg: SearchPermitMessage) { + fn send_if_still_running( + msg_sender: &mpsc::WeakUnboundedSender, + msg: SearchPermitMessage, + ) { if let Some(sender) = msg_sender.upgrade() { sender .send(msg) @@ -389,27 +530,35 @@ impl SearchPermit { impl Drop for SearchPermit { fn drop(&mut self) { match &self.mode { - SearchPermitMode2::Async { memory_allocation, warmup_slot_freed, msg_sender, .. } => { - Self::send_if_still_running(msg_sender, SearchPermitMessage::Drop { - memory_size: *memory_allocation, - warmup_slot_freed: *warmup_slot_freed, - }); + SearchPermitMode2::Async { + memory_allocation, + warmup_slot_freed, + msg_sender, + .. + } => { + Self::send_if_still_running( + msg_sender, + SearchPermitMessage::Drop { + memory_size: *memory_allocation, + warmup_slot_freed: *warmup_slot_freed, + }, + ); } SearchPermitMode2::Sync { sync_permit } => { let mut state = sync_permit.state.lock().unwrap(); state.current_permits = state.current_permits.saturating_sub(1); - state.current_memory = state.current_memory.saturating_sub(sync_permit.memory_allocation); + state.current_memory = state + .current_memory + .saturating_sub(sync_permit.memory_allocation); } } } } -#[derive(Debug)] pub struct SearchPermitFuture { mode: SearchPermitFutureMode, } -#[derive(Debug)] enum SearchPermitFutureMode { Async(oneshot::Receiver), Sync(Option), @@ -439,7 +588,9 @@ impl Future for SearchPermitFuture { let receiver = Pin::new(receiver); match receiver.poll(cx) { Poll::Ready(Ok(search_permit)) => Poll::Ready(search_permit), - Poll::Ready(Err(_)) => panic!("Failed to acquire permit. This should never happen! Please, report on https://github.com/quickwit-oss/quickwit/issues."), + Poll::Ready(Err(_)) => panic!( + "Failed to acquire permit. This should never happen! Please, report on https://github.com/quickwit-oss/quickwit/issues." + ), Poll::Pending => Poll::Pending, } } @@ -466,13 +617,14 @@ impl Drop for SyncSearchPermit { // Decrement the memory and permit counters when the permit is dropped let mut state_guard = self.state.lock().unwrap(); state_guard.current_permits = state_guard.current_permits.saturating_sub(1); - state_guard.current_memory = state_guard.current_memory.saturating_sub(self.memory_allocation); + state_guard.current_memory = state_guard + .current_memory + .saturating_sub(self.memory_allocation); } } #[cfg(test)] mod tests { - use std::iter::repeat_n; use std::time::Duration; use futures::StreamExt; @@ -486,7 +638,7 @@ mod tests { let permit_provider = SearchPermitProvider::new(1, ByteSize::mb(100)); let mut all_futures = Vec::new(); let first_batch_of_permits = permit_provider - .get_permits(repeat_n(ByteSize::mb(10), 10)) + .get_permits(vec![ByteSize::mb(10); 10]) .await; assert_eq!(first_batch_of_permits.len(), 10); all_futures.extend( @@ -497,7 +649,7 @@ mod tests { ); let second_batch_of_permits = permit_provider - .get_permits(repeat_n(ByteSize::mb(10), 10)) + .get_permits(vec![ByteSize::mb(10); 10]) .await; assert_eq!(second_batch_of_permits.len(), 10); all_futures.extend( @@ -508,7 +660,7 @@ mod tests { ); // not super useful, considering what join set does, but still a tiny bit more sound. - all_futures.shuffle(&mut rand::thread_rng()); + all_futures.shuffle(&mut rand::rng()); let mut join_set = JoinSet::new(); for (res, fut) in all_futures { @@ -531,6 +683,69 @@ mod tests { } } + #[tokio::test] + async fn test_search_permit_order_with_concurrent_search() { + let permit_provider = SearchPermitProvider::new(4, ByteSize::mb(100)); + let mut all_futures = Vec::new(); + let first_batch_of_permits = permit_provider.get_permits(vec![ByteSize::mb(10); 8]).await; + assert_eq!(first_batch_of_permits.len(), 8); + all_futures.extend( + first_batch_of_permits + .into_iter() + .enumerate() + .map(move |(i, fut)| ((1, i), fut)), + ); + + let second_batch_of_permits = permit_provider.get_permits(vec![ByteSize::mb(10); 2]).await; + all_futures.extend( + second_batch_of_permits + .into_iter() + .enumerate() + .map(move |(i, fut)| ((2, i), fut)), + ); + + let third_batch_of_permits = permit_provider.get_permits(vec![ByteSize::mb(10); 6]).await; + all_futures.extend( + third_batch_of_permits + .into_iter() + .enumerate() + .map(move |(i, fut)| ((3, i), fut)), + ); + + // not super useful, considering what join set does, but still a tiny bit more sound. + all_futures.shuffle(&mut rand::rng()); + + let mut join_set = JoinSet::new(); + for (res, fut) in all_futures { + join_set.spawn(async move { + let permit = fut.await; + (res, permit) + }); + } + let mut ordered_result: Vec<(usize, usize)> = Vec::with_capacity(20); + while let Some(Ok(((batch_id, order), _permit))) = join_set.join_next().await { + ordered_result.push((batch_id, order)); + } + + let mut counters = [0; 4]; + let expected_result: Vec<(usize, usize)> = [ + 1, 1, 1, 1, // initial 4 permits + 2, 2, 1, 1, 1, 1, 3, 3, 3, 3, 3, 3, + ] + .into_iter() + .map(|batch_id| { + let order = counters[batch_id]; + counters[batch_id] += 1; + (batch_id, order) + }) + .collect(); + + // for the first 4 permits, the order is not well defined as they are all granted at once, + // and we poll futures in a random order. We sort them to fix that artifact + ordered_result[..4].sort(); + assert_eq!(ordered_result, expected_result); + } + #[tokio::test] async fn test_search_permit_early_drops() { let permit_provider = SearchPermitProvider::new(1, ByteSize::mb(100)); @@ -541,7 +756,7 @@ mod tests { .next() .unwrap(); let permit_fut2 = permit_provider - .get_permits([ByteSize::mb(10)]) + .get_permits(vec![ByteSize::mb(10)]) .await .into_iter() .next() @@ -549,20 +764,20 @@ mod tests { drop(permit_fut1); let permit = permit_fut2.await; assert_eq!(permit.memory_allocation, ByteSize::mb(10).as_u64()); - assert_eq!(*permit_provider.actor_stopped.borrow(), false); + assert!(!permit_provider.actor_join_handle.is_finished()); let _permit_fut3 = permit_provider - .get_permits([ByteSize::mb(10)]) + .get_permits(vec![ByteSize::mb(10)]) .await .into_iter() .next() .unwrap(); - let mut actor_stopped = permit_provider.actor_stopped.clone(); - drop(permit_provider); - { - actor_stopped.changed().await.unwrap(); - assert!(*actor_stopped.borrow()); - } + let SearchPermitProvider { + message_sender, + actor_join_handle, + } = permit_provider; + drop(message_sender); + Arc::into_inner(actor_join_handle).unwrap().await.unwrap(); } /// Tries to wait for a permit @@ -576,7 +791,7 @@ mod tests { async fn test_memory_budget() { let permit_provider = SearchPermitProvider::new(100, ByteSize::mb(100)); let mut permit_futs = permit_provider - .get_permits(repeat_n(ByteSize::mb(10), 14)) + .get_permits(vec![ByteSize::mb(10); 14]) .await; let mut remaining_permit_futs = permit_futs.split_off(10).into_iter(); assert_eq!(remaining_permit_futs.len(), 4); @@ -587,14 +802,14 @@ mod tests { .await; // the next permit is blocked by the memory budget let next_blocked_permit_fut = remaining_permit_futs.next().unwrap(); - try_get(next_blocked_permit_fut).await.unwrap_err(); + try_get(next_blocked_permit_fut).await.err().unwrap(); // if we drop one of the permits, we can get a new one permits.drain(0..1); let next_permit_fut = remaining_permit_futs.next().unwrap(); let _new_permit = try_get(next_permit_fut).await.unwrap(); // the next permit is blocked again by the memory budget let next_blocked_permit_fut = remaining_permit_futs.next().unwrap(); - try_get(next_blocked_permit_fut).await.unwrap_err(); + try_get(next_blocked_permit_fut).await.err().unwrap(); // by setting a more accurate memory usage after a completed warmup, we can get more permits permits[0].update_memory_usage(ByteSize::mb(4)); permits[1].update_memory_usage(ByteSize::mb(6)); @@ -602,12 +817,87 @@ mod tests { try_get(next_permit_fut).await.unwrap(); } + #[tokio::test] + async fn test_get_permits_with_offload_threshold_max_returns_all() { + let permit_provider = SearchPermitProvider::new(100, ByteSize::mb(100)); + let permits = permit_provider + .get_permits_with_offload(vec![ByteSize::mb(1); 8], usize::MAX) + .await; + assert_eq!(permits.len(), 8); + } + + #[tokio::test] + async fn test_get_permits_with_offload_threshold_zero_returns_none() { + let permit_provider = SearchPermitProvider::new(100, ByteSize::mb(100)); + let permits = permit_provider + .get_permits_with_offload(vec![ByteSize::mb(1); 5], 0) + .await; + assert!(permits.is_empty()); + let permit_actor = permit_provider.stop_and_unwrap().await; + assert!(permit_actor.permits_requests.is_empty()); + } + + #[tokio::test] + async fn test_get_permits_with_offload_truncates_to_threshold() { + let permit_provider = SearchPermitProvider::new(100, ByteSize::mb(100)); + let permits = permit_provider + .get_permits_with_offload(vec![ByteSize::mb(1); 10], 4) + .await; + assert_eq!(permits.len(), 4); + } + + #[tokio::test] + async fn test_get_permits_with_offload_futures_resolve_in_order() { + // We use a search permit provider with a capacity of 1 to make sure that the permits are + // resolved in order. + let permit_provider = SearchPermitProvider::new(1, ByteSize::mb(100)); + let permits = permit_provider + .get_permits_with_offload(vec![ByteSize::mb(1); 4], 10) + .await; + assert_eq!(permits.len(), 4); + let mut futs: Vec<_> = permits + .into_iter() + .enumerate() + .map(|(i, permit_fut)| async move { + permit_fut.await; + i + }) + .collect(); + futs.shuffle(&mut rand::rng()); + let mut join_set = JoinSet::new(); + for fut in futs { + join_set.spawn(fut); + } + let mut results = Vec::new(); + while let Some(result) = join_set.join_next().await { + results.push(result.unwrap()); + } + assert_eq!(results, vec![0, 1, 2, 3]); + } + + #[tokio::test] + async fn test_get_permits_with_offload_pending_consumed_frees_capacity() { + let permit_provider = SearchPermitProvider::new(100, ByteSize::mb(100)); + // First call: 4 splits, threshold 6. + let first_permits = permit_provider + .get_permits_with_offload(vec![ByteSize::mb(1); 4], 6) + .await; + assert_eq!(first_permits.len(), 4); + // Consume all permits from the first batch (they resolve and get dropped). + for permit_fut in first_permits { + let _permit = permit_fut.await; + } + // Second call: the consumed permits no longer count as pending. + let second_permits = permit_provider + .get_permits_with_offload(vec![ByteSize::mb(1); 5], 6) + .await; + assert_eq!(second_permits.len(), 5); + } + #[tokio::test] async fn test_warmup_slot() { let permit_provider = SearchPermitProvider::new(10, ByteSize::mb(100)); - let mut permit_futs = permit_provider - .get_permits(repeat_n(ByteSize::mb(1), 16)) - .await; + let mut permit_futs = permit_provider.get_permits(vec![ByteSize::mb(1); 16]).await; let mut remaining_permit_futs = permit_futs.split_off(10).into_iter(); assert_eq!(remaining_permit_futs.len(), 6); // we should be able to obtain 10 permits right away @@ -617,14 +907,14 @@ mod tests { .await; // the next permit is blocked by the warmup slots let next_blocked_permit_fut = remaining_permit_futs.next().unwrap(); - try_get(next_blocked_permit_fut).await.unwrap_err(); + try_get(next_blocked_permit_fut).await.err().unwrap(); // if we drop one of the permits, we can get a new one permits.drain(0..1); let next_permit_fut = remaining_permit_futs.next().unwrap(); permits.push(try_get(next_permit_fut).await.unwrap()); // the next permit is blocked again by the warmup slots let next_blocked_permit_fut = remaining_permit_futs.next().unwrap(); - try_get(next_blocked_permit_fut).await.unwrap_err(); + try_get(next_blocked_permit_fut).await.err().unwrap(); // we can explicitly free the warmup slot on a permit permits[0].free_warmup_slot(); let next_permit_fut = remaining_permit_futs.next().unwrap(); @@ -632,7 +922,7 @@ mod tests { // dropping that same permit does not free up another slot permits.drain(0..1); let next_blocked_permit_fut = remaining_permit_futs.next().unwrap(); - try_get(next_blocked_permit_fut).await.unwrap_err(); + try_get(next_blocked_permit_fut).await.err().unwrap(); // but dropping a permit for which the slot wasn't explicitly free does free up a slot permits.drain(0..1); let next_blocked_permit_fut = remaining_permit_futs.next().unwrap(); diff --git a/quickwit/quickwit-search/src/search_stream/collector.rs b/quickwit/quickwit-search/src/search_stream/collector.rs deleted file mode 100644 index 33e3526de0f..00000000000 --- a/quickwit/quickwit-search/src/search_stream/collector.rs +++ /dev/null @@ -1,241 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::HashMap; -use std::hash::Hash; -use std::marker::PhantomData; - -use tantivy::collector::{Collector, SegmentCollector}; -use tantivy::columnar::{DynamicColumn, HasAssociatedColumnType}; -use tantivy::fastfield::Column; -use tantivy::{DocId, Score, SegmentOrdinal, SegmentReader}; - -use crate::filters::{TimestampFilter, TimestampFilterBuilder}; - -#[derive(Clone)] -pub struct FastFieldSegmentCollector { - fast_field_values: Vec, - column_opt: Option>, - timestamp_filter_opt: Option, -} - -impl FastFieldSegmentCollector { - pub fn new( - column_opt: Option>, - timestamp_filter_opt: Option, - ) -> Self { - Self { - fast_field_values: Vec::new(), - column_opt, - timestamp_filter_opt, - } - } - - fn accept_document(&self, doc_id: DocId) -> bool { - if let Some(ref timestamp_filter) = self.timestamp_filter_opt { - return timestamp_filter.contains_doc_timestamp(doc_id); - } - true - } -} - -impl SegmentCollector for FastFieldSegmentCollector { - type Fruit = Vec; - - fn collect(&mut self, doc_id: DocId, _score: Score) { - let Some(column) = self.column_opt.as_ref() else { - return; - }; - if !self.accept_document(doc_id) { - return; - } - self.fast_field_values.extend(column.values_for_doc(doc_id)); - } - - fn harvest(self) -> Vec { - self.fast_field_values - } -} - -#[derive(Clone)] -pub struct FastFieldCollector { - pub fast_field_to_collect: String, - pub timestamp_filter_builder_opt: Option, - pub _marker: PhantomData, -} - -impl Collector for FastFieldCollector -where DynamicColumn: Into>> -{ - type Child = FastFieldSegmentCollector; - type Fruit = Vec; - - fn for_segment( - &self, - _segment_ord: SegmentOrdinal, - segment_reader: &SegmentReader, - ) -> tantivy::Result { - let timestamp_filter_opt = - if let Some(timestamp_filter_builder) = &self.timestamp_filter_builder_opt { - timestamp_filter_builder.build(segment_reader)? - } else { - None - }; - - let column_opt: Option> = segment_reader - .fast_fields() - .column_opt::(&self.fast_field_to_collect)?; - - Ok(FastFieldSegmentCollector::new( - column_opt, - timestamp_filter_opt, - )) - } - - fn requires_scoring(&self) -> bool { - // We do not need BM25 scoring in Quickwit. - false - } - - fn merge_fruits(&self, segment_fruits: Vec>) -> tantivy::Result { - Ok(segment_fruits.into_iter().flatten().collect::>()) - } -} - -#[derive(Clone)] -pub struct PartionnedFastFieldCollector { - pub fast_field_to_collect: String, - pub partition_by_fast_field: String, - pub timestamp_filter_builder_opt: Option, - pub _marker: PhantomData<(Item, PartitionItem)>, -} - -#[derive(Debug, Eq, PartialEq)] -pub struct PartitionValues { - pub partition_value: PartitionItem, - pub fast_field_values: Vec, -} - -impl Collector - for PartionnedFastFieldCollector -where - DynamicColumn: Into>>, - DynamicColumn: Into>>, -{ - type Child = PartitionedFastFieldSegmentCollector; - type Fruit = Vec>; - - fn for_segment( - &self, - _segment_ord: SegmentOrdinal, - segment_reader: &SegmentReader, - ) -> tantivy::Result { - let timestamp_filter_opt = - if let Some(timestamp_filter_builder) = &self.timestamp_filter_builder_opt { - timestamp_filter_builder.build(segment_reader)? - } else { - None - }; - let column_opt: Option> = segment_reader - .fast_fields() - .column_opt(&self.fast_field_to_collect)?; - - let partition_column_opt = segment_reader - .fast_fields() - .column_opt(self.partition_by_fast_field.as_str())?; - - Ok(PartitionedFastFieldSegmentCollector::new( - column_opt, - partition_column_opt, - timestamp_filter_opt, - )) - } - - fn requires_scoring(&self) -> bool { - // We do not need BM25 scoring in Quickwit. - false - } - - fn merge_fruits( - &self, - segment_fruits: Vec>>, - ) -> tantivy::Result { - Ok(segment_fruits - .into_iter() - .flat_map(|e| e.into_iter()) - .map(|(partition_value, values)| PartitionValues { - partition_value, - fast_field_values: values, - }) - .collect()) - } -} - -#[derive(Clone)] -pub struct PartitionedFastFieldSegmentCollector { - fast_field_values: HashMap>, - fast_field_reader: Option>, - partition_by_fast_field_reader: Option>, - timestamp_filter_opt: Option, -} - -impl PartitionedFastFieldSegmentCollector { - pub fn new( - fast_field_reader: Option>, - partition_by_fast_field_reader: Option>, - timestamp_filter_opt: Option, - ) -> Self { - Self { - fast_field_values: HashMap::default(), - fast_field_reader, - partition_by_fast_field_reader, - timestamp_filter_opt, - } - } - - fn accept_document(&self, doc_id: DocId) -> bool { - if let Some(ref timestamp_filter) = self.timestamp_filter_opt { - return timestamp_filter.contains_doc_timestamp(doc_id); - } - true - } -} - -impl - SegmentCollector for PartitionedFastFieldSegmentCollector -{ - type Fruit = HashMap>; - - fn collect(&mut self, doc_id: DocId, _score: Score) { - let Some(column) = self.fast_field_reader.as_ref() else { - return; - }; - let Some(partition_column) = self.partition_by_fast_field_reader.as_ref() else { - return; - }; - if !self.accept_document(doc_id) { - return; - } - if let Some(partition) = partition_column.first(doc_id) { - self.fast_field_values - .entry(partition) - .or_default() - .extend(column.values_for_doc(doc_id)); - } - } - - fn harvest(self) -> Self::Fruit { - self.fast_field_values - } -} diff --git a/quickwit/quickwit-search/src/search_stream/leaf.rs b/quickwit/quickwit-search/src/search_stream/leaf.rs deleted file mode 100644 index d5a8311f6bc..00000000000 --- a/quickwit/quickwit-search/src/search_stream/leaf.rs +++ /dev/null @@ -1,792 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::HashSet; -use std::hash::Hash; -use std::marker::PhantomData; -use std::sync::Arc; - -use futures::{FutureExt, StreamExt}; -use quickwit_common::pretty::PrettySample; -use quickwit_doc_mapper::{DocMapper, FastFieldWarmupInfo, WarmupInfo}; -use quickwit_proto::search::{ - LeafSearchStreamResponse, OutputFormat, SearchRequest, SearchStreamRequest, - SplitIdAndFooterOffsets, -}; -use quickwit_storage::{ByteRangeCache, Storage}; -use tantivy::columnar::{DynamicColumn, HasAssociatedColumnType}; -use tantivy::fastfield::Column; -use tantivy::query::Query; -use tantivy::schema::{Field, Schema, Type}; -use tantivy::{DateTime, ReloadPolicy, Searcher}; -use tokio_stream::wrappers::UnboundedReceiverStream; -use tracing::*; - -use super::FastFieldCollector; -use super::collector::{PartionnedFastFieldCollector, PartitionValues}; -use crate::filters::{TimestampFilterBuilder, create_timestamp_filter_builder}; -use crate::leaf::{open_index_with_caches, rewrite_start_end_time_bounds, warmup}; -use crate::service::SearcherContext; -use crate::{Result, SearchError}; - -/// `leaf` step of search stream. -// Note: we return a stream of a result with a tonic::Status error -// to be compatible with the stream coming from the grpc client. -// It would be better to have a SearchError but we need then -// to process stream in grpc_adapter.rs to change SearchError -// to tonic::Status as tonic::Status is required by the stream result -// signature defined by proto generated code. -#[instrument(skip_all, fields(index = request.index_id))] -pub async fn leaf_search_stream( - searcher_context: Arc, - request: SearchStreamRequest, - storage: Arc, - splits: Vec, - doc_mapper: Arc, -) -> UnboundedReceiverStream> { - info!(split_offsets = ?PrettySample::new(&splits, 5)); - let (result_sender, result_receiver) = tokio::sync::mpsc::unbounded_channel(); - let span = info_span!("leaf_search_stream",); - tokio::spawn( - async move { - let mut stream = - leaf_search_results_stream(searcher_context, request, storage, splits, doc_mapper) - .await; - while let Some(item) = stream.next().await { - if let Err(error) = result_sender.send(item) { - error!( - "Failed to send leaf search stream result. Stop sending. Cause: {}", - error - ); - break; - } - } - } - .instrument(span), - ); - UnboundedReceiverStream::new(result_receiver) -} - -async fn leaf_search_results_stream( - searcher_context: Arc, - request: SearchStreamRequest, - storage: Arc, - splits: Vec, - doc_mapper: Arc, -) -> impl futures::Stream> + Sync + Send + 'static { - let max_num_concurrent_split_streams = searcher_context - .searcher_config - .max_num_concurrent_split_streams; - futures::stream::iter(splits) - .map(move |split| { - leaf_search_stream_single_split( - searcher_context.clone(), - split, - doc_mapper.clone(), - request.clone(), - storage.clone(), - ) - .shared() - }) - .buffer_unordered(max_num_concurrent_split_streams) -} - -/// Apply a leaf search on a single split. -#[instrument(skip_all, fields(split_id = %split.split_id))] -async fn leaf_search_stream_single_split( - searcher_context: Arc, - split: SplitIdAndFooterOffsets, - doc_mapper: Arc, - mut stream_request: SearchStreamRequest, - storage: Arc, -) -> crate::Result { - // TODO: Should we track the memory here using the SearchPermitProvider? - let _leaf_split_stream_permit = searcher_context - .split_stream_semaphore - .acquire() - .await - .expect("Failed to acquire permit. This should never happen! Please, report on https://github.com/quickwit-oss/quickwit/issues."); - rewrite_start_end_time_bounds( - &mut stream_request.start_timestamp, - &mut stream_request.end_timestamp, - &split, - ); - - let cache = - ByteRangeCache::with_infinite_capacity(&quickwit_storage::STORAGE_METRICS.shortlived_cache); - let (index, _) = open_index_with_caches( - &searcher_context, - storage, - &split, - Some(doc_mapper.tokenizer_manager()), - Some(cache), - ) - .await?; - let split_schema = index.schema(); - - let request_fields = Arc::new(SearchStreamRequestFields::from_request( - &stream_request, - &split_schema, - doc_mapper.as_ref(), - )?); - - let output_format = OutputFormat::try_from(stream_request.output_format) - .map_err(|_| SearchError::Internal("invalid output format specified".to_string()))?; - - if request_fields.partition_by_fast_field.is_some() - && output_format != OutputFormat::ClickHouseRowBinary - { - return Err(SearchError::Internal( - "invalid output format specified, only ClickHouseRowBinary is allowed when providing \ - a partitioned-by field" - .to_string(), - )); - } - - let search_request = Arc::new(SearchRequest::try_from(stream_request.clone())?); - let query_ast = serde_json::from_str(&search_request.query_ast) - .map_err(|err| SearchError::InvalidQuery(err.to_string()))?; - let (query, mut warmup_info) = doc_mapper.query(split_schema.clone(), &query_ast, false)?; - let reader = index - .reader_builder() - .reload_policy(ReloadPolicy::Manual) - .try_into()?; - let searcher = reader.searcher(); - - let timestamp_filter_builder_opt: Option = - create_timestamp_filter_builder( - request_fields.timestamp_field_name(), - search_request.start_timestamp, - search_request.end_timestamp, - ); - - let requires_scoring = search_request - .sort_fields - .iter() - .any(|sort| sort.field_name == "_score"); - - let fast_fields = request_fields - .fast_fields_for_request(timestamp_filter_builder_opt.as_ref()) - .into_iter() - .map(|name| FastFieldWarmupInfo { - name, - with_subfields: false, - }) - .collect(); - let stream_warmup_info = WarmupInfo { - fast_fields, - // TODO no test fail if this line get removed - field_norms: requires_scoring, - ..Default::default() - }; - warmup_info.merge(stream_warmup_info); - warmup_info.simplify(); - - warmup(&searcher, &warmup_info).await?; - - let span = info_span!( - "collect_fast_field", - split_id = %split.split_id, - request_fields=%request_fields, - ); - - let _ = span.enter(); - let m_request_fields = request_fields.clone(); - let collect_handle = crate::search_thread_pool().run_cpu_intensive(move || { - let mut buffer = Vec::new(); - match m_request_fields.fast_field_types() { - (Type::I64, None) => { - let collected_values = collect_values::( - &m_request_fields, - timestamp_filter_builder_opt, - &searcher, - &query, - )?; - super::serialize::(&collected_values, &mut buffer, output_format).map_err( - |_| { - SearchError::Internal("error when serializing i64 during export".to_owned()) - }, - )?; - } - (Type::U64, None) => { - let collected_values = collect_values::( - &m_request_fields, - timestamp_filter_builder_opt, - &searcher, - &query, - )?; - super::serialize::(&collected_values, &mut buffer, output_format).map_err( - |_| { - SearchError::Internal("error when serializing u64 during export".to_owned()) - }, - )?; - } - (Type::Date, None) => { - let collected_values = collect_values::( - &m_request_fields, - timestamp_filter_builder_opt, - &searcher, - &query, - )?; - // It may seem overkill and expensive considering DateTime is just a wrapper - // over the i64, but the compiler is smarter than it looks and the code - // below actually is zero-cost: No allocation and no copy happens. - let collected_values_as_micros = collected_values - .into_iter() - .map(|date_time| date_time.into_timestamp_micros()) - .collect::>(); - // We serialize Date as i64 microseconds. - super::serialize::(&collected_values_as_micros, &mut buffer, output_format) - .map_err(|_| { - SearchError::Internal("error when serializing i64 during export".to_owned()) - })?; - } - (Type::I64, Some(Type::I64)) => { - let collected_values = collect_partitioned_values::( - &m_request_fields, - timestamp_filter_builder_opt, - &searcher, - &query, - )?; - super::serialize_partitions::(collected_values.as_slice(), &mut buffer) - .map_err(|_| { - SearchError::Internal("error when serializing i64 during export".to_owned()) - })?; - } - (Type::U64, Some(Type::U64)) => { - let collected_values = collect_partitioned_values::( - &m_request_fields, - timestamp_filter_builder_opt, - &searcher, - &query, - )?; - super::serialize_partitions::(collected_values.as_slice(), &mut buffer) - .map_err(|_| { - SearchError::Internal("error when serializing i64 during export".to_owned()) - })?; - } - (fast_field_type, None) => { - return Err(SearchError::Internal(format!( - "search stream does not support fast field of type `{fast_field_type:?}`" - ))); - } - (fast_field_type, Some(partition_fast_field_type)) => { - return Err(SearchError::Internal(format!( - "search stream does not support the combination of fast field type \ - `{fast_field_type:?}` and partition fast field type \ - `{partition_fast_field_type:?}`" - ))); - } - }; - Result::>::Ok(buffer) - }); - let buffer = collect_handle.await.map_err(|_| { - error!(split_id = %split.split_id, request_fields=%request_fields, "failed to collect fast field"); - SearchError::Internal(format!("error when collecting fast field values for split {}", split.split_id)) - })??; - Ok(LeafSearchStreamResponse { - data: buffer, - split_id: split.split_id, - }) -} - -fn collect_values( - request_fields: &SearchStreamRequestFields, - timestamp_filter_builder_opt: Option, - searcher: &Searcher, - query: &dyn Query, -) -> crate::Result> -where - DynamicColumn: Into>>, -{ - let collector = FastFieldCollector:: { - fast_field_to_collect: request_fields.fast_field_name().to_string(), - timestamp_filter_builder_opt, - _marker: PhantomData, - }; - let result = searcher.search(query, &collector)?; - Ok(result) -} - -fn collect_partitioned_values< - Item: HasAssociatedColumnType, - TPartitionValue: HasAssociatedColumnType + Eq + Hash, ->( - request_fields: &SearchStreamRequestFields, - timestamp_filter_builder_opt: Option, - searcher: &Searcher, - query: &dyn Query, -) -> crate::Result>> -where - DynamicColumn: Into>> + Into>>, -{ - let collector = PartionnedFastFieldCollector:: { - fast_field_to_collect: request_fields.fast_field_name().to_string(), - partition_by_fast_field: request_fields - .partition_by_fast_field_name() - .expect("`partition_by_fast_field` is not defined. This should never happen! Please, report on https://github.com/quickwit-oss/quickwit/issues.") - .to_string(), - timestamp_filter_builder_opt, - _marker: PhantomData, - }; - let result = searcher.search(query, &collector)?; - Ok(result) -} - -#[derive(Debug)] -// TODO move to owned values, implement Send + Sync -struct SearchStreamRequestFields { - fast_field: Field, - partition_by_fast_field: Option, - timestamp_field_name: Option, - schema: Schema, -} - -impl std::fmt::Display for SearchStreamRequestFields { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "fast_field: {},", self.fast_field_name())?; - write!( - f, - "timestamp_field: {},", - self.timestamp_field_name().unwrap_or("None") - )?; - write!( - f, - "partition_by_fast_field: {}", - self.partition_by_fast_field_name().unwrap_or("None") - ) - } -} - -impl SearchStreamRequestFields { - pub fn from_request( - stream_request: &SearchStreamRequest, - schema: &Schema, - doc_mapper: &DocMapper, - ) -> crate::Result { - let fast_field = schema.get_field(&stream_request.fast_field)?; - - if !Self::is_fast_field(schema, &fast_field) { - return Err(SearchError::InvalidQuery(format!( - "field `{}` is not a fast field", - &stream_request.fast_field - ))); - } - - let timestamp_field_name = doc_mapper.timestamp_field_name().map(ToString::to_string); - let partition_by_fast_field = stream_request - .partition_by_field - .as_deref() - .and_then(|field_name| schema.get_field(field_name).ok()); - - if partition_by_fast_field.is_some() - && !Self::is_fast_field(schema, &partition_by_fast_field.unwrap()) - { - return Err(SearchError::InvalidQuery(format!( - "field `{}` is not a fast field", - &stream_request.partition_by_field.as_deref().unwrap() - ))); - } - - Ok(SearchStreamRequestFields { - schema: schema.to_owned(), - fast_field, - partition_by_fast_field, - timestamp_field_name, - }) - } - - pub fn fast_field_types(&self) -> (Type, Option) { - ( - self.schema - .get_field_entry(self.fast_field) - .field_type() - .value_type(), - self.partition_by_fast_field - .map(|field| self.schema.get_field_entry(field).field_type().value_type()), - ) - } - - fn fast_fields_for_request( - &self, - timestamp_filter_builder_opt: Option<&TimestampFilterBuilder>, - ) -> HashSet { - let mut set = HashSet::new(); - set.insert(self.fast_field_name().to_string()); - if let Some(timestamp_filter_builder) = timestamp_filter_builder_opt { - set.insert(timestamp_filter_builder.timestamp_field_name.clone()); - } - if let Some(partition_by_fast_field) = self.partition_by_fast_field_name() { - set.insert(partition_by_fast_field.to_string()); - } - set - } - - pub fn timestamp_field_name(&self) -> Option<&str> { - self.timestamp_field_name.as_deref() - } - - pub fn fast_field_name(&self) -> &str { - self.schema.get_field_name(self.fast_field) - } - - pub fn partition_by_fast_field_name(&self) -> Option<&str> { - self.partition_by_fast_field - .map(|field| self.schema.get_field_name(field)) - } - - fn is_fast_field(schema: &Schema, field: &Field) -> bool { - schema.get_field_entry(*field).is_fast() - } -} - -#[cfg(test)] -mod tests { - use std::collections::HashMap; - use std::convert::TryInto; - use std::str::from_utf8; - - use itertools::Itertools; - use quickwit_indexing::TestSandbox; - use quickwit_metastore::{ListSplitsRequestExt, MetastoreServiceStreamSplitsExt}; - use quickwit_proto::metastore::{ListSplitsRequest, MetastoreService}; - use quickwit_query::query_ast::qast_json_helper; - use serde_json::json; - use tantivy::time::{Duration, OffsetDateTime}; - - use super::*; - use crate::extract_split_and_footer_offsets; - - #[tokio::test] - async fn test_leaf_search_stream_to_csv_output_with_filtering() -> anyhow::Result<()> { - let index_id = "single-node-simple"; - let doc_mapping_yaml = r#" - field_mappings: - - name: body - type: text - - name: ts - type: datetime - fast: true - timestamp_field: ts - "#; - let test_sandbox = TestSandbox::create(index_id, doc_mapping_yaml, "", &["body"]).await?; - - let mut docs = Vec::new(); - let mut filtered_timestamp_values = Vec::new(); - let start_timestamp = 72057595; - let end_timestamp = start_timestamp + 20; - for i in 0..30 { - let timestamp = start_timestamp + (i + 1) as i64; - let body = format!("info @ t:{timestamp}"); - docs.push(json!({"body": body, "ts": timestamp})); - if timestamp < end_timestamp { - filtered_timestamp_values.push(timestamp); - } - } - test_sandbox.add_documents(docs).await?; - - let request = SearchStreamRequest { - index_id: index_id.to_string(), - query_ast: qast_json_helper("info", &["body"]), - snippet_fields: Vec::new(), - start_timestamp: None, - end_timestamp: Some(end_timestamp), - fast_field: "ts".to_string(), - output_format: 0, - partition_by_field: None, - }; - let splits = test_sandbox - .metastore() - .list_splits(ListSplitsRequest::try_from_index_uid(test_sandbox.index_uid()).unwrap()) - .await? - .collect_splits() - .await - .unwrap(); - let splits_offsets = splits - .into_iter() - .map(|split| extract_split_and_footer_offsets(&split.split_metadata)) - .collect(); - let searcher_context = Arc::new(SearcherContext::for_test()); - let mut single_node_stream = leaf_search_stream( - searcher_context, - request, - test_sandbox.storage(), - splits_offsets, - test_sandbox.doc_mapper(), - ) - .await; - let res = single_node_stream.next().await.expect("no leaf result")?; - assert_eq!( - from_utf8(&res.data)?, - format!( - "{}\n", - filtered_timestamp_values - .iter() - .map(|timestamp_secs| (timestamp_secs * 1_000_000).to_string()) - .join("\n") - ) - ); - test_sandbox.assert_quit().await; - Ok(()) - } - - #[tokio::test] - async fn test_leaf_search_stream_filtering_with_datetime() -> anyhow::Result<()> { - let index_id = "single-node-simple-datetime"; - let doc_mapping_yaml = r#" - field_mappings: - - name: body - type: text - - name: ts - type: datetime - input_formats: - - "unix_timestamp" - fast: true - timestamp_field: ts - "#; - let test_sandbox = TestSandbox::create(index_id, doc_mapping_yaml, "", &["body"]).await?; - let mut docs = Vec::new(); - let mut filtered_timestamp_values = Vec::new(); - let start_date = OffsetDateTime::now_utc(); - let num_days = 20; - for i in 0..30 { - let dt = start_date.checked_add(Duration::days(i + 1)).unwrap(); - let body = format!("info @ t:{}", i + 1); - docs.push(json!({"body": body, "ts": dt.unix_timestamp()})); - if i + 1 < num_days { - let ts_secs = dt.unix_timestamp() * 1_000_000; - filtered_timestamp_values.push(ts_secs.to_string()); - } - } - test_sandbox.add_documents(docs).await?; - - let end_timestamp = start_date - .checked_add(Duration::days(num_days)) - .unwrap() - .unix_timestamp(); - let request = SearchStreamRequest { - index_id: index_id.to_string(), - query_ast: qast_json_helper("info", &["body"]), - snippet_fields: Vec::new(), - start_timestamp: None, - end_timestamp: Some(end_timestamp), - fast_field: "ts".to_string(), - output_format: 0, - partition_by_field: None, - }; - let splits = test_sandbox - .metastore() - .list_splits(ListSplitsRequest::try_from_index_uid(test_sandbox.index_uid()).unwrap()) - .await? - .collect_splits() - .await?; - let splits_offsets = splits - .into_iter() - .map(|split| extract_split_and_footer_offsets(&split.split_metadata)) - .collect(); - let searcher_context = Arc::new(SearcherContext::for_test()); - let mut single_node_stream = leaf_search_stream( - searcher_context, - request, - test_sandbox.storage(), - splits_offsets, - test_sandbox.doc_mapper(), - ) - .await; - let res = single_node_stream.next().await.expect("no leaf result")?; - assert_eq!( - from_utf8(&res.data)?, - format!("{}\n", filtered_timestamp_values.join("\n")) - ); - test_sandbox.assert_quit().await; - Ok(()) - } - - #[tokio::test] - async fn test_leaf_search_stream_with_string_fast_field_should_return_proper_error() - -> anyhow::Result<()> { - let index_id = "single-node-simple-string-fast-field"; - let doc_mapping_yaml = r#" - field_mappings: - - name: body - type: text - - name: app - type: text - tokenizer: raw - fast: true - "#; - let test_sandbox = TestSandbox::create(index_id, doc_mapping_yaml, "{}", &["body"]).await?; - - test_sandbox - .add_documents(vec![json!({"body": "body", "app": "my-app"})]) - .await?; - - let request = SearchStreamRequest { - index_id: index_id.to_string(), - query_ast: qast_json_helper("info", &["body"]), - snippet_fields: Vec::new(), - start_timestamp: None, - end_timestamp: None, - fast_field: "app".to_string(), - output_format: 0, - partition_by_field: None, - }; - let splits = test_sandbox - .metastore() - .list_splits(ListSplitsRequest::try_from_index_uid(test_sandbox.index_uid()).unwrap()) - .await? - .collect_splits() - .await?; - let splits_offsets = splits - .into_iter() - .map(|split| extract_split_and_footer_offsets(&split.split_metadata)) - .collect(); - let searcher_context = Arc::new(SearcherContext::for_test()); - let mut single_node_stream = leaf_search_stream( - searcher_context, - request, - test_sandbox.storage(), - splits_offsets, - test_sandbox.doc_mapper(), - ) - .await; - let res = single_node_stream.next().await.expect("no leaf result"); - let error_message = res.unwrap_err().to_string(); - assert!(error_message.contains("search stream does not support fast field of type `Str`"),); - test_sandbox.assert_quit().await; - Ok(()) - } - - #[tokio::test] - async fn test_leaf_search_stream_to_partitioned_clickhouse_binary_output_with_filtering() - -> anyhow::Result<()> { - let index_id = "single-node-simple-2"; - let doc_mapping_yaml = r#" - field_mappings: - - name: body - type: text - - name: ts - type: datetime - fast: true - - name: partition_by_fast_field - type: u64 - fast: true - - name: fast_field - type: u64 - fast: true - timestamp_field: ts - "#; - let test_sandbox = TestSandbox::create(index_id, doc_mapping_yaml, "", &["body"]).await?; - - let mut docs = Vec::new(); - let partition_by_fast_field_values = [1, 2, 3, 4, 5]; - let mut expected_output_tmp: HashMap> = HashMap::new(); - let start_timestamp = 72057595; - let end_timestamp: i64 = start_timestamp + 20; - for i in 0..30 { - let timestamp = start_timestamp + (i + 1) as i64; - let body = format!("info @ t:{timestamp}"); - let partition_number = partition_by_fast_field_values[i % 5]; - let fast_field: u64 = (i * 2).try_into().unwrap(); - docs.push(json!({ - "body": body, - "ts": timestamp, - "partition_by_fast_field": partition_number, - "fast_field": fast_field, - })); - if timestamp < end_timestamp { - if let Some(values_for_partition) = expected_output_tmp.get_mut(&partition_number) { - values_for_partition.push(fast_field) - } else { - expected_output_tmp.insert(partition_number, vec![fast_field]); - } - } - } - test_sandbox.add_documents(docs).await?; - let mut expected_output: Vec> = expected_output_tmp - .iter() - .map(|(key, value)| PartitionValues { - partition_value: *key, - fast_field_values: value.to_vec(), - }) - .collect(); - - let request = SearchStreamRequest { - index_id: index_id.to_string(), - query_ast: qast_json_helper("info", &["body"]), - snippet_fields: Vec::new(), - start_timestamp: None, - end_timestamp: Some(end_timestamp), - fast_field: "fast_field".to_string(), - output_format: 1, - partition_by_field: Some(String::from("partition_by_fast_field")), - }; - let splits = test_sandbox - .metastore() - .list_splits(ListSplitsRequest::try_from_index_uid(test_sandbox.index_uid()).unwrap()) - .await? - .collect_splits() - .await?; - let splits_offsets = splits - .into_iter() - .map(|split| extract_split_and_footer_offsets(&split.split_metadata)) - .collect(); - let searcher_context = Arc::new(SearcherContext::for_test()); - let mut single_node_stream = leaf_search_stream( - searcher_context, - request, - test_sandbox.storage(), - splits_offsets, - test_sandbox.doc_mapper(), - ) - .await; - let res = single_node_stream.next().await.expect("no leaf result")?; - let mut deserialized_output = deserialize_partitions(res.data); - expected_output.sort_by(|l, r| l.partition_value.cmp(&r.partition_value)); - deserialized_output.sort_by(|l, r| l.partition_value.cmp(&r.partition_value)); - assert_eq!(expected_output, deserialized_output); - test_sandbox.assert_quit().await; - Ok(()) - } - - fn deserialize_partitions(buffer: Vec) -> Vec> { - // Note: this function is only meant to be used with valid payloads for testing purposes - let mut cursor = 0; - let mut partitions_values = Vec::new(); - while cursor < buffer.len() { - let partition_slice: [u8; 8] = buffer[cursor..cursor + 8].try_into().unwrap(); - let partition = u64::from_le_bytes(partition_slice); - cursor += 8; - - let payload_size_slice: [u8; 8] = buffer[cursor..cursor + 8].try_into().unwrap(); - let payload_size = u64::from_le_bytes(payload_size_slice); - let nb_values: usize = (payload_size / 8).try_into().unwrap(); - cursor += 8; - - let mut partition_value = PartitionValues { - partition_value: partition, - fast_field_values: Vec::with_capacity(nb_values), - }; - - for _ in 0..nb_values { - let value_slice: [u8; 8] = buffer[cursor..cursor + 8].try_into().unwrap(); - let value = u64::from_le_bytes(value_slice); - cursor += 8; - partition_value.fast_field_values.push(value); - } - partitions_values.push(partition_value); - } - partitions_values - } -} diff --git a/quickwit/quickwit-search/src/search_stream/mod.rs b/quickwit/quickwit-search/src/search_stream/mod.rs deleted file mode 100644 index dba519d822e..00000000000 --- a/quickwit/quickwit-search/src/search_stream/mod.rs +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod collector; -mod leaf; -mod root; - -use std::fmt::Display; -use std::io; -use std::io::Write; - -pub use collector::FastFieldCollector; -pub use leaf::leaf_search_stream; -use quickwit_proto::search::OutputFormat; -pub use root::root_search_stream; -use tantivy::columnar::MonotonicallyMappableToU64; - -use self::collector::PartitionValues; - -pub trait ToLittleEndian { - fn to_le_bytes(&self) -> [u8; 8]; -} - -impl ToLittleEndian for u64 { - fn to_le_bytes(&self) -> [u8; 8] { - u64::to_le_bytes(*self) - } -} - -impl ToLittleEndian for i64 { - fn to_le_bytes(&self) -> [u8; 8] { - i64::to_le_bytes(*self) - } -} - -impl ToLittleEndian for f64 { - fn to_le_bytes(&self) -> [u8; 8] { - f64::to_le_bytes(*self) - } -} - -/// Serialize the values into the `buffer` as bytes. -/// -/// Please note that the `buffer` is always cleared. -pub fn serialize( - values: &[T], - buffer: &mut Vec, - format: OutputFormat, -) -> io::Result<()> { - match format { - OutputFormat::Csv => serialize_csv(values, buffer), - OutputFormat::ClickHouseRowBinary => serialize_click_house_row_binary(values, buffer), - } -} - -pub fn serialize_partitions< - TFastValue: MonotonicallyMappableToU64, - TPartitionFastValue: MonotonicallyMappableToU64, ->( - p_values: &[PartitionValues], - buffer: &mut Vec, -) -> io::Result<()> { - let buf_size = helpers::partitions_size_in_bytes(p_values); - buffer.clear(); - buffer.reserve_exact(buf_size); - for partition in p_values { - let values_byte_size = std::mem::size_of::() * partition.fast_field_values.len(); - - buffer.extend(partition.partition_value.to_u64().to_le_bytes()); - buffer.extend(values_byte_size.to_le_bytes()); - - for value in &partition.fast_field_values { - buffer.extend(value.to_u64().to_le_bytes()); - } - } - Ok(()) -} - -fn serialize_csv(values: &[T], buffer: &mut Vec) -> io::Result<()> { - buffer.clear(); - for value in values { - writeln!(buffer, "{value}")?; - } - Ok(()) -} - -fn serialize_click_house_row_binary( - values: &[T], - buffer: &mut Vec, -) -> io::Result<()> { - buffer.clear(); - buffer.reserve_exact(8 * values.len()); - for value in values { - buffer.extend(value.to_le_bytes()); - } - Ok(()) -} - -mod helpers { - use super::collector::PartitionValues; - - #[inline(always)] - pub fn partitions_size_in_bytes( - partitions: &[PartitionValues], - ) -> usize { - let mut size = 0; - for partition in partitions { - size += partition_size_in_bytes(partition); - } - size - } - - #[inline(always)] - fn partition_size_in_bytes( - partition: &PartitionValues, - ) -> usize { - std::mem::size_of::() * partition.fast_field_values.len() - + std::mem::size_of::() - + std::mem::size_of::() - } -} - -#[cfg(test)] -mod tests { - use crate::search_stream::collector::PartitionValues; - use crate::search_stream::{serialize_click_house_row_binary, serialize_csv}; - - #[test] - fn test_serialize_row_binary() { - let mut buffer = Vec::new(); - serialize_click_house_row_binary::(&[-10i64], &mut buffer).unwrap(); - assert_eq!(buffer, (-10i64).to_le_bytes()); - - let mut buffer = Vec::new(); - serialize_click_house_row_binary::(&[-10f64], &mut buffer).unwrap(); - assert_eq!(buffer, (-10f64).to_le_bytes()); - } - - #[test] - fn test_serialize_csv() { - let mut buffer = Vec::new(); - serialize_csv::(&[-10i64], &mut buffer).unwrap(); - assert_eq!(buffer, "-10\n".as_bytes()); - } - - #[test] - fn test_serialize_partitions() { - let mut buffer = Vec::new(); - let partition_1 = PartitionValues { - partition_value: 1u64, - fast_field_values: vec![3u64, 4u64], - }; - let partition_2 = PartitionValues { - partition_value: 2u64, - fast_field_values: vec![5u64], - }; - super::serialize_partitions::(&[partition_1, partition_2], &mut buffer).unwrap(); - let expected_buffer: Vec = vec![ - 1u64.to_le_bytes(), - 16usize.to_le_bytes(), - 3u64.to_le_bytes(), - 4u64.to_le_bytes(), - 2u64.to_le_bytes(), - 8usize.to_le_bytes(), - 5u64.to_le_bytes(), - ] - .into_iter() - .flatten() - .collect(); - assert_eq!(buffer, expected_buffer); - } -} diff --git a/quickwit/quickwit-search/src/search_stream/root.rs b/quickwit/quickwit-search/src/search_stream/root.rs deleted file mode 100644 index 5f76a9d29cd..00000000000 --- a/quickwit/quickwit-search/src/search_stream/root.rs +++ /dev/null @@ -1,386 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::HashSet; - -use bytes::Bytes; -use futures::{StreamExt, TryStreamExt}; -use quickwit_common::uri::Uri; -use quickwit_config::build_doc_mapper; -use quickwit_doc_mapper::tag_pruning::extract_tags_from_query; -use quickwit_metastore::IndexMetadataResponseExt; -use quickwit_proto::metastore::{IndexMetadataRequest, MetastoreService, MetastoreServiceClient}; -use quickwit_proto::search::{LeafSearchStreamRequest, SearchRequest, SearchStreamRequest}; -use quickwit_query::query_ast::QueryAst; -use tokio_stream::StreamMap; -use tracing::*; - -use crate::cluster_client::ClusterClient; -use crate::root::{SearchJob, refine_start_end_timestamp_from_ast}; -use crate::{SearchError, list_relevant_splits}; - -/// Perform a distributed search stream. -#[instrument(skip(metastore, cluster_client))] -pub async fn root_search_stream( - mut search_stream_request: SearchStreamRequest, - mut metastore: MetastoreServiceClient, - cluster_client: ClusterClient, -) -> crate::Result>> { - // TODO: building a search request should not be necessary for listing splits. - // This needs some refactoring: relevant splits, metadata_map, jobs... - let index_metadata_request = - IndexMetadataRequest::for_index_id(search_stream_request.index_id.clone()); - let index_metadata = metastore - .index_metadata(index_metadata_request) - .await? - .deserialize_index_metadata()?; - let index_uid = index_metadata.index_uid.clone(); - let index_config = index_metadata.into_index_config(); - - let doc_mapper = build_doc_mapper(&index_config.doc_mapping, &index_config.search_settings) - .map_err(|err| { - SearchError::Internal(format!("failed to build doc mapper. cause: {err}")) - })?; - - let query_ast: QueryAst = serde_json::from_str(&search_stream_request.query_ast) - .map_err(|err| SearchError::InvalidQuery(err.to_string()))?; - let query_ast_resolved = query_ast.parse_user_query(doc_mapper.default_search_fields())?; - let tags_filter_ast = extract_tags_from_query(query_ast_resolved.clone()); - - if let Some(timestamp_field) = doc_mapper.timestamp_field_name() { - refine_start_end_timestamp_from_ast( - &query_ast_resolved, - timestamp_field, - &mut search_stream_request.start_timestamp, - &mut search_stream_request.end_timestamp, - ); - } - - // Validates the query by effectively building it against the current schema. - doc_mapper.query(doc_mapper.schema(), &query_ast_resolved, true)?; - search_stream_request.query_ast = serde_json::to_string(&query_ast_resolved)?; - - let search_request = SearchRequest::try_from(search_stream_request.clone())?; - let split_metadatas = list_relevant_splits( - vec![index_uid], - search_request.start_timestamp, - search_request.end_timestamp, - tags_filter_ast, - &mut metastore, - ) - .await?; - - let doc_mapper_str = serde_json::to_string(&doc_mapper).map_err(|err| { - SearchError::Internal(format!("failed to serialize doc mapper: cause {err}")) - })?; - - let index_uri: &Uri = &index_config.index_uri; - let leaf_search_jobs: Vec = split_metadatas.iter().map(SearchJob::from).collect(); - let assigned_leaf_search_jobs = cluster_client - .search_job_placer - .assign_jobs(leaf_search_jobs, &HashSet::default()) - .await?; - - let mut stream_map: StreamMap = StreamMap::new(); - for (leaf_ord, (client, client_jobs)) in assigned_leaf_search_jobs.enumerate() { - let leaf_request: LeafSearchStreamRequest = jobs_to_leaf_request( - &search_stream_request, - &doc_mapper_str, - index_uri.as_ref(), - client_jobs, - ); - let leaf_stream = cluster_client - .leaf_search_stream(leaf_request, client) - .await; - stream_map.insert(leaf_ord, leaf_stream); - } - Ok(stream_map - .map(|(_leaf_ord, result)| result) - .map_ok(|leaf_response| Bytes::from(leaf_response.data))) -} - -fn jobs_to_leaf_request( - request: &SearchStreamRequest, - doc_mapper_str: &str, - index_uri: &str, // TODO make Uri - jobs: Vec, -) -> LeafSearchStreamRequest { - LeafSearchStreamRequest { - request: Some(request.clone()), - split_offsets: jobs.into_iter().map(Into::into).collect(), - doc_mapper: doc_mapper_str.to_string(), - index_uri: index_uri.to_string(), - } -} - -#[cfg(test)] -mod tests { - - use quickwit_common::ServiceStream; - use quickwit_indexing::MockSplitBuilder; - use quickwit_metastore::{IndexMetadata, ListSplitsResponseExt}; - use quickwit_proto::metastore::{ - IndexMetadataResponse, ListSplitsResponse, MockMetastoreService, - }; - use quickwit_proto::search::OutputFormat; - use quickwit_query::query_ast::qast_json_helper; - use tokio_stream::wrappers::UnboundedReceiverStream; - - use super::*; - use crate::{MockSearchService, SearchJobPlacer, searcher_pool_for_test}; - - #[tokio::test] - async fn test_root_search_stream_single_split() -> anyhow::Result<()> { - let request = quickwit_proto::search::SearchStreamRequest { - index_id: "test-index".to_string(), - query_ast: qast_json_helper("test", &["body"]), - fast_field: "timestamp".to_string(), - output_format: OutputFormat::Csv as i32, - ..Default::default() - }; - let mut mock_metastore = MockMetastoreService::new(); - let index_metadata = IndexMetadata::for_test("test-index", "ram:///test-index"); - let index_uid = index_metadata.index_uid.clone(); - mock_metastore.expect_index_metadata().returning(move |_| { - Ok(IndexMetadataResponse::try_from_index_metadata(&index_metadata).unwrap()) - }); - mock_metastore.expect_list_splits().returning(move |_| { - let splits = vec![ - MockSplitBuilder::new("split1") - .with_index_uid(&index_uid) - .build(), - ]; - let splits = ListSplitsResponse::try_from_splits(splits).unwrap(); - Ok(ServiceStream::from(vec![Ok(splits)])) - }); - let mut mock_search_service = MockSearchService::new(); - let (result_sender, result_receiver) = tokio::sync::mpsc::unbounded_channel(); - result_sender.send(Ok(quickwit_proto::search::LeafSearchStreamResponse { - data: b"123".to_vec(), - split_id: "split_1".to_string(), - }))?; - result_sender.send(Ok(quickwit_proto::search::LeafSearchStreamResponse { - data: b"456".to_vec(), - split_id: "split_1".to_string(), - }))?; - mock_search_service.expect_leaf_search_stream().return_once( - |_leaf_search_req: quickwit_proto::search::LeafSearchStreamRequest| { - Ok(UnboundedReceiverStream::new(result_receiver)) - }, - ); - // The test will hang on indefinitely if we don't drop the receiver. - drop(result_sender); - - let searcher_pool = searcher_pool_for_test([("127.0.0.1:1001", mock_search_service)]); - let search_job_placer = SearchJobPlacer::new(searcher_pool); - let cluster_client = ClusterClient::new(search_job_placer.clone()); - let result: Vec = root_search_stream( - request, - MetastoreServiceClient::from_mock(mock_metastore), - cluster_client, - ) - .await? - .try_collect() - .await?; - assert_eq!(result.len(), 2); - assert_eq!(&result[0], &b"123"[..]); - assert_eq!(&result[1], &b"456"[..]); - Ok(()) - } - - #[tokio::test] - async fn test_root_search_stream_single_split_partitioned() -> anyhow::Result<()> { - let request = quickwit_proto::search::SearchStreamRequest { - index_id: "test-index".to_string(), - query_ast: qast_json_helper("test", &["body"]), - fast_field: "timestamp".to_string(), - output_format: OutputFormat::Csv as i32, - partition_by_field: Some("timestamp".to_string()), - ..Default::default() - }; - let mut mock_metastore = MockMetastoreService::new(); - let index_metadata = IndexMetadata::for_test("test-index", "ram:///test-index"); - let index_uid = index_metadata.index_uid.clone(); - mock_metastore.expect_index_metadata().returning(move |_| { - Ok(IndexMetadataResponse::try_from_index_metadata(&index_metadata).unwrap()) - }); - mock_metastore.expect_list_splits().returning(move |_| { - let splits = vec![ - MockSplitBuilder::new("split1") - .with_index_uid(&index_uid) - .build(), - ]; - let splits = ListSplitsResponse::try_from_splits(splits).unwrap(); - Ok(ServiceStream::from(vec![Ok(splits)])) - }); - let mut mock_search_service = MockSearchService::new(); - let (result_sender, result_receiver) = tokio::sync::mpsc::unbounded_channel(); - result_sender.send(Ok(quickwit_proto::search::LeafSearchStreamResponse { - data: b"123".to_vec(), - split_id: "1".to_string(), - }))?; - result_sender.send(Ok(quickwit_proto::search::LeafSearchStreamResponse { - data: b"456".to_vec(), - split_id: "2".to_string(), - }))?; - mock_search_service.expect_leaf_search_stream().return_once( - |_leaf_search_req: quickwit_proto::search::LeafSearchStreamRequest| { - Ok(UnboundedReceiverStream::new(result_receiver)) - }, - ); - // The test will hang on indefinitely if we don't drop the sender. - drop(result_sender); - - let searcher_pool = searcher_pool_for_test([("127.0.0.1:1001", mock_search_service)]); - let search_job_placer = SearchJobPlacer::new(searcher_pool); - let cluster_client = ClusterClient::new(search_job_placer.clone()); - let stream = root_search_stream( - request, - MetastoreServiceClient::from_mock(mock_metastore), - cluster_client, - ) - .await?; - let result: Vec<_> = stream.try_collect().await?; - assert_eq!(result.len(), 2); - assert_eq!(&result[0], &b"123"[..]); - assert_eq!(&result[1], &b"456"[..]); - Ok(()) - } - - #[tokio::test] - async fn test_root_search_stream_single_split_with_error() -> anyhow::Result<()> { - let request = quickwit_proto::search::SearchStreamRequest { - index_id: "test-index".to_string(), - query_ast: qast_json_helper("test", &["body"]), - fast_field: "timestamp".to_string(), - output_format: OutputFormat::Csv as i32, - ..Default::default() - }; - let mut mock_metastore = MockMetastoreService::new(); - let index_metadata = IndexMetadata::for_test("test-index", "ram:///test-index"); - let index_uid = index_metadata.index_uid.clone(); - mock_metastore.expect_index_metadata().returning(move |_| { - Ok(IndexMetadataResponse::try_from_index_metadata(&index_metadata).unwrap()) - }); - mock_metastore.expect_list_splits().returning(move |_| { - let splits = vec![ - MockSplitBuilder::new("split1") - .with_index_uid(&index_uid) - .build(), - MockSplitBuilder::new("split2") - .with_index_uid(&index_uid) - .build(), - ]; - let splits = ListSplitsResponse::try_from_splits(splits).unwrap(); - Ok(ServiceStream::from(vec![Ok(splits)])) - }); - let mut mock_search_service = MockSearchService::new(); - let (result_sender, result_receiver) = tokio::sync::mpsc::unbounded_channel(); - result_sender.send(Ok(quickwit_proto::search::LeafSearchStreamResponse { - data: b"123".to_vec(), - split_id: "split1".to_string(), - }))?; - result_sender.send(Err(SearchError::Internal("error".to_string())))?; - mock_search_service - .expect_leaf_search_stream() - .withf(|request| request.split_offsets.len() == 2) // First request. - .return_once( - |_leaf_search_req: quickwit_proto::search::LeafSearchStreamRequest| { - Ok(UnboundedReceiverStream::new(result_receiver)) - }, - ); - mock_search_service - .expect_leaf_search_stream() - .withf(|request| request.split_offsets.len() == 1) // Retry request on the failed split. - .return_once( - |_leaf_search_req: quickwit_proto::search::LeafSearchStreamRequest| { - Err(SearchError::Internal("error".to_string())) - }, - ); - // The test will hang on indefinitely if we don't drop the sender. - drop(result_sender); - - let searcher_pool = searcher_pool_for_test([("127.0.0.1:1001", mock_search_service)]); - let search_job_placer = SearchJobPlacer::new(searcher_pool); - let cluster_client = ClusterClient::new(search_job_placer.clone()); - let stream = root_search_stream( - request, - MetastoreServiceClient::from_mock(mock_metastore), - cluster_client, - ) - .await?; - let result: Result, SearchError> = stream.try_collect().await; - assert_eq!(result.is_err(), true); - assert_eq!(result.unwrap_err().to_string(), "internal error: `error`"); - Ok(()) - } - - #[tokio::test] - async fn test_root_search_stream_with_invalid_query() -> anyhow::Result<()> { - let mut mock_metastore = MockMetastoreService::new(); - let index_metadata = IndexMetadata::for_test("test-index", "ram:///test-index"); - let index_uid = index_metadata.index_uid.clone(); - mock_metastore.expect_index_metadata().returning(move |_| { - Ok(IndexMetadataResponse::try_from_index_metadata(&index_metadata).unwrap()) - }); - mock_metastore.expect_list_splits().returning(move |_| { - let splits = vec![ - MockSplitBuilder::new("split") - .with_index_uid(&index_uid) - .build(), - ]; - let splits = ListSplitsResponse::try_from_splits(splits).unwrap(); - Ok(ServiceStream::from(vec![Ok(splits)])) - }); - - let searcher_pool = searcher_pool_for_test([("127.0.0.1:1001", MockSearchService::new())]); - let search_job_placer = SearchJobPlacer::new(searcher_pool); - let metastore = MetastoreServiceClient::from_mock(mock_metastore); - assert!( - root_search_stream( - quickwit_proto::search::SearchStreamRequest { - index_id: "test-index".to_string(), - query_ast: qast_json_helper(r#"invalid_field:"test""#, &[]), - fast_field: "timestamp".to_string(), - output_format: OutputFormat::Csv as i32, - partition_by_field: Some("timestamp".to_string()), - ..Default::default() - }, - metastore.clone(), - ClusterClient::new(search_job_placer.clone()), - ) - .await - .is_err() - ); - - assert!( - root_search_stream( - quickwit_proto::search::SearchStreamRequest { - index_id: "test-index".to_string(), - query_ast: qast_json_helper("test", &["invalid_field"]), - fast_field: "timestamp".to_string(), - output_format: OutputFormat::Csv as i32, - partition_by_field: Some("timestamp".to_string()), - ..Default::default() - }, - metastore, - ClusterClient::new(search_job_placer.clone()), - ) - .await - .is_err() - ); - - Ok(()) - } -} diff --git a/quickwit/quickwit-search/src/service.rs b/quickwit/quickwit-search/src/service.rs index eaed0337d5f..86041fd62a5 100644 --- a/quickwit/quickwit-search/src/service.rs +++ b/quickwit/quickwit-search/src/service.rs @@ -12,13 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::pin::Pin; use std::str::FromStr; use std::sync::Arc; use std::time::{Duration, Instant}; use async_trait::async_trait; -use bytes::Bytes; use quickwit_common::uri::Uri; use quickwit_config::SearcherConfig; use quickwit_doc_mapper::DocMapper; @@ -26,20 +24,18 @@ use quickwit_proto::metastore::MetastoreServiceClient; use quickwit_proto::search::{ FetchDocsRequest, FetchDocsResponse, GetKvRequest, Hit, LeafListFieldsRequest, LeafListTermsRequest, LeafListTermsResponse, LeafSearchRequest, LeafSearchResponse, - LeafSearchStreamRequest, LeafSearchStreamResponse, ListFieldsRequest, ListFieldsResponse, - ListTermsRequest, ListTermsResponse, PutKvRequest, ReportSplitsRequest, ReportSplitsResponse, - ScrollRequest, SearchPlanResponse, SearchRequest, SearchResponse, SearchStreamRequest, - SnippetRequest, + ListFieldsRequest, ListFieldsResponse, ListTermsRequest, ListTermsResponse, PutKvRequest, + ReportSplitsRequest, ReportSplitsResponse, ScrollRequest, SearchPlanResponse, SearchRequest, + SearchResponse, SnippetRequest, }; use quickwit_storage::{ MemorySizedCache, QuickwitCache, SplitCache, StorageCache, StorageResolver, }; use tantivy::aggregation::AggregationLimitsGuard; -use tokio::sync::Semaphore; -use tokio_stream::wrappers::UnboundedReceiverStream; +use crate::invoker::LambdaLeafSearchInvoker; use crate::leaf::multi_index_leaf_search; -use crate::leaf_cache::LeafSearchCache; +use crate::leaf_cache::{LeafSearchCache, PredicateCacheImpl}; use crate::list_fields::{leaf_list_fields, root_list_fields}; use crate::list_fields_cache::ListFieldsCache; use crate::list_terms::{leaf_list_terms, root_list_terms}; @@ -47,7 +43,6 @@ use crate::metrics_trackers::LeafSearchMetricsFuture; use crate::root::fetch_docs_phase; use crate::scroll_context::{MiniKV, ScrollContext, ScrollKeyAndStartOffset}; use crate::search_permit_provider::SearchPermitProvider; -use crate::search_stream::{leaf_search_stream, root_search_stream}; use crate::{ClusterClient, SearchError, fetch_docs, root_search, search_plan}; #[derive(Clone)] @@ -89,18 +84,6 @@ pub trait SearchService: 'static + Send + Sync { /// This methods takes `PartialHit`s and returns `Hit`s. async fn fetch_docs(&self, request: FetchDocsRequest) -> crate::Result; - /// Performs a root search returning a receiver for streaming - async fn root_search_stream( - &self, - request: SearchStreamRequest, - ) -> crate::Result> + Send>>>; - - /// Performs a leaf search on a given set of splits and returns a stream. - async fn leaf_search_stream( - &self, - request: LeafSearchStreamRequest, - ) -> crate::Result>>; - /// Root search API. /// This RPC identifies the set of splits on which the query should run on, /// and dispatches the multiple calls to `LeafSearch`. @@ -170,7 +153,8 @@ impl SearchServiceImpl { } } -pub fn deserialize_doc_mapper(doc_mapper_str: &str) -> crate::Result> { +/// Deserializes a JSON-encoded doc mapper string into an `Arc`. +pub(crate) fn deserialize_doc_mapper(doc_mapper_str: &str) -> crate::Result> { let doc_mapper = serde_json::from_str::>(doc_mapper_str).map_err(|err| { SearchError::Internal(format!("failed to deserialize doc mapper: `{err}`")) })?; @@ -204,17 +188,18 @@ impl SearchService for SearchServiceImpl { .map(|req| req.split_offsets.len()) .sum::(); - LeafSearchMetricsFuture { + let tracked_future = LeafSearchMetricsFuture { tracked: multi_index_leaf_search( self.searcher_context.clone(), leaf_search_request, - &self.storage_resolver, + self.storage_resolver.clone(), ), start: Instant::now(), targeted_splits: num_splits, status: None, - } - .await + }; + let timeout = self.searcher_context.searcher_config.request_timeout(); + tokio::time::timeout(timeout, tracked_future).await? } async fn fetch_docs( @@ -239,40 +224,6 @@ impl SearchService for SearchServiceImpl { Ok(fetch_docs_response) } - async fn root_search_stream( - &self, - stream_request: SearchStreamRequest, - ) -> crate::Result> + Send>>> { - let data = root_search_stream( - stream_request, - self.metastore.clone(), - self.cluster_client.clone(), - ) - .await?; - Ok(Box::pin(data)) - } - - async fn leaf_search_stream( - &self, - leaf_stream_request: LeafSearchStreamRequest, - ) -> crate::Result>> { - let stream_request = leaf_stream_request - .request - .ok_or_else(|| SearchError::Internal("no search request".to_string()))?; - let index_uri = Uri::from_str(&leaf_stream_request.index_uri)?; - let storage = self.storage_resolver.resolve(&index_uri).await?; - let doc_mapper = deserialize_doc_mapper(&leaf_stream_request.doc_mapper)?; - let leaf_receiver = leaf_search_stream( - self.searcher_context.clone(), - stream_request, - storage, - leaf_stream_request.split_offsets, - doc_mapper, - ) - .await; - Ok(leaf_receiver) - } - async fn root_list_terms( &self, list_terms_request: ListTermsRequest, @@ -423,15 +374,15 @@ pub(crate) async fn scroll( partial_hits.last().cloned().unwrap_or_default(), ); - if let Some(scroll_ttl_secs) = scroll_request.scroll_ttl_secs { - if scroll_context_modified { - scroll_context.clear_cache_if_unneeded(); - let payload = scroll_context.serialize(); - let scroll_ttl = Duration::from_secs(scroll_ttl_secs as u64); - cluster_client - .put_kv(&scroll_key, &payload, scroll_ttl) - .await; - } + if let Some(scroll_ttl_secs) = scroll_request.scroll_ttl_secs + && scroll_context_modified + { + scroll_context.clear_cache_if_unneeded(); + let payload = scroll_context.serialize(); + let scroll_ttl = Duration::from_secs(scroll_ttl_secs as u64); + cluster_client + .put_kv(&scroll_key, &payload, scroll_ttl) + .await; } Ok(SearchResponse { @@ -457,23 +408,25 @@ pub struct SearcherContext { pub search_permit_provider: SearchPermitProvider, /// Split footer cache - now supports sharing via Arc. pub split_footer_cache: MemorySizedCache, - /// Counting semaphore to limit concurrent split stream requests. - pub split_stream_semaphore: Semaphore, - /// Recent sub-query cache. + /// Per-split and per-query cache. pub leaf_search_cache: LeafSearchCache, + /// Per-split and per-predicate cache. + pub predicate_cache: Arc, /// Search split cache. `None` if no split cache is configured. pub split_cache_opt: Option>, /// List fields cache. Caches the list fields response for a given split. pub list_fields_cache: ListFieldsCache, /// The aggregation limits are passed to limit the memory usage. + /// This object is shared across all request. pub aggregation_limit: AggregationLimitsGuard, + /// Optional Lambda invoker for offloading leaf search to serverless functions. + pub lambda_invoker: Option>, } impl std::fmt::Debug for SearcherContext { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { f.debug_struct("SearcherContext") .field("searcher_config", &self.searcher_config) - .field("split_stream_semaphore", &self.split_stream_semaphore) .finish() } } @@ -483,43 +436,59 @@ impl SearcherContext { #[cfg(test)] pub fn for_test() -> SearcherContext { let searcher_config = SearcherConfig::default(); - SearcherContext::new(searcher_config, None) + SearcherContext::new_without_invoker(searcher_config, None) + } + + /// Creates a new searcher context without a lambda invoker. + pub fn new_without_invoker( + searcher_config: SearcherConfig, + split_cache_opt: Option>, + ) -> Self { + Self::new( + searcher_config, + split_cache_opt, + None::>, + ) } /// Creates a new searcher context, given a searcher config, and an optional `SplitCache`. - pub fn new(searcher_config: SearcherConfig, split_cache_opt: Option>) -> Self { - let capacity_in_bytes = searcher_config.split_footer_cache_capacity.as_u64() as usize; - let global_split_footer_cache = MemorySizedCache::with_capacity_in_bytes( - capacity_in_bytes, + pub fn new( + searcher_config: SearcherConfig, + split_cache_opt: Option>, + lambda_invoker: Option, + ) -> Self { + let global_split_footer_cache = MemorySizedCache::from_config( + &searcher_config.split_footer_cache, &quickwit_storage::STORAGE_METRICS.split_footer_cache, ); let leaf_search_split_semaphore = SearchPermitProvider::new( searcher_config.max_num_concurrent_split_searches, searcher_config.warmup_memory_budget, ); - let split_stream_semaphore = - Semaphore::new(searcher_config.max_num_concurrent_split_streams); - let fast_field_cache_capacity = searcher_config.fast_field_cache_capacity.as_u64() as usize; - let storage_long_term_cache = Arc::new(QuickwitCache::new(fast_field_cache_capacity)); - let leaf_search_cache = - LeafSearchCache::new(searcher_config.partial_request_cache_capacity.as_u64() as usize); - let list_fields_cache = - ListFieldsCache::new(searcher_config.partial_request_cache_capacity.as_u64() as usize); + let storage_long_term_cache = + Arc::new(QuickwitCache::new(&searcher_config.fast_field_cache)); + let leaf_search_cache = LeafSearchCache::new(&searcher_config.partial_request_cache); + let predicate_cache = PredicateCacheImpl::new(&searcher_config.predicate_cache); + let list_fields_cache = ListFieldsCache::new(&searcher_config.partial_request_cache); let aggregation_limit = AggregationLimitsGuard::new( Some(searcher_config.aggregation_memory_limit.as_u64()), Some(searcher_config.aggregation_bucket_limit), ); + let lambda_invoker = + lambda_invoker.map(|invoker| Arc::new(invoker) as Arc); + Self { searcher_config, fast_fields_cache: storage_long_term_cache, + predicate_cache: predicate_cache.into(), search_permit_provider: leaf_search_split_semaphore, split_footer_cache: global_split_footer_cache, - split_stream_semaphore, leaf_search_cache, list_fields_cache, split_cache_opt, aggregation_limit, + lambda_invoker, } } diff --git a/quickwit/quickwit-search/src/tests.rs b/quickwit/quickwit-search/src/tests.rs index 83d5abad8a1..c8d851d06cb 100644 --- a/quickwit/quickwit-search/src/tests.rs +++ b/quickwit/quickwit-search/src/tests.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::cmp::Ordering; use std::collections::{BTreeMap, BTreeSet}; use assert_json_diff::{assert_json_eq, assert_json_include}; @@ -19,10 +20,9 @@ use quickwit_config::SearcherConfig; use quickwit_doc_mapper::DocMapper; use quickwit_doc_mapper::tag_pruning::extract_tags_from_query; use quickwit_indexing::TestSandbox; -use quickwit_opentelemetry::otlp::TraceId; use quickwit_proto::search::{ LeafListTermsResponse, ListTermsRequest, SearchRequest, SortByValue, SortField, SortOrder, - SortValue, + SortValue, TraceId, }; use quickwit_query::query_ast::{ QueryAst, qast_helper, qast_json_helper, query_ast_from_user_text, @@ -260,24 +260,6 @@ async fn test_slop_queries() { test_sandbox.assert_quit().await; } -// TODO remove me once `Iterator::is_sorted_by_key` is stabilized. -fn is_reverse_sorted>(mut it: I) -> bool -where E: Ord { - let mut previous_el = if let Some(first_el) = it.next() { - first_el - } else { - // The empty list is sorted! - return true; - }; - for next_el in it { - if next_el > previous_el { - return false; - } - previous_el = next_el; - } - true -} - #[tokio::test] async fn test_single_node_several_splits() -> anyhow::Result<()> { let index_id = "single-node-several-splits"; @@ -325,7 +307,7 @@ async fn test_single_node_several_splits() -> anyhow::Result<()> { .as_ref() .map(|partial_hit| (partial_hit.split_id.as_str(), partial_hit.doc_id as i32)) }); - assert!(is_reverse_sorted(hit_keys)); + assert!(hit_keys.is_sorted_by(|left, right| left.cmp(right) == Ordering::Greater)); assert!(single_node_result.elapsed_time_micros > 10); assert!(single_node_result.elapsed_time_micros < 1_000_000); test_sandbox.assert_quit().await; @@ -1046,10 +1028,10 @@ async fn test_search_util(test_sandbox: &TestSandbox, query: &str) -> Vec { max_hits: 100, ..Default::default() }); - let searcher_context: Arc = - Arc::new(SearcherContext::new(SearcherConfig::default(), None)); - - let agg_limits = searcher_context.get_aggregation_limits(); + let searcher_context: Arc = Arc::new(SearcherContext::new_without_invoker( + SearcherConfig::default(), + None, + )); let search_response = single_doc_mapping_leaf_search( searcher_context, @@ -1057,7 +1039,6 @@ async fn test_search_util(test_sandbox: &TestSandbox, query: &str) -> Vec { test_sandbox.storage(), splits_offsets, test_sandbox.doc_mapper(), - agg_limits, ) .await .unwrap(); @@ -1646,11 +1627,12 @@ async fn test_single_node_range_queries() -> anyhow::Result<()> { Ok(()) } +#[allow(deprecated)] fn collect_str_terms(response: LeafListTermsResponse) -> Vec { response .terms .into_iter() - .map(|term| Term::wrap(term).value().as_str().unwrap().to_string()) + .map(|term| Term::wrap(&term).value().as_str().unwrap().to_string()) .collect() } @@ -1686,7 +1668,10 @@ async fn test_single_node_list_terms() -> anyhow::Result<()> { .into_iter() .map(|split| extract_split_and_footer_offsets(&split.split_metadata)) .collect(); - let searcher_context = Arc::new(SearcherContext::new(SearcherConfig::default(), None)); + let searcher_context = Arc::new(SearcherContext::new_without_invoker( + SearcherConfig::default(), + None, + )); { let request = ListTermsRequest { diff --git a/quickwit/quickwit-search/src/top_k_collector.rs b/quickwit/quickwit-search/src/top_k_collector.rs index f36eb6370e2..5d63d9598e2 100644 --- a/quickwit/quickwit-search/src/top_k_collector.rs +++ b/quickwit/quickwit-search/src/top_k_collector.rs @@ -350,7 +350,8 @@ impl Clone for TopKComputer { } impl TopKComputer -where D: Ord + Copy + Debug + MinValue +where + D: Ord + Copy + Debug + MinValue, { /// Create a new `TopKComputer`. pub fn new(top_n: usize) -> Self { diff --git a/quickwit/quickwit-serve/Cargo.toml b/quickwit/quickwit-serve/Cargo.toml index 0d31bfd3db1..2721aa719f3 100644 --- a/quickwit/quickwit-serve/Cargo.toml +++ b/quickwit/quickwit-serve/Cargo.toml @@ -23,15 +23,13 @@ futures-util = { workspace = true } glob = { workspace = true } hex = { workspace = true } http = { workspace = true } +http-body = { workspace = true } http-serde = { workspace = true } humantime = { workspace = true } -hyper = { workspace = true } -hyper-rustls = { workspace = true } hyper-util = {workspace = true} itertools = { workspace = true } mime_guess = { workspace = true } once_cell = { workspace = true } -opentelemetry = { workspace = true } percent-encoding = { workspace = true } pprof = { workspace = true, optional = true } prost = { workspace = true } @@ -49,13 +47,14 @@ tokio = { workspace = true } tokio-rustls = { workspace = true } tokio-stream = { workspace = true } tokio-util = { workspace = true } +tonic = { workspace = true } tonic-health = { workspace = true } tonic-reflection = { workspace = true } tower = { workspace = true, features = ["limit"] } tower-http = { workspace = true } tracing = { workspace = true } utoipa = { workspace = true } -warp = { workspace = true } +warp = { workspace = true, features = ["server"] } zstd = { workspace = true } quickwit-actors = { workspace = true } @@ -74,6 +73,7 @@ quickwit-opentelemetry = { workspace = true } quickwit-proto = { workspace = true } quickwit-query = { workspace = true } quickwit-search = { workspace = true } +quickwit-lambda-client = { workspace = true, optional = true } quickwit-storage = { workspace = true } quickwit-telemetry = { workspace = true } @@ -116,3 +116,6 @@ sqs-for-tests = [ "quickwit-indexing/sqs", "quickwit-indexing/sqs-test-helpers" ] +lambda = [ + "quickwit-lambda-client" +] diff --git a/quickwit/quickwit-serve/src/delete_task_api/handler.rs b/quickwit/quickwit-serve/src/delete_task_api/handler.rs index f1e305bd004..33c295df008 100644 --- a/quickwit/quickwit-serve/src/delete_task_api/handler.rs +++ b/quickwit/quickwit-serve/src/delete_task_api/handler.rs @@ -173,7 +173,7 @@ pub async fn post_delete_request( let query_ast: QueryAst = serde_json::from_str(&delete_search_request.query_ast) .map_err(|err| JanitorError::InvalidDeleteQuery(err.to_string()))?; doc_mapper - .query(doc_mapper.schema(), &query_ast, true) + .query(doc_mapper.schema(), query_ast, true, None) .map_err(|error| JanitorError::InvalidDeleteQuery(error.to_string()))?; let delete_task = metastore.create_delete_task(delete_query).await?; Ok(delete_task) diff --git a/quickwit/quickwit-serve/src/developer_api/heap_prof.rs b/quickwit/quickwit-serve/src/developer_api/heap_prof.rs index a1d3cd3d224..a6b833bc4ac 100644 --- a/quickwit/quickwit-serve/src/developer_api/heap_prof.rs +++ b/quickwit/quickwit-serve/src/developer_api/heap_prof.rs @@ -35,7 +35,7 @@ pub fn heap_prof_handlers() async fn start_profiler_handler( params: ProfilerQueryParams, - ) -> Result, warp::Rejection> { + ) -> Result { start_profiling(params.min_alloc_size, params.backtrace_every); let response = warp::reply::with_status("Heap profiling started", warp::http::StatusCode::OK) @@ -43,8 +43,7 @@ pub fn heap_prof_handlers() Ok(response) } - async fn stop_profiler_handler() - -> Result, warp::Rejection> { + async fn stop_profiler_handler() -> Result { stop_profiling(); let response = warp::reply::with_status("Heap profiling stopped", warp::http::StatusCode::OK) diff --git a/quickwit/quickwit-serve/src/developer_api/pprof.rs b/quickwit/quickwit-serve/src/developer_api/pprof.rs index b4209453822..582ec066976 100644 --- a/quickwit/quickwit-serve/src/developer_api/pprof.rs +++ b/quickwit/quickwit-serve/src/developer_api/pprof.rs @@ -127,16 +127,15 @@ pub fn pprof_handlers() -> impl Filter>) { let handle = quickwit_common::thread_pool::run_cpu_intensive(move || { let mut state = profiler_state.lock().unwrap(); - if let Some(profiler) = state.profiler_guard.take() { - if let Ok(report) = profiler + if let Some(profiler) = state.profiler_guard.take() + && let Ok(report) = profiler .report() .frames_post_processor(frames_post_processor) .build() - { - let mut buffer = Vec::new(); - if report.flamegraph(&mut buffer).is_ok() { - state.flamegraph_data = Some(buffer); - } + { + let mut buffer = Vec::new(); + if report.flamegraph(&mut buffer).is_ok() { + state.flamegraph_data = Some(buffer); } } }); diff --git a/quickwit/quickwit-serve/src/developer_api/server.rs b/quickwit/quickwit-serve/src/developer_api/server.rs index fd43e89b978..680953f0509 100644 --- a/quickwit/quickwit-serve/src/developer_api/server.rs +++ b/quickwit/quickwit-serve/src/developer_api/server.rs @@ -76,10 +76,14 @@ impl DeveloperService for DeveloperApiServer { let cluster_snapshot = self.cluster.snapshot().await; + // We must redact sensitive information such as credentials. + let mut node_config = (*self.node_config).clone(); + node_config.redact(); + let mut debug_info = json!({ "build_info": BuildInfo::get(), "runtime_info": RuntimeInfo::get(), - "node_config": self.node_config, + "node_config": node_config, "cluster_membership_info": json!({ "ready_nodes": cluster_snapshot.ready_nodes, "live_nodes": cluster_snapshot.live_nodes, @@ -87,23 +91,23 @@ impl DeveloperService for DeveloperApiServer { "chitchat_state": cluster_snapshot.chitchat_state_snapshot.node_states, }) }); - if let Some(control_plane_mailbox) = &self.control_plane_mailbox_opt { - if roles.is_empty() || roles.contains(&QuickwitService::ControlPlane) { - debug_info["control_plane"] = match control_plane_mailbox.ask(GetDebugInfo).await { - Ok(debug_info) => debug_info, - Err(error) => { - json!({"error": error.to_string()}) - } - }; - } + if let Some(control_plane_mailbox) = &self.control_plane_mailbox_opt + && (roles.is_empty() || roles.contains(&QuickwitService::ControlPlane)) + { + debug_info["control_plane"] = match control_plane_mailbox.ask(GetDebugInfo).await { + Ok(debug_info) => debug_info, + Err(error) => { + json!({"error": error.to_string()}) + } + }; } if let Some(ingest_router) = &self.ingest_router_opt { debug_info["ingest_router"] = ingest_router.debug_info().await; } - if let Some(ingester) = &self.ingester_opt { - if roles.is_empty() || roles.contains(&QuickwitService::Indexer) { - debug_info["ingester"] = ingester.debug_info().await; - } + if let Some(ingester) = &self.ingester_opt + && (roles.is_empty() || roles.contains(&QuickwitService::Indexer)) + { + debug_info["ingester"] = ingester.debug_info().await; }; let debug_info_json = serde_json::to_vec(&debug_info).map_err(|error| { let message = format!("failed to JSON serialize debug info: {error}"); @@ -137,7 +141,10 @@ mod tests { .await .unwrap(); - let node_config = Arc::new(NodeConfig::for_test()); + let mut node_config = NodeConfig::for_test(); + node_config.metastore_uri = + quickwit_common::uri::Uri::for_test("postgresql://username:password@db"); + let node_config = Arc::new(node_config); let developer_api_server = DeveloperApiServer { node_config, @@ -155,6 +162,11 @@ mod tests { assert!(debug_info["node_config"].is_object()); assert!(debug_info["cluster_membership_info"].is_object()); + assert_eq!( + debug_info["node_config"]["metastore_uri"], + "postgresql://username:***redacted***@db" + ); + // TODO: Test control plane and ingester debug info. } } diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/filter.rs b/quickwit/quickwit-serve/src/elasticsearch_api/filter.rs index 3104ded1b68..5f8ddd69f9f 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/filter.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/filter.rs @@ -50,7 +50,7 @@ pub(crate) fn elasticsearch_filter() -> impl Filter + Clone { warp::path!("_elastic" / "_search") .and(warp::get().or(warp::post()).unify()) - .and(serde_qs::warp::query(serde_qs::Config::default())) + .and(warp::query()) } #[utoipa::path( @@ -74,7 +74,7 @@ pub(crate) fn elastic_bulk_filter( content_length_limit.as_u64(), )) .and(get_body_bytes()) - .and(serde_qs::warp::query(serde_qs::Config::default())) + .and(warp::query()) } #[utoipa::path( @@ -98,9 +98,7 @@ pub(crate) fn elastic_index_bulk_filter( content_length_limit.as_u64(), )) .and(get_body_bytes()) - .and(serde_qs::warp::query::( - serde_qs::Config::default(), - )) + .and(warp::query::()) } /// Like the warp json filter, but accepts an empty body and interprets it as `T::default`. @@ -138,7 +136,7 @@ pub(crate) fn elastic_index_field_capabilities_filter() -> impl Filter< warp::path!("_elastic" / String / "_field_caps") .and_then(extract_index_id_patterns) .and(warp::get().or(warp::post()).unify()) - .and(serde_qs::warp::query(serde_qs::Config::default())) + .and(warp::query()) .and(json_or_empty()) } @@ -154,7 +152,7 @@ pub(crate) fn elastic_field_capabilities_filter() -> impl Filter< warp::path!("_elastic" / "_field_caps") .and_then(extract_index_id_patterns_default) .and(warp::get().or(warp::post()).unify()) - .and(serde_qs::warp::query(serde_qs::Config::default())) + .and(warp::query()) .and(json_or_empty()) } @@ -173,7 +171,7 @@ pub(crate) fn elastic_index_count_filter() warp::path!("_elastic" / String / "_count") .and_then(extract_index_id_patterns) .and(warp::get().or(warp::post()).unify()) - .and(serde_qs::warp::query(serde_qs::Config::default())) + .and(warp::query()) .and(json_or_empty()) } @@ -183,7 +181,7 @@ pub(crate) fn elastic_delete_index_filter() warp::path!("_elastic" / String) .and(warp::delete()) .and_then(extract_index_id_patterns) - .and(serde_qs::warp::query(serde_qs::Config::default())) + .and(warp::query()) } // No support for any query parameters for now. @@ -212,7 +210,7 @@ pub(crate) fn elastic_index_cat_indices_filter() warp::path!("_elastic" / "_cat" / "indices" / String) .and_then(extract_index_id_patterns) .and(warp::get()) - .and(serde_qs::warp::query(serde_qs::Config::default())) + .and(warp::query()) } #[utoipa::path(get, tag = "Search", path = "/_cat/indices")] @@ -220,7 +218,7 @@ pub(crate) fn elastic_cat_indices_filter() -> impl Filter + Clone { warp::path!("_elastic" / "_cat" / "indices") .and(warp::get()) - .and(serde_qs::warp::query(serde_qs::Config::default())) + .and(warp::query()) } #[utoipa::path(get, tag = "Search", path = "/{index}/_search")] @@ -229,7 +227,7 @@ pub(crate) fn elastic_index_search_filter() warp::path!("_elastic" / String / "_search") .and_then(extract_index_id_patterns) .and(warp::get().or(warp::post()).unify()) - .and(serde_qs::warp::query(serde_qs::Config::default())) + .and(warp::query()) .and(json_or_empty()) } @@ -240,7 +238,7 @@ pub(crate) fn elastic_multi_search_filter() .and(warp::body::content_length_limit(BODY_LENGTH_LIMIT.as_u64())) .and(warp::body::bytes()) .and(warp::post()) - .and(serde_qs::warp::query(serde_qs::Config::default())) + .and(warp::query()) } fn merge_scroll_body_params( @@ -253,13 +251,22 @@ fn merge_scroll_body_params( } } +pub(crate) fn elastic_nodes_filter() -> impl Filter + Clone { + warp::path!("_elastic" / "_nodes" / "http").and(warp::get()) +} + +pub(crate) fn elastic_search_shards_filter() +-> impl Filter + Clone { + warp::path!("_elastic" / String / "_search_shards").and(warp::get()) +} + #[utoipa::path(post, tag = "Search", path = "/_search/scroll")] pub(crate) fn elastic_scroll_filter() -> impl Filter + Clone { warp::path!("_elastic" / "_search" / "scroll") .and(warp::body::content_length_limit(BODY_LENGTH_LIMIT.as_u64())) .and(warp::get().or(warp::post()).unify()) - .and(serde_qs::warp::query(serde_qs::Config::default())) + .and(warp::query()) .and(json_or_empty()) .map( |scroll_query_params: ScrollQueryParams, scroll_body: ScrollQueryParams| { @@ -267,3 +274,20 @@ pub(crate) fn elastic_scroll_filter() }, ) } + +pub(crate) fn elastic_delete_scroll_filter() -> impl Filter + Clone +{ + warp::path!("_elastic" / "_search" / "scroll").and(warp::delete()) +} + +pub(crate) fn elastic_aliases_filter() -> impl Filter + Clone { + warp::path!("_elastic" / "_aliases").and(warp::get()) +} + +pub(crate) fn elastic_index_mapping_filter() +-> impl Filter + Clone { + warp::path!("_elastic" / String / "_mapping") + .or(warp::path!("_elastic" / String / "_mappings")) + .unify() + .and(warp::get()) +} diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/mod.rs b/quickwit/quickwit-serve/src/elasticsearch_api/mod.rs index dd189c834b6..683f152ba6e 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/mod.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/mod.rs @@ -29,19 +29,25 @@ use quickwit_ingest::IngestServiceClient; use quickwit_proto::ingest::router::IngestRouterServiceClient; use quickwit_proto::metastore::MetastoreServiceClient; use quickwit_search::SearchService; -use rest_handler::es_compat_cluster_health_handler; pub use rest_handler::{ es_compat_cat_indices_handler, es_compat_cluster_info_handler, es_compat_delete_index_handler, - es_compat_index_cat_indices_handler, es_compat_index_count_handler, - es_compat_index_field_capabilities_handler, es_compat_index_multi_search_handler, - es_compat_index_search_handler, es_compat_index_stats_handler, es_compat_resolve_index_handler, - es_compat_scroll_handler, es_compat_search_handler, es_compat_stats_handler, + es_compat_delete_scroll_handler, es_compat_index_cat_indices_handler, + es_compat_index_count_handler, es_compat_index_field_capabilities_handler, + es_compat_index_multi_search_handler, es_compat_index_search_handler, + es_compat_index_stats_handler, es_compat_resolve_index_handler, es_compat_scroll_handler, + es_compat_search_handler, es_compat_stats_handler, +}; +use rest_handler::{ + es_compat_cluster_health_handler, es_compat_nodes_handler, es_compat_search_shards_handler, }; use serde::{Deserialize, Serialize}; use warp::hyper::StatusCode; use warp::{Filter, Rejection}; use crate::elasticsearch_api::model::ElasticsearchError; +use crate::elasticsearch_api::rest_handler::{ + es_compat_aliases_handler, es_compat_index_mapping_handler, +}; use crate::rest::recover_fn; use crate::rest_api_response::RestApiResponse; use crate::{BodyFormat, BuildInfo}; @@ -63,7 +69,8 @@ pub fn elastic_api_handlers( enable_ingest_v2: bool, ) -> impl Filter + Clone { let ingest_content_length_limit = node_config.ingest_api_config.content_length_limit; - es_compat_cluster_info_handler(node_config, BuildInfo::get()) + es_compat_cluster_info_handler(node_config.clone(), BuildInfo::get()) + .or(es_compat_nodes_handler(node_config.clone())) .or(es_compat_search_handler(search_service.clone())) .or(es_compat_bulk_handler( ingest_service.clone(), @@ -83,6 +90,7 @@ pub fn elastic_api_handlers( .or(es_compat_index_search_handler(search_service.clone())) .or(es_compat_index_count_handler(search_service.clone())) .or(es_compat_scroll_handler(search_service.clone())) + .or(es_compat_delete_scroll_handler()) .or(es_compat_index_multi_search_handler(search_service.clone())) .or(es_compat_index_field_capabilities_handler( search_service.clone(), @@ -95,7 +103,17 @@ pub fn elastic_api_handlers( .or(es_compat_index_cat_indices_handler(metastore.clone())) .or(es_compat_cat_indices_handler(metastore.clone())) .or(es_compat_resolve_index_handler(metastore.clone())) + .or(es_compat_aliases_handler()) + .or(es_compat_index_mapping_handler( + metastore.clone(), + search_service.clone(), + )) + .or(es_compat_search_shards_handler(node_config)) .recover(recover_fn) + .with(warp::reply::with::header( + "X-Elastic-Product", + "Elasticsearch", + )) .boxed() // Register newly created handlers here. } @@ -221,7 +239,10 @@ mod tests { .reply(&es_search_api_handler) .await; assert_eq!(resp.status(), 200); - assert!(resp.headers().get("x-elastic-product").is_none(),); + assert_eq!( + resp.headers().get("x-elastic-product").unwrap(), + "Elasticsearch" + ); let string_body = String::from_utf8(resp.body().to_vec()).unwrap(); let es_msearch_response: serde_json::Value = serde_json::from_str(&string_body).unwrap(); let responses = es_msearch_response @@ -518,9 +539,13 @@ mod tests { "cluster_name" : config.cluster_id, "version" : { "distribution" : "quickwit", - "number" : build_info.version, + "number" : "7.17.0", "build_hash" : build_info.commit_hash, "build_date" : build_info.build_date, + "build_snapshot" : false, + "lucene_version" : "8.11.1", + "minimum_wire_compatibility_version" : "6.8.0", + "minimum_index_compatibility_version" : "6.0.0-beta1", } }); assert_json_include!(actual: resp_json, expected: expected_response_json); diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/model/cat_indices.rs b/quickwit/quickwit-serve/src/elasticsearch_api/model/cat_indices.rs index 37c56e73d47..22e451624b6 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/model/cat_indices.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/model/cat_indices.rs @@ -83,8 +83,13 @@ impl CatIndexQueryParams { if self.v.is_some() { return Err(unsupported_parameter_error("v")); } - if self.s.is_some() { - return Err(unsupported_parameter_error("s")); + if let Some(sort_by) = &self.s { + if sort_by.len() > 1 { + return Err(unsupported_parameter_error("s")); + } + if sort_by[0] != "index" && sort_by[0] != "index:asc" { + return Err(unsupported_parameter_error("s")); + } } Ok(()) } @@ -194,12 +199,16 @@ impl From for ElasticsearchResolveIndexEntryResponse { } fn serialize_u64_as_string(value: &u64, serializer: S) -> Result -where S: Serializer { +where + S: Serializer, +{ serializer.serialize_str(&value.to_string()) } fn ser_es_format(bytes: &u64, serializer: S) -> Result -where S: Serializer { +where + S: Serializer, +{ serializer.serialize_str(&format_byte_size(*bytes)) } @@ -305,4 +314,42 @@ mod tests { // Add more test cases as needed } + + #[test] + fn test_cat_index_query_params_validate_s_parameter() { + let params = CatIndexQueryParams { + format: Some("json".to_string()), + s: Some(vec!["index:asc".to_string()]), + ..Default::default() + }; + assert!(params.validate().is_ok()); + + let params = CatIndexQueryParams { + format: Some("json".to_string()), + s: Some(vec!["index".to_string()]), + ..Default::default() + }; + assert!(params.validate().is_ok()); + + let params = CatIndexQueryParams { + format: Some("json".to_string()), + s: Some(vec!["index:desc".to_string()]), + ..Default::default() + }; + assert!(params.validate().is_err()); + + let params = CatIndexQueryParams { + format: Some("json".to_string()), + s: Some(vec!["index:asc".to_string(), "docs.count".to_string()]), + ..Default::default() + }; + assert!(params.validate().is_err()); + + let params = CatIndexQueryParams { + format: Some("json".to_string()), + s: None, + ..Default::default() + }; + assert!(params.validate().is_ok()); + } } diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/model/error.rs b/quickwit/quickwit-serve/src/elasticsearch_api/model/error.rs index 713a3d8e5ce..c11807fb4fb 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/model/error.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/model/error.rs @@ -22,8 +22,6 @@ use quickwit_search::SearchError; use serde::{Deserialize, Serialize}; use warp::hyper::StatusCode; -use crate::convert_status_code_to_legacy_http; - #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ElasticsearchError { #[serde(with = "http_serde::status_code")] @@ -71,7 +69,7 @@ impl From for ElasticsearchError { additional_details: Default::default(), }; ElasticsearchError { - status: crate::convert_status_code_to_legacy_http(status), + status, error: reason, } } @@ -79,9 +77,7 @@ impl From for ElasticsearchError { impl From for ElasticsearchError { fn from(ingest_service_error: IngestServiceError) -> Self { - let status = crate::convert_status_code_to_legacy_http( - ingest_service_error.error_code().http_status_code(), - ); + let status = ingest_service_error.error_code().http_status_code(); let reason = ErrorCause { reason: Some(ingest_service_error.to_string()), @@ -113,7 +109,7 @@ impl From for ElasticsearchError { additional_details: Default::default(), }; ElasticsearchError { - status: crate::convert_status_code_to_legacy_http(status), + status, error: reason, } } @@ -133,7 +129,7 @@ impl From for ElasticsearchError { additional_details: Default::default(), }; ElasticsearchError { - status: convert_status_code_to_legacy_http(status), + status, error: reason, } } diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/model/field_capability.rs b/quickwit/quickwit-serve/src/elasticsearch_api/model/field_capability.rs index 896e79d5a6b..9aefdc83762 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/model/field_capability.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/model/field_capability.rs @@ -15,7 +15,10 @@ use std::collections::HashMap; use quickwit_proto::search::{ListFieldType, ListFieldsEntryResponse, ListFieldsResponse}; +use quickwit_query::ElasticQueryDsl; +use quickwit_query::query_ast::QueryAst; use serde::{Deserialize, Serialize}; +use warp::hyper::StatusCode; use super::ElasticsearchError; use super::search_query_params::*; @@ -120,12 +123,6 @@ impl FieldCapabilityEntryResponse { } } -#[derive(Serialize, Deserialize, Debug)] -struct FieldCapabilityEntry { - searchable: bool, - aggregatable: bool, -} - pub fn convert_to_es_field_capabilities_response( resp: ListFieldsResponse, ) -> FieldCapabilityResponse { @@ -179,16 +176,227 @@ pub fn convert_to_es_field_capabilities_response( FieldCapabilityResponse { indices, fields } } +/// Parses an Elasticsearch index_filter JSON value into a Quickwit QueryAst. +/// +/// Returns `Ok(None)` if the index_filter is null. +/// Returns `Ok(Some(QueryAst))` if the index_filter is valid. +/// Returns `Err` if the index_filter is invalid or cannot be converted (including empty object). +#[allow(clippy::result_large_err)] +pub fn parse_index_filter_to_query_ast( + index_filter: serde_json::Value, +) -> Result, ElasticsearchError> { + if index_filter.is_null() { + return Ok(None); + } + + // Parse ES Query DSL to internal QueryAst + let elastic_query_dsl: ElasticQueryDsl = + serde_json::from_value(index_filter).map_err(|err| { + ElasticsearchError::new( + StatusCode::BAD_REQUEST, + format!("Invalid index_filter: {err}"), + None, + ) + })?; + + let query_ast: QueryAst = elastic_query_dsl.try_into().map_err(|err: anyhow::Error| { + ElasticsearchError::new( + StatusCode::BAD_REQUEST, + format!("Failed to convert index_filter: {err}"), + None, + ) + })?; + + Ok(Some(query_ast)) +} + #[allow(clippy::result_large_err)] pub fn build_list_field_request_for_es_api( index_id_patterns: Vec, search_params: FieldCapabilityQueryParams, - _search_body: FieldCapabilityRequestBody, + search_body: FieldCapabilityRequestBody, ) -> Result { + let query_ast = parse_index_filter_to_query_ast(search_body.index_filter)?; + let query_ast_json = query_ast + .map(|ast| serde_json::to_string(&ast).expect("QueryAst should be JSON serializable")); + Ok(quickwit_proto::search::ListFieldsRequest { index_id_patterns, fields: search_params.fields.unwrap_or_default(), start_timestamp: search_params.start_timestamp, end_timestamp: search_params.end_timestamp, + query_ast: query_ast_json, }) } + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn test_build_list_field_request_empty_index_filter() { + let result = build_list_field_request_for_es_api( + vec!["test_index".to_string()], + FieldCapabilityQueryParams::default(), + FieldCapabilityRequestBody::default(), + ) + .unwrap(); + + assert_eq!(result.index_id_patterns, vec!["test_index".to_string()]); + assert!(result.query_ast.is_none()); + } + + #[test] + fn test_build_list_field_request_with_term_index_filter() { + let search_body = FieldCapabilityRequestBody { + index_filter: json!({ + "term": { + "status": "active" + } + }), + runtime_mappings: serde_json::Value::Null, + }; + + let result = build_list_field_request_for_es_api( + vec!["test_index".to_string()], + FieldCapabilityQueryParams::default(), + search_body, + ) + .unwrap(); + + assert_eq!(result.index_id_patterns, vec!["test_index".to_string()]); + assert!(result.query_ast.is_some()); + + // Verify the query_ast is valid JSON + let query_ast: serde_json::Value = + serde_json::from_str(&result.query_ast.unwrap()).unwrap(); + assert!(query_ast.is_object()); + } + + #[test] + fn test_build_list_field_request_with_bool_index_filter() { + let search_body = FieldCapabilityRequestBody { + index_filter: json!({ + "bool": { + "must": [ + { "term": { "status": "active" } } + ], + "filter": [ + { "range": { "age": { "gte": 18 } } } + ] + } + }), + runtime_mappings: serde_json::Value::Null, + }; + + let result = build_list_field_request_for_es_api( + vec!["test_index".to_string()], + FieldCapabilityQueryParams::default(), + search_body, + ) + .unwrap(); + + assert!(result.query_ast.is_some()); + } + + #[test] + fn test_build_list_field_request_with_invalid_index_filter() { + let search_body = FieldCapabilityRequestBody { + index_filter: json!({ + "invalid_query_type": { + "field": "value" + } + }), + runtime_mappings: serde_json::Value::Null, + }; + + let result = build_list_field_request_for_es_api( + vec!["test_index".to_string()], + FieldCapabilityQueryParams::default(), + search_body, + ); + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert_eq!(err.status, StatusCode::BAD_REQUEST); + } + + #[test] + fn test_build_list_field_request_with_null_index_filter() { + let search_body = FieldCapabilityRequestBody { + index_filter: serde_json::Value::Null, + runtime_mappings: serde_json::Value::Null, + }; + + let result = build_list_field_request_for_es_api( + vec!["test_index".to_string()], + FieldCapabilityQueryParams::default(), + search_body, + ) + .unwrap(); + + assert!(result.query_ast.is_none()); + } + + #[test] + fn test_build_list_field_request_preserves_other_params() { + let search_params = FieldCapabilityQueryParams { + fields: Some(vec!["field1".to_string(), "field2".to_string()]), + start_timestamp: Some(1000), + end_timestamp: Some(2000), + ..Default::default() + }; + + let search_body = FieldCapabilityRequestBody { + index_filter: json!({ "match_all": {} }), + runtime_mappings: serde_json::Value::Null, + }; + + let result = build_list_field_request_for_es_api( + vec!["test_index".to_string()], + search_params, + search_body, + ) + .unwrap(); + + assert_eq!( + result.fields, + vec!["field1".to_string(), "field2".to_string()] + ); + assert_eq!(result.start_timestamp, Some(1000)); + assert_eq!(result.end_timestamp, Some(2000)); + assert!(result.query_ast.is_some()); + } + + #[test] + fn test_parse_index_filter_to_query_ast_null() { + let result = parse_index_filter_to_query_ast(serde_json::Value::Null).unwrap(); + assert!(result.is_none()); + } + + #[test] + fn test_parse_index_filter_to_query_ast_empty_object() { + // Empty object {} should return error to match ES behavior + let result = parse_index_filter_to_query_ast(json!({})); + assert!(result.is_err()); + } + + #[test] + fn test_parse_index_filter_to_query_ast_valid_term() { + let result = parse_index_filter_to_query_ast(json!({ + "term": { "status": "active" } + })) + .unwrap(); + assert!(result.is_some()); + } + + #[test] + fn test_parse_index_filter_to_query_ast_invalid() { + let result = parse_index_filter_to_query_ast(json!({ + "invalid_query_type": { "field": "value" } + })); + assert!(result.is_err()); + } +} diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/model/mappings.rs b/quickwit/quickwit-serve/src/elasticsearch_api/model/mappings.rs new file mode 100644 index 00000000000..c639ed3daa9 --- /dev/null +++ b/quickwit/quickwit-serve/src/elasticsearch_api/model/mappings.rs @@ -0,0 +1,309 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; + +use quickwit_doc_mapper::{FieldMappingEntry, FieldMappingType}; +use quickwit_metastore::IndexMetadata; +use quickwit_proto::search::{ListFieldType, ListFieldsResponse}; +use serde::ser::SerializeMap; +use serde::{Serialize, Serializer}; + +/// Top-level response for `GET /{index}/_mapping(s)`. +/// +/// Serializes as `{ "": { "mappings": { "properties": { ... } } } }`. +pub(crate) struct ElasticsearchMappingsResponse { + indices: HashMap, +} + +impl Serialize for ElasticsearchMappingsResponse { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(Some(self.indices.len()))?; + for (index_id, mappings) in &self.indices { + map.serialize_entry(index_id, mappings)?; + } + map.end() + } +} + +#[derive(Debug, Serialize)] +struct IndexMappings { + mappings: MappingProperties, +} + +#[derive(Debug, Serialize)] +struct MappingProperties { + properties: HashMap, +} + +#[derive(Debug, Serialize)] +#[serde(untagged)] +enum FieldMapping { + Leaf { + #[serde(rename = "type")] + typ: &'static str, + }, + Object { + #[serde(rename = "type")] + typ: &'static str, + properties: HashMap, + }, +} + +impl ElasticsearchMappingsResponse { + pub fn from_doc_mapping( + indexes_metadata: Vec, + list_fields_response: Option<&ListFieldsResponse>, + ) -> Self { + let indices = indexes_metadata + .into_iter() + .map(|index_metadata| { + let field_mappings = &index_metadata.index_config.doc_mapping.field_mappings; + let mut properties = build_properties(field_mappings); + if let Some(list_fields) = list_fields_response { + merge_dynamic_fields(&mut properties, list_fields); + } + let index_id = index_metadata.index_id().to_string(); + ( + index_id, + IndexMappings { + mappings: MappingProperties { properties }, + }, + ) + }) + .collect(); + Self { indices } + } +} + +fn build_properties(field_mappings: &[FieldMappingEntry]) -> HashMap { + let mut properties = HashMap::with_capacity(field_mappings.len()); + for entry in field_mappings { + if let Some(field_mapping) = field_mapping_from_entry(entry) { + properties.insert(entry.name.clone(), field_mapping); + } + } + properties +} + +fn field_mapping_from_entry(entry: &FieldMappingEntry) -> Option { + match &entry.mapping_type { + // Quickwit text fields behave like ES keyword fields: they support exact + // match, prefix, and regexp queries. Reporting them as "keyword" enables + // downstream connectors (e.g. Trino ES connector) to push down filters and + // LIKE predicates, which they only do for keyword-typed fields. + FieldMappingType::Text(..) => Some(FieldMapping::Leaf { typ: "keyword" }), + FieldMappingType::I64(..) => Some(FieldMapping::Leaf { typ: "long" }), + FieldMappingType::U64(..) => Some(FieldMapping::Leaf { typ: "long" }), + FieldMappingType::F64(..) => Some(FieldMapping::Leaf { typ: "double" }), + FieldMappingType::Bool(..) => Some(FieldMapping::Leaf { typ: "boolean" }), + FieldMappingType::DateTime(..) => Some(FieldMapping::Leaf { typ: "date" }), + FieldMappingType::IpAddr(..) => Some(FieldMapping::Leaf { typ: "ip" }), + FieldMappingType::Bytes(..) => Some(FieldMapping::Leaf { typ: "binary" }), + FieldMappingType::Json(..) => Some(FieldMapping::Leaf { typ: "object" }), + FieldMappingType::Object(options) => { + let properties = build_properties(&options.field_mappings); + Some(FieldMapping::Object { + typ: "object", + properties, + }) + } + FieldMappingType::Concatenate(_) => Some(FieldMapping::Leaf { typ: "keyword" }), + } +} + +/// Merges dynamic fields from a `ListFieldsResponse` into the properties map. +/// +/// Fields already present in the map (from explicit doc mappings) are skipped, +/// as are internal fields (prefixed with `_`). +fn merge_dynamic_fields( + properties: &mut HashMap, + list_fields_response: &ListFieldsResponse, +) { + for field_entry in &list_fields_response.fields { + let field_name = &field_entry.field_name; + if field_name.starts_with('_') { + continue; + } + if properties.contains_key(field_name) { + continue; + } + let Ok(field_type) = ListFieldType::try_from(field_entry.field_type) else { + continue; + }; + if let Some(es_type) = es_type_from_list_field_type(field_type) { + properties.insert(field_name.clone(), FieldMapping::Leaf { typ: es_type }); + } + } +} + +fn es_type_from_list_field_type(field_type: ListFieldType) -> Option<&'static str> { + match field_type { + ListFieldType::Str => Some("keyword"), + ListFieldType::U64 | ListFieldType::I64 => Some("long"), + ListFieldType::F64 => Some("double"), + ListFieldType::Bool => Some("boolean"), + ListFieldType::Date => Some("date"), + ListFieldType::Bytes => Some("binary"), + ListFieldType::IpAddr => Some("ip"), + ListFieldType::Facet | ListFieldType::Json => None, + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn test_field_mapping_from_entry_bool() { + let entry_json = json!({ "name": "active", "type": "bool" }); + let entry: FieldMappingEntry = serde_json::from_value(entry_json).unwrap(); + let mapping = field_mapping_from_entry(&entry).unwrap(); + let serialized = serde_json::to_value(&mapping).unwrap(); + assert_eq!(serialized, json!({ "type": "boolean" })); + } + + #[test] + fn test_field_mapping_from_entry_text() { + let entry_json = json!({ "name": "message", "type": "text" }); + let entry: FieldMappingEntry = serde_json::from_value(entry_json).unwrap(); + let mapping = field_mapping_from_entry(&entry).unwrap(); + let serialized = serde_json::to_value(&mapping).unwrap(); + assert_eq!(serialized, json!({ "type": "keyword" })); + } + + #[test] + fn test_field_mapping_from_entry_i64() { + let entry_json = json!({ "name": "count", "type": "i64" }); + let entry: FieldMappingEntry = serde_json::from_value(entry_json).unwrap(); + let mapping = field_mapping_from_entry(&entry).unwrap(); + let serialized = serde_json::to_value(&mapping).unwrap(); + assert_eq!(serialized, json!({ "type": "long" })); + } + + #[test] + fn test_field_mapping_from_entry_object() { + let entry_json = json!({ + "name": "nested", + "type": "object", + "field_mappings": [ + { "name": "id", "type": "u64" }, + { "name": "label", "type": "text" } + ] + }); + let entry: FieldMappingEntry = serde_json::from_value(entry_json).unwrap(); + let mapping = field_mapping_from_entry(&entry).unwrap(); + let serialized = serde_json::to_value(&mapping).unwrap(); + assert_eq!( + serialized, + json!({ + "type": "object", + "properties": { + "id": { "type": "long" }, + "label": { "type": "keyword" } + } + }) + ); + } + + #[test] + fn test_field_mapping_from_entry_concatenate_exposed_as_keyword() { + let entry_json = json!({ + "name": "concat_field", + "type": "concatenate", + "concatenate_fields": ["field_a", "field_b"] + }); + let entry: FieldMappingEntry = serde_json::from_value(entry_json).unwrap(); + let mapping = field_mapping_from_entry(&entry).unwrap(); + let serialized = serde_json::to_value(&mapping).unwrap(); + assert_eq!(serialized, json!({ "type": "keyword" })); + } + + #[test] + fn test_build_properties_all_leaf_types() { + let entries: Vec = serde_json::from_value(json!([ + { "name": "title", "type": "text" }, + { "name": "count", "type": "i64" }, + { "name": "unsigned", "type": "u64" }, + { "name": "score", "type": "f64" }, + { "name": "active", "type": "bool" }, + { "name": "created_at", "type": "datetime" }, + { "name": "ip_field", "type": "ip" }, + { "name": "data", "type": "bytes" }, + { "name": "payload", "type": "json" }, + { + "name": "metadata", + "type": "object", + "field_mappings": [ + { "name": "source", "type": "text" } + ] + } + ])) + .unwrap(); + + let props = build_properties(&entries); + let to_json = |fm: &FieldMapping| serde_json::to_value(fm).unwrap(); + + assert_eq!(to_json(&props["title"]), json!({ "type": "keyword" })); + assert_eq!(to_json(&props["count"]), json!({ "type": "long" })); + assert_eq!(to_json(&props["unsigned"]), json!({ "type": "long" })); + assert_eq!(to_json(&props["score"]), json!({ "type": "double" })); + assert_eq!(to_json(&props["active"]), json!({ "type": "boolean" })); + assert_eq!(to_json(&props["created_at"]), json!({ "type": "date" })); + assert_eq!(to_json(&props["ip_field"]), json!({ "type": "ip" })); + assert_eq!(to_json(&props["data"]), json!({ "type": "binary" })); + assert_eq!(to_json(&props["payload"]), json!({ "type": "object" })); + + let meta = to_json(&props["metadata"]); + assert_eq!(meta["type"], "object"); + assert_eq!(meta["properties"]["source"]["type"], "keyword"); + } + + #[test] + fn test_merge_dynamic_fields_skips_existing_and_internal() { + use quickwit_proto::search::ListFieldsEntryResponse; + + let mut properties = HashMap::new(); + properties.insert("title".to_string(), FieldMapping::Leaf { typ: "text" }); + + let list_fields = ListFieldsResponse { + fields: vec![ + ListFieldsEntryResponse { + field_name: "title".to_string(), + field_type: ListFieldType::Str as i32, + ..Default::default() + }, + ListFieldsEntryResponse { + field_name: "_timestamp".to_string(), + field_type: ListFieldType::Date as i32, + ..Default::default() + }, + ListFieldsEntryResponse { + field_name: "dynamic_field".to_string(), + field_type: ListFieldType::Str as i32, + ..Default::default() + }, + ], + }; + + merge_dynamic_fields(&mut properties, &list_fields); + + assert_eq!(properties.len(), 2); + assert!(properties.contains_key("title")); + assert!(properties.contains_key("dynamic_field")); + assert!(!properties.contains_key("_timestamp")); + } +} diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/model/mod.rs b/quickwit/quickwit-serve/src/elasticsearch_api/model/mod.rs index ae1f6aa1a41..4351a26b65b 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/model/mod.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/model/mod.rs @@ -17,6 +17,7 @@ mod bulk_query_params; mod cat_indices; mod error; mod field_capability; +mod mappings; mod multi_search; mod scroll; mod search_body; @@ -35,6 +36,7 @@ pub use field_capability::{ FieldCapabilityQueryParams, FieldCapabilityRequestBody, FieldCapabilityResponse, build_list_field_request_for_es_api, convert_to_es_field_capabilities_response, }; +pub(crate) use mappings::ElasticsearchMappingsResponse; pub use multi_search::{ MultiSearchHeader, MultiSearchQueryParams, MultiSearchResponse, MultiSearchSingleResponse, }; diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/model/multi_search.rs b/quickwit/quickwit-serve/src/elasticsearch_api/model/multi_search.rs index 0dbc9dab5a2..758a35f7d17 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/model/multi_search.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/model/multi_search.rs @@ -25,6 +25,7 @@ use crate::simple_list::{from_simple_list, to_simple_list}; // Multi search doc: https://www.elastic.co/guide/en/elasticsearch/reference/current/search-multi-search.html +#[serde_as] #[serde_with::skip_serializing_none] #[derive(Default, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -50,6 +51,10 @@ pub struct MultiSearchQueryParams { pub ignore_throttled: Option, #[serde(default)] pub ignore_unavailable: Option, + /// List of indexes to search. + #[serde_as(deserialize_as = "OneOrMany<_, PreferMany>")] + #[serde(default, rename = "index")] + pub indexes: Vec, #[serde(default)] pub max_concurrent_searches: Option, #[serde(default)] @@ -90,8 +95,8 @@ pub struct MultiSearchHeader { #[serde(default)] pub ignore_unavailable: Option, #[serde_as(deserialize_as = "OneOrMany<_, PreferMany>")] - #[serde(default)] - pub index: Vec, + #[serde(default, rename = "index")] + pub indexes: Vec, #[serde(default)] pub preference: Option, #[serde(default)] @@ -100,6 +105,26 @@ pub struct MultiSearchHeader { pub routing: Option>, } +impl MultiSearchHeader { + pub fn apply_query_param_defaults(&mut self, defaults: &MultiSearchQueryParams) { + if self.allow_no_indices.is_none() { + self.allow_no_indices = defaults.allow_no_indices; + } + if self.expand_wildcards.is_none() { + self.expand_wildcards = defaults.expand_wildcards.clone(); + } + if self.ignore_unavailable.is_none() { + self.ignore_unavailable = defaults.ignore_unavailable; + } + if self.indexes.is_empty() { + self.indexes = defaults.indexes.clone(); + } + if self.routing.is_none() { + self.routing = defaults.routing.clone(); + } + } +} + #[derive(Serialize)] pub struct MultiSearchResponse { pub responses: Vec, diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/model/search_body.rs b/quickwit/quickwit-serve/src/elasticsearch_api/model/search_body.rs index d2850349aa2..d80389af196 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/model/search_body.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/model/search_body.rs @@ -134,11 +134,25 @@ impl<'de> Visitor<'de> for FieldSortVecVisitor { type Value = Vec; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("An array containing the sort fields.") + formatter.write_str("A string, array, or object containing the sort fields.") + } + + fn visit_str(self, field_name: &str) -> Result, E> + where + E: serde::de::Error, + { + let order = default_elasticsearch_sort_order(field_name); + Ok(vec![SortField { + field: field_name.to_string(), + order, + date_format: None, + }]) } fn visit_seq(self, mut seq: A) -> Result, A::Error> - where A: serde::de::SeqAccess<'de> { + where + A: serde::de::SeqAccess<'de>, + { let mut sort_fields: Vec = Vec::new(); while let Some(field_sort) = seq.next_element::()? { sort_fields.push(field_sort.into()); @@ -147,7 +161,9 @@ impl<'de> Visitor<'de> for FieldSortVecVisitor { } fn visit_map(self, mut map: M) -> Result, M::Error> - where M: MapAccess<'de> { + where + M: MapAccess<'de>, + { let mut sort_fields: Vec = Vec::new(); while let Some((field_sort_key, field_sort_params)) = map.next_entry::()? @@ -168,7 +184,9 @@ impl<'de> Visitor<'de> for FieldSortVecVisitor { /// ES accepts structs to describe the sort field. /// In that case the order of apparition in the JSON object matters. fn deserialize_field_sorts<'de, D>(deserializer: D) -> Result>, D::Error> -where D: Deserializer<'de> { +where + D: Deserializer<'de>, +{ deserializer.deserialize_any(FieldSortVecVisitor).map(Some) } @@ -231,6 +249,20 @@ mod tests { assert_eq!(field_sorts[1].order, SortOrder::Asc); } + #[test] + fn test_sort_field_str() { + let json = r#" + { + "sort": "timestamp" + } + "#; + let search_body: SearchBody = serde_json::from_str(json).unwrap(); + let field_sorts = search_body.sort.unwrap(); + assert_eq!(field_sorts.len(), 1); + assert_eq!(field_sorts[0].field, "timestamp"); + assert_eq!(field_sorts[0].order, SortOrder::Asc); + } + #[test] fn test_sort_default_orders() { let json = r#" diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/model/search_query_params.rs b/quickwit/quickwit-serve/src/elasticsearch_api/model/search_query_params.rs index c4a4a124439..3098f23b964 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/model/search_query_params.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/model/search_query_params.rs @@ -115,6 +115,8 @@ pub struct SearchQueryParams { #[serde(default)] pub scroll: Option, #[serde(default)] + pub search_type: Option, + #[serde(default)] pub seq_no_primary_term: Option, #[serde(default)] pub size: Option, diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs b/quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs index ebc1b5ef622..9abce6c61e6 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs @@ -27,7 +27,7 @@ use quickwit_common::truncate_str; use quickwit_config::{NodeConfig, validate_index_id_pattern}; use quickwit_index_management::IndexService; use quickwit_metastore::*; -use quickwit_proto::metastore::MetastoreServiceClient; +use quickwit_proto::metastore::{IndexMetadataRequest, MetastoreService, MetastoreServiceClient}; use quickwit_proto::search::{ CountHits, ListFieldsResponse, PartialHit, ScrollRequest, SearchResponse, SortByValue, SortDatetimeFormat, @@ -39,18 +39,20 @@ use quickwit_search::{ AggregationResults, SearchError, SearchService, list_all_splits, resolve_index_patterns, }; use serde::{Deserialize, Serialize}; -use serde_json::json; +use serde_json::{Map, Value, json}; use warp::hyper::StatusCode; use warp::reply::with_status; use warp::{Filter, Rejection}; use super::filter::{ - elastic_cat_indices_filter, elastic_cluster_health_filter, elastic_cluster_info_filter, - elastic_delete_index_filter, elastic_field_capabilities_filter, - elastic_index_cat_indices_filter, elastic_index_count_filter, - elastic_index_field_capabilities_filter, elastic_index_search_filter, - elastic_index_stats_filter, elastic_multi_search_filter, elastic_resolve_index_filter, - elastic_scroll_filter, elastic_stats_filter, elasticsearch_filter, + elastic_aliases_filter, elastic_cat_indices_filter, elastic_cluster_health_filter, + elastic_cluster_info_filter, elastic_delete_index_filter, elastic_delete_scroll_filter, + elastic_field_capabilities_filter, elastic_index_cat_indices_filter, + elastic_index_count_filter, elastic_index_field_capabilities_filter, + elastic_index_mapping_filter, elastic_index_search_filter, elastic_index_stats_filter, + elastic_multi_search_filter, elastic_nodes_filter, elastic_resolve_index_filter, + elastic_scroll_filter, elastic_search_shards_filter, elastic_stats_filter, + elasticsearch_filter, }; use super::model::{ CatIndexQueryParams, DeleteQueryParams, ElasticsearchCatIndexResponse, ElasticsearchError, @@ -62,6 +64,7 @@ use super::model::{ build_list_field_request_for_es_api, convert_to_es_field_capabilities_response, }; use super::{TrackTotalHits, make_elastic_api_response}; +use crate::elasticsearch_api::model::ElasticsearchMappingsResponse; use crate::format::BodyFormat; use crate::rest::recover_fn; use crate::rest_api_response::{RestApiError, RestApiResponse}; @@ -80,11 +83,17 @@ pub fn es_compat_cluster_info_handler( warp::reply::json(&json!({ "name" : config.node_id, "cluster_name" : config.cluster_id, + "cluster_uuid" : config.cluster_id, + "tagline" : "You Know, for Search", "version" : { "distribution" : "quickwit", - "number" : build_info.version, + "number" : "7.17.0", "build_hash" : build_info.commit_hash, "build_date" : build_info.build_date, + "build_snapshot" : false, + "lucene_version" : "8.11.1", + "minimum_wire_compatibility_version" : "6.8.0", + "minimum_index_compatibility_version" : "6.0.0-beta1", } })) }, @@ -92,6 +101,118 @@ pub fn es_compat_cluster_info_handler( .boxed() } +/// GET _elastic/_nodes/http +pub fn es_compat_nodes_handler( + node_config: Arc, +) -> impl Filter + Clone { + elastic_nodes_filter() + .and(with_arg(node_config)) + .then(|config: Arc| async move { + let advertise_addr = std::net::SocketAddr::new( + config.grpc_advertise_addr.ip(), + config.rest_config.listen_addr.port(), + ); + warp::reply::json(&json!({ + "nodes": { + config.node_id.as_str(): { + "roles": ["data", "ingest"], + "http": { + "publish_address": advertise_addr.to_string() + } + } + } + })) + }) + .boxed() +} + +/// GET _elastic/{index}/_search_shards +pub fn es_compat_search_shards_handler( + node_config: Arc, +) -> impl Filter + Clone { + elastic_search_shards_filter() + .and(with_arg(node_config)) + .then(|index_id: String, config: Arc| async move { + warp::reply::json(&json!({ + "shards": [[{ + "index": index_id, + "shard": 0, + "primary": true, + "node": config.node_id.as_str() + }]] + })) + }) + .boxed() +} + +/// GET _elastic/_aliases +pub fn es_compat_aliases_handler() +-> impl Filter + Clone { + elastic_aliases_filter() + .then(|| async { Ok(Value::Object(Map::new())) }) + .map(|result| make_elastic_api_response(result, BodyFormat::default())) + .recover(recover_fn) + .boxed() +} + +/// GET _elastic/{index}/_mapping or _elastic/{index}/_mappings +pub fn es_compat_index_mapping_handler( + metastore: MetastoreServiceClient, + search_service: Arc, +) -> impl Filter + Clone { + elastic_index_mapping_filter() + .and(with_arg(metastore)) + .and(with_arg(search_service)) + .then(es_compat_index_mapping) + .map(|result| make_elastic_api_response(result, BodyFormat::default())) + .recover(recover_fn) +} + +async fn get_index_metadata( + index_id: String, + metastore: MetastoreServiceClient, +) -> Result { + let index_metadata_request = IndexMetadataRequest::for_index_id(index_id); + let index_metadata = metastore + .index_metadata(index_metadata_request) + .await? + .deserialize_index_metadata()?; + Ok(index_metadata) +} + +async fn es_compat_index_mapping( + index_id: String, + mut metastore: MetastoreServiceClient, + search_service: Arc, +) -> Result { + let indexes_metadata = if index_id.contains('*') || index_id.contains(',') { + let patterns: Vec = index_id.split(',').map(|s| s.trim().to_string()).collect(); + resolve_index_patterns(&patterns, &mut metastore).await? + } else { + vec![get_index_metadata(index_id.clone(), metastore).await?] + }; + let index_id_patterns: Vec = indexes_metadata + .iter() + .map(|m| m.index_id().to_string()) + .collect(); + let list_fields_request = quickwit_proto::search::ListFieldsRequest { + index_id_patterns, + fields: Vec::new(), + start_timestamp: None, + end_timestamp: None, + query_ast: None, + }; + let list_fields_response = search_service + .root_list_fields(list_fields_request) + .await + .ok(); + let response = ElasticsearchMappingsResponse::from_doc_mapping( + indexes_metadata, + list_fields_response.as_ref(), + ); + Ok(response) +} + /// GET or POST _elastic/_search pub fn es_compat_search_handler( _search_service: Arc, @@ -302,6 +423,24 @@ pub fn es_compat_scroll_handler( .boxed() } +/// DELETE _elastic/_search/scroll +/// +/// Clears a scroll context. Quickwit manages scroll lifetime via TTL, +/// so this is a no-op that returns success. +pub fn es_compat_delete_scroll_handler() +-> impl Filter + Clone { + elastic_delete_scroll_filter() + .then(|| async { + Ok::<_, ElasticsearchError>(json!({ + "succeeded": true, + "num_freed": 0 + })) + }) + .map(|result| make_elastic_api_response(result, BodyFormat::default())) + .recover(recover_fn) + .boxed() +} + #[allow(clippy::result_large_err)] fn build_request_for_es_api( index_id_patterns: Vec, @@ -358,6 +497,7 @@ fn build_request_for_es_api( let max_hits = search_params.size.or(search_body.size).unwrap_or(10); let start_offset = search_params.from.or(search_body.from).unwrap_or(0); + let ignore_missing_indexes = search_params.ignore_unavailable.unwrap_or(false); let count_hits = match search_params .track_total_hits .or(search_body.track_total_hits) @@ -410,6 +550,8 @@ fn build_request_for_es_api( scroll_ttl_secs, search_after, count_hits, + ignore_missing_indexes, + skip_aggregation_finalization: false, }, has_doc_id_field, )) @@ -489,7 +631,8 @@ async fn es_compat_index_count( search_body: SearchBody, search_service: Arc, ) -> Result { - let search_params: SearchQueryParams = search_params.into(); + let mut search_params: SearchQueryParams = search_params.into(); + search_params.track_total_hits = Some(TrackTotalHits::Track(true)); let (search_request, _append_shard_doc) = build_request_for_es_api(index_id_patterns, search_params, search_body)?; let search_response: SearchResponse = search_service.root_search(search_request).await?; @@ -814,26 +957,28 @@ async fn es_compat_index_multi_search( let mut payload_lines = str_lines(str_payload); while let Some(line) = payload_lines.next() { - let request_header = serde_json::from_str::(line).map_err(|err| { - SearchError::InvalidArgument(format!( - "failed to parse request header `{}...`: {}", - truncate_str(line, 20), - err - )) - })?; - if request_header.index.is_empty() { + let mut request_header = + serde_json::from_str::(line).map_err(|err| { + SearchError::InvalidArgument(format!( + "failed to parse request header `{}...`: {}", + truncate_str(line, 20), + err + )) + })?; + request_header.apply_query_param_defaults(&multi_search_params); + if request_header.indexes.is_empty() { return Err(ElasticsearchError::from(SearchError::InvalidArgument( "`_msearch` request header must define at least one index".to_string(), ))); } - for index in &request_header.index { + for index in &request_header.indexes { validate_index_id_pattern(index, true).map_err(|err| { SearchError::InvalidArgument(format!( "request header contains an invalid index: {err}" )) })?; } - let index_ids_patterns = request_header.index.clone(); + let index_ids_patterns = request_header.indexes.clone(); let search_body = payload_lines .next() .ok_or_else(|| { @@ -1000,10 +1145,10 @@ fn convert_to_es_search_response( _source_includes: Option>, allow_partial_results: bool, ) -> Result { - if !allow_partial_results || resp.num_successful_splits == 0 { - if let Some(search_error) = SearchError::from_split_errors(&resp.failed_splits) { - return Err(ElasticsearchError::from(search_error)); - } + if (!allow_partial_results || resp.num_successful_splits == 0) + && let Some(search_error) = SearchError::from_split_errors(&resp.failed_splits) + { + return Err(ElasticsearchError::from(search_error)); } let hits: Vec = resp .hits diff --git a/quickwit/quickwit-serve/src/format.rs b/quickwit/quickwit-serve/src/format.rs index 7b53f92f9c8..38b0a0872c9 100644 --- a/quickwit/quickwit-serve/src/format.rs +++ b/quickwit/quickwit-serve/src/format.rs @@ -63,7 +63,9 @@ impl fmt::Display for BodyFormat { impl Serialize for BodyFormat { fn serialize(&self, serializer: S) -> Result - where S: Serializer { + where + S: Serializer, + { serializer.serialize_str(&self.to_string()) } } @@ -80,8 +82,7 @@ struct FormatQueryString { pub(crate) fn extract_format_from_qs() -> impl Filter + Clone { - serde_qs::warp::query::(serde_qs::Config::default()) - .map(|format_qs: FormatQueryString| format_qs.format) + warp::query::().map(|format_qs: FormatQueryString| format_qs.format) } #[derive(Debug, Error)] diff --git a/quickwit/quickwit-serve/src/grpc.rs b/quickwit/quickwit-serve/src/grpc.rs index c136d8436a8..698c9e07d71 100644 --- a/quickwit/quickwit-serve/src/grpc.rs +++ b/quickwit/quickwit-serve/src/grpc.rs @@ -23,6 +23,7 @@ use quickwit_config::service::QuickwitService; use quickwit_proto::developer::DeveloperServiceClient; use quickwit_proto::indexing::IndexingServiceClient; use quickwit_proto::jaeger::storage::v1::span_reader_plugin_server::SpanReaderPluginServer; +use quickwit_proto::jaeger::storage::v2::trace_reader_server::TraceReaderServer; use quickwit_proto::opentelemetry::proto::collector::logs::v1::logs_service_server::LogsServiceServer; use quickwit_proto::opentelemetry::proto::collector::trace::v1::trace_service_server::TraceServiceServer; use quickwit_proto::search::search_service_server::SearchServiceServer; @@ -162,7 +163,9 @@ pub(crate) async fn start_grpc_server( enabled_grpc_services.insert("otlp-traces"); let trace_service = TraceServiceServer::new(otlp_traces_service) .accept_compressed(CompressionEncoding::Gzip) - .accept_compressed(CompressionEncoding::Zstd); + .accept_compressed(CompressionEncoding::Zstd) + .max_decoding_message_size(grpc_config.max_message_size.0 as usize) + .max_encoding_message_size(grpc_config.max_message_size.0 as usize); Some(trace_service) } else { None @@ -172,7 +175,9 @@ pub(crate) async fn start_grpc_server( enabled_grpc_services.insert("otlp-logs"); let logs_service = LogsServiceServer::new(otlp_logs_service) .accept_compressed(CompressionEncoding::Gzip) - .accept_compressed(CompressionEncoding::Zstd); + .accept_compressed(CompressionEncoding::Zstd) + .max_decoding_message_size(grpc_config.max_message_size.0 as usize) + .max_encoding_message_size(grpc_config.max_message_size.0 as usize); Some(logs_service) } else { None @@ -203,6 +208,15 @@ pub(crate) async fn start_grpc_server( } else { None }; + + // Mount gRPC jaeger v2 service (TraceReader) if present. + let jaeger_v2_grpc_service = if let Some(jaeger_service) = services.jaeger_service_opt.clone() { + enabled_grpc_services.insert("jaeger-v2"); + Some(TraceReaderServer::new(jaeger_service)) + } else { + None + }; + let developer_grpc_service = { enabled_grpc_services.insert("developer"); file_descriptor_sets.push(quickwit_proto::developer::DEVELOPER_FILE_DESCRIPTOR_SET); @@ -230,6 +244,7 @@ pub(crate) async fn start_grpc_server( .add_optional_service(ingest_router_grpc_service) .add_optional_service(ingester_grpc_service) .add_optional_service(jaeger_grpc_service) + .add_optional_service(jaeger_v2_grpc_service) .add_optional_service(metastore_grpc_service) .add_optional_service(otlp_log_grpc_service) .add_optional_service(otlp_trace_grpc_service) diff --git a/quickwit/quickwit-serve/src/health_check_api/handler.rs b/quickwit/quickwit-serve/src/health_check_api/handler.rs index a1a27a50f54..71ba487cc57 100644 --- a/quickwit/quickwit-serve/src/health_check_api/handler.rs +++ b/quickwit/quickwit-serve/src/health_check_api/handler.rs @@ -75,17 +75,17 @@ async fn get_liveness( ) -> impl warp::Reply { let mut is_live = true; - if let Some(indexer_service) = indexer_service_opt { - if !indexer_service.ask(Healthz).await.unwrap_or(false) { - error!("indexer service is unhealthy"); - is_live = false; - } + if let Some(indexer_service) = indexer_service_opt + && !indexer_service.ask(Healthz).await.unwrap_or(false) + { + error!("indexer service is unhealthy"); + is_live = false; } - if let Some(janitor_service) = janitor_service_opt { - if !janitor_service.ask(Healthz).await.unwrap_or(false) { - error!("janitor service is unhealthy"); - is_live = false; - } + if let Some(janitor_service) = janitor_service_opt + && !janitor_service.ask(Healthz).await.unwrap_or(false) + { + error!("janitor service is unhealthy"); + is_live = false; } let status_code = if is_live { StatusCode::OK diff --git a/quickwit/quickwit-serve/src/index_api/index_resource.rs b/quickwit/quickwit-serve/src/index_api/index_resource.rs index 29634af7881..01fc4858016 100644 --- a/quickwit/quickwit-serve/src/index_api/index_resource.rs +++ b/quickwit/quickwit-serve/src/index_api/index_resource.rs @@ -23,11 +23,10 @@ use quickwit_index_management::{IndexService, IndexServiceError}; use quickwit_metastore::{ IndexMetadata, IndexMetadataResponseExt, ListIndexesMetadataResponseExt, ListSplitsQuery, ListSplitsRequestExt, MetastoreServiceStreamSplitsExt, Split, SplitInfo, SplitState, - UpdateIndexRequestExt, }; use quickwit_proto::metastore::{ IndexMetadataRequest, ListIndexesMetadataRequest, ListSplitsRequest, MetastoreError, - MetastoreResult, MetastoreService, MetastoreServiceClient, UpdateIndexRequest, + MetastoreResult, MetastoreService, MetastoreServiceClient, }; use quickwit_proto::types::IndexId; use serde::{Deserialize, Serialize}; @@ -81,7 +80,7 @@ pub fn list_indexes_metadata_handler( ) -> impl Filter + Clone { warp::path!("indexes") .and(warp::get()) - .and(serde_qs::warp::query(serde_qs::Config::default())) + .and(warp::query()) .and(with_arg(metastore)) .then(list_indexes_metadata) .and(extract_format_from_qs()) @@ -238,7 +237,7 @@ pub fn create_index_handler( ) -> impl Filter + Clone { warp::path!("indexes") .and(warp::post()) - .and(serde_qs::warp::query(serde_qs::Config::default())) + .and(warp::query()) .and(extract_config_format()) .and(warp::body::content_length_limit(1024 * 1024)) .and(warp::filters::body::bytes()) @@ -294,7 +293,7 @@ pub struct UpdateQueryParams { } fn update_index_qp() -> impl Filter + Clone { - serde_qs::warp::query::(serde_qs::Config::default()) + warp::query::() } pub fn update_index_handler( @@ -392,16 +391,10 @@ pub async fn update_index( ) .map_err(IndexServiceError::InvalidConfig)?; - let update_request = UpdateIndexRequest::try_from_updates( - index_uid, - &new_index_config.doc_mapping, - &new_index_config.indexing_settings, - &new_index_config.ingest_settings, - &new_index_config.search_settings, - &new_index_config.retention_policy_opt, - )?; - let update_resp = metastore.update_index(update_request).await?; - Ok(update_resp.deserialize_index_metadata()?) + let index_metadata = index_service + .update_index(index_uid, new_index_config) + .await?; + Ok(index_metadata) } pub fn clear_index_handler( @@ -449,7 +442,7 @@ pub fn delete_index_handler( ) -> impl Filter + Clone { warp::path!("indexes" / String) .and(warp::delete()) - .and(serde_qs::warp::query(serde_qs::Config::default())) + .and(warp::query()) .and(with_arg(index_service)) .then(delete_index) .and(extract_format_from_qs()) diff --git a/quickwit/quickwit-serve/src/index_api/rest_handler.rs b/quickwit/quickwit-serve/src/index_api/rest_handler.rs index 1b6e3a7a649..bc4c8d9b105 100644 --- a/quickwit/quickwit-serve/src/index_api/rest_handler.rs +++ b/quickwit/quickwit-serve/src/index_api/rest_handler.rs @@ -1140,7 +1140,7 @@ mod tests { use quickwit_indexing::source::sqs_queue::test_helpers::start_mock_sqs_get_queue_attributes_endpoint; let metastore = metastore_for_test(); - let (queue_url, _guard) = start_mock_sqs_get_queue_attributes_endpoint(); + let (queue_url, _guard) = start_mock_sqs_get_queue_attributes_endpoint().await; let index_service = IndexService::new(metastore.clone(), StorageResolver::unconfigured()); let mut node_config = NodeConfig::for_test(); node_config.default_index_root_uri = Uri::for_test("file:///default-index-root-uri"); diff --git a/quickwit/quickwit-serve/src/index_api/source_resource.rs b/quickwit/quickwit-serve/src/index_api/source_resource.rs index 39eb09b8894..66f3069b228 100644 --- a/quickwit/quickwit-serve/src/index_api/source_resource.rs +++ b/quickwit/quickwit-serve/src/index_api/source_resource.rs @@ -111,7 +111,7 @@ pub struct UpdateQueryParams { } fn update_source_qp() -> impl Filter + Clone { - serde_qs::warp::query::(serde_qs::Config::default()) + warp::query::() } pub fn update_source_handler( diff --git a/quickwit/quickwit-serve/src/index_api/split_resource.rs b/quickwit/quickwit-serve/src/index_api/split_resource.rs index a439328ffa1..a062186e551 100644 --- a/quickwit/quickwit-serve/src/index_api/split_resource.rs +++ b/quickwit/quickwit-serve/src/index_api/split_resource.rs @@ -141,7 +141,7 @@ pub fn list_splits_handler( ) -> impl Filter + Clone { warp::path!("indexes" / String / "splits") .and(warp::get()) - .and(serde_qs::warp::query(serde_qs::Config::default())) + .and(warp::query()) .and(with_arg(metastore)) .then(list_splits) .and(extract_format_from_qs()) diff --git a/quickwit/quickwit-serve/src/ingest_api/rest_handler.rs b/quickwit/quickwit-serve/src/ingest_api/rest_handler.rs index a12d154c63c..9040c8fd700 100644 --- a/quickwit/quickwit-serve/src/ingest_api/rest_handler.rs +++ b/quickwit/quickwit-serve/src/ingest_api/rest_handler.rs @@ -98,9 +98,7 @@ fn ingest_filter( config.content_length_limit.as_u64(), )) .and(get_body_bytes()) - .and(serde_qs::warp::query::( - serde_qs::Config::default(), - )) + .and(warp::query::()) } fn ingest_handler( diff --git a/quickwit/quickwit-serve/src/jaeger_api/model.rs b/quickwit/quickwit-serve/src/jaeger_api/model.rs index bf6e386806b..1a69c6c63fc 100644 --- a/quickwit/quickwit-serve/src/jaeger_api/model.rs +++ b/quickwit/quickwit-serve/src/jaeger_api/model.rs @@ -325,7 +325,9 @@ impl From for JaegerError { } fn serialize_bytes_to_hex(bytes: &Vec, s: S) -> Result -where S: serde::Serializer { +where + S: serde::Serializer, +{ s.serialize_str(&format!("{:0>16}", hex::encode(bytes))) } diff --git a/quickwit/quickwit-serve/src/jaeger_api/rest_handler.rs b/quickwit/quickwit-serve/src/jaeger_api/rest_handler.rs index 45d3d5d42f6..def8a4c6ca7 100644 --- a/quickwit/quickwit-serve/src/jaeger_api/rest_handler.rs +++ b/quickwit/quickwit-serve/src/jaeger_api/rest_handler.rs @@ -139,7 +139,7 @@ pub fn jaeger_traces_search_handler( ) -> impl Filter + Clone { jaeger_api_path_filter() .and(warp::path!("traces")) - .and(serde_qs::warp::query(serde_qs::Config::default())) + .and(warp::query()) .and(require(jaeger_service_opt)) .then(jaeger_traces_search) .map(|result| make_jaeger_api_response(result, BodyFormat::default())) diff --git a/quickwit/quickwit-serve/src/lib.rs b/quickwit/quickwit-serve/src/lib.rs index 91afc4a4dd7..3ad5bd72aa7 100644 --- a/quickwit/quickwit-serve/src/lib.rs +++ b/quickwit/quickwit-serve/src/lib.rs @@ -59,7 +59,7 @@ use itertools::Itertools; use once_cell::sync::Lazy; use quickwit_actors::{ActorExitStatus, Mailbox, SpawnContext, Universe}; use quickwit_cluster::{ - Cluster, ClusterChange, ClusterChangeStream, ListenerHandle, start_cluster_service, + Cluster, ClusterChange, ClusterChangeStream, ClusterNode, ListenerHandle, start_cluster_service, }; use quickwit_common::pubsub::{EventBroker, EventSubscriptionHandle}; use quickwit_common::rate_limiter::RateLimiterSettings; @@ -82,8 +82,10 @@ use quickwit_indexing::models::ShardPositionsService; use quickwit_indexing::start_indexing_service; use quickwit_ingest::{ GetMemoryCapacity, IngestRequest, IngestRouter, IngestServiceClient, Ingester, IngesterPool, - LocalShardsUpdate, get_idle_shard_timeout, setup_local_shards_update_listener, - start_ingest_api_service, wait_for_ingester_decommission, wait_for_ingester_status, + IngesterPoolEntry, LocalShardsUpdate, get_idle_shard_timeout, + setup_ingester_capacity_update_listener, setup_local_shards_update_listener, + start_ingest_api_service, try_get_ingester_status, wait_for_ingester_decommission, + wait_for_ingester_status, }; use quickwit_jaeger::JaegerService; use quickwit_janitor::{JanitorService, start_janitor_service}; @@ -112,6 +114,7 @@ use quickwit_search::{ use quickwit_storage::{SplitCache, StorageResolver}; use tcp_listener::TcpListenerResolver; use tokio::sync::oneshot; +use tonic::codec::CompressionEncoding; use tonic_health::ServingStatus; use tonic_health::server::HealthReporter; use tower::ServiceBuilder; @@ -148,6 +151,7 @@ fn get_metastore_client_max_concurrency() -> usize { quickwit_common::get_from_env( METASTORE_CLIENT_MAX_CONCURRENCY_ENV_KEY, DEFAULT_METASTORE_CLIENT_MAX_CONCURRENCY, + false, ) } @@ -257,11 +261,6 @@ async fn balance_channel_for_service( BalanceChannel::from_stream(service_change_stream) } -fn convert_status_code_to_legacy_http(status_code: http::StatusCode) -> warp::http::StatusCode { - warp::http::StatusCode::from_u16(status_code.as_u16()) - .unwrap_or(warp::http::StatusCode::INTERNAL_SERVER_ERROR) -} - async fn start_ingest_client_if_needed( node_config: &NodeConfig, universe: &Universe, @@ -383,13 +382,12 @@ fn start_shard_positions_service( // We spawn a task here, because we need the ingester to be ready before spawning the // the `ShardPositionsService`. If we don't, all the events we emit too early will be dismissed. tokio::spawn(async move { - if let Some(ingester) = ingester_opt { - if wait_for_ingester_status(ingester, IngesterStatus::Ready) + if let Some(ingester) = &ingester_opt + && wait_for_ingester_status(ingester, IngesterStatus::Ready, Duration::from_secs(300)) .await .is_err() - { - warn!("ingester failed to reach ready status"); - } + { + warn!("ingester failed to reach ready status"); } ShardPositionsService::spawn(&spawn_ctx, event_broker, cluster); }); @@ -411,10 +409,10 @@ async fn shutdown_signal_handler( shutdown_signal.await; // We must decommission the ingester first before terminating the indexing pipelines that // may consume from it. We also need to keep the gRPC server running while doing so. - if let Some(ingester) = ingester_opt { - if let Err(error) = wait_for_ingester_decommission(ingester).await { - error!("failed to decommission ingester gracefully: {:?}", error); - } + if let Some(ingester) = &ingester_opt + && let Err(error) = wait_for_ingester_decommission(ingester, Duration::from_secs(300)).await + { + error!("failed to decommission ingester gracefully: {:?}", error); } let actor_exit_statuses = universe.quit().await; @@ -561,10 +559,10 @@ pub async fn serve_quickwit( // Setup the indexer pool to track cluster changes. setup_indexer_pool( - &node_config, cluster.change_stream(), - indexer_pool, indexing_service_opt.clone(), + indexer_pool, + node_config.grpc_config.max_message_size, ); // Setup ingest service v2. @@ -635,10 +633,38 @@ pub async fn serve_quickwit( None }; - let searcher_context = Arc::new(SearcherContext::new( - node_config.searcher_config.clone(), - split_cache_opt, - )); + // Initialize Lambda invoker if enabled and searcher service is running + let searcher_context = if node_config.is_service_enabled(QuickwitService::Searcher) { + if let Some(lambda_config) = &node_config.searcher_config.lambda { + #[cfg(feature = "lambda")] + { + info!("initializing AWS Lambda invoker for search"); + warn!("offloading to lambda is EXPERIMENTAL. Use at your own risk"); + let invoker = + quickwit_lambda_client::try_get_or_deploy_invoker(lambda_config).await?; + Arc::new(SearcherContext::new( + node_config.searcher_config.clone(), + split_cache_opt, + Some(invoker), + )) + } + #[cfg(not(feature = "lambda"))] + { + let _ = lambda_config; + bail!("lambda support is statically disabled, but enabled in configuration"); + } + } else { + Arc::new(SearcherContext::new_without_invoker( + node_config.searcher_config.clone(), + split_cache_opt, + )) + } + } else { + Arc::new(SearcherContext::new_without_invoker( + node_config.searcher_config.clone(), + split_cache_opt, + )) + }; let (search_job_placer, search_service) = setup_searcher( &node_config, @@ -909,8 +935,12 @@ async fn setup_ingest_v2( ingester_pool.clone(), replication_factor, event_broker.clone(), + node_config.availability_zone.clone(), ); ingest_router.subscribe(); + setup_ingester_capacity_update_listener(cluster.clone(), event_broker.clone()) + .await + .forever(); let ingest_router_service = IngestRouterServiceClient::tower() .stack_layer(INGEST_GRPC_SERVER_METRICS_LAYER.clone()) @@ -953,64 +983,128 @@ async fn setup_ingest_v2( } else { None }; - // Setup ingester pool change stream. - let ingester_opt_clone = ingester_opt.clone(); - let max_message_size = node_config.grpc_config.max_message_size; - let ingester_change_stream = cluster.change_stream().filter_map(move |cluster_change| { - let ingester_opt_clone_clone = ingester_opt_clone.clone(); + setup_ingester_pool( + cluster.change_stream(), + ingester_opt.clone(), + ingester_pool, + grpc_compression_encoding_opt, + node_config.grpc_config.max_message_size, + ); + Ok((ingest_router, ingest_router_service, ingester_opt)) +} + +fn setup_ingester_pool( + cluster_change_stream: ClusterChangeStream, + ingester_opt: Option, + ingester_pool: IngesterPool, + grpc_compression_encoding_opt: Option, + grpc_max_message_size: ByteSize, +) { + let ingester_change_stream = cluster_change_stream.filter_map(move |cluster_change| { + let ingester_opt_clone = ingester_opt.clone(); Box::pin(async move { match cluster_change { ClusterChange::Add(node) if node.is_indexer() => { - let chitchat_id = node.chitchat_id(); - info!( - node_id = chitchat_id.node_id, - generation_id = chitchat_id.generation_id, - "adding node `{}` to ingester pool", - chitchat_id.node_id, + let change = build_ingester_insert_change( + &node, + ingester_opt_clone, + grpc_max_message_size, + grpc_compression_encoding_opt, ); - let node_id: NodeId = node.node_id().into(); - - if node.is_self_node() { - // Here, since the service is available locally, we bypass the network stack - // and use the instance directly. However, we still want client-side - // metrics, so we use both metrics layers. - let ingester = ingester_opt_clone_clone - .expect("ingester service should be initialized"); - let ingester_service = ingester_service_layer_stack( - IngesterServiceClient::tower() - .stack_layer(INGEST_GRPC_CLIENT_METRICS_LAYER.clone()), - ) - .build(ingester); - Some(Change::Insert(node_id, ingester_service)) - } else { - let ingester_service = IngesterServiceClient::tower() - .stack_layer(INGEST_GRPC_CLIENT_METRICS_LAYER.clone()) - .stack_layer(TimeoutLayer::new(GRPC_INGESTER_SERVICE_TIMEOUT)) - .build_from_channel( - node.grpc_advertise_addr(), - node.channel(), - max_message_size, - grpc_compression_encoding_opt, - ); - Some(Change::Insert(node_id, ingester_service)) - } + Some(change) } - ClusterChange::Remove(node) if node.is_indexer() => { - let chitchat_id = node.chitchat_id(); - info!( - node_id = chitchat_id.node_id, - generation_id = chitchat_id.generation_id, - "removing node `{}` from ingester pool", - chitchat_id.node_id, + // only update the ingester pool when the ingester status changes, to avoid + // unnecessary churn + ClusterChange::Update { previous, updated } + if updated.is_indexer() + && previous.ingester_status() != updated.ingester_status() => + { + let change = build_ingester_insert_change( + &updated, + ingester_opt_clone, + grpc_max_message_size, + grpc_compression_encoding_opt, ); - Some(Change::Remove(node.node_id().into())) + Some(change) + } + ClusterChange::Remove(node) if node.is_indexer() => { + let change = build_ingester_remove_change(&node); + Some(change) } _ => None, } }) }); ingester_pool.listen_for_changes(ingester_change_stream); - Ok((ingest_router, ingest_router_service, ingester_opt)) +} + +fn build_ingester_insert_change( + node: &ClusterNode, + ingester_opt: Option, + grpc_max_message_size: ByteSize, + grpc_compression_encoding_opt: Option, +) -> Change { + let chitchat_id = node.chitchat_id(); + info!( + node_id = chitchat_id.node_id, + generation_id = chitchat_id.generation_id, + "adding/updating node `{}` with ingester status `{}` to ingester pool", + chitchat_id.node_id, + node.ingester_status(), + ); + let node_id: NodeId = node.node_id().into(); + let ingester_service = build_ingester_service( + node, + ingester_opt, + grpc_max_message_size, + grpc_compression_encoding_opt, + ); + let pool_entry = IngesterPoolEntry { + client: ingester_service, + status: node.ingester_status(), + availability_zone: node.availability_zone().map(|az| az.to_string()), + }; + Change::Insert(node_id, pool_entry) +} + +fn build_ingester_remove_change(node: &ClusterNode) -> Change { + let chitchat_id = node.chitchat_id(); + info!( + node_id = chitchat_id.node_id, + generation_id = chitchat_id.generation_id, + "removing node `{}` from ingester pool", + chitchat_id.node_id, + ); + let node_id: NodeId = node.node_id().into(); + Change::Remove(node_id) +} + +fn build_ingester_service( + node: &ClusterNode, + ingester_opt: Option, + max_message_size: ByteSize, + grpc_compression_encoding_opt: Option, +) -> IngesterServiceClient { + if node.is_self_node() { + // Here, since the service is available locally, we bypass the network stack + // and use the instance directly. However, we still want client-side + // metrics, so we use both metrics layers. + let ingester = ingester_opt.expect("ingester service should be initialized"); + let service = ingester_service_layer_stack( + IngesterServiceClient::tower().stack_layer(INGEST_GRPC_CLIENT_METRICS_LAYER.clone()), + ) + .build(ingester); + return service; + } + IngesterServiceClient::tower() + .stack_layer(INGEST_GRPC_CLIENT_METRICS_LAYER.clone()) + .stack_layer(TimeoutLayer::new(GRPC_INGESTER_SERVICE_TIMEOUT)) + .build_from_channel( + node.grpc_advertise_addr(), + node.channel(), + max_message_size, + grpc_compression_encoding_opt, + ) } async fn setup_searcher( @@ -1022,6 +1116,7 @@ async fn setup_searcher( ) -> anyhow::Result<(SearchJobPlacer, Arc)> { let searcher_pool = SearcherPool::default(); let search_job_placer = SearchJobPlacer::new(searcher_pool.clone()); + let search_service = start_searcher_service( metastore, storage_resolver, @@ -1133,90 +1228,26 @@ async fn setup_control_plane( } fn setup_indexer_pool( - node_config: &NodeConfig, cluster_change_stream: ClusterChangeStream, - indexer_pool: IndexerPool, indexing_service_opt: Option>, + indexer_pool: IndexerPool, + grpc_max_message_size: ByteSize, ) { - let max_message_size = node_config.grpc_config.max_message_size; let indexer_change_stream = cluster_change_stream.filter_map(move |cluster_change| { let indexing_service_clone_opt = indexing_service_opt.clone(); Box::pin(async move { - match &cluster_change { + match cluster_change { ClusterChange::Add(node) if node.is_indexer() => { - let chitchat_id = node.chitchat_id(); - info!( - node_id = chitchat_id.node_id, - generation_id = chitchat_id.generation_id, - "adding node `{}` to indexer pool", - chitchat_id.node_id, + let change = build_indexer_insert_change( + &node, + indexing_service_clone_opt, + grpc_max_message_size, ); - } - _ => {} - }; - match cluster_change { - ClusterChange::Add(node) | ClusterChange::Update(node) if node.is_indexer() => { - let node_id = node.node_id().to_owned(); - let indexing_tasks = node.indexing_tasks().to_vec(); - let indexing_capacity = node.indexing_capacity(); - - if node.is_self_node() { - // Here, since the service is available locally, we bypass the network stack - // and use the mailbox directly. However, we still want client-side metrics, - // so we use both metrics layers. - let indexing_service_mailbox = indexing_service_clone_opt - .expect("indexing service should be initialized"); - // These layers apply to all the RPCs of the indexing service. - let shared_layers = ServiceBuilder::new() - .layer(INDEXING_GRPC_CLIENT_METRICS_LAYER.clone()) - .layer(INDEXING_GRPC_SERVER_METRICS_LAYER.clone()) - .into_inner(); - let client = IndexingServiceClient::tower() - .stack_layer(shared_layers) - .build_from_mailbox(indexing_service_mailbox); - let change = Change::Insert( - node_id.clone(), - IndexerNodeInfo { - node_id, - generation_id: node.chitchat_id().generation_id, - client, - indexing_tasks, - indexing_capacity, - }, - ); - Some(change) - } else { - let client = IndexingServiceClient::tower() - .stack_layer(INDEXING_GRPC_CLIENT_METRICS_LAYER.clone()) - .stack_layer(TimeoutLayer::new(GRPC_INDEXING_SERVICE_TIMEOUT)) - .build_from_channel( - node.grpc_advertise_addr(), - node.channel(), - max_message_size, - None, - ); - let change = Change::Insert( - node_id.clone(), - IndexerNodeInfo { - node_id, - generation_id: node.chitchat_id().generation_id, - client, - indexing_tasks, - indexing_capacity, - }, - ); - Some(change) - } + Some(change) } ClusterChange::Remove(node) if node.is_indexer() => { - let chitchat_id = node.chitchat_id(); - info!( - node_id = chitchat_id.node_id, - generation_id = chitchat_id.generation_id, - "removing node `{}` from indexer pool", - chitchat_id.node_id, - ); - Some(Change::Remove(node.node_id().to_owned())) + let change = build_indexer_remove_change(&node); + Some(change) } _ => None, } @@ -1225,6 +1256,75 @@ fn setup_indexer_pool( indexer_pool.listen_for_changes(indexer_change_stream); } +fn build_indexer_insert_change( + node: &ClusterNode, + indexing_service_opt: Option>, + grpc_max_message_size: ByteSize, +) -> Change { + let chitchat_id = node.chitchat_id(); + info!( + node_id = chitchat_id.node_id, + generation_id = chitchat_id.generation_id, + "adding node `{}` with ingester status `{}` to indexer pool", + chitchat_id.node_id, + node.ingester_status() + ); + let node_id: NodeId = node.node_id().into(); + let client = build_indexing_service(node, indexing_service_opt, grpc_max_message_size); + Change::Insert( + node_id.clone(), + IndexerNodeInfo { + node_id, + generation_id: chitchat_id.generation_id, + client, + indexing_tasks: node.indexing_tasks().to_vec(), + indexing_capacity: node.indexing_capacity(), + }, + ) +} + +fn build_indexer_remove_change(node: &ClusterNode) -> Change { + let chitchat_id = node.chitchat_id(); + info!( + node_id = chitchat_id.node_id, + generation_id = chitchat_id.generation_id, + "removing node `{}` from indexer pool", + chitchat_id.node_id, + ); + let node_id: NodeId = node.node_id().into(); + Change::Remove(node_id) +} + +fn build_indexing_service( + node: &ClusterNode, + indexing_service_opt: Option>, + max_message_size: ByteSize, +) -> IndexingServiceClient { + if node.is_self_node() { + // Here, since the service is available locally, we bypass the network stack + // and use the mailbox directly. However, we still want client-side metrics, + // so we use both metrics layers. + let indexing_service_mailbox = + indexing_service_opt.expect("indexing service should be initialized"); + let shared_layers = ServiceBuilder::new() + .layer(INDEXING_GRPC_CLIENT_METRICS_LAYER.clone()) + .layer(INDEXING_GRPC_SERVER_METRICS_LAYER.clone()) + .into_inner(); + return IndexingServiceClient::tower() + .stack_layer(shared_layers) + .build_from_mailbox(indexing_service_mailbox); + } + IndexingServiceClient::tower() + .stack_layer(INDEXING_GRPC_CLIENT_METRICS_LAYER.clone()) + .stack_layer(TimeoutLayer::new(GRPC_INDEXING_SERVICE_TIMEOUT)) + .build_from_channel( + node.grpc_advertise_addr(), + node.channel(), + max_message_size, + None, + ) +} + fn require( val_opt: Option, ) -> impl Filter + Clone { @@ -1273,19 +1373,12 @@ async fn node_readiness_reporting_task( }; info!("REST server is ready"); - if let Some(ingester) = ingester_opt { - if let Err(error) = wait_for_ingester_status(ingester, IngesterStatus::Ready).await { - error!("failed to initialize ingester: {:?}", error); - info!("shutting down"); - return; - } - } let mut interval = tokio::time::interval(READINESS_REPORTING_INTERVAL); loop { interval.tick().await; - let new_node_ready = match metastore.check_connectivity().await { + let metastore_is_available = match metastore.check_connectivity().await { Ok(()) => { debug!(metastore_endpoints=?metastore.endpoints(), "metastore service is available"); true @@ -1295,6 +1388,23 @@ async fn node_readiness_reporting_task( false } }; + let ingester_is_available = if let Some(ingester) = &ingester_opt { + match try_get_ingester_status(ingester).await { + Ok(status) => { + status == IngesterStatus::Initializing || status != IngesterStatus::Failed + } + Err(error) => { + // If we couldn't get the ingester status, it's not looking good, so we set the + // node to not ready. + error!(%error, "failed to get ingester status"); + false + } + } + } else { + true + }; + let new_node_ready = metastore_is_available && ingester_is_available; + if new_node_ready != node_ready { node_ready = new_node_ready; cluster.set_self_node_readiness(node_ready).await; @@ -1355,30 +1465,15 @@ async fn check_cluster_configuration( Ok(()) } -pub mod lambda_search_api { - pub use crate::elasticsearch_api::{ - es_compat_cat_indices_handler, es_compat_index_cat_indices_handler, - es_compat_index_count_handler, es_compat_index_field_capabilities_handler, - es_compat_index_multi_search_handler, es_compat_index_search_handler, - es_compat_index_stats_handler, es_compat_resolve_index_handler, es_compat_scroll_handler, - es_compat_search_handler, es_compat_stats_handler, - }; - pub use crate::index_api::get_index_metadata_handler; - pub use crate::rest::recover_fn; - pub use crate::search_api::{search_get_handler, search_post_handler}; -} - #[cfg(test)] mod tests { use quickwit_cluster::{ChannelTransport, ClusterNode, create_cluster_for_test}; - use quickwit_common::ServiceStream; use quickwit_common::uri::Uri; + use quickwit_common::{ServiceStream, assert_eventually}; use quickwit_config::SearcherConfig; use quickwit_metastore::{IndexMetadata, metastore_for_test}; - use quickwit_proto::indexing::IndexingTask; use quickwit_proto::ingest::ingester::{MockIngesterService, ObservationMessage}; use quickwit_proto::metastore::{ListIndexesMetadataResponse, MockMetastoreService}; - use quickwit_proto::types::{IndexUid, PipelineUid}; use quickwit_search::Job; use tokio::sync::watch; use tonic::transport::{Channel, Server}; @@ -1490,16 +1585,14 @@ mod tests { metastore_readiness_tx.send(true).unwrap(); ingester_status_tx.send(IngesterStatus::Ready).unwrap(); - tokio::time::sleep(Duration::from_millis(25)).await; - assert!(cluster.is_self_node_ready().await); + assert_eventually!(cluster.is_self_node_ready().await); let request = tonic::Request::new(HealthCheckRequest::default()); let response = health_client.check(request).await.unwrap().into_inner(); assert_eq!(response.status(), ServingStatus::Serving.into()); metastore_readiness_tx.send(false).unwrap(); - tokio::time::sleep(Duration::from_millis(25)).await; - assert!(!cluster.is_self_node_ready().await); + assert_eventually!(!cluster.is_self_node_ready().await); let request = tonic::Request::new(HealthCheckRequest::default()); let response = health_client.check(request).await.unwrap().into_inner(); @@ -1517,53 +1610,32 @@ mod tests { ClusterChangeStream::new_unbounded(); let indexer_pool = IndexerPool::default(); setup_indexer_pool( - &node_config, cluster_change_stream, - indexer_pool.clone(), Some(indexing_service_mailbox), + indexer_pool.clone(), + node_config.grpc_config.max_message_size, ); - let new_indexer_node = - ClusterNode::for_test("test-indexer-node", 1, true, &["indexer"], &[]).await; - cluster_change_stream_tx - .send(ClusterChange::Add(new_indexer_node)) - .unwrap(); - tokio::time::sleep(Duration::from_millis(1)).await; - - assert_eq!(indexer_pool.len(), 1); - - let new_indexer_node_info = indexer_pool.get("test-indexer-node").unwrap(); - assert!(new_indexer_node_info.indexing_tasks.is_empty()); - - let new_indexing_task = IndexingTask { - pipeline_uid: Some(PipelineUid::for_test(0u128)), - index_uid: Some(IndexUid::for_test("test-index", 0)), - source_id: "test-source".to_string(), - shard_ids: Vec::new(), - params_fingerprint: 0, - }; - let updated_indexer_node = ClusterNode::for_test( + // adding a indexer node refreshes the indexer pool + let new_indexer_node = ClusterNode::for_test( "test-indexer-node", 1, true, &["indexer"], - &[new_indexing_task.clone()], + &[], + IngesterStatus::Ready, ) .await; cluster_change_stream_tx - .send(ClusterChange::Update(updated_indexer_node.clone())) + .send(ClusterChange::Add(new_indexer_node.clone())) .unwrap(); tokio::time::sleep(Duration::from_millis(1)).await; - let updated_indexer_node_info = indexer_pool.get("test-indexer-node").unwrap(); - assert_eq!(updated_indexer_node_info.indexing_tasks.len(), 1); - assert_eq!( - updated_indexer_node_info.indexing_tasks[0], - new_indexing_task - ); + assert_eq!(indexer_pool.len(), 1); + // removing an indexer node refreshes the indexer pool cluster_change_stream_tx - .send(ClusterChange::Remove(updated_indexer_node)) + .send(ClusterChange::Remove(new_indexer_node)) .unwrap(); tokio::time::sleep(Duration::from_millis(1)).await; @@ -1573,7 +1645,10 @@ mod tests { #[tokio::test] async fn test_setup_searcher() { let node_config = NodeConfig::for_test(); - let searcher_context = Arc::new(SearcherContext::new(SearcherConfig::default(), None)); + let searcher_context = Arc::new(SearcherContext::new_without_invoker( + SearcherConfig::default(), + None, + )); let metastore = metastore_for_test(); let (change_stream, change_stream_tx) = ClusterChangeStream::new_unbounded(); let storage_resolver = StorageResolver::unconfigured(); @@ -1603,7 +1678,15 @@ mod tests { .await .unwrap_err(); - let self_node = ClusterNode::for_test("node-1", 1337, true, &["searcher"], &[]).await; + let self_node = ClusterNode::for_test( + "node-1", + 1337, + true, + &["searcher"], + &[], + IngesterStatus::Ready, + ) + .await; change_stream_tx .send(ClusterChange::Add(self_node.clone())) .unwrap(); @@ -1619,7 +1702,15 @@ mod tests { .send(ClusterChange::Remove(self_node)) .unwrap(); - let node = ClusterNode::for_test("node-1", 1337, false, &["searcher"], &[]).await; + let node = ClusterNode::for_test( + "node-1", + 1337, + false, + &["searcher"], + &[], + IngesterStatus::Ready, + ) + .await; change_stream_tx.send(ClusterChange::Add(node)).unwrap(); tokio::time::sleep(Duration::from_millis(1)).await; @@ -1629,4 +1720,96 @@ mod tests { .unwrap(); assert!(!searcher_client.is_local()); } + + #[tokio::test] + async fn test_setup_ingester_pool() { + let (cluster_change_stream, cluster_change_stream_tx) = + ClusterChangeStream::new_unbounded(); + let ingester_pool = IngesterPool::default(); + setup_ingester_pool( + cluster_change_stream, + None::, + ingester_pool.clone(), + None, + ByteSize::mib(20), + ); + + // Add an indexer node with IngesterStatus::Initializing. + let new_node = ClusterNode::for_test( + "test-ingester-node", + 1, + false, + &["indexer"], + &[], + IngesterStatus::Initializing, + ) + .await; + cluster_change_stream_tx + .send(ClusterChange::Add(new_node.clone())) + .unwrap(); + tokio::time::sleep(Duration::from_millis(1)).await; + + assert_eq!(ingester_pool.len(), 1); + let pool_entry = ingester_pool + .get(&NodeId::from("test-ingester-node")) + .unwrap(); + assert_eq!(pool_entry.status, IngesterStatus::Initializing); + + // Update the node: ingester status transitions from Initializing to Ready. + let updated_node = ClusterNode::for_test( + "test-ingester-node", + 1, + false, + &["indexer"], + &[], + IngesterStatus::Ready, + ) + .await; + cluster_change_stream_tx + .send(ClusterChange::Update { + previous: new_node.clone(), + updated: updated_node.clone(), + }) + .unwrap(); + tokio::time::sleep(Duration::from_millis(1)).await; + + assert_eq!(ingester_pool.len(), 1); + let pool_entry = ingester_pool + .get(&NodeId::from("test-ingester-node")) + .unwrap(); + assert_eq!(pool_entry.status, IngesterStatus::Ready); + + // Update the node: ingester status transitions from Ready to Decommissioning. + let updated_node_2 = ClusterNode::for_test( + "test-ingester-node", + 1, + false, + &["indexer"], + &[], + IngesterStatus::Decommissioning, + ) + .await; + cluster_change_stream_tx + .send(ClusterChange::Update { + previous: updated_node.clone(), + updated: updated_node_2.clone(), + }) + .unwrap(); + tokio::time::sleep(Duration::from_millis(1)).await; + + // The node should still be in the pool with updated status. + assert_eq!(ingester_pool.len(), 1); + let pool_entry = ingester_pool + .get(&NodeId::from("test-ingester-node")) + .unwrap(); + assert_eq!(pool_entry.status, IngesterStatus::Decommissioning); + + // Remove the node. + cluster_change_stream_tx + .send(ClusterChange::Remove(updated_node)) + .unwrap(); + tokio::time::sleep(Duration::from_millis(1)).await; + + assert!(ingester_pool.is_empty()); + } } diff --git a/quickwit/quickwit-serve/src/load_shield.rs b/quickwit/quickwit-serve/src/load_shield.rs index 477c6e73d79..568ffabd53b 100644 --- a/quickwit/quickwit-serve/src/load_shield.rs +++ b/quickwit/quickwit-serve/src/load_shield.rs @@ -38,9 +38,9 @@ impl LoadShield { let max_in_flight_env_key = format!("QW_{endpoint_group_uppercase}_MAX_IN_FLIGHT"); let max_concurrency_env_key = format!("QW_{endpoint_group_uppercase}_MAX_CONCURRENCY"); let max_in_flight_opt: Option = - quickwit_common::get_from_env_opt(&max_in_flight_env_key); + quickwit_common::get_from_env_opt(&max_in_flight_env_key, false); let max_concurrency_opt: Option = - quickwit_common::get_from_env_opt(&max_concurrency_env_key); + quickwit_common::get_from_env_opt(&max_concurrency_env_key, false); let in_flight_semaphore_opt = max_in_flight_opt.map(Semaphore::new); let concurrency_semaphore_opt = max_concurrency_opt.map(Semaphore::new); let pending_gauge = crate::metrics::SERVE_METRICS diff --git a/quickwit/quickwit-serve/src/openapi.rs b/quickwit/quickwit-serve/src/openapi.rs index f54c4147c92..a7677b70c37 100644 --- a/quickwit/quickwit-serve/src/openapi.rs +++ b/quickwit/quickwit-serve/src/openapi.rs @@ -300,13 +300,13 @@ mod openapi_schema_tests { } RefOr::T(schema) => { for (_, content) in &schema.content { - if let RefOr::Ref(r) = &content.schema { - if !schema_lookup.contains(&r.ref_location) { - resolve_once.push(CheckResolve::new( - r.ref_location.clone(), - schema_item.clone(), - )); - } + if let RefOr::Ref(r) = &content.schema + && !schema_lookup.contains(&r.ref_location) + { + resolve_once.push(CheckResolve::new( + r.ref_location.clone(), + schema_item.clone(), + )); } } schema_lookup.insert(path); @@ -383,17 +383,15 @@ mod openapi_schema_tests { } } - if let Some(ref props) = object.additional_properties { - if let AdditionalProperties::RefOr(ref r) = **props { - match r { - RefOr::Ref(r) => resolve_once.push(CheckResolve::new( - r.ref_location.clone(), - parent_location.to_owned(), - )), - RefOr::T(schema) => { - resolve_schema(resolve_once, parent_location, schema) - } - } + if let Some(ref props) = object.additional_properties + && let AdditionalProperties::RefOr(ref r) = **props + { + match r { + RefOr::Ref(r) => resolve_once.push(CheckResolve::new( + r.ref_location.clone(), + parent_location.to_owned(), + )), + RefOr::T(schema) => resolve_schema(resolve_once, parent_location, schema), } } } @@ -459,28 +457,28 @@ mod openapi_schema_tests { } } - if let Some(ref props) = object.additional_properties { - if let AdditionalProperties::RefOr(ref r) = **props { - match r { - RefOr::Ref(r) => { - if !schemas_lookup.contains(&r.ref_location) { - errors.push(( - parent_location.to_string(), - method.to_string(), - path, - String::new(), - )); - } + if let Some(ref props) = object.additional_properties + && let AdditionalProperties::RefOr(ref r) = **props + { + match r { + RefOr::Ref(r) => { + if !schemas_lookup.contains(&r.ref_location) { + errors.push(( + parent_location.to_string(), + method.to_string(), + path, + String::new(), + )); } - RefOr::T(schema) => check_schema( - method, - path, - schemas_lookup, - errors, - parent_location, - schema, - ), } + RefOr::T(schema) => check_schema( + method, + path, + schemas_lookup, + errors, + parent_location, + schema, + ), } } } diff --git a/quickwit/quickwit-serve/src/otlp_api/rest_handler.rs b/quickwit/quickwit-serve/src/otlp_api/rest_handler.rs index 1654a840dad..4ec47c15847 100644 --- a/quickwit/quickwit-serve/src/otlp_api/rest_handler.rs +++ b/quickwit/quickwit-serve/src/otlp_api/rest_handler.rs @@ -25,7 +25,6 @@ use quickwit_proto::opentelemetry::proto::collector::trace::v1::{ use quickwit_proto::types::IndexId; use quickwit_proto::{ServiceError, ServiceErrorCode, tonic}; use serde::{self, Serialize}; -use tracing::error; use warp::{Filter, Rejection}; use crate::decompression::get_body_bytes; diff --git a/quickwit/quickwit-serve/src/rate_modulator.rs b/quickwit/quickwit-serve/src/rate_modulator.rs index 3052d6e7853..2533b2be0d7 100644 --- a/quickwit/quickwit-serve/src/rate_modulator.rs +++ b/quickwit/quickwit-serve/src/rate_modulator.rs @@ -25,7 +25,8 @@ pub struct RateModulator { } impl RateModulator -where R: Rate +where + R: Rate, { /// Creates a new [`RateModulator`] instance. /// @@ -48,7 +49,8 @@ where R: Rate } impl Rate for RateModulator -where R: Rate +where + R: Rate, { fn work(&self) -> u64 { let memory_usage_ratio = self.memory_capacity.usage_ratio(); diff --git a/quickwit/quickwit-serve/src/rest.rs b/quickwit/quickwit-serve/src/rest.rs index 1f3a33bb456..6ab2b42f413 100644 --- a/quickwit/quickwit-serve/src/rest.rs +++ b/quickwit/quickwit-serve/src/rest.rs @@ -13,23 +13,26 @@ // limitations under the License. use std::fmt::Formatter; -use std::pin::Pin; +use std::io; use std::sync::Arc; +use hyper_util::rt::{TokioExecutor, TokioIo}; +use hyper_util::server::conn::auto::Builder; +use hyper_util::service::TowerToHyperService; use quickwit_common::tower::BoxFutureInfaillible; use quickwit_config::{disable_ingest_v1, enable_ingest_v2}; use quickwit_search::SearchService; -use tokio::net::TcpListener; +use tokio::io::{AsyncRead, AsyncWrite}; +use tokio::net::{TcpListener, TcpStream}; +use tokio_rustls::TlsAcceptor; +use tokio_util::either::Either; use tower::ServiceBuilder; -use tower::make::Shared; use tower_http::compression::CompressionLayer; use tower_http::compression::predicate::{NotForContentType, Predicate, SizeAbove}; -use tower_http::cors::CorsLayer; +use tower_http::cors::{AllowOrigin, CorsLayer}; use tracing::{error, info}; use warp::filters::log::Info; use warp::hyper::http::HeaderValue; -use warp::hyper::server::accept::Accept; -use warp::hyper::server::conn::AddrIncoming; use warp::hyper::{Method, StatusCode, http}; use warp::{Filter, Rejection, Reply, redirect}; @@ -49,7 +52,6 @@ use crate::otlp_api::otlp_ingest_api_handlers; use crate::rest_api_response::{RestApiError, RestApiResponse}; use crate::search_api::{ search_get_handler, search_plan_get_handler, search_plan_post_handler, search_post_handler, - search_stream_handler, }; use crate::template_api::index_template_api_handlers; use crate::ui_handler::ui_handler; @@ -76,17 +78,6 @@ impl std::fmt::Display for TooManyRequests { } } -#[derive(Debug)] -pub struct InternalError(pub String); - -impl warp::reject::Reject for InternalError {} - -impl std::fmt::Display for InternalError { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - write!(f, "internal error: {}", self.0) - } -} - /// Env variable key to define the minimum size above which a response should be compressed. /// If unset, no compression is applied. const QW_MINIMUM_COMPRESSION_SIZE_KEY: &str = "QW_MINIMUM_COMPRESSION_SIZE"; @@ -98,12 +89,12 @@ struct CompressionPredicate { impl CompressionPredicate { fn from_env() -> CompressionPredicate { - let minimum_compression_size_opt: Option = quickwit_common::get_from_env_opt::( - QW_MINIMUM_COMPRESSION_SIZE_KEY, - ) - .map(|minimum_compression_size: usize| { - u16::try_from(minimum_compression_size).unwrap_or(u16::MAX) - }); + let minimum_compression_size_opt: Option = + quickwit_common::get_from_env_opt::(QW_MINIMUM_COMPRESSION_SIZE_KEY, false).map( + |minimum_compression_size: usize| { + u16::try_from(minimum_compression_size).unwrap_or(u16::MAX) + }, + ); let size_above_opt = minimum_compression_size_opt.map(SizeAbove::new); CompressionPredicate { size_above_opt } } @@ -111,7 +102,9 @@ impl CompressionPredicate { impl Predicate for CompressionPredicate { fn should_compress(&self, response: &http::Response) -> bool - where B: warp::hyper::body::HttpBody { + where + B: http_body::Body, + { if let Some(size_above) = self.size_above_opt { size_above.should_compress(response) } else { @@ -120,6 +113,20 @@ impl Predicate for CompressionPredicate { } } +async fn apply_tls_if_necessary( + tcp_stream: TcpStream, + tls_acceptor_opt: &Option, +) -> io::Result { + let Some(tls_acceptor) = &tls_acceptor_opt else { + return Ok(Either::Right(tcp_stream)); + }; + let tls_stream_res = tls_acceptor + .accept(tcp_stream) + .await + .inspect_err(|err| error!("failed to perform tls handshake: {err:#}"))?; + Ok(Either::Left(tls_stream_res)) +} + /// Starts REST services. pub(crate) async fn start_rest_server( tcp_listener: TcpListener, @@ -220,32 +227,54 @@ pub(crate) async fn start_rest_server( "starting REST server listening on {rest_listen_addr}" ); - let incoming = AddrIncoming::from_listener(tcp_listener)?; + let service = TowerToHyperService::new(service); - let maybe_tls_incoming = + let server = Builder::new(TokioExecutor::new()); + let graceful = hyper_util::server::graceful::GracefulShutdown::new(); + let mut shutdown_signal = std::pin::pin!(shutdown_signal); + readiness_trigger.await; + + let tls_acceptor_opt: Option = if let Some(tls_config) = &quickwit_services.node_config.rest_config.tls { let rustls_config = tls::make_rustls_config(tls_config)?; - EitherIncoming::Left(tls::TlsAcceptor::new(rustls_config, incoming)) + Some(TlsAcceptor::from(rustls_config)) } else { - EitherIncoming::Right(incoming) + None }; - // `graceful_shutdown()` seems to be blocking in presence of existing connections. - // The following approach of dropping the serve supposedly is not bullet proof, but it seems to - // work in our unit test. - // - // See more of the discussion here: - // https://github.com/hyperium/hyper/issues/2386 - - let serve_fut = async move { + loop { tokio::select! { - res = warp::hyper::Server::builder(maybe_tls_incoming).serve(Shared::new(service)) => { res } - _ = shutdown_signal => { Ok(()) } + tcp_accept_res = tcp_listener.accept() => { + let tcp_stream = match tcp_accept_res { + Ok((tcp_stream, _remote_addr)) => tcp_stream, + Err(err) => { + error!("failed to accept connection: {err:#}"); + continue; + } + }; + + let Ok(tcp_or_tls_stream) = apply_tls_if_necessary(tcp_stream, &tls_acceptor_opt).await else { + continue; + }; + + let serve_fut = server.serve_connection_with_upgrades(TokioIo::new(tcp_or_tls_stream), service.clone()); + let serve_with_shutdown_fut = graceful.watch(serve_fut.into_owned()); + tokio::spawn(async move { + if let Err(err) = serve_with_shutdown_fut.await { + error!("failed to serve connection: {err:#}"); + } + }); + }, + _ = &mut shutdown_signal => { + info!("REST server shutdown signal received"); + break; + } } - }; + } + + graceful.shutdown().await; + info!("gracefully shutdown"); - let (serve_res, _trigger_res) = tokio::join!(serve_fut, readiness_trigger); - serve_res?; Ok(()) } @@ -256,7 +285,6 @@ fn search_routes( .or(search_post_handler(search_service.clone())) .or(search_plan_get_handler(search_service.clone())) .or(search_plan_post_handler(search_service.clone())) - .or(search_stream_handler(search_service)) .recover(recover_fn) .boxed() } @@ -454,6 +482,25 @@ fn get_status_with_error(rejection: Rejection) -> Result CorsLayer { + let debug_mode = quickwit_common::get_bool_from_env("QW_ENABLE_CORS_DEBUG", false); + if debug_mode { + info!("CORS debug mode is enabled, localhost and 127.0.0.1 origins will be allowed"); + return CorsLayer::new() + .allow_methods([ + Method::GET, + Method::POST, + Method::PUT, + Method::PATCH, + Method::DELETE, + ]) + .allow_origin(AllowOrigin::predicate(|origin, _parts| { + [b"https://localhost:", b"https://127.0.0.1:"] + .iter() + .any(|prefix| origin.as_bytes().starts_with(*prefix)) + })) + .allow_headers([http::header::CONTENT_TYPE]); + } + let mut cors = CorsLayer::new().allow_methods([ Method::GET, Method::POST, @@ -482,158 +529,37 @@ fn build_cors(cors_origins: &[String]) -> CorsLayer { mod tls { // most of this module is copied from hyper-tls examples, licensed under Apache 2.0, MIT or ISC - use std::future::Future; - use std::pin::Pin; use std::sync::Arc; - use std::task::{Context, Poll, ready}; use std::vec::Vec; use std::{fs, io}; use quickwit_config::TlsConfig; - use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; + use rustls::pki_types::{CertificateDer, PrivateKeyDer}; use tokio_rustls::rustls::ServerConfig; - use warp::hyper::server::accept::Accept; - use warp::hyper::server::conn::{AddrIncoming, AddrStream}; fn io_error(error: String) -> io::Error { io::Error::other(error) } // Load public certificate from file. - fn load_certs(filename: &str) -> io::Result> { + fn load_certs(filename: &str) -> io::Result>> { // Open certificate file. - let certfile = fs::read(filename) + let certfile = fs::File::open(filename) .map_err(|error| io_error(format!("failed to open {filename}: {error}")))?; - + let mut reader = io::BufReader::new(certfile); // Load and return certificate. - let certs = rustls_pemfile::certs(&mut certfile.as_ref()) - .map_err(|_| io_error("failed to load certificate".to_string()))?; - Ok(certs.into_iter().map(rustls::Certificate).collect()) + rustls_pemfile::certs(&mut reader).collect() } // Load private key from file. - fn load_private_key(filename: &str) -> io::Result { + fn load_private_key(filename: &str) -> io::Result> { // Open keyfile. - let keyfile = fs::read(filename) + let keyfile = fs::File::open(filename) .map_err(|error| io_error(format!("failed to open {filename}: {error}")))?; + let mut reader = io::BufReader::new(keyfile); // Load and return a single private key. - let keys = rustls_pemfile::pkcs8_private_keys(&mut keyfile.as_ref()) - .map_err(|_| io_error("failed to load private key".to_string()))?; - - if keys.len() != 1 { - return Err(io_error(format!( - "expected a single private key, got {}", - keys.len() - ))); - } - - Ok(rustls::PrivateKey(keys[0].clone())) - } - - pub struct TlsAcceptor { - config: Arc, - incoming: AddrIncoming, - } - - impl TlsAcceptor { - pub fn new(config: Arc, incoming: AddrIncoming) -> TlsAcceptor { - TlsAcceptor { config, incoming } - } - } - - impl Accept for TlsAcceptor { - type Conn = TlsStream; - type Error = io::Error; - - fn poll_accept( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll>> { - let pin = self.get_mut(); - match ready!(Pin::new(&mut pin.incoming).poll_accept(cx)) { - Some(Ok(sock)) => Poll::Ready(Some(Ok(TlsStream::new(sock, pin.config.clone())))), - Some(Err(e)) => Poll::Ready(Some(Err(e))), - None => Poll::Ready(None), - } - } - } - - enum State { - Handshaking(tokio_rustls::Accept), - Streaming(tokio_rustls::server::TlsStream), - } - - // tokio_rustls::server::TlsStream doesn't expose constructor methods, - // so we have to TlsAcceptor::accept and handshake to have access to it - // TlsStream implements AsyncRead/AsyncWrite handshaking tokio_rustls::Accept first - pub struct TlsStream { - state: State, - } - - impl TlsStream { - fn new(stream: AddrStream, config: Arc) -> TlsStream { - let accept = tokio_rustls::TlsAcceptor::from(config).accept(stream); - TlsStream { - state: State::Handshaking(accept), - } - } - } - - impl AsyncRead for TlsStream { - fn poll_read( - self: Pin<&mut Self>, - cx: &mut Context, - buf: &mut ReadBuf, - ) -> Poll> { - let pin = self.get_mut(); - match pin.state { - State::Handshaking(ref mut accept) => match ready!(Pin::new(accept).poll(cx)) { - Ok(mut stream) => { - let result = Pin::new(&mut stream).poll_read(cx, buf); - pin.state = State::Streaming(stream); - result - } - Err(err) => Poll::Ready(Err(err)), - }, - State::Streaming(ref mut stream) => Pin::new(stream).poll_read(cx, buf), - } - } - } - - impl AsyncWrite for TlsStream { - fn poll_write( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &[u8], - ) -> Poll> { - let pin = self.get_mut(); - match pin.state { - State::Handshaking(ref mut accept) => match ready!(Pin::new(accept).poll(cx)) { - Ok(mut stream) => { - let result = Pin::new(&mut stream).poll_write(cx, buf); - pin.state = State::Streaming(stream); - result - } - Err(err) => Poll::Ready(Err(err)), - }, - State::Streaming(ref mut stream) => Pin::new(stream).poll_write(cx, buf), - } - } - - fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - match self.state { - State::Handshaking(_) => Poll::Ready(Ok(())), - State::Streaming(ref mut stream) => Pin::new(stream).poll_flush(cx), - } - } - - fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - match self.state { - State::Handshaking(_) => Poll::Ready(Ok(())), - State::Streaming(ref mut stream) => Pin::new(stream).poll_shutdown(cx), - } - } + rustls_pemfile::private_key(&mut reader).map(|key| key.unwrap()) } pub fn make_rustls_config(config: &TlsConfig) -> anyhow::Result> { @@ -647,7 +573,6 @@ mod tls { } let mut cfg = rustls::ServerConfig::builder() - .with_safe_defaults() .with_no_client_auth() .with_single_cert(certs, key) .map_err(|error| io_error(error.to_string()))?; @@ -657,50 +582,6 @@ mod tls { } } -enum EitherIncoming { - Left(L), - Right(R), -} - -impl EitherIncoming { - pub fn as_pin_mut(self: Pin<&mut Self>) -> EitherIncoming, Pin<&mut R>> { - // SAFETY: `get_unchecked_mut` is fine because we don't move anything. - // We can use `new_unchecked` because the `inner` parts are guaranteed - // to be pinned, as they come from `self` which is pinned, and we never - // offer an unpinned `&mut A` or `&mut B` through `Pin<&mut Self>`. We - // also don't have an implementation of `Drop`, nor manual `Unpin`. - unsafe { - match self.get_unchecked_mut() { - EitherIncoming::Left(inner) => EitherIncoming::Left(Pin::new_unchecked(inner)), - EitherIncoming::Right(inner) => EitherIncoming::Right(Pin::new_unchecked(inner)), - } - } - } -} - -impl Accept for EitherIncoming -where - L: Accept, - R: Accept, -{ - type Conn = tokio_util::either::Either; - type Error = E; - - fn poll_accept( - self: Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll>> { - match self.as_pin_mut() { - EitherIncoming::Left(l) => l - .poll_accept(cx) - .map(|opt| opt.map(|res| res.map(tokio_util::either::Either::Left))), - EitherIncoming::Right(r) => r - .poll_accept(cx) - .map(|opt| opt.map(|res| res.map(tokio_util::either::Either::Right))), - } - } -} - #[cfg(test)] mod tests { use std::future::Future; diff --git a/quickwit/quickwit-serve/src/rest_api_response.rs b/quickwit/quickwit-serve/src/rest_api_response.rs index 3efd8158971..0bf56f831f0 100644 --- a/quickwit/quickwit-serve/src/rest_api_response.rs +++ b/quickwit/quickwit-serve/src/rest_api_response.rs @@ -40,9 +40,7 @@ pub(crate) fn into_rest_api_response( body_format: BodyFormat, ) -> RestApiResponse { let rest_api_result = result.map_err(|error| RestApiError { - status_code: crate::convert_status_code_to_legacy_http( - error.error_code().http_status_code(), - ), + status_code: error.error_code().http_status_code(), message: error.to_string(), }); let status_code = match &rest_api_result { diff --git a/quickwit/quickwit-serve/src/search_api/grpc_adapter.rs b/quickwit/quickwit-serve/src/search_api/grpc_adapter.rs index 65bb4eb329b..c5250ee2465 100644 --- a/quickwit/quickwit-serve/src/search_api/grpc_adapter.rs +++ b/quickwit/quickwit-serve/src/search_api/grpc_adapter.rs @@ -15,14 +15,12 @@ use std::sync::Arc; use async_trait::async_trait; -use futures::TryStreamExt; use quickwit_proto::error::convert_to_grpc_result; use quickwit_proto::search::{ - GetKvRequest, GetKvResponse, LeafListFieldsRequest, LeafSearchStreamRequest, - LeafSearchStreamResponse, ListFieldsRequest, ListFieldsResponse, ReportSplitsRequest, - ReportSplitsResponse, search_service_server as grpc, + GetKvRequest, GetKvResponse, LeafListFieldsRequest, ListFieldsRequest, ListFieldsResponse, + ReportSplitsRequest, ReportSplitsResponse, search_service_server as grpc, }; -use quickwit_proto::{GrpcServiceError, set_parent_span_from_request_metadata, tonic}; +use quickwit_proto::{set_parent_span_from_request_metadata, tonic}; use quickwit_search::SearchService; use tracing::instrument; @@ -70,29 +68,6 @@ impl grpc::SearchService for GrpcSearchAdapter { convert_to_grpc_result(fetch_docs_result) } - type LeafSearchStreamStream = std::pin::Pin< - Box< - dyn futures::Stream> - + Send - + Sync, - >, - >; - #[instrument(name = "search_adapter:leaf_search_stream", skip(self, request))] - async fn leaf_search_stream( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - set_parent_span_from_request_metadata(request.metadata()); - let leaf_search_request = request.into_inner(); - let leaf_search_result = self - .0 - .leaf_search_stream(leaf_search_request) - .await - .map_err(|error| error.into_grpc_status())? - .map_err(|error| error.into_grpc_status()); - Ok(tonic::Response::new(Box::pin(leaf_search_result))) - } - #[instrument(skip(self, request))] async fn root_list_terms( &self, diff --git a/quickwit/quickwit-serve/src/search_api/mod.rs b/quickwit/quickwit-serve/src/search_api/mod.rs index df4a15b0a66..ef1f643257c 100644 --- a/quickwit/quickwit-serve/src/search_api/mod.rs +++ b/quickwit/quickwit-serve/src/search_api/mod.rs @@ -19,130 +19,5 @@ pub use self::grpc_adapter::GrpcSearchAdapter; pub use self::rest_handler::{ SearchApi, SearchRequestQueryString, SortBy, search_get_handler, search_plan_get_handler, search_plan_post_handler, search_post_handler, search_request_from_api_request, - search_stream_handler, }; pub(crate) use self::rest_handler::{extract_index_id_patterns, extract_index_id_patterns_default}; - -#[cfg(test)] -mod tests { - use std::net::SocketAddr; - use std::sync::Arc; - - use bytesize::ByteSize; - use futures::TryStreamExt; - use quickwit_common::ServiceStream; - use quickwit_indexing::MockSplitBuilder; - use quickwit_metastore::{IndexMetadata, IndexMetadataResponseExt, ListSplitsResponseExt}; - use quickwit_proto::metastore::{ - IndexMetadataResponse, ListSplitsResponse, MetastoreServiceClient, MockMetastoreService, - }; - use quickwit_proto::search::OutputFormat; - use quickwit_proto::search::search_service_server::SearchServiceServer; - use quickwit_proto::tonic; - use quickwit_query::query_ast::qast_json_helper; - use quickwit_search::{ - ClusterClient, MockSearchService, SearchError, SearchJobPlacer, SearchService, - SearcherPool, create_search_client_from_grpc_addr, root_search_stream, - }; - use tokio_stream::wrappers::UnboundedReceiverStream; - use tonic::transport::Server; - - use crate::search_api::GrpcSearchAdapter; - - async fn start_test_server( - address: SocketAddr, - search_service: Arc, - ) -> anyhow::Result<()> { - let search_grpc_adapter = GrpcSearchAdapter::from(search_service); - tokio::spawn(async move { - Server::builder() - .add_service(SearchServiceServer::new(search_grpc_adapter)) - .serve(address) - .await?; - Result::<_, anyhow::Error>::Ok(()) - }); - Ok(()) - } - - #[tokio::test] - async fn test_serve_search_stream_with_a_leaf_error_on_leaf_node() -> anyhow::Result<()> { - // This test aims at checking the client gRPC implementation. - let request = quickwit_proto::search::SearchStreamRequest { - index_id: "test-index".to_string(), - query_ast: qast_json_helper("test", &["body"]), - snippet_fields: Vec::new(), - start_timestamp: None, - end_timestamp: None, - fast_field: "timestamp".to_string(), - output_format: OutputFormat::Csv as i32, - partition_by_field: None, - }; - let mut mock_metastore = MockMetastoreService::new(); - let index_metadata = IndexMetadata::for_test("test-index", "ram:///indexes/test-index"); - let index_uid = index_metadata.index_uid.clone(); - mock_metastore.expect_index_metadata().returning(move |_| { - Ok(IndexMetadataResponse::try_from_index_metadata(&index_metadata).unwrap()) - }); - mock_metastore.expect_list_splits().returning(move |_| { - let splits = vec![ - MockSplitBuilder::new("split_1") - .with_index_uid(&index_uid) - .build(), - MockSplitBuilder::new("split_2") - .with_index_uid(&index_uid) - .build(), - ]; - let splits = ListSplitsResponse::try_from_splits(splits).unwrap(); - Ok(ServiceStream::from(vec![Ok(splits)])) - }); - let mut mock_search_service = MockSearchService::new(); - let (result_sender, result_receiver) = tokio::sync::mpsc::unbounded_channel(); - result_sender.send(Ok(quickwit_proto::search::LeafSearchStreamResponse { - data: b"123".to_vec(), - split_id: "split_1".to_string(), - }))?; - result_sender.send(Err(SearchError::Internal("Error on `split2`".to_string())))?; - mock_search_service - .expect_leaf_search_stream() - .withf(|request| request.split_offsets.len() == 2) // First request. - .return_once( - |_leaf_search_req: quickwit_proto::search::LeafSearchStreamRequest| { - Ok(UnboundedReceiverStream::new(result_receiver)) - }, - ); - mock_search_service - .expect_leaf_search_stream() - .withf(|request| request.split_offsets.len() == 1) // Retry request on the failing split. - .return_once( - |_leaf_search_req: quickwit_proto::search::LeafSearchStreamRequest| { - Err(SearchError::Internal("error again on `split2`".to_string())) - }, - ); - // The test will hang on indefinitely if we don't drop the sender. - drop(result_sender); - - let grpc_addr: SocketAddr = "127.0.0.1:10001".parse()?; - start_test_server(grpc_addr, Arc::new(mock_search_service)).await?; - - let searcher_pool = SearcherPool::default(); - searcher_pool.insert( - grpc_addr, - create_search_client_from_grpc_addr(grpc_addr, ByteSize::mib(1)), - ); - let search_job_placer = SearchJobPlacer::new(searcher_pool); - let cluster_client = ClusterClient::new(search_job_placer.clone()); - let stream = root_search_stream( - request, - MetastoreServiceClient::from_mock(mock_metastore), - cluster_client, - ) - .await?; - let search_stream_result: Result, SearchError> = stream.try_collect().await; - let search_error = search_stream_result.unwrap_err(); - assert_eq!( - search_error.to_string(), - "internal error: `internal error: `error again on `split2```" - ); - Ok(()) - } -} diff --git a/quickwit/quickwit-serve/src/search_api/rest_handler.rs b/quickwit/quickwit-serve/src/search_api/rest_handler.rs index f8a120cc083..ef2a433b3fb 100644 --- a/quickwit/quickwit-serve/src/search_api/rest_handler.rs +++ b/quickwit/quickwit-serve/src/search_api/rest_handler.rs @@ -15,20 +15,15 @@ use std::convert::TryFrom; use std::sync::Arc; -use futures::stream::StreamExt; use percent_encoding::percent_decode_str; use quickwit_config::validate_index_id_pattern; -use quickwit_proto::ServiceError; -use quickwit_proto::search::{CountHits, OutputFormat, SortField, SortOrder}; -use quickwit_proto::types::IndexId; +use quickwit_proto::search::{CountHits, SortField, SortOrder}; use quickwit_query::query_ast::query_ast_from_user_text; use quickwit_search::{SearchError, SearchPlanResponseRest, SearchResponseRest, SearchService}; -use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_json::Value as JsonValue; use tracing::info; -use warp::hyper::header::{CONTENT_TYPE, HeaderValue}; -use warp::hyper::{HeaderMap, StatusCode}; -use warp::{Filter, Rejection, Reply, reply}; +use warp::{Filter, Rejection}; use crate::rest_api_response::into_rest_api_response; use crate::simple_list::{from_simple_list, to_simple_list}; @@ -39,13 +34,11 @@ use crate::{BodyFormat, with_arg}; paths( search_get_handler, search_post_handler, - search_stream_handler, search_plan_get_handler, search_plan_post_handler, ), components(schemas( BodyFormat, - OutputFormat, SearchRequestQueryString, SearchResponseRest, SearchPlanResponseRest, @@ -126,14 +119,18 @@ impl From for SortBy { } pub fn sort_by_mini_dsl<'de, D>(deserializer: D) -> Result -where D: Deserializer<'de> { +where + D: Deserializer<'de>, +{ let sort_by_mini_dsl = String::deserialize(deserializer)?; Ok(SortBy::from(sort_by_mini_dsl)) } impl Serialize for SortBy { fn serialize(&self, serializer: S) -> Result - where S: Serializer { + where + S: Serializer, + { let mut sort_by_mini_dsl = String::new(); for sort_field in &self.sort_fields { @@ -150,25 +147,6 @@ fn default_max_hits() -> u64 { 20 } -// Deserialize a string field and return and error if it's empty. -// We have 2 issues with this implementation: -// - this is not generic and thus nos sustainable and we may need to -// use an external crate for validation in the future like -// this one https://github.com/Keats/validator. -// - the error does not mention the field name and this is not user friendly. There -// is an external crate that can help https://github.com/dtolnay/path-to-error but -// I did not find a way to plug it to serde_qs. -// Conclusion: the best way I found to reject a user query that contains an empty -// string on an mandatory field is this serializer. -fn deserialize_non_empty_string<'de, D>(deserializer: D) -> Result -where D: Deserializer<'de> { - let value = String::deserialize(deserializer)?; - if value.is_empty() { - return Err(de::Error::custom("expected a non-empty string field")); - } - Ok(value) -} - /// This struct represents the QueryString passed to /// the rest API. #[derive( @@ -243,7 +221,9 @@ mod count_hits_from_bool { use serde::{self, Deserialize, Deserializer, Serializer}; pub fn serialize(count_hits: &CountHits, serializer: S) -> Result - where S: Serializer { + where + S: Serializer, + { if count_hits == &CountHits::Underestimate { serializer.serialize_bool(false) } else { @@ -252,7 +232,9 @@ mod count_hits_from_bool { } pub fn deserialize<'de, D>(deserializer: D) -> Result - where D: Deserializer<'de> { + where + D: Deserializer<'de>, + { let count_all = Option::::deserialize(deserializer)?.unwrap_or(true); Ok(if count_all { CountHits::CountAll @@ -290,6 +272,8 @@ pub fn search_request_from_api_request( scroll_ttl_secs: None, search_after: None, count_hits: search_request.count_all.into(), + ignore_missing_indexes: false, + skip_aggregation_finalization: false, }; Ok(search_request) } @@ -306,12 +290,11 @@ async fn search_endpoint( .root_search(search_request) .await .and_then(|search_response| { - if !allow_failed_splits || search_response.num_successful_splits == 0 { - if let Some(search_error) = + if (!allow_failed_splits || search_response.num_successful_splits == 0) + && let Some(search_error) = SearchError::from_split_errors(&search_response.failed_splits[..]) - { - return Err(search_error); - } + { + return Err(search_error); } Ok(search_response) })?; @@ -324,7 +307,7 @@ fn search_get_filter() warp::path!(String / "search") .and_then(extract_index_id_patterns) .and(warp::get()) - .and(serde_qs::warp::query(serde_qs::Config::default())) + .and(warp::query()) } fn search_post_filter() @@ -341,7 +324,7 @@ fn search_plan_get_filter() warp::path!(String / "search-plan") .and_then(extract_index_id_patterns) .and(warp::get()) - .and(serde_qs::warp::query(serde_qs::Config::default())) + .and(warp::query()) } fn search_plan_post_filter() @@ -428,27 +411,6 @@ pub fn search_post_handler( .then(search) } -#[utoipa::path( - get, - tag = "Search", - path = "/{index_id}/search/stream", - responses( - (status = 200, description = "Successfully executed search.") - ), - params( - SearchStreamRequestQueryString, - ("index_id" = String, Path, description = "The index ID to search."), - ) -)] -/// Stream Search Index -pub fn search_stream_handler( - search_service: Arc, -) -> impl Filter + Clone { - search_stream_filter() - .and(with_arg(search_service)) - .then(search_stream) -} - #[utoipa::path( get, tag = "Search", @@ -495,133 +457,9 @@ pub fn search_plan_post_handler( .then(search_plan) } -/// This struct represents the search stream query passed to -/// the REST API. -#[derive(Deserialize, Debug, Eq, PartialEq, utoipa::IntoParams)] -#[into_params(parameter_in = Query)] -#[serde(deny_unknown_fields)] -struct SearchStreamRequestQueryString { - /// Query text. The query language is that of tantivy. - pub query: String, - // Fields to search on. - #[param(rename = "search_field")] - #[serde(default)] - #[serde(rename(deserialize = "search_field"))] - #[serde(deserialize_with = "from_simple_list")] - pub search_fields: Option>, - /// Fields to extract snippet on - #[serde(default)] - #[serde(rename(deserialize = "snippet_fields"))] // TODO: Was this supposed to be `snippet_field`? - CF - #[serde(deserialize_with = "from_simple_list")] - pub snippet_fields: Option>, - /// If set, restricts search to documents with a `timestamp >= start_timestamp`. - pub start_timestamp: Option, - /// If set, restricts search to documents with a `timestamp < end_timestamp``. - pub end_timestamp: Option, - /// The fast field to extract. - #[serde(deserialize_with = "deserialize_non_empty_string")] - pub fast_field: String, - /// The requested output format. - #[serde(default)] - pub output_format: OutputFormat, - #[serde(default)] - pub partition_by_field: Option, -} - -async fn search_stream_endpoint( - index_id: IndexId, - search_request: SearchStreamRequestQueryString, - search_service: &dyn SearchService, -) -> Result { - let query_ast = query_ast_from_user_text(&search_request.query, search_request.search_fields); - let query_ast_json = serde_json::to_string(&query_ast)?; - let request = quickwit_proto::search::SearchStreamRequest { - index_id, - query_ast: query_ast_json, - snippet_fields: search_request.snippet_fields.unwrap_or_default(), - start_timestamp: search_request.start_timestamp, - end_timestamp: search_request.end_timestamp, - fast_field: search_request.fast_field, - output_format: search_request.output_format as i32, - partition_by_field: search_request.partition_by_field, - }; - let mut data = search_service.root_search_stream(request).await?; - let (mut sender, body) = warp::hyper::Body::channel(); - tokio::spawn(async move { - while let Some(result) = data.next().await { - match result { - Ok(bytes) => { - if sender.send_data(bytes).await.is_err() { - sender.abort(); - break; - } - } - Err(error) => { - // Add trailer to signal to the client that there is an error. Only works - // if the request is made with an http2 client that can read it... and - // actually this seems pretty rare, for example `curl` will not show this - // trailer. Thus we also call `sender.abort()` so that the - // client will see something wrong happened. But he will - // need to look at the logs to understand that. - tracing::error!(error=?error, "error when streaming search results"); - let header_value_str = - format!("Error when streaming search results: {error:?}."); - let header_value = HeaderValue::from_str(header_value_str.as_str()) - .unwrap_or_else(|_| HeaderValue::from_static("Search stream error")); - let mut trailers = HeaderMap::new(); - trailers.insert("X-Stream-Error", header_value); - let _ = sender.send_trailers(trailers).await; - sender.abort(); - break; - } - }; - } - }); - Ok(body) -} - -fn make_streaming_reply(result: Result) -> impl Reply { - let status_code: StatusCode; - let body = match result { - Ok(body) => { - status_code = StatusCode::OK; - warp::reply::Response::new(body) - } - Err(error) => { - status_code = - crate::convert_status_code_to_legacy_http(error.error_code().http_status_code()); - warp::reply::Response::new(warp::hyper::Body::from(error.to_string())) - } - }; - reply::with_status(body, status_code) -} - -async fn search_stream( - index_id: IndexId, - request: SearchStreamRequestQueryString, - search_service: Arc, -) -> impl warp::Reply { - info!(index_id=%index_id,request=?request, "search_stream"); - let content_type = match request.output_format { - OutputFormat::ClickHouseRowBinary => "application/octet-stream", - OutputFormat::Csv => "text/csv", - }; - let reply = - make_streaming_reply(search_stream_endpoint(index_id, request, &*search_service).await); - reply::with_header(reply, CONTENT_TYPE, content_type) -} - -fn search_stream_filter() --> impl Filter + Clone { - warp::path!(String / "search" / "stream") - .and(warp::get()) - .and(serde_qs::warp::query(serde_qs::Config::default())) -} - #[cfg(test)] mod tests { use assert_json_diff::{assert_json_eq, assert_json_include}; - use bytes::Bytes; use mockall::predicate; use quickwit_search::{MockSearchService, SearchError}; use serde_json::{Value as JsonValue, json}; @@ -635,7 +473,6 @@ mod tests { let mock_search_service_in_arc = Arc::new(mock_search_service); search_get_handler(mock_search_service_in_arc.clone()) .or(search_post_handler(mock_search_service_in_arc.clone())) - .or(search_stream_handler(mock_search_service_in_arc.clone())) .or(search_plan_get_handler(mock_search_service_in_arc.clone())) .or(search_plan_post_handler(mock_search_service_in_arc.clone())) .recover(recover_fn) @@ -1038,7 +875,7 @@ mod tests { .unwrap() .as_str() .unwrap() - .contains("unknown field `end_unix_timestamp`") + .contains("Invalid query string") ); } @@ -1163,110 +1000,6 @@ mod tests { Ok(()) } - #[tokio::test] - async fn test_rest_search_stream_api() { - let mut mock_search_service = MockSearchService::new(); - mock_search_service - .expect_root_search_stream() - .return_once(|_| { - Ok(Box::pin(futures::stream::iter(vec![ - Ok(Bytes::from("first row\n")), - Ok(Bytes::from("second row")), - ]))) - }); - let rest_search_stream_api_handler = search_handler(mock_search_service); - let response = warp::test::request() - .path( - "/my-index/search/stream?query=obama&search_field=body&fast_field=external_id&\ - output_format=csv", - ) - .reply(&rest_search_stream_api_handler) - .await; - assert_eq!(response.status(), 200); - let body = String::from_utf8_lossy(response.body()); - assert_eq!(body, "first row\nsecond row"); - } - - #[tokio::test] - async fn test_rest_search_stream_api_csv() { - let (index, req) = warp::test::request() - .path("/my-index/search/stream?query=obama&fast_field=external_id&output_format=csv") - .filter(&super::search_stream_filter()) - .await - .unwrap(); - assert_eq!(&index, "my-index"); - assert_eq!( - &req, - &super::SearchStreamRequestQueryString { - query: "obama".to_string(), - search_fields: None, - snippet_fields: None, - start_timestamp: None, - end_timestamp: None, - fast_field: "external_id".to_string(), - output_format: OutputFormat::Csv, - partition_by_field: None, - } - ); - } - - #[tokio::test] - async fn test_rest_search_stream_api_click_house_row_binary() { - let (index, req) = warp::test::request() - .path( - "/my-index/search/stream?query=obama&fast_field=external_id&\ - output_format=click_house_row_binary", - ) - .filter(&super::search_stream_filter()) - .await - .unwrap(); - assert_eq!(&index, "my-index"); - assert_eq!( - &req, - &super::SearchStreamRequestQueryString { - query: "obama".to_string(), - search_fields: None, - snippet_fields: None, - start_timestamp: None, - end_timestamp: None, - fast_field: "external_id".to_string(), - output_format: OutputFormat::ClickHouseRowBinary, - partition_by_field: None, - } - ); - } - - #[tokio::test] - async fn test_rest_search_stream_api_error() { - let rejection = warp::test::request() - .path( - "/my-index/search/stream?query=obama&fast_field=external_id&\ - output_format=ClickHouseRowBinary", - ) - .filter(&super::search_stream_filter()) - .await - .unwrap_err(); - let parse_error = rejection.find::().unwrap(); - assert_eq!( - parse_error.to_string(), - "unknown variant `ClickHouseRowBinary`, expected `csv` or `click_house_row_binary`" - ); - } - - #[tokio::test] - async fn test_rest_search_stream_api_error_empty_fastfield() { - let rejection = warp::test::request() - .path( - "/my-index/search/stream?query=obama&fast_field=&\ - output_format=click_house_row_binary", - ) - .filter(&super::search_stream_filter()) - .await - .unwrap_err(); - let parse_error = rejection.find::().unwrap(); - assert_eq!(parse_error.to_string(), "expected a non-empty string field"); - } - #[tokio::test] async fn test_rest_search_api_route_serialize_results_with_snippet() -> anyhow::Result<()> { let mut mock_search_service = MockSearchService::new(); diff --git a/quickwit/quickwit-serve/src/ui_handler.rs b/quickwit/quickwit-serve/src/ui_handler.rs index 76743a73047..f1a57129726 100644 --- a/quickwit/quickwit-serve/src/ui_handler.rs +++ b/quickwit/quickwit-serve/src/ui_handler.rs @@ -25,7 +25,7 @@ use crate::rest::recover_fn; /// Regular expression to identify which path should serve an asset file. /// If not matched, the server serves the `index.html` file. -const PATH_PATTERN: &str = r"(^static|\.(png|json|txt|ico|js|map)$)"; +const PATH_PATTERN: &str = r"(^static|\.(png|json|txt|ico|js|map|css|woff2|ttf)$)"; const UI_INDEX_FILE_NAME: &str = "index.html"; @@ -60,7 +60,7 @@ async fn serve_impl(path: &str) -> Result, Rejection> { let asset = Asset::get(path_to_file).ok_or_else(warp::reject::not_found)?; let mime = mime_guess::from_path(path_to_file).first_or_octet_stream(); - let mut res = Response::new(asset.data.into()); + let mut res = Response::new(asset.data.into_owned().into()); res.headers_mut().insert( "content-type", HeaderValue::from_str(mime.as_ref()).unwrap(), diff --git a/quickwit/quickwit-storage/Cargo.toml b/quickwit/quickwit-storage/Cargo.toml index 4ba84c90e01..59ffcc4fa8d 100644 --- a/quickwit/quickwit-storage/Cargo.toml +++ b/quickwit/quickwit-storage/Cargo.toml @@ -22,10 +22,11 @@ http-body-util = { workspace = true} hyper = { workspace = true } lru = { workspace = true } md5 = { workspace = true } +mini-moka = { workspace = true } mockall = { workspace = true, optional = true } once_cell = { workspace = true } pin-project = { workspace = true } -rand = { workspace = true } +quick_cache = { workspace = true } regex = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } @@ -54,10 +55,10 @@ quickwit-config = { workspace = true } quickwit-proto = { workspace = true } opendal = { workspace = true, optional = true } -reqsign = { workspace = true, optional = true } reqwest = { workspace = true, optional = true } [dev-dependencies] +http = { workspace = true } mockall = { workspace = true } proptest = { workspace = true } tokio = { workspace = true } @@ -86,8 +87,6 @@ integration-testsuite = [ "azure_core/azurite_workaround", "azure_storage_blobs/azurite_workaround", "gcs", # Stands for Google cloud storage. - "dep:reqsign", - "reqsign/services-google", "dep:reqwest", ] testsuite = ["mockall"] diff --git a/quickwit/quickwit-storage/src/bundle_storage.rs b/quickwit/quickwit-storage/src/bundle_storage.rs index 3eddd474896..ea1134e804d 100644 --- a/quickwit/quickwit-storage/src/bundle_storage.rs +++ b/quickwit/quickwit-storage/src/bundle_storage.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use quickwit_common::tantivy4java_debug; use std::collections::HashMap; use std::convert::TryInto; use std::fmt::Debug; @@ -19,7 +20,6 @@ use std::ops::Range; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::{fmt, io}; -use quickwit_common::tantivy4java_debug; use anyhow::Context; use async_trait::async_trait; @@ -28,7 +28,6 @@ use quickwit_common::uri::Uri; use serde::{Deserialize, Serialize}; use tantivy::HasLen; use tantivy::directory::FileSlice; -use thiserror::Error; use tokio::io::{AsyncRead, AsyncWriteExt}; use tracing::error; @@ -77,14 +76,19 @@ impl BundleStorage { ) -> anyhow::Result<(FileSlice, Self)> { tantivy4java_debug!("🔍 QUICKWIT DEBUG: BundleStorage::open_from_split_data called"); tantivy4java_debug!("🔍 QUICKWIT DEBUG: Bundle filepath: {:?}", bundle_filepath); - tantivy4java_debug!("🔍 QUICKWIT DEBUG: Split data size: {} bytes", split_data.len()); + tantivy4java_debug!( + "🔍 QUICKWIT DEBUG: Split data size: {} bytes", + split_data.len() + ); tantivy4java_debug!("🔍 QUICKWIT DEBUG: Storage URI: {}", storage.uri()); - + let (hotcache, metadata) = BundleStorageFileOffsets::open_from_split_data(split_data)?; - - tantivy4java_debug!("🔍 QUICKWIT DEBUG: BundleStorageFileOffsets::open_from_split_data completed"); + + tantivy4java_debug!( + "🔍 QUICKWIT DEBUG: BundleStorageFileOffsets::open_from_split_data completed" + ); tantivy4java_debug!("🔍 QUICKWIT DEBUG: Hotcache size: {} bytes", hotcache.len()); - + Ok(( hotcache, BundleStorage { @@ -101,14 +105,6 @@ impl BundleStorage { } } -#[derive(Debug, Error)] -#[error("corrupted data. error: {error:?}")] -pub struct CorruptedData { - #[from] - #[source] - pub error: serde_json::Error, -} - const SPLIT_HOTBYTES_FOOTER_LENGTH_NUM_BYTES: usize = std::mem::size_of::(); const BUNDLE_METADATA_LENGTH_NUM_BYTES: usize = std::mem::size_of::(); @@ -158,9 +154,14 @@ impl BundleStorageFileOffsets { /// [Files, FileMetadata, FileMetadata Len, HotCache, HotCache Len] /// Returns (Hotcache, Self) fn open_from_split_data(file: FileSlice) -> anyhow::Result<(FileSlice, Self)> { - tantivy4java_debug!("🔍 QUICKWIT DEBUG: BundleStorageFileOffsets::open_from_split_data called"); - tantivy4java_debug!("🔍 QUICKWIT DEBUG: Input file slice size: {} bytes", file.len()); - + tantivy4java_debug!( + "🔍 QUICKWIT DEBUG: BundleStorageFileOffsets::open_from_split_data called" + ); + tantivy4java_debug!( + "🔍 QUICKWIT DEBUG: Input file slice size: {} bytes", + file.len() + ); + let (bundle_and_hotcache_bytes, hotcache_num_bytes_data) = file.split_from_end(SPLIT_HOTBYTES_FOOTER_LENGTH_NUM_BYTES); let hotcache_num_bytes: u32 = u32::from_le_bytes( @@ -170,16 +171,28 @@ impl BundleStorageFileOffsets { .try_into() .unwrap(), ); - - tantivy4java_debug!("🔍 QUICKWIT DEBUG: Hotcache num bytes read from footer: {}", hotcache_num_bytes); - tantivy4java_debug!("🔍 QUICKWIT DEBUG: Bundle and hotcache bytes size: {}", bundle_and_hotcache_bytes.len()); - + + tantivy4java_debug!( + "🔍 QUICKWIT DEBUG: Hotcache num bytes read from footer: {}", + hotcache_num_bytes + ); + tantivy4java_debug!( + "🔍 QUICKWIT DEBUG: Bundle and hotcache bytes size: {}", + bundle_and_hotcache_bytes.len() + ); + let (bundle, hotcache) = bundle_and_hotcache_bytes.split_from_end(hotcache_num_bytes as usize); - - tantivy4java_debug!("🔍 QUICKWIT DEBUG: Bundle size after split: {} bytes", bundle.len()); - tantivy4java_debug!("🔍 QUICKWIT DEBUG: Hotcache size after split: {} bytes", hotcache.len()); - + + tantivy4java_debug!( + "🔍 QUICKWIT DEBUG: Bundle size after split: {} bytes", + bundle.len() + ); + tantivy4java_debug!( + "🔍 QUICKWIT DEBUG: Hotcache size after split: {} bytes", + hotcache.len() + ); + Ok((hotcache, Self::open(bundle)?)) } @@ -188,21 +201,37 @@ impl BundleStorageFileOffsets { /// [Files, FileMetadata, FileMetadata Len] pub fn open(file: FileSlice) -> anyhow::Result { let thread_id = std::thread::current().id(); - tantivy4java_debug!("QUICKWIT DEBUG: [Thread {:?}] BundleStorageFileOffsets::open - file size: {}", thread_id, file.len()); - tantivy4java_debug!("QUICKWIT DEBUG: [Thread {:?}] BundleStorageFileOffsets::open - file slice len: {}", thread_id, file.len()); - tantivy4java_debug!("QUICKWIT DEBUG: [Thread {:?}] BundleStorageFileOffsets::open - call stack trace:", thread_id); - + tantivy4java_debug!( + "QUICKWIT DEBUG: [Thread {:?}] BundleStorageFileOffsets::open - file size: {}", + thread_id, + file.len() + ); + tantivy4java_debug!( + "QUICKWIT DEBUG: [Thread {:?}] BundleStorageFileOffsets::open - file slice len: {}", + thread_id, + file.len() + ); + tantivy4java_debug!( + "QUICKWIT DEBUG: [Thread {:?}] BundleStorageFileOffsets::open - call stack trace:", + thread_id + ); + // Print complete stack trace to understand the call path - no filters, increased to 30 frames let backtrace = std::backtrace::Backtrace::force_capture(); let backtrace_str = format!("{}", backtrace); for (i, line) in backtrace_str.lines().take(30).enumerate() { - tantivy4java_debug!("QUICKWIT DEBUG: [Thread {:?}] {}: {}", thread_id, i, line.trim()); + tantivy4java_debug!( + "QUICKWIT DEBUG: [Thread {:?}] {}: {}", + thread_id, + i, + line.trim() + ); } - + tantivy4java_debug!("QUICKWIT DEBUG: Splitting file from end for metadata length..."); let (tantivy_files_data, num_bytes_file_metadata) = file.split_from_end(BUNDLE_METADATA_LENGTH_NUM_BYTES); - + tantivy4java_debug!("QUICKWIT DEBUG: Reading footer num bytes..."); let footer_num_bytes: u32 = u32::from_le_bytes( num_bytes_file_metadata @@ -217,16 +246,24 @@ impl BundleStorageFileOffsets { let mut bundle_storage_file_offsets_data = tantivy_files_data .slice_from_end(footer_num_bytes as usize) .read_bytes()?; - tantivy4java_debug!("QUICKWIT DEBUG: Bundle storage file offsets data size: {}", bundle_storage_file_offsets_data.len()); - + tantivy4java_debug!( + "QUICKWIT DEBUG: Bundle storage file offsets data size: {}", + bundle_storage_file_offsets_data.len() + ); + tantivy4java_debug!("QUICKWIT DEBUG: Calling try_read_component..."); - let result = BundleStorageFileOffsetsVersions::try_read_component(&mut bundle_storage_file_offsets_data); - + let result = BundleStorageFileOffsetsVersions::try_read_component( + &mut bundle_storage_file_offsets_data, + ); + match &result { - Ok(file_offsets) => tantivy4java_debug!("QUICKWIT DEBUG: Successfully read component with {} files", file_offsets.files.len()), + Ok(file_offsets) => tantivy4java_debug!( + "QUICKWIT DEBUG: Successfully read component with {} files", + file_offsets.files.len() + ), Err(e) => tantivy4java_debug!("QUICKWIT DEBUG: try_read_component failed: {}", e), } - + result } diff --git a/quickwit/quickwit-storage/src/cache/base_cache.rs b/quickwit/quickwit-storage/src/cache/base_cache.rs new file mode 100644 index 00000000000..c7a4f7faceb --- /dev/null +++ b/quickwit/quickwit-storage/src/cache/base_cache.rs @@ -0,0 +1,454 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::borrow::Borrow; +use std::hash::Hash; +use std::sync::{Arc, Weak}; +use std::time::Duration; + +use bytesize::ByteSize; +use lru::LruCache; +use mini_moka::sync::Cache as MokaCache; +use quick_cache::unsync::Cache as QuickCache; +use quickwit_config::CachePolicy; +use tokio::time::Instant; +use tracing::{error, warn}; + +use crate::OwnedBytes; +use crate::cache::stored_item::{StoredItem, ValueLen}; +use crate::metrics::SingleCacheMetrics; + +/// We do not evict anything that has been accessed in the last 60s. +/// +/// The goal is to behave better on scan access patterns, without being as aggressive as +/// using a MRU strategy. +/// +/// TLDR is: +/// +/// If two items have been access in the last 60s it is not really worth considering the +/// latter too be more recent than the previous and do an eviction. +/// The difference is not significant enough to raise the probability of its future access. +/// +/// On the other hand, for very large queries involving enough data to saturate the cache, +/// we are facing a scanning pattern. If variations of this query is repeated over and over +/// a regular LRU eviction policy would yield a hit rate of 0. +pub(crate) const LRU_MIN_TIME_SINCE_LAST_ACCESS: Duration = Duration::from_secs(60); + +// A fake entry inside a cache, which the cache believe to be of the given size +#[derive(Clone)] +pub(crate) struct FakeCacheEntry(pub usize); + +impl ValueLen for FakeCacheEntry { + fn len(&self) -> usize { + self.0 + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub(crate) enum Capacity { + Unlimited, + InBytes(usize), +} + +impl Capacity { + fn exceeds_capacity(&self, num_bytes: usize) -> bool { + match *self { + Capacity::Unlimited => false, + Capacity::InBytes(capacity_in_bytes) => num_bytes > capacity_in_bytes, + } + } +} + +pub(crate) enum AnyCache { + Lru(Lru), + S3Fifo(S3Fifo), + TinyLfu(TinyLfu), +} + +impl + AnyCache +{ + pub fn from_policy_and_capacity( + policy: CachePolicy, + capacity: ByteSize, + cache_metrics: SingleCacheMetrics, + ) -> Self { + match policy { + CachePolicy::Lru => AnyCache::Lru(Lru::with_capacity( + Capacity::InBytes(capacity.as_u64().try_into().unwrap_or(usize::MAX)), + cache_metrics, + )), + CachePolicy::S3Fifo => { + AnyCache::S3Fifo(S3Fifo::with_capacity(capacity.as_u64(), cache_metrics)) + } + CachePolicy::TinyLfu => { + AnyCache::TinyLfu(TinyLfu::with_capacity(capacity.as_u64(), cache_metrics)) + } + } + } + pub fn unbounded(cache_metrics: SingleCacheMetrics) -> Self { + AnyCache::Lru(Lru::with_capacity(Capacity::Unlimited, cache_metrics)) + } + + pub fn get(&mut self, cache_key: &Q) -> Option + where + K: Borrow, + Arc: Borrow, + Q: Hash + Eq + ?Sized, + { + match self { + AnyCache::Lru(lru) => lru.get(cache_key), + AnyCache::S3Fifo(s3fifo) => s3fifo.get(cache_key), + AnyCache::TinyLfu(tiny_lfu) => tiny_lfu.get(cache_key), + } + } + pub fn put(&mut self, key: K, value: V) { + match self { + AnyCache::Lru(lru) => lru.put(key, value), + AnyCache::S3Fifo(s3fifo) => s3fifo.put(key, value), + AnyCache::TinyLfu(tiny_lfu) => tiny_lfu.put(key, value), + } + } +} + +pub struct Lru { + lru_cache: LruCache>, + num_items: usize, + num_bytes: u64, + capacity: Capacity, + cache_metrics: SingleCacheMetrics, +} + +impl Drop for Lru { + fn drop(&mut self) { + // we don't count this toward evicted entries, as we are clearing the whole cache + self.cache_metrics.in_cache_count.sub(self.num_items as i64); + self.cache_metrics + .in_cache_num_bytes + .sub(self.num_bytes as i64); + } +} + +impl Lru { + /// Creates a new NeedMutSliceCache with the given capacity. + fn with_capacity(capacity: Capacity, cache_metrics: SingleCacheMetrics) -> Self { + Lru { + // The limit will be decided by the amount of memory in the cache, + // not the number of items in the cache. + // Enforcing this limit is done in the `NeedMutCache` impl. + lru_cache: LruCache::unbounded(), + num_items: 0, + num_bytes: 0, + capacity, + cache_metrics, + } + } + + fn record_item(&mut self, num_bytes: u64) { + self.num_items += 1; + self.num_bytes += num_bytes; + self.cache_metrics.in_cache_count.inc(); + self.cache_metrics.in_cache_num_bytes.add(num_bytes as i64); + } + + fn drop_item(&mut self, num_bytes: u64) { + self.num_items -= 1; + self.num_bytes -= num_bytes; + self.cache_metrics.in_cache_count.dec(); + self.cache_metrics.in_cache_num_bytes.sub(num_bytes as i64); + self.cache_metrics.evict_num_items.inc(); + self.cache_metrics.evict_num_bytes.inc_by(num_bytes); + } + + pub fn get(&mut self, cache_key: &Q) -> Option + where + K: Borrow, + Q: Hash + Eq + ?Sized, + { + let item_opt = self.lru_cache.get_mut(cache_key); + if let Some(item) = item_opt { + self.cache_metrics.hits_num_items.inc(); + self.cache_metrics.hits_num_bytes.inc_by(item.len() as u64); + Some(item.payload()) + } else { + self.cache_metrics.misses_num_items.inc(); + None + } + } + + /// Attempt to put the given amount of data in the cache. + /// This may fail silently if the owned_bytes slice is larger than the cache + /// capacity. + fn put(&mut self, key: K, bytes: V) { + if self.capacity.exceeds_capacity(bytes.len()) { + // The value does not fit in the cache. We simply don't store it. + if self.capacity != Capacity::InBytes(0) { + warn!( + capacity_in_bytes = ?self.capacity, + len = bytes.len(), + "Downloaded a byte slice larger than the cache capacity." + ); + } + return; + } + if let Some(previous_data) = self.lru_cache.pop(&key) { + self.drop_item(previous_data.len() as u64); + } + + let now = Instant::now(); + while self + .capacity + .exceeds_capacity(self.num_bytes as usize + bytes.len()) + { + if let Some((_, candidate_for_eviction)) = self.lru_cache.peek_lru() { + let time_since_last_access = + now.duration_since(candidate_for_eviction.last_access_time()); + if time_since_last_access < LRU_MIN_TIME_SINCE_LAST_ACCESS { + // It is not worth doing an eviction. + // TODO: It is sub-optimal that we might have needlessly evicted items in this + // loop before just returning. + return; + } + } + if let Some((_, bytes)) = self.lru_cache.pop_lru() { + self.drop_item(bytes.len() as u64); + } else { + error!( + "Logical error. Even after removing all of the items in the cache the \ + capacity is insufficient. This case is guarded against and should never \ + happen." + ); + return; + } + } + self.record_item(bytes.len() as u64); + self.lru_cache.put(key, StoredItem::new(bytes, now)); + } +} + +// actually, quick_cache is a Clock-PRO, not a S3-fifo contrary to what quick-cache and Moka's +// readme says. While both are clearly distinct (one being clock-based, the other being fifo +// based), they are not too disimilar in term of strenght/weaknesses. +pub struct S3Fifo { + cache: + QuickCache, + capacity: u64, + cache_metrics: SingleCacheMetrics, +} + +impl Drop for S3Fifo { + fn drop(&mut self) { + // we don't count this toward evicted entries, as we are clearing the whole cache + self.cache_metrics + .in_cache_count + .sub(self.cache.len() as i64); + self.cache_metrics + .in_cache_num_bytes + .sub(self.cache.weight() as i64); + } +} + +struct QuickCacheWeighter; +impl quick_cache::Weighter for QuickCacheWeighter { + fn weight(&self, _key: &K, value: &V) -> u64 { + value.len() as u64 + } +} + +struct QuickCacheLifecycle; +#[derive(Default)] +struct QuickCacheQueryEffect { + count: u64, + bytes: u64, +} +impl quick_cache::Lifecycle for QuickCacheLifecycle { + type RequestState = QuickCacheQueryEffect; + + fn begin_request(&self) -> Self::RequestState { + QuickCacheQueryEffect::default() + } + fn on_evict(&self, state: &mut Self::RequestState, _key: K, val: V) { + state.count += 1; + state.bytes += val.len() as u64; + } +} + +impl S3Fifo { + /// Creates a new NeedMutSliceCache with the given capacity. + fn with_capacity(capacity: u64, cache_metrics: SingleCacheMetrics) -> Self { + S3Fifo { + cache: QuickCache::with( + (capacity / (128 * 1024)) as usize, + capacity, + QuickCacheWeighter, + quick_cache::DefaultHashBuilder::new(), + QuickCacheLifecycle, + ), + capacity, + cache_metrics, + } + } + + pub fn get(&mut self, cache_key: &Q) -> Option + where + K: Borrow, + Q: Hash + Eq + ?Sized, + { + let item_opt = self.cache.get(cache_key); + if let Some(item) = item_opt { + self.cache_metrics.hits_num_items.inc(); + self.cache_metrics.hits_num_bytes.inc_by(item.len() as u64); + Some(item.clone()) + } else { + self.cache_metrics.misses_num_items.inc(); + None + } + } + + /// Attempt to put the given amount of data in the cache. + /// This may fail silently if the owned_bytes slice is larger than the cache + /// capacity. + fn put(&mut self, key: K, value: V) { + if self.capacity < value.len() as u64 { + // The value does not fit in the cache. We simply don't store it. + if self.capacity != 0 { + warn!( + capacity_in_bytes = ?self.capacity, + len = value.len(), + "Downloaded a byte slice larger than the cache capacity." + ); + } + return; + } + + self.cache_metrics.in_cache_count.inc(); + self.cache_metrics + .in_cache_num_bytes + .add(value.len() as i64); + let evicted = self.cache.insert_with_lifecycle(key, value); + self.cache_metrics.in_cache_count.sub(evicted.count as i64); + self.cache_metrics + .in_cache_num_bytes + .sub(evicted.bytes as i64); + self.cache_metrics.evict_num_items.inc_by(evicted.count); + self.cache_metrics.evict_num_bytes.inc_by(evicted.bytes); + } +} + +// We don't make this value Clone to ensure each item is dropped only once +struct CapacityTracker { + item: V, + cache_metrics: Weak, +} + +impl Drop for CapacityTracker { + fn drop(&mut self) { + if let Some(cache_metrics) = self.cache_metrics.upgrade() { + cache_metrics.in_cache_count.dec(); + cache_metrics.in_cache_num_bytes.sub(self.item.len() as i64); + cache_metrics.evict_num_items.inc(); + cache_metrics.evict_num_bytes.inc_by(self.item.len() as u64); + } + } +} + +pub struct TinyLfu { + // this field is put first so it's dropped before the cache + // we use that to not count removed entries as "evicted", by + // calling CapacityTracker's Drop only after its Weak has expired. + cache_metrics: Arc, + + // we store an Arc because moka does a lot of internal cloning, and it's hard to not double + // evict/forget to count eviction otherwise + cache: MokaCache>>, + capacity: u64, +} + +impl Drop for TinyLfu { + fn drop(&mut self) { + // we don't count this toward evicted entries, as we are clearing the whole cache + self.cache_metrics + .in_cache_count + .sub(self.cache.entry_count() as i64); + self.cache_metrics + .in_cache_num_bytes + .sub(self.cache.weighted_size() as i64); + } +} + +impl + TinyLfu +{ + /// Creates a new NeedMutSliceCache with the given capacity. + fn with_capacity(capacity: u64, cache_metrics: SingleCacheMetrics) -> Self { + TinyLfu { + cache: MokaCache::builder() + .max_capacity(capacity) + .weigher(|_k, v: &Arc>| { + v.item.len().try_into().unwrap_or(u32::MAX) + }) + .build(), + capacity, + cache_metrics: cache_metrics.into(), + } + } + + pub fn get(&mut self, cache_key: &Q) -> Option + where + Arc: Borrow, + Q: Hash + Eq + ?Sized, + { + let item_opt = self.cache.get(cache_key); + if let Some(item) = item_opt { + self.cache_metrics.hits_num_items.inc(); + self.cache_metrics + .hits_num_bytes + .inc_by(item.item.len() as u64); + Some(item.item.clone()) + } else { + self.cache_metrics.misses_num_items.inc(); + None + } + } + + /// Attempt to put the given amount of data in the cache. + /// This may fail silently if the owned_bytes slice is larger than the cache + /// capacity. + fn put(&mut self, key: K, value: V) { + if self.capacity < value.len() as u64 { + // The value does not fit in the cache. We simply don't store it. + if self.capacity != 0 { + warn!( + capacity_in_bytes = ?self.capacity, + len = value.len(), + "Downloaded a byte slice larger than the cache capacity." + ); + } + return; + } + + self.cache_metrics.in_cache_count.inc(); + self.cache_metrics + .in_cache_num_bytes + .add(value.len() as i64); + self.cache.insert( + key, + CapacityTracker { + item: value, + cache_metrics: Arc::downgrade(&self.cache_metrics), + } + .into(), + ); + } +} diff --git a/quickwit/quickwit-storage/src/cache/byte_range_cache.rs b/quickwit/quickwit-storage/src/cache/byte_range_cache.rs index 81e6b4d94f7..9d6c69d7fed 100644 --- a/quickwit/quickwit-storage/src/cache/byte_range_cache.rs +++ b/quickwit/quickwit-storage/src/cache/byte_range_cache.rs @@ -21,7 +21,7 @@ use std::sync::{Arc, Mutex}; use tantivy::directory::OwnedBytes; -use crate::metrics::CacheMetrics; +use crate::metrics::{CacheMetrics, SingleCacheMetrics}; #[derive(Clone, PartialOrd, Ord, PartialEq, Eq)] struct CacheKey<'a, T: ToOwned + ?Sized> { @@ -58,7 +58,7 @@ struct NeedMutByteRangeCache { // this is hardly significant as items can get merged if they overlap num_items: u64, num_bytes: u64, - cache_counters: &'static CacheMetrics, + cache_counters: &'static SingleCacheMetrics, } impl NeedMutByteRangeCache { @@ -67,7 +67,7 @@ impl NeedMutByteRangeCache { cache: BTreeMap::new(), num_items: 0, num_bytes: 0, - cache_counters, + cache_counters: &cache_counters.cache_metrics, } } @@ -393,7 +393,10 @@ impl ByteRangeCache { /// - 1GB+ for heavy workloads with large splits pub fn with_capacity(capacity_bytes: u64, cache_counters: &'static CacheMetrics) -> Self { use quickwit_common::tantivy4java_debug; - tantivy4java_debug!("QUICKWIT DEBUG: 📦 Creating ByteRangeCache with capacity {} MB", capacity_bytes / 1024 / 1024); + tantivy4java_debug!( + "QUICKWIT DEBUG: 📦 Creating ByteRangeCache with capacity {} MB", + capacity_bytes / 1024 / 1024 + ); let need_mut_byte_range_cache = NeedMutByteRangeCache::with_infinite_capacity(cache_counters); let inner = Inner { @@ -429,30 +432,42 @@ impl ByteRangeCache { need_mut_byte_range_cache_locked.clear(); drop(need_mut_byte_range_cache_locked); self.inner_arc.num_stored_bytes.store(0, Ordering::Relaxed); - tantivy4java_debug!("QUICKWIT DEBUG: 🧹 CACHE CLEAR: Freed {} bytes from ByteRangeCache (L1)", bytes_before); + tantivy4java_debug!( + "QUICKWIT DEBUG: 🧹 CACHE CLEAR: Freed {} bytes from ByteRangeCache (L1)", + bytes_before + ); } /// If available, returns the cached view of the slice. pub fn get_slice(&self, path: &Path, byte_range: Range) -> Option { let range_size = byte_range.end - byte_range.start; - let result = self.inner_arc + let result = self + .inner_arc .need_mut_byte_range_cache .lock() .unwrap() .get_slice(path, byte_range.clone()); - + // Use tantivy4java_debug! macro for consistent debugging use quickwit_common::tantivy4java_debug; - + match &result { Some(_) => { - tantivy4java_debug!("QUICKWIT DEBUG: 🎯 CACHE HIT: {} bytes for path '{}'", range_size, path.display()); - }, + tantivy4java_debug!( + "QUICKWIT DEBUG: 🎯 CACHE HIT: {} bytes for path '{}'", + range_size, + path.display() + ); + } None => { - tantivy4java_debug!("QUICKWIT DEBUG: ❌ CACHE MISS: {} bytes for path '{}' - WILL DOWNLOAD", range_size, path.display()); + tantivy4java_debug!( + "QUICKWIT DEBUG: ❌ CACHE MISS: {} bytes for path '{}' - WILL DOWNLOAD", + range_size, + path.display() + ); } } - + result } @@ -464,7 +479,11 @@ impl ByteRangeCache { pub fn put_slice(&self, path: PathBuf, byte_range: Range, bytes: OwnedBytes) { use quickwit_common::tantivy4java_debug; let range_size = byte_range.end - byte_range.start; - tantivy4java_debug!("QUICKWIT DEBUG: 💾 CACHE PUT: {} bytes for path '{}'", range_size, path.display()); + tantivy4java_debug!( + "QUICKWIT DEBUG: 💾 CACHE PUT: {} bytes for path '{}'", + range_size, + path.display() + ); // Check capacity constraints if let Some(capacity) = self.inner_arc.capacity_bytes { @@ -473,7 +492,8 @@ impl ByteRangeCache { if range_size as u64 > capacity { tantivy4java_debug!( "QUICKWIT DEBUG: ⏭️ CACHE SKIP: {} bytes exceeds cache capacity of {} MB, not storing", - range_size, capacity / 1024 / 1024 + range_size, + capacity / 1024 / 1024 ); return; } @@ -486,7 +506,10 @@ impl ByteRangeCache { // This is a simple but effective policy - data is safe in L2 disk cache tantivy4java_debug!( "QUICKWIT DEBUG: ⚠️ CACHE CAPACITY: {} + {} = {} exceeds {} MB limit, clearing cache", - current_bytes, range_size, new_total, capacity / 1024 / 1024 + current_bytes, + range_size, + new_total, + capacity / 1024 / 1024 ); let mut need_mut_byte_range_cache_locked = self.inner_arc.need_mut_byte_range_cache.lock().unwrap(); diff --git a/quickwit/quickwit-storage/src/cache/memory_sized_cache.rs b/quickwit/quickwit-storage/src/cache/memory_sized_cache.rs index 59b8273a4fd..707d633501f 100644 --- a/quickwit/quickwit-storage/src/cache/memory_sized_cache.rs +++ b/quickwit/quickwit-storage/src/cache/memory_sized_cache.rs @@ -16,192 +16,93 @@ use std::borrow::Borrow; use std::hash::Hash; use std::ops::Range; use std::path::{Path, PathBuf}; -use std::sync::Mutex; -use std::time::Duration; +use std::sync::{Arc, Mutex}; -use lru::LruCache; -use tokio::time::Instant; -use tracing::{error, warn}; +use quickwit_config::CacheConfig; use crate::OwnedBytes; +use crate::cache::base_cache::{AnyCache, FakeCacheEntry}; use crate::cache::slice_address::{SliceAddress, SliceAddressKey, SliceAddressRef}; -use crate::cache::stored_item::StoredItem; use crate::metrics::CacheMetrics; -/// We do not evict anything that has been accessed in the last 60s. -/// -/// The goal is to behave better on scan access patterns, without being as aggressive as -/// using a MRU strategy. -/// -/// TLDR is: -/// -/// If two items have been access in the last 60s it is not really worth considering the -/// latter too be more recent than the previous and do an eviction. -/// The difference is not significant enough to raise the probability of its future access. -/// -/// On the other hand, for very large queries involving enough data to saturate the cache, -/// we are facing a scanning pattern. If variations of this query is repeated over and over -/// a regular LRU eviction policy would yield a hit rate of 0. -const MIN_TIME_SINCE_LAST_ACCESS: Duration = Duration::from_secs(60); - -#[derive(Clone, Copy, Debug, PartialEq)] -enum Capacity { - Unlimited, - InBytes(usize), +struct CacheState { + cache: AnyCache, + virtual_caches: Vec>, } -impl Capacity { - fn exceeds_capacity(&self, num_bytes: usize) -> bool { - match *self { - Capacity::Unlimited => false, - Capacity::InBytes(capacity_in_bytes) => num_bytes > capacity_in_bytes, +impl CacheState { + fn from_config(cache_config: &CacheConfig, cache_counters: &'static CacheMetrics) -> Self { + let cache = AnyCache::from_policy_and_capacity( + cache_config.policy(), + cache_config.capacity(), + cache_counters.cache_metrics.clone(), + ); + let virtual_caches = cache_config + .virtual_caches + .iter() + .cloned() + .map(|mut virtual_cache_config| { + AnyCache::from_policy_and_capacity( + virtual_cache_config.policy_for_virtual_cache(cache_config.policy()), + virtual_cache_config.capacity_for_virtual_cache(cache_config.capacity()), + cache_counters.virtual_cache(&virtual_cache_config), + ) + }) + .collect(); + CacheState { + cache, + virtual_caches, } } -} - -struct NeedMutMemorySizedCache { - lru_cache: LruCache, - num_items: usize, - num_bytes: u64, - capacity: Capacity, - cache_counters: &'static CacheMetrics, -} - -impl Drop for NeedMutMemorySizedCache { - fn drop(&mut self) { - // we don't count this toward evicted entries, as we are clearing the whole cache - self.cache_counters - .in_cache_count - .sub(self.num_items as i64); - self.cache_counters - .in_cache_num_bytes - .sub(self.num_bytes as i64); - } -} -impl NeedMutMemorySizedCache { - /// Creates a new NeedMutSliceCache with the given capacity. - fn with_capacity(capacity: Capacity, cache_counters: &'static CacheMetrics) -> Self { - NeedMutMemorySizedCache { - // The limit will be decided by the amount of memory in the cache, - // not the number of items in the cache. - // Enforcing this limit is done in the `NeedMutCache` impl. - lru_cache: LruCache::unbounded(), - num_items: 0, - num_bytes: 0, - capacity, - cache_counters, + fn infinite(cache_counters: &'static CacheMetrics) -> Self { + CacheState { + cache: AnyCache::unbounded(cache_counters.cache_metrics.clone()), + // there is no point in having virtual caches for an unbounded cache + virtual_caches: Vec::new(), } } - pub fn record_item(&mut self, num_bytes: u64) { - self.num_items += 1; - self.num_bytes += num_bytes; - self.cache_counters.in_cache_count.inc(); - self.cache_counters.in_cache_num_bytes.add(num_bytes as i64); - } - - pub fn drop_item(&mut self, num_bytes: u64) { - self.num_items -= 1; - self.num_bytes -= num_bytes; - self.cache_counters.in_cache_count.dec(); - self.cache_counters.in_cache_num_bytes.sub(num_bytes as i64); - self.cache_counters.evict_num_items.inc(); - self.cache_counters.evict_num_bytes.inc_by(num_bytes); - } - pub fn get(&mut self, cache_key: &Q) -> Option where K: Borrow, + Arc: Borrow, Q: Hash + Eq + ?Sized, { - let item_opt = self.lru_cache.get_mut(cache_key); - if let Some(item) = item_opt { - self.cache_counters.hits_num_items.inc(); - self.cache_counters.hits_num_bytes.inc_by(item.len() as u64); - Some(item.payload()) - } else { - self.cache_counters.misses_num_items.inc(); - None + for virtual_cache in &mut self.virtual_caches { + // we simulate an access on all virtual caches + virtual_cache.get(cache_key); } + self.cache.get(cache_key) } - /// Attempt to put the given amount of data in the cache. - /// This may fail silently if the owned_bytes slice is larger than the cache - /// capacity. fn put(&mut self, key: K, bytes: OwnedBytes) { - if self.capacity.exceeds_capacity(bytes.len()) { - // The value does not fit in the cache. We simply don't store it. - if self.capacity != Capacity::InBytes(0) { - warn!( - capacity_in_bytes = ?self.capacity, - len = bytes.len(), - "Downloaded a byte slice larger than the cache capacity." - ); - } - return; - } - if let Some(previous_data) = self.lru_cache.pop(&key) { - self.drop_item(previous_data.len() as u64); + for virtual_cache in &mut self.virtual_caches { + // we simulate an access on all virtual caches + virtual_cache.put(key.clone(), FakeCacheEntry(bytes.len())); } - let now = Instant::now(); - while self - .capacity - .exceeds_capacity(self.num_bytes as usize + bytes.len()) - { - if let Some((_, candidate_for_eviction)) = self.lru_cache.peek_lru() { - let time_since_last_access = - now.duration_since(candidate_for_eviction.last_access_time()); - if time_since_last_access < MIN_TIME_SINCE_LAST_ACCESS { - // It is not worth doing an eviction. - // TODO: It is sub-optimal that we might have needlessly evicted items in this - // loop before just returning. - return; - } - } - if let Some((_, bytes)) = self.lru_cache.pop_lru() { - self.drop_item(bytes.len() as u64); - } else { - error!( - "Logical error. Even after removing all of the items in the cache the \ - capacity is insufficient. This case is guarded against and should never \ - happen." - ); - return; - } - } - self.record_item(bytes.len() as u64); - self.lru_cache.put(key, StoredItem::new(bytes, now)); + self.cache.put(key, bytes) } } /// A simple in-resident memory slice cache. pub struct MemorySizedCache { - inner: Mutex>, + inner: Mutex>, } -impl MemorySizedCache { +impl MemorySizedCache { /// Creates an slice cache with the given capacity. - pub fn with_capacity_in_bytes( - capacity_in_bytes: usize, - cache_counters: &'static CacheMetrics, - ) -> Self { + pub fn from_config(cache_config: &CacheConfig, cache_counters: &'static CacheMetrics) -> Self { MemorySizedCache { - inner: Mutex::new(NeedMutMemorySizedCache::with_capacity( - Capacity::InBytes(capacity_in_bytes), - cache_counters, - )), + inner: Mutex::new(CacheState::from_config(cache_config, cache_counters)), } } /// Creates a slice cache that never removes any entry. pub fn with_infinite_capacity(cache_counters: &'static CacheMetrics) -> Self { MemorySizedCache { - inner: Mutex::new(NeedMutMemorySizedCache::with_capacity( - Capacity::Unlimited, - cache_counters, - )), + inner: Mutex::new(CacheState::infinite(cache_counters)), } } @@ -209,6 +110,7 @@ impl MemorySizedCache { pub fn get(&self, cache_key: &Q) -> Option where K: Borrow, + Arc: Borrow, Q: Hash + Eq + ?Sized, { self.inner.lock().unwrap().get(cache_key) @@ -240,14 +142,19 @@ impl MemorySizedCache { #[cfg(test)] mod tests { + use bytesize::ByteSize; use super::*; + use crate::cache::base_cache::LRU_MIN_TIME_SINCE_LAST_ACCESS; use crate::metrics::CACHE_METRICS_FOR_TESTS; #[tokio::test] async fn test_cache_edge_condition() { tokio::time::pause(); - let cache = MemorySizedCache::::with_capacity_in_bytes(5, &CACHE_METRICS_FOR_TESTS); + let cache = MemorySizedCache::::from_config( + &ByteSize::b(5).into(), + &CACHE_METRICS_FOR_TESTS, + ); { let data = OwnedBytes::new(&b"abc"[..]); cache.put("3".to_string(), data); @@ -266,7 +173,7 @@ mod tests { // Eviction should not happen, because all items in cache are too young. assert!(cache.get(&"5".to_string()).is_none()); } - tokio::time::advance(super::MIN_TIME_SINCE_LAST_ACCESS.mul_f32(1.1f32)).await; + tokio::time::advance(LRU_MIN_TIME_SINCE_LAST_ACCESS.mul_f32(1.1f32)).await; { let data = OwnedBytes::new(&b"fghij"[..]); cache.put("5".to_string(), data); @@ -275,7 +182,7 @@ mod tests { assert!(cache.get(&"2".to_string()).is_none()); assert!(cache.get(&"3".to_string()).is_none()); } - tokio::time::advance(super::MIN_TIME_SINCE_LAST_ACCESS.mul_f32(1.1f32)).await; + tokio::time::advance(LRU_MIN_TIME_SINCE_LAST_ACCESS.mul_f32(1.1f32)).await; { let data = OwnedBytes::new(&b"klmnop"[..]); cache.put("6".to_string(), data); @@ -304,10 +211,21 @@ mod tests { #[test] fn test_cache() { - let cache = MemorySizedCache::with_capacity_in_bytes(10_000, &CACHE_METRICS_FOR_TESTS); + let cache = + MemorySizedCache::from_config(&ByteSize::kb(10).into(), &CACHE_METRICS_FOR_TESTS); assert!(cache.get(&"hello.seg").is_none()); let data = OwnedBytes::new(&b"werwer"[..]); cache.put("hello.seg", data); assert_eq!(cache.get(&"hello.seg").unwrap(), &b"werwer"[..]); } + + #[test] + fn test_cache_no_cache() { + let cache = + MemorySizedCache::from_config(&CacheConfig::no_cache(), &CACHE_METRICS_FOR_TESTS); + assert!(cache.get(&"hello.seg").is_none()); + let data = OwnedBytes::new(&b"werwer"[..]); + cache.put("hello.seg", data); + assert!(cache.get(&"hello.seg").is_none()); + } } diff --git a/quickwit/quickwit-storage/src/cache/mod.rs b/quickwit/quickwit-storage/src/cache/mod.rs index fe2a45d2f4f..f73a96b90f4 100644 --- a/quickwit/quickwit-storage/src/cache/mod.rs +++ b/quickwit/quickwit-storage/src/cache/mod.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +mod base_cache; mod byte_range_cache; mod memory_sized_cache; mod quickwit_cache; diff --git a/quickwit/quickwit-storage/src/cache/quickwit_cache.rs b/quickwit/quickwit-storage/src/cache/quickwit_cache.rs index dabb5c6ef98..20441bd5fd1 100644 --- a/quickwit/quickwit-storage/src/cache/quickwit_cache.rs +++ b/quickwit/quickwit-storage/src/cache/quickwit_cache.rs @@ -17,6 +17,7 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use async_trait::async_trait; +use quickwit_config::CacheConfig; use crate::OwnedBytes; use crate::cache::{MemorySizedCache, StorageCache}; @@ -37,16 +38,15 @@ impl From)>> for QuickwitCache { } impl QuickwitCache { - /// Creates a [`QuickwitCache`] with a cache on fast fields - /// with a capacity of `fast_field_cache_capacity`. - pub fn new(fast_field_cache_capacity: usize) -> Self { + /// Creates a [`QuickwitCache`] with a cache on fast fields. + pub fn new(cache_config: &CacheConfig) -> Self { let mut quickwit_cache = QuickwitCache::empty(); let fast_field_cache_counters: &'static CacheMetrics = &crate::STORAGE_METRICS.fast_field_cache; quickwit_cache.add_route( ".fast", - Arc::new(SimpleCache::with_capacity_in_bytes( - fast_field_cache_capacity, + Arc::new(SimpleCache::from_config( + cache_config, fast_field_cache_counters, )), ); @@ -117,15 +117,9 @@ struct SimpleCache { } impl SimpleCache { - fn with_capacity_in_bytes( - capacity_in_bytes: usize, - cache_counters: &'static CacheMetrics, - ) -> Self { + fn from_config(cache_config: &CacheConfig, cache_counters: &'static CacheMetrics) -> Self { SimpleCache { - slice_cache: MemorySizedCache::with_capacity_in_bytes( - capacity_in_bytes, - cache_counters, - ), + slice_cache: MemorySizedCache::from_config(cache_config, cache_counters), } } } diff --git a/quickwit/quickwit-storage/src/cache/slice_address.rs b/quickwit/quickwit-storage/src/cache/slice_address.rs index 7961c1bd070..b64e59d2148 100644 --- a/quickwit/quickwit-storage/src/cache/slice_address.rs +++ b/quickwit/quickwit-storage/src/cache/slice_address.rs @@ -36,11 +36,11 @@ pub(crate) struct SliceAddressRef<'a> { } pub(crate) trait SliceAddressKey { - fn key(&self) -> SliceAddressRef; + fn key(&self) -> SliceAddressRef<'_>; } impl SliceAddressKey for SliceAddress { - fn key(&self) -> SliceAddressRef { + fn key(&self) -> SliceAddressRef<'_> { SliceAddressRef { path: self.path.as_path(), byte_range: self.byte_range.clone(), @@ -49,25 +49,31 @@ impl SliceAddressKey for SliceAddress { } impl SliceAddressKey for SliceAddressRef<'_> { - fn key(&self) -> SliceAddressRef { + fn key(&self) -> SliceAddressRef<'_> { self.clone() } } +impl<'a> Borrow for std::sync::Arc { + fn borrow(&self) -> &(dyn SliceAddressKey + 'a) { + &**self + } +} + impl<'a> Borrow for SliceAddress { fn borrow(&self) -> &(dyn SliceAddressKey + 'a) { self } } -impl PartialEq for (dyn SliceAddressKey + '_) { +impl PartialEq for dyn SliceAddressKey + '_ { fn eq(&self, other: &Self) -> bool { self.key().eq(&other.key()) } } -impl Eq for (dyn SliceAddressKey + '_) {} +impl Eq for dyn SliceAddressKey + '_ {} -impl Hash for (dyn SliceAddressKey + '_) { +impl Hash for dyn SliceAddressKey + '_ { fn hash(&self, state: &mut H) { self.key().hash(state) } diff --git a/quickwit/quickwit-storage/src/cache/stored_item.rs b/quickwit/quickwit-storage/src/cache/stored_item.rs index dafaa3a1f65..8d1af95c8fd 100644 --- a/quickwit/quickwit-storage/src/cache/stored_item.rs +++ b/quickwit/quickwit-storage/src/cache/stored_item.rs @@ -17,13 +17,13 @@ use tokio::time::Instant; /// It is a bit overkill to put this in its own module, but I /// wanted to ensure that no one would access payload without updating `last_access_time`. -pub(super) struct StoredItem { +pub(super) struct StoredItem { last_access_time: Instant, - payload: OwnedBytes, + payload: V, } -impl StoredItem { - pub fn new(payload: OwnedBytes, now: Instant) -> Self { +impl StoredItem { + pub fn new(payload: V, now: Instant) -> Self { StoredItem { last_access_time: now, payload, @@ -31,8 +31,8 @@ impl StoredItem { } } -impl StoredItem { - pub fn payload(&mut self) -> OwnedBytes { +impl StoredItem { + pub fn payload(&mut self) -> V { self.last_access_time = Instant::now(); self.payload.clone() } @@ -45,3 +45,13 @@ impl StoredItem { self.last_access_time } } + +pub(crate) trait ValueLen { + fn len(&self) -> usize; +} + +impl ValueLen for OwnedBytes { + fn len(&self) -> usize { + OwnedBytes::len(self) + } +} diff --git a/quickwit/quickwit-storage/src/debouncer.rs b/quickwit/quickwit-storage/src/debouncer.rs index 180faaf3520..60e9a3cf299 100644 --- a/quickwit/quickwit-storage/src/debouncer.rs +++ b/quickwit/quickwit-storage/src/debouncer.rs @@ -74,10 +74,10 @@ impl AsyncDebouncer { // explicit scope to drop the lock let weak_fut_opt = { self.cache.lock().unwrap().get(&key).cloned() }; - if let Some(weak_future) = weak_fut_opt { - if let Some(future) = weak_future.upgrade() { - return future.await; - } + if let Some(weak_future) = weak_fut_opt + && let Some(future) = weak_future.upgrade() + { + return future.await; } let fut = Box::pin(build_a_future()) as BoxFuture<'static, V>; diff --git a/quickwit/quickwit-storage/src/error.rs b/quickwit/quickwit-storage/src/error.rs index e3873d91bd8..603df666353 100644 --- a/quickwit/quickwit-storage/src/error.rs +++ b/quickwit/quickwit-storage/src/error.rs @@ -102,7 +102,9 @@ pub type StorageResult = Result; impl StorageError { /// Add some context to the wrapper error. pub fn add_context(self, ctx: C) -> Self - where C: fmt::Display + Send + Sync + 'static { + where + C: fmt::Display + Send + Sync + 'static, + { StorageError { kind: self.kind, source: Arc::new(anyhow::anyhow!("{ctx}").context(self.source)), diff --git a/quickwit/quickwit-storage/src/file_descriptor_cache.rs b/quickwit/quickwit-storage/src/file_descriptor_cache.rs index 93b3658d608..4626e71bee9 100644 --- a/quickwit/quickwit-storage/src/file_descriptor_cache.rs +++ b/quickwit/quickwit-storage/src/file_descriptor_cache.rs @@ -23,12 +23,12 @@ use tantivy::directory::OwnedBytes; use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use ulid::Ulid; -use crate::metrics::CacheMetrics; +use crate::metrics::SingleCacheMetrics; pub struct FileDescriptorCache { fd_cache: Mutex>, fd_semaphore: Arc, - fd_cache_metrics: CacheMetrics, + fd_cache_metrics: SingleCacheMetrics, } #[derive(Clone)] @@ -68,7 +68,7 @@ impl FileDescriptorCache { fn new( max_fd_limit: NonZeroU32, fd_cache_capacity: NonZeroU32, - fd_cache_metrics: CacheMetrics, + fd_cache_metrics: SingleCacheMetrics, ) -> FileDescriptorCache { assert!(max_fd_limit.get() > fd_cache_capacity.get()); let fd_cache = Mutex::new(lru::LruCache::new( @@ -88,7 +88,10 @@ impl FileDescriptorCache { Self::new( NonZeroU32::new(max_fd_limit).unwrap(), fd_cache_capacity, - crate::STORAGE_METRICS.fd_cache_metrics.clone(), + crate::STORAGE_METRICS + .fd_cache_metrics + .cache_metrics + .clone(), ) } @@ -185,7 +188,7 @@ mod tests { #[tokio::test] async fn test_fd_cache_big_cache() { - let cache_metrics = CacheMetrics::for_component("fdtest"); + let cache_metrics = CacheMetrics::for_component("fdtest").cache_metrics; let fd_cache = FileDescriptorCache::new( NonZeroU32::new(20).unwrap(), NonZeroU32::new(10).unwrap(), @@ -227,7 +230,7 @@ mod tests { // opening the file several times. #[tokio::test] async fn test_fd_cache_small_cache() { - let cache_metrics = CacheMetrics::for_component("fdtest2"); + let cache_metrics = CacheMetrics::for_component("fdtest2").cache_metrics; let fd_cache = FileDescriptorCache::new( NonZeroU32::new(20).unwrap(), NonZeroU32::new(10).unwrap(), diff --git a/quickwit/quickwit-storage/src/lib.rs b/quickwit/quickwit-storage/src/lib.rs index 1d7ff35fc27..401eec7d118 100644 --- a/quickwit/quickwit-storage/src/lib.rs +++ b/quickwit/quickwit-storage/src/lib.rs @@ -69,9 +69,9 @@ pub use self::local_file_storage::{LocalFileStorage, LocalFileStorageFactory}; #[cfg(feature = "azure")] pub use self::object_storage::{AzureBlobStorage, AzureBlobStorageFactory}; pub use self::object_storage::{ - MultiPartPolicy, S3CompatibleObjectStorage, S3CompatibleObjectStorageFactory, - OBJECT_STORAGE_REQUEST_COUNT, OBJECT_STORAGE_BYTES_FETCHED, - get_object_storage_request_count, get_object_storage_bytes_fetched, reset_object_storage_request_stats, + MultiPartPolicy, OBJECT_STORAGE_BYTES_FETCHED, OBJECT_STORAGE_REQUEST_COUNT, + S3CompatibleObjectStorage, S3CompatibleObjectStorageFactory, get_object_storage_bytes_fetched, + get_object_storage_request_count, reset_object_storage_request_stats, }; #[cfg(feature = "gcs")] pub use self::opendal_storage::GoogleCloudStorageFactory; @@ -385,9 +385,28 @@ pub(crate) mod test_suite { #[cfg(feature = "integration-testsuite")] pub async fn storage_test_multi_part_upload(storage: &mut dyn Storage) -> anyhow::Result<()> { let test_path = Path::new("hello_large.txt"); - let test_buffer = vec![0u8; 15_000_000]; - storage.put(test_path, Box::new(test_buffer)).await?; + + let mut test_buffer = Vec::with_capacity(15_000_000); + for i in 0..15_000_000u32 { + test_buffer.push((i % 256) as u8); + } + + storage + .put(test_path, Box::new(test_buffer.clone())) + .await?; + assert_eq!(storage.file_num_bytes(test_path).await?, 15_000_000); + + let downloaded_data = storage.get_all(test_path).await?; + + assert_eq!(test_buffer.len(), downloaded_data.len(), "Length mismatch"); + // dont use assert_eq since we dont want large buffers to be printed + // if assert fails + assert!( + test_buffer.as_slice() == downloaded_data.as_slice(), + "Content mismatch - data corruption detected!" + ); + Ok(()) } } diff --git a/quickwit/quickwit-storage/src/local_file_storage.rs b/quickwit/quickwit-storage/src/local_file_storage.rs index 504c5a4ce3a..ec5fdbbd633 100644 --- a/quickwit/quickwit-storage/src/local_file_storage.rs +++ b/quickwit/quickwit-storage/src/local_file_storage.rs @@ -244,10 +244,10 @@ impl Storage for LocalFileStorage { async fn delete(&self, path: &Path) -> StorageResult<()> { self.delete_single_file(path).await?; - if let Some(parent) = path.parent() { - if let Err(error) = delete_all_dirs_if_empty(&self.root, parent).await { - warn!(error=?error, path=%path.display(), "failed to delete directory"); - } + if let Some(parent) = path.parent() + && let Err(error) = delete_all_dirs_if_empty(&self.root, parent).await + { + warn!(error=?error, path=%path.display(), "failed to delete directory"); } Ok(()) } diff --git a/quickwit/quickwit-storage/src/metrics.rs b/quickwit/quickwit-storage/src/metrics.rs index 43ef588e192..8b439bfaeaa 100644 --- a/quickwit/quickwit-storage/src/metrics.rs +++ b/quickwit/quickwit-storage/src/metrics.rs @@ -14,16 +14,21 @@ // See https://prometheus.io/docs/practices/naming/ +use std::collections::HashMap; +use std::sync::RwLock; + use once_cell::sync::Lazy; use quickwit_common::metrics::{ GaugeGuard, Histogram, IntCounter, IntCounterVec, IntGauge, new_counter, new_counter_vec, new_gauge, new_histogram_vec, }; +use quickwit_config::CacheConfig; /// Counters associated to storage operations. pub struct StorageMetrics { pub shortlived_cache: CacheMetrics, pub partial_request_cache: CacheMetrics, + pub predicate_cache: CacheMetrics, pub fd_cache_metrics: CacheMetrics, pub fast_field_cache: CacheMetrics, pub split_footer_cache: CacheMetrics, @@ -92,6 +97,7 @@ impl Default for StorageMetrics { fast_field_cache: CacheMetrics::for_component("fastfields"), fd_cache_metrics: CacheMetrics::for_component("fd"), partial_request_cache: CacheMetrics::for_component("partial_request"), + predicate_cache: CacheMetrics::for_component("predicate"), searcher_split_cache: CacheMetrics::for_component("searcher_split"), shortlived_cache: CacheMetrics::for_component("shortlived"), split_footer_cache: CacheMetrics::for_component("splitfooter"), @@ -158,9 +164,14 @@ impl Default for StorageMetrics { } /// Counters associated to a cache. -#[derive(Clone)] pub struct CacheMetrics { pub component_name: String, + pub cache_metrics: SingleCacheMetrics, + virtual_caches_metrics: RwLock>, +} + +#[derive(Clone)] +pub struct SingleCacheMetrics { pub in_cache_count: IntGauge, pub in_cache_num_bytes: IntGauge, pub hits_num_items: IntCounter, @@ -173,51 +184,122 @@ pub struct CacheMetrics { impl CacheMetrics { pub fn for_component(component_name: &str) -> Self { const CACHE_METRICS_NAMESPACE: &str = "cache"; + let labels = [("component_name", component_name)]; CacheMetrics { component_name: component_name.to_string(), + cache_metrics: SingleCacheMetrics { + in_cache_count: new_gauge( + "in_cache_count", + "Count of in cache by component", + CACHE_METRICS_NAMESPACE, + &labels, + ), + in_cache_num_bytes: new_gauge( + "in_cache_num_bytes", + "Number of bytes in cache by component", + CACHE_METRICS_NAMESPACE, + &labels, + ), + hits_num_items: new_counter( + "cache_hits_total", + "Number of cache hits by component", + CACHE_METRICS_NAMESPACE, + &labels, + ), + hits_num_bytes: new_counter( + "cache_hits_bytes", + "Number of cache hits in bytes by component", + CACHE_METRICS_NAMESPACE, + &labels, + ), + misses_num_items: new_counter( + "cache_misses_total", + "Number of cache misses by component", + CACHE_METRICS_NAMESPACE, + &labels, + ), + evict_num_items: new_counter( + "cache_evict_total", + "Number of cache entry evicted by component", + CACHE_METRICS_NAMESPACE, + &labels, + ), + evict_num_bytes: new_counter( + "cache_evict_bytes", + "Number of cache entry evicted in bytes by component", + CACHE_METRICS_NAMESPACE, + &labels, + ), + }, + virtual_caches_metrics: RwLock::default(), + } + } + + pub fn virtual_cache(&self, config: &CacheConfig) -> SingleCacheMetrics { + if let Some(virtual_cache_metrics) = self.virtual_caches_metrics.read().unwrap().get(config) + { + return virtual_cache_metrics.clone(); + } + + const CACHE_METRICS_NAMESPACE: &str = "cache"; + let capacity = config.capacity().as_u64().to_string(); + let policy = config.policy().to_string(); + let labels = [ + ("component_name", self.component_name.as_str()), + ("capacity", &capacity), + ("policy", &policy), + ]; + let new_virtual_cache_metrics = SingleCacheMetrics { in_cache_count: new_gauge( - "in_cache_count", + "virtual_in_cache_count", "Count of in cache by component", CACHE_METRICS_NAMESPACE, - &[("component_name", component_name)], + &labels, ), in_cache_num_bytes: new_gauge( - "in_cache_num_bytes", + "virtual_in_cache_num_bytes", "Number of bytes in cache by component", CACHE_METRICS_NAMESPACE, - &[("component_name", component_name)], + &labels, ), hits_num_items: new_counter( - "cache_hits_total", + "virtual_cache_hits_total", "Number of cache hits by component", CACHE_METRICS_NAMESPACE, - &[("component_name", component_name)], + &labels, ), hits_num_bytes: new_counter( - "cache_hits_bytes", + "virtual_cache_hits_bytes", "Number of cache hits in bytes by component", CACHE_METRICS_NAMESPACE, - &[("component_name", component_name)], + &labels, ), misses_num_items: new_counter( - "cache_misses_total", + "virtual_cache_misses_total", "Number of cache misses by component", CACHE_METRICS_NAMESPACE, - &[("component_name", component_name)], + &labels, ), evict_num_items: new_counter( - "cache_evict_total", + "virtual_cache_evict_total", "Number of cache entry evicted by component", CACHE_METRICS_NAMESPACE, - &[("component_name", component_name)], + &labels, ), evict_num_bytes: new_counter( - "cache_evict_bytes", + "virtual_cache_evict_bytes", "Number of cache entry evicted in bytes by component", CACHE_METRICS_NAMESPACE, - &[("component_name", component_name)], + &labels, ), - } + }; + + self.virtual_caches_metrics + .write() + .unwrap() + .entry(config.clone()) + .or_insert(new_virtual_cache_metrics) + .clone() } } diff --git a/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs b/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs index 80f00f5dda8..2be0437fee9 100644 --- a/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs +++ b/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs @@ -20,7 +20,7 @@ use std::sync::Arc; use std::sync::atomic::Ordering; use std::{fmt, io}; -use super::s3_compatible_storage::{OBJECT_STORAGE_REQUEST_COUNT, OBJECT_STORAGE_BYTES_FETCHED}; +use super::s3_compatible_storage::{OBJECT_STORAGE_BYTES_FETCHED, OBJECT_STORAGE_REQUEST_COUNT}; use async_trait::async_trait; use azure_core::error::ErrorKind; @@ -115,7 +115,11 @@ impl AzureBlobStorage { let blob_endpoint = if endpoint_url.contains(&storage_account_name) { endpoint_url.clone() } else { - let constructed = format!("{}/{}", endpoint_url.trim_end_matches('/'), storage_account_name); + let constructed = format!( + "{}/{}", + endpoint_url.trim_end_matches('/'), + storage_account_name + ); constructed }; @@ -245,7 +249,6 @@ impl AzureBlobStorage { path: &Path, range_opt: Option>, ) -> StorageResult> { - let name = self.blob_name(path); let capacity = range_opt.as_ref().map(Range::len).unwrap_or(0); @@ -311,7 +314,7 @@ impl AzureBlobStorage { chunk_range(0..total_len as usize, part_len as usize).map(into_u64_range); let blob_client = self.container_client.blob_client(name); - let mut upload_blocks_stream_result = tokio_stream::iter(multipart_ranges.enumerate()) + let upload_blocks_stream = tokio_stream::iter(multipart_ranges.enumerate()) .map(|(num, range)| { let moved_blob_client = blob_client.clone(); let moved_payload = payload.clone(); @@ -321,7 +324,8 @@ impl AzureBlobStorage { .inc_by(range.end - range.start); async move { retry(&self.retry_params, || async { - let block_id = format!("block:{num}"); + // zero pad block ids to make them sortable as strings + let block_id = format!("block:{:05}", num); let (data, hash_digest) = extract_range_data_and_hash(moved_payload.box_clone(), range.clone()) .await?; @@ -338,16 +342,22 @@ impl AzureBlobStorage { }) .buffer_unordered(self.multipart_policy.max_concurrent_uploads()); - // Concurrently upload block with limit. - let mut block_list = BlockList::default(); - while let Some(put_block_result) = upload_blocks_stream_result.next().await { - match put_block_result { - Ok(block_id) => block_list - .blocks - .push(BlobBlockType::new_uncommitted(block_id)), - Err(error) => return Err(error.into()), - } - } + // Collect and sort block ids to preserve part order for put_block_list. + // Azure docs: "The put block list operation enforces the order in which blocks + // are to be combined to create a blob". + // https://docs.microsoft.com/en-us/rest/api/storageservices/put-block-list + let mut block_ids: Vec = upload_blocks_stream + .try_collect() + .await + .map_err(StorageError::from)?; + block_ids.sort_unstable(); + + let block_list = BlockList { + blocks: block_ids + .into_iter() + .map(BlobBlockType::new_uncommitted) + .collect(), + }; // Commit all uploaded blocks. blob_client @@ -480,19 +490,16 @@ impl Storage for AzureBlobStorage { let result = self.get_to_vec(path, Some(range.clone())).await; - let final_result = result - .map(OwnedBytes::new) - .map_err(|err| { - err.add_context(format!( - "failed to fetch slice {:?} for object: {}/{}", - range, - self.uri, - path.display(), - )) - }); + let final_result = result.map(OwnedBytes::new).map_err(|err| { + err.add_context(format!( + "failed to fetch slice {:?} for object: {}/{}", + range, + self.uri, + path.display(), + )) + }); - if final_result.is_ok() { - } + if final_result.is_ok() {} final_result } @@ -608,7 +615,6 @@ async fn download_all( chunk_stream: &mut Pageable, output: &mut Vec, ) -> Result<(), AzureErrorWrapper> { - output.clear(); let mut chunk_count = 0; @@ -618,9 +624,7 @@ async fn download_all( } { chunk_count += 1; let chunk_response = match chunk_result { - Ok(response) => { - response - } + Ok(response) => response, Err(e) => { return Err(e.into()); } diff --git a/quickwit/quickwit-storage/src/object_storage/error.rs b/quickwit/quickwit-storage/src/object_storage/error.rs index 9f5b2e71530..ffc02216971 100644 --- a/quickwit/quickwit-storage/src/object_storage/error.rs +++ b/quickwit/quickwit-storage/src/object_storage/error.rs @@ -26,7 +26,8 @@ use aws_sdk_s3::operation::upload_part::UploadPartError; use crate::{StorageError, StorageErrorKind}; impl From> for StorageError -where E: std::error::Error + ToStorageErrorKind + Send + Sync + 'static +where + E: std::error::Error + ToStorageErrorKind + Send + Sync + 'static, { fn from(error: SdkError) -> StorageError { let error_kind = match &error { @@ -54,10 +55,16 @@ where E: std::error::Error + ToStorageErrorKind + Send + Sync + 'static let source = if let SdkError::DispatchFailure(failure) = &error { // For DNS lookup failures, try to extract the hostname from the error let error_str = format!("{}", DisplayErrorContext(&error)); - if error_str.contains("dns error") && error_str.contains("failed to lookup address information") { + if error_str.contains("dns error") + && error_str.contains("failed to lookup address information") + { // Try to extract hostname from the error context if let Some(hostname) = extract_hostname_from_dispatch_failure(&failure) { - anyhow::anyhow!("dns error: failed to lookup address information for hostname '{}': {}", hostname, error_str) + anyhow::anyhow!( + "dns error: failed to lookup address information for hostname '{}': {}", + hostname, + error_str + ) } else { anyhow::anyhow!("dns error (hostname not extractable): {}", error_str) } @@ -149,17 +156,17 @@ impl ToStorageErrorKind for HeadObjectError { fn extract_hostname_from_dispatch_failure(failure: &impl std::fmt::Debug) -> Option { // Convert the failure to string to extract hostname let error_string = format!("{:?}", failure); - + // Look for patterns that might contain the hostname // Common patterns in hyper/AWS SDK errors for DNS failures: - // - "tcp connect error: dns error: failed to lookup address information" + // - "tcp connect error: dns error: failed to lookup address information" // - Connection attempts often have the hostname embedded - + // Try to find hostname in various patterns if let Some(hostname) = extract_hostname_from_error_string(&error_string) { return Some(hostname); } - + // If we can't extract it from the error string, return None None } @@ -176,7 +183,7 @@ fn extract_hostname_from_error_string(error_str: &str) -> Option { } } } - + // Pattern 2: Look for hostname in connection error messages if let Some(start) = error_str.find("connection: ") { let substr = &error_str[start + 12..]; @@ -187,7 +194,7 @@ fn extract_hostname_from_error_string(error_str: &str) -> Option { } } } - + // Pattern 3: Look for "Host: " pattern if let Some(start) = error_str.find("Host: ") { let substr = &error_str[start + 6..]; @@ -198,12 +205,12 @@ fn extract_hostname_from_error_string(error_str: &str) -> Option { } } } - + // Pattern 4: Extract from URL-like patterns s3://bucket or https://endpoint if let Some(hostname) = extract_hostname_from_url_pattern(error_str) { return Some(hostname); } - + None } @@ -214,34 +221,36 @@ fn extract_hostname_from_url_pattern(error_str: &str) -> Option { if let Some(start) = error_str.find(prefix) { let url_start = start + prefix.len(); let substr = &error_str[url_start..]; - + // Find the end of hostname (before path or query) - let hostname_end = substr.find(&['/', '?', ' ', '\n', '\r', ')', '"'][..]) + let hostname_end = substr + .find(&['/', '?', ' ', '\n', '\r', ')', '"'][..]) .unwrap_or(substr.len()); - + let hostname = &substr[..hostname_end]; if !hostname.is_empty() && !hostname.contains("Unknown") { return Some(hostname.to_string()); } } } - + // Look for s3:// patterns if let Some(start) = error_str.find("s3://") { let url_start = start + 5; let substr = &error_str[url_start..]; - + // Find the end of bucket name - let bucket_end = substr.find(&['/', ' ', '\n', '\r', ')', '"'][..]) + let bucket_end = substr + .find(&['/', ' ', '\n', '\r', ')', '"'][..]) .unwrap_or(substr.len()); - + let bucket = &substr[..bucket_end]; if !bucket.is_empty() { - // For s3://, we might want to show the bucket, but the actual hostname + // For s3://, we might want to show the bucket, but the actual hostname // would be the S3 endpoint. Let's return the s3:// URL for context. return Some(format!("s3://{}", bucket)); } } - + None } diff --git a/quickwit/quickwit-storage/src/object_storage/mod.rs b/quickwit/quickwit-storage/src/object_storage/mod.rs index aa9c93b84bb..73730c62789 100644 --- a/quickwit/quickwit-storage/src/object_storage/mod.rs +++ b/quickwit/quickwit-storage/src/object_storage/mod.rs @@ -17,8 +17,8 @@ mod error; mod s3_compatible_storage; pub use self::s3_compatible_storage::S3CompatibleObjectStorage; pub use self::s3_compatible_storage::{ - OBJECT_STORAGE_REQUEST_COUNT, OBJECT_STORAGE_BYTES_FETCHED, - get_object_storage_request_count, get_object_storage_bytes_fetched, reset_object_storage_request_stats, + OBJECT_STORAGE_BYTES_FETCHED, OBJECT_STORAGE_REQUEST_COUNT, get_object_storage_bytes_fetched, + get_object_storage_request_count, reset_object_storage_request_stats, }; pub use self::s3_compatible_storage_resolver::S3CompatibleObjectStorageFactory; diff --git a/quickwit/quickwit-storage/src/object_storage/s3_compatible_storage.rs b/quickwit/quickwit-storage/src/object_storage/s3_compatible_storage.rs index 050a89eb293..d23233c2598 100644 --- a/quickwit/quickwit-storage/src/object_storage/s3_compatible_storage.rs +++ b/quickwit/quickwit-storage/src/object_storage/s3_compatible_storage.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use quickwit_common::tantivy4java_debug; use std::collections::HashMap; use std::ops::Range; use std::path::{Path, PathBuf}; @@ -19,7 +20,6 @@ use std::pin::Pin; use std::sync::atomic::{AtomicU64, Ordering}; use std::task::{Context, Poll}; use std::{fmt, io}; -use quickwit_common::tantivy4java_debug; use anyhow::{Context as AnyhhowContext, anyhow}; use async_trait::async_trait; @@ -57,7 +57,8 @@ use crate::{ /// Semaphore to limit the number of concurrent requests to the object store. Some object stores /// (R2, SeaweedFs...) return errors when too many concurrent requests are emitted. static REQUEST_SEMAPHORE: Lazy = Lazy::new(|| { - let num_permits: usize = quickwit_common::get_from_env("QW_S3_MAX_CONCURRENCY", 10_000usize); + let num_permits: usize = + quickwit_common::get_from_env("QW_S3_MAX_CONCURRENCY", 10_000usize, false); Semaphore::new(num_permits) }); @@ -134,11 +135,8 @@ fn get_credentials_provider( ) { (Some(access_key_id), Some(secret_access_key), session_token) => { info!("using S3 credentials defined in storage config"); - let credentials = Credentials::from_keys( - access_key_id, - secret_access_key, - session_token.clone() - ); + let credentials = + Credentials::from_keys(access_key_id, secret_access_key, session_token.clone()); let credentials_provider = SharedCredentialsProvider::new(credentials); Some(credentials_provider) } @@ -204,7 +202,8 @@ impl S3CompatibleObjectStorage { uri: &Uri, credentials_provider: SharedCredentialsProvider, ) -> Result { - let s3_client = create_s3_client_with_provider(s3_storage_config, Some(credentials_provider)).await; + let s3_client = + create_s3_client_with_provider(s3_storage_config, Some(credentials_provider)).await; Self::from_uri_and_client(s3_storage_config, uri, s3_client).await } @@ -219,10 +218,14 @@ impl S3CompatibleObjectStorage { StorageResolverError::InvalidUri(message) })?; // Conditional debug logging - tantivy4java_debug!("QUICKWIT DEBUG: S3 storage created from URI: '{}', bucket: '{}', prefix: '{}'", - uri.as_str(), bucket, prefix.display()); + tantivy4java_debug!( + "QUICKWIT DEBUG: S3 storage created from URI: '{}', bucket: '{}', prefix: '{}'", + uri.as_str(), + bucket, + prefix.display() + ); tantivy4java_debug!("QUICKWIT DEBUG: S3 storage instance created"); - + // Add stack trace to understand what's calling this tantivy4java_debug!("QUICKWIT DEBUG: S3 storage creation stack trace:"); // let backtrace_str = format!("{}", backtrace); @@ -314,7 +317,7 @@ async fn compute_md5(mut read: T) -> io::Resu let read_len = read.read(&mut buf).await?; checksum.consume(&buf[..read_len]); if read_len == 0 { - return Ok(checksum.compute()); + return Ok(checksum.finalize()); } } } @@ -324,9 +327,16 @@ impl S3CompatibleObjectStorage { // FIXME: This may not work on Windows. let key_path = self.prefix.join(relative_path); let key_str = key_path.to_string_lossy().to_string(); - tantivy4java_debug!("QUICKWIT DEBUG: key() called with relative_path: '{}', prefix: '{}', result: '{}'", - relative_path.display(), self.prefix.display(), key_str); - tantivy4java_debug!("QUICKWIT DEBUG: key() called on storage instance: {:p}", self); + tantivy4java_debug!( + "QUICKWIT DEBUG: key() called with relative_path: '{}', prefix: '{}', result: '{}'", + relative_path.display(), + self.prefix.display(), + key_str + ); + tantivy4java_debug!( + "QUICKWIT DEBUG: key() called on storage instance: {:p}", + self + ); key_str } @@ -883,21 +893,37 @@ impl Storage for S3CompatibleObjectStorage { tantivy4java_debug!("QUICKWIT DEBUG: ===== S3 GET_SLICE REQUEST ====="); tantivy4java_debug!("QUICKWIT DEBUG: get_slice path: '{}'", path.display()); - tantivy4java_debug!("QUICKWIT DEBUG: get_slice range: {} to {} ({} bytes)", range.start, range.end, range_size); + tantivy4java_debug!( + "QUICKWIT DEBUG: get_slice range: {} to {} ({} bytes)", + range.start, + range.end, + range_size + ); tantivy4java_debug!("QUICKWIT DEBUG: get_slice key: '{}'", key); tantivy4java_debug!("QUICKWIT DEBUG: get_slice storage instance: {:p}", self); - + // Track large vs small requests for hotcache analysis - if range_size > 1_000_000 { // > 1MB - tantivy4java_debug!("QUICKWIT DEBUG: 🔥 LARGE REQUEST: {} MB - POTENTIAL HOTCACHE MISS!", range_size / 1_000_000); - } else if range_size > 100_000 { // > 100KB - tantivy4java_debug!("QUICKWIT DEBUG: ⚠️ MEDIUM REQUEST: {} KB", range_size / 1000); + if range_size > 1_000_000 { + // > 1MB + tantivy4java_debug!( + "QUICKWIT DEBUG: 🔥 LARGE REQUEST: {} MB - POTENTIAL HOTCACHE MISS!", + range_size / 1_000_000 + ); + } else if range_size > 100_000 { + // > 100KB + tantivy4java_debug!( + "QUICKWIT DEBUG: ⚠️ MEDIUM REQUEST: {} KB", + range_size / 1000 + ); } else { - tantivy4java_debug!("QUICKWIT DEBUG: ✅ SMALL REQUEST: {} bytes - LIKELY HOTCACHE", range_size); + tantivy4java_debug!( + "QUICKWIT DEBUG: ✅ SMALL REQUEST: {} bytes - LIKELY HOTCACHE", + range_size + ); } - + let result = self.get_to_vec(path, Some(range.clone())).await; - + // match &result { // Ok(bytes) => { // tantivy4java_debug!("QUICKWIT DEBUG: [Thread {:?}] get_slice SUCCESS: received {} bytes", thread_id, bytes.len()); @@ -906,18 +932,16 @@ impl Storage for S3CompatibleObjectStorage { // tantivy4java_debug!("QUICKWIT DEBUG: [Thread {:?}] get_slice FAILED: {}", thread_id, err); // } // } - - result - .map(OwnedBytes::new) - .map_err(|err| { - tantivy4java_debug!("QUICKWIT DEBUG: get_slice error occurred: {:?}", err); - err.add_context(format!( - "failed to fetch slice {:?} for object: {}/{}", - range, - self.uri, - path.display(), - )) - }) + + result.map(OwnedBytes::new).map_err(|err| { + tantivy4java_debug!("QUICKWIT DEBUG: get_slice error occurred: {:?}", err); + err.add_context(format!( + "failed to fetch slice {:?} for object: {}/{}", + range, + self.uri, + path.display(), + )) + }) } #[instrument(level = "debug", skip(self, range), fields(range.start = range.start, range.end = range.end))] @@ -946,9 +970,9 @@ impl Storage for S3CompatibleObjectStorage { tantivy4java_debug!("QUICKWIT DEBUG: get_all path: '{}'", path.display()); tantivy4java_debug!("QUICKWIT DEBUG: get_all key: '{}'", key); tantivy4java_debug!("QUICKWIT DEBUG: get_all storage instance: {:p}", self); - + let result = self.get_to_vec(path, None).await; - + // match &result { // Ok(bytes) => { // tantivy4java_debug!("QUICKWIT DEBUG: [Thread {:?}] get_all SUCCESS: received {} bytes", thread_id, bytes.len()); @@ -957,16 +981,14 @@ impl Storage for S3CompatibleObjectStorage { // tantivy4java_debug!("QUICKWIT DEBUG: [Thread {:?}] get_all FAILED: {}", thread_id, err); // } // } - - let bytes = result - .map(OwnedBytes::new) - .map_err(|err| { - err.add_context(format!( - "failed to fetch object: {}/{}", - self.uri, - path.display() - )) - })?; + + let bytes = result.map(OwnedBytes::new).map_err(|err| { + err.add_context(format!( + "failed to fetch object: {}/{}", + self.uri, + path.display() + )) + })?; tracing::Span::current().record("num_bytes_fetched", bytes.len()); Ok(bytes) } @@ -981,22 +1003,31 @@ impl Storage for S3CompatibleObjectStorage { tantivy4java_debug!("QUICKWIT DEBUG: input path: '{}'", path.display()); tantivy4java_debug!("QUICKWIT DEBUG: storage URI: '{}'", self.uri.as_str()); tantivy4java_debug!("QUICKWIT DEBUG: storage bucket: '{}'", bucket); - tantivy4java_debug!("QUICKWIT DEBUG: storage prefix: '{}'", self.prefix.display()); + tantivy4java_debug!( + "QUICKWIT DEBUG: storage prefix: '{}'", + self.prefix.display() + ); tantivy4java_debug!("QUICKWIT DEBUG: computed key: '{}'", key); tantivy4java_debug!("QUICKWIT DEBUG: storage instance: {:p}", self); - + // Try to get AWS config details for debugging - tantivy4java_debug!("QUICKWIT DEBUG: AWS credentials: [using environment/default provider]"); - + tantivy4java_debug!( + "QUICKWIT DEBUG: AWS credentials: [using environment/default provider]" + ); + // if let Some(region) = self.s3_client.config().region() { // tantivy4java_debug!("QUICKWIT DEBUG: AWS region: '{}'", region.as_ref()); // } else { // tantivy4java_debug!("QUICKWIT DEBUG: AWS region: not set"); // } - + tantivy4java_debug!("QUICKWIT DEBUG: ====================================="); let head_object_output = aws_retry(&self.retry_params, || async { - tantivy4java_debug!("QUICKWIT DEBUG: Sending HEAD object request to S3 with bucket='{}' key='{}'", bucket, key); + tantivy4java_debug!( + "QUICKWIT DEBUG: Sending HEAD object request to S3 with bucket='{}' key='{}'", + bucket, + key + ); self.s3_client .head_object() .bucket(&bucket) @@ -1007,12 +1038,19 @@ impl Storage for S3CompatibleObjectStorage { .await .map_err(|e| { tantivy4java_debug!("QUICKWIT DEBUG: HEAD object request FAILED: {:?}", e); - tantivy4java_debug!("QUICKWIT DEBUG: Failed request was: bucket='{}' key='{}'", bucket, key); + tantivy4java_debug!( + "QUICKWIT DEBUG: Failed request was: bucket='{}' key='{}'", + bucket, + key + ); e })?; let size = head_object_output.content_length().unwrap_or(0) as u64; - tantivy4java_debug!("QUICKWIT DEBUG: HEAD object request SUCCEEDED, content_length: {}", size); + tantivy4java_debug!( + "QUICKWIT DEBUG: HEAD object request SUCCEEDED, content_length: {}", + size + ); Ok(size) } @@ -1445,10 +1483,17 @@ mod tests { ..Default::default() }; - let credentials = Credentials::new("custom_key", "custom_secret", Some("custom_token".to_string()), None, "test_provider"); + let credentials = Credentials::new( + "custom_key", + "custom_secret", + Some("custom_token".to_string()), + None, + "test_provider", + ); let custom_provider = SharedCredentialsProvider::new(credentials); - let _s3_client = create_s3_client_with_provider(&s3_storage_config, Some(custom_provider)).await; + let _s3_client = + create_s3_client_with_provider(&s3_storage_config, Some(custom_provider)).await; // Just verify the client can be created without panicking } @@ -1477,14 +1522,21 @@ mod tests { ..Default::default() }; let uri = Uri::for_test("s3://test-bucket/test-prefix"); - let credentials = Credentials::new("provider_key", "provider_secret", Some("provider_token".to_string()), None, "test_provider"); + let credentials = Credentials::new( + "provider_key", + "provider_secret", + Some("provider_token".to_string()), + None, + "test_provider", + ); let custom_provider = SharedCredentialsProvider::new(credentials); let storage = S3CompatibleObjectStorage::from_uri_with_credentials_provider( &s3_storage_config, &uri, - custom_provider - ).await; + custom_provider, + ) + .await; assert!(storage.is_ok()); let storage = storage.unwrap(); diff --git a/quickwit/quickwit-storage/src/object_storage/s3_compatible_storage_resolver.rs b/quickwit/quickwit-storage/src/object_storage/s3_compatible_storage_resolver.rs index 9cda9b50487..b9386fbd1a9 100644 --- a/quickwit/quickwit-storage/src/object_storage/s3_compatible_storage_resolver.rs +++ b/quickwit/quickwit-storage/src/object_storage/s3_compatible_storage_resolver.rs @@ -72,7 +72,11 @@ impl StorageFactory for S3CompatibleObjectStorageFactory { .s3_client .get_or_init(|| async { if let Some(ref credentials_provider) = self.credentials_provider { - create_s3_client_with_provider(&self.storage_config, Some(credentials_provider.clone())).await + create_s3_client_with_provider( + &self.storage_config, + Some(credentials_provider.clone()), + ) + .await } else { create_s3_client(&self.storage_config).await } @@ -112,7 +116,10 @@ mod tests { let factory = S3CompatibleObjectStorageFactory::new(config.clone()); assert_eq!(factory.storage_config.access_key_id, config.access_key_id); - assert_eq!(factory.storage_config.secret_access_key, config.secret_access_key); + assert_eq!( + factory.storage_config.secret_access_key, + config.secret_access_key + ); assert_eq!(factory.storage_config.region, config.region); assert!(factory.credentials_provider.is_none()); } @@ -124,12 +131,18 @@ mod tests { ..Default::default() }; - let credentials = Credentials::new("provider_key", "provider_secret", Some("provider_token".to_string()), None, "test_provider"); + let credentials = Credentials::new( + "provider_key", + "provider_secret", + Some("provider_token".to_string()), + None, + "test_provider", + ); let credentials_provider = SharedCredentialsProvider::new(credentials); let factory = S3CompatibleObjectStorageFactory::new_with_credentials_provider( config.clone(), - credentials_provider.clone() + credentials_provider.clone(), ); assert_eq!(factory.storage_config.region, config.region); @@ -174,12 +187,18 @@ mod tests { ..Default::default() }; - let credentials = Credentials::new("provider_key", "provider_secret", Some("provider_token".to_string()), None, "test_provider"); + let credentials = Credentials::new( + "provider_key", + "provider_secret", + Some("provider_token".to_string()), + None, + "test_provider", + ); let credentials_provider = SharedCredentialsProvider::new(credentials); let factory = S3CompatibleObjectStorageFactory::new_with_credentials_provider( config, - credentials_provider + credentials_provider, ); let uri = Uri::for_test("s3://test-bucket/test-prefix"); diff --git a/quickwit/quickwit-storage/src/opendal_storage/base.rs b/quickwit/quickwit-storage/src/opendal_storage/base.rs index ee573d68c84..f4a203eb9a9 100644 --- a/quickwit/quickwit-storage/src/opendal_storage/base.rs +++ b/quickwit/quickwit-storage/src/opendal_storage/base.rs @@ -79,6 +79,7 @@ impl Storage for OpendalStorage { } async fn put(&self, path: &Path, payload: Box) -> StorageResult<()> { + crate::STORAGE_METRICS.object_storage_put_total.inc(); let path = path.as_os_str().to_string_lossy(); let mut payload_reader = payload.byte_stream().await?.into_async_read(); @@ -91,6 +92,9 @@ impl Storage for OpendalStorage { .compat_write(); tokio::io::copy(&mut payload_reader, &mut storage_writer).await?; storage_writer.get_mut().close().await?; + crate::STORAGE_METRICS + .object_storage_upload_num_bytes + .inc_by(payload.len()); Ok(()) } @@ -103,7 +107,10 @@ impl Storage for OpendalStorage { .into_futures_async_read(..) .await? .compat(); - tokio::io::copy(&mut storage_reader, output).await?; + let num_bytes_copied = tokio::io::copy(&mut storage_reader, output).await?; + crate::STORAGE_METRICS + .object_storage_download_num_bytes + .inc_by(num_bytes_copied); output.flush().await?; Ok(()) } @@ -115,6 +122,7 @@ impl Storage for OpendalStorage { // Unlike other object store implementations, in flight requests are // recorded before issuing the query to the object store. let _inflight_guards = object_storage_get_slice_in_flight_guards(size); + crate::STORAGE_METRICS.object_storage_get_total.inc(); let storage_content = self.op.read_with(&path).range(range).await?.to_vec(); Ok(OwnedBytes::new(storage_content)) } @@ -144,6 +152,12 @@ impl Storage for OpendalStorage { async fn delete(&self, path: &Path) -> StorageResult<()> { let path = path.as_os_str().to_string_lossy(); + crate::STORAGE_METRICS + .object_storage_delete_requests_total + .inc(); + let _timer = crate::STORAGE_METRICS + .object_storage_delete_request_duration + .start_timer(); self.op.delete(&path).await?; Ok(()) } @@ -154,11 +168,15 @@ impl Storage for OpendalStorage { #[cfg(feature = "integration-testsuite")] { let storage_info = self.op.info(); - if storage_info.name().starts_with("sample-bucket") - && storage_info.scheme() == opendal::Scheme::Gcs - { + if storage_info.name().starts_with("sample-bucket") && storage_info.scheme() == "gcs" { let mut bulk_error = BulkDeleteError::default(); for (index, path) in paths.iter().enumerate() { + crate::STORAGE_METRICS + .object_storage_bulk_delete_requests_total + .inc(); + let _timer = crate::STORAGE_METRICS + .object_storage_bulk_delete_request_duration + .start_timer(); let result = self.op.delete(&path.as_os_str().to_string_lossy()).await; if let Err(err) = result { let storage_error_kind = err.kind(); diff --git a/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs b/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs index f722c6545a8..9f174f0a744 100644 --- a/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs +++ b/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs @@ -57,22 +57,6 @@ pub mod test_config_helpers { /// URL of the local GCP emulator. pub const LOCAL_GCP_EMULATOR_ENDPOINT: &str = "http://127.0.0.1:4443"; - - /// reqsign::GoogleTokenLoad implementation for testing. - #[derive(Debug)] - pub struct DummyTokenLoader; - - #[async_trait] - impl reqsign::GoogleTokenLoad for DummyTokenLoader { - async fn load(&self, _: reqwest::Client) -> anyhow::Result> { - Ok(Some(reqsign::GoogleToken::new( - "dummy", - 86400, - "https://www.googleapis.com/auth/devstorage.full_control", - ))) - } - } - /// Creates a storage connecting to a local emulated google cloud storage. pub fn new_emulated_google_cloud_storage( uri: &Uri, @@ -83,7 +67,8 @@ pub mod test_config_helpers { .bucket(&bucket) .root(&root.to_string_lossy()) .endpoint(LOCAL_GCP_EMULATOR_ENDPOINT) - .customized_token_loader(Box::new(DummyTokenLoader)); + .allow_anonymous() // Disable authentication for fake GCS server + .disable_vm_metadata(); // Disable GCE metadata server requests let store = OpendalStorage::new_google_cloud_storage(uri.clone(), cfg)?; Ok(store) } diff --git a/quickwit/quickwit-storage/src/payload.rs b/quickwit/quickwit-storage/src/payload.rs index 65f4c8cc69f..266a327cdeb 100644 --- a/quickwit/quickwit-storage/src/payload.rs +++ b/quickwit/quickwit-storage/src/payload.rs @@ -53,7 +53,8 @@ pub trait PutPayloadClone { } impl PutPayloadClone for T -where T: 'static + PutPayload + Clone +where + T: 'static + PutPayload + Clone, { fn box_clone(&self) -> Box { Box::new(self.clone()) diff --git a/quickwit/quickwit-storage/src/split_cache/mod.rs b/quickwit/quickwit-storage/src/split_cache/mod.rs index 193e6735d01..7417471af58 100644 --- a/quickwit/quickwit-storage/src/split_cache/mod.rs +++ b/quickwit/quickwit-storage/src/split_cache/mod.rs @@ -73,10 +73,10 @@ impl SplitCache { // This file is a temporary file that was being downloaded, when Quickwit was // stopped (killed for instance) in a way that prevented // their cleanup. It is important to remove it. - if let Err(io_err) = std::fs::remove_file(&path) { - if io_err.kind() != io::ErrorKind::NotFound { - error!(path=?path, "failed to remove temporary file"); - } + if let Err(io_err) = std::fs::remove_file(&path) + && io_err.kind() != io::ErrorKind::NotFound + { + error!(path=?path, "failed to remove temporary file"); } } "split" => { @@ -217,7 +217,7 @@ impl SplitCacheBackingStorage { } fn record_hit_metrics(&self, result_opt: Option<&OwnedBytes>) { - let split_metrics = &crate::STORAGE_METRICS.searcher_split_cache; + let split_metrics = &crate::STORAGE_METRICS.searcher_split_cache.cache_metrics; if let Some(result) = result_opt { split_metrics.hits_num_items.inc(); split_metrics.hits_num_bytes.inc_by(result.len() as u64); diff --git a/quickwit/quickwit-storage/src/split_cache/split_table.rs b/quickwit/quickwit-storage/src/split_cache/split_table.rs index 7be1c242db0..3546aeff23f 100644 --- a/quickwit/quickwit-storage/src/split_cache/split_table.rs +++ b/quickwit/quickwit-storage/src/split_cache/split_table.rs @@ -154,18 +154,22 @@ impl SplitTable { self.on_disk_bytes -= num_bytes; crate::metrics::STORAGE_METRICS .searcher_split_cache + .cache_metrics .in_cache_count .dec(); crate::metrics::STORAGE_METRICS .searcher_split_cache + .cache_metrics .in_cache_num_bytes .sub(num_bytes as i64); crate::metrics::STORAGE_METRICS .searcher_split_cache + .cache_metrics .evict_num_items .inc(); crate::metrics::STORAGE_METRICS .searcher_split_cache + .cache_metrics .evict_num_bytes .inc_by(num_bytes); &mut self.on_disk_splits @@ -173,10 +177,10 @@ impl SplitTable { }; let is_in_queue = split_queue.remove(&split_info.split_key); assert!(is_in_queue); - if let Status::Downloading { alive_token } = &split_info.status { - if alive_token.strong_count() == 0 { - return None; - } + if let Status::Downloading { alive_token } = &split_info.status + && alive_token.strong_count() == 0 + { + return None; } Some(split_info) } @@ -189,12 +193,11 @@ impl SplitTable { } let mut splits_to_remove = Vec::new(); for split in &self.downloading_splits { - if let Some(split_info) = self.split_to_status.get(&split.split_ulid) { - if let Status::Downloading { alive_token } = &split_info.status { - if alive_token.strong_count() == 0 { - splits_to_remove.push(split.split_ulid); - } - } + if let Some(split_info) = self.split_to_status.get(&split.split_ulid) + && let Status::Downloading { alive_token } = &split_info.status + && alive_token.strong_count() == 0 + { + splits_to_remove.push(split.split_ulid); } } for split in splits_to_remove { @@ -219,10 +222,12 @@ impl SplitTable { self.on_disk_bytes += num_bytes; crate::metrics::STORAGE_METRICS .searcher_split_cache + .cache_metrics .in_cache_count .inc(); crate::metrics::STORAGE_METRICS .searcher_split_cache + .cache_metrics .in_cache_num_bytes .add(num_bytes as i64); self.on_disk_splits.insert(split_info.split_key) diff --git a/quickwit/quickwit-storage/tests/azure_storage.rs b/quickwit/quickwit-storage/tests/azure_storage.rs index c54720d5bbd..c5d4937220a 100644 --- a/quickwit/quickwit-storage/tests/azure_storage.rs +++ b/quickwit/quickwit-storage/tests/azure_storage.rs @@ -45,7 +45,7 @@ async fn azure_storage_test_suite() -> anyhow::Result<()> { object_storage.set_policy(MultiPartPolicy { // On azure, block size is limited between 64KB and 100MB. - target_part_num_bytes: 5 * 1_024 * 1_024, // 5MB + target_part_num_bytes: 5 * 1_024 * 1_024, // 5MiB max_num_parts: 10_000, multipart_threshold_num_bytes: 10_000_000, max_object_num_bytes: 5_000_000_000_000, diff --git a/quickwit/quickwit-storage/tests/google_cloud_storage.rs b/quickwit/quickwit-storage/tests/google_cloud_storage.rs index 17fa37f8c7a..4b916b44f4f 100644 --- a/quickwit/quickwit-storage/tests/google_cloud_storage.rs +++ b/quickwit/quickwit-storage/tests/google_cloud_storage.rs @@ -25,21 +25,14 @@ mod gcp_storage_test_suite { use quickwit_common::setup_logging_for_tests; use quickwit_common::uri::Uri; use quickwit_storage::test_config_helpers::{ - DummyTokenLoader, LOCAL_GCP_EMULATOR_ENDPOINT, new_emulated_google_cloud_storage, + LOCAL_GCP_EMULATOR_ENDPOINT, new_emulated_google_cloud_storage, }; - use reqsign::GoogleTokenLoad; - pub async fn sign_gcs_request(req: &mut reqwest::Request) -> anyhow::Result<()> { - let client = reqwest::Client::new(); - let token = DummyTokenLoader - .load(client.clone()) - .await? - .ok_or_else(|| anyhow::anyhow!("Failed to obtain authentication token"))?; - - let signer = reqsign::GoogleSigner::new("storage"); - signer.sign(req, &token)?; - - Ok(()) + pub fn sign_gcs_request(req: &mut reqwest::Request) { + req.headers_mut().insert( + reqwest::header::AUTHORIZATION, + reqwest::header::HeaderValue::from_str("Bearer dummy").unwrap(), + ); } async fn create_gcs_bucket(bucket_name: &str) -> anyhow::Result<()> { @@ -53,7 +46,7 @@ mod gcp_storage_test_suite { .header(reqwest::header::CONTENT_TYPE, "application/json") .build()?; - sign_gcs_request(&mut request).await?; + sign_gcs_request(&mut request); let response = client.execute(request).await?; diff --git a/quickwit/quickwit-storage/tests/s3_storage.rs b/quickwit/quickwit-storage/tests/s3_storage.rs index cb7b559dca4..3b87c2915d8 100644 --- a/quickwit/quickwit-storage/tests/s3_storage.rs +++ b/quickwit/quickwit-storage/tests/s3_storage.rs @@ -168,8 +168,7 @@ pub mod s3_storage_test_suite { return; } - let bucket_uri = - append_random_suffix("s3://quickwit-integration-tests/test-session-token"); + let bucket_uri = append_random_suffix("s3://quickwit-integration-tests/test-session-token"); let test_runtime = test_runtime_singleton(); test_runtime.block_on(run_s3_storage_test_suite(s3_storage_config, &bucket_uri)); } @@ -187,10 +186,16 @@ pub mod s3_storage_test_suite { // or use mock credentials for testing let credentials = if let (Ok(access_key), Ok(secret_key)) = ( std::env::var("AWS_ACCESS_KEY_ID"), - std::env::var("AWS_SECRET_ACCESS_KEY") + std::env::var("AWS_SECRET_ACCESS_KEY"), ) { let session_token = std::env::var("AWS_SESSION_TOKEN").ok(); - Credentials::new(access_key, secret_key, session_token, None, "integration_test") + Credentials::new( + access_key, + secret_key, + session_token, + None, + "integration_test", + ) } else { // Use mock credentials for unit testing eprintln!("Using mock credentials for provider test"); @@ -198,7 +203,8 @@ pub mod s3_storage_test_suite { }; let credentials_provider = SharedCredentialsProvider::new(credentials); - let bucket_uri = append_random_suffix("s3://quickwit-integration-tests/test-credentials-provider"); + let bucket_uri = + append_random_suffix("s3://quickwit-integration-tests/test-credentials-provider"); let storage_uri = Uri::from_str(&bucket_uri).unwrap(); let s3_storage_config = S3StorageConfig { @@ -233,7 +239,7 @@ pub mod s3_storage_test_suite { // Create credentials provider let credentials = if let (Ok(access_key), Ok(secret_key)) = ( std::env::var("AWS_ACCESS_KEY_ID"), - std::env::var("AWS_SECRET_ACCESS_KEY") + std::env::var("AWS_SECRET_ACCESS_KEY"), ) { let session_token = std::env::var("AWS_SESSION_TOKEN").ok(); Credentials::new(access_key, secret_key, session_token, None, "factory_test") @@ -252,13 +258,19 @@ pub mod s3_storage_test_suite { credentials_provider, ); - let bucket_uri = append_random_suffix("s3://quickwit-integration-tests/test-factory-provider"); + let bucket_uri = + append_random_suffix("s3://quickwit-integration-tests/test-factory-provider"); let storage_uri = Uri::from_str(&bucket_uri).unwrap(); let storage = factory.resolve(&storage_uri).await.unwrap(); // Verify the storage was created successfully - assert!(storage.uri().as_str().contains("quickwit-integration-tests")); + assert!( + storage + .uri() + .as_str() + .contains("quickwit-integration-tests") + ); }); } } diff --git a/quickwit/quickwit-telemetry/src/payload.rs b/quickwit/quickwit-telemetry/src/payload.rs index c8be92e73d6..2b7f484b283 100644 --- a/quickwit/quickwit-telemetry/src/payload.rs +++ b/quickwit/quickwit-telemetry/src/payload.rs @@ -119,7 +119,6 @@ pub enum QuickwitFeature { Jaeger, Otlp, PostgresqMetastore, - AwsLambda, } fn hashed_host_username() -> String { diff --git a/quickwit/quickwit-ui/.eslintignore b/quickwit/quickwit-ui/.eslintignore deleted file mode 100644 index dd87e2d73f9..00000000000 --- a/quickwit/quickwit-ui/.eslintignore +++ /dev/null @@ -1,2 +0,0 @@ -node_modules -build diff --git a/quickwit/quickwit-ui/.eslintrc b/quickwit/quickwit-ui/.eslintrc deleted file mode 100644 index 569d85fd08c..00000000000 --- a/quickwit/quickwit-ui/.eslintrc +++ /dev/null @@ -1,12 +0,0 @@ -{ - "root": true, - "parser": "@typescript-eslint/parser", - "plugins": ["@typescript-eslint"], - "extends": [ - "eslint:recommended", - "plugin:@typescript-eslint/eslint-recommended", - "plugin:@typescript-eslint/recommended" - ], - "rules": { - } -} diff --git a/quickwit/quickwit-ui/.prettierignore b/quickwit/quickwit-ui/.prettierignore deleted file mode 100644 index 378eac25d31..00000000000 --- a/quickwit/quickwit-ui/.prettierignore +++ /dev/null @@ -1 +0,0 @@ -build diff --git a/quickwit/quickwit-ui/.prettierrc.json b/quickwit/quickwit-ui/.prettierrc.json deleted file mode 100644 index 0967ef424bc..00000000000 --- a/quickwit/quickwit-ui/.prettierrc.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/quickwit/quickwit-ui/Makefile b/quickwit/quickwit-ui/Makefile index 3457b35f546..4ca9676adea 100644 --- a/quickwit/quickwit-ui/Makefile +++ b/quickwit/quickwit-ui/Makefile @@ -4,7 +4,7 @@ build: yarn build install: - yarn install + yarn install --frozen-lockfile --network-timeout 300000 start: yarn start diff --git a/quickwit/quickwit-ui/README.md b/quickwit/quickwit-ui/README.md index 6a1eb379e58..3e6ffa9cd00 100644 --- a/quickwit/quickwit-ui/README.md +++ b/quickwit/quickwit-ui/README.md @@ -1,12 +1,12 @@ -# Getting Started with Create React App +# quickwit-ui + -This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app). ## Prerequisites -`node` and `npm` need to be installed on your system. +`node` and `yarn` need to be installed on your system. The project then relies on misc nodejs tools that can be installed locally by -running `npm install`. +running `yarn`. ## Available Scripts @@ -24,14 +24,18 @@ You will also see any lint errors in the console. ### `yarn test` -Launches the test runner in the interactive watch mode.\ -See the section about [running tests](https://facebook.github.io/create-react-app/docs/running-tests) for more information. +Launches the test runner. ### `yarn e2e-test` Launches the e2e test runner with [cypress](https://www.cypress.io/). To make them work, you need to start a searcher beforehand with `cargo r run --service searcher --config config/quickwit.yaml`. +### `yarn format` + +Re-writes files with the correct formatting if needed.\ +You might want to configure your IDE to do that [automatically](https://biomejs.dev/guides/editors/first-party-extensions/). + ### `yarn build` Builds the app for production to the `build` folder.\ @@ -39,21 +43,3 @@ It correctly bundles React in production mode and optimizes the build for the be The build is minified and the filenames include the hashes.\ Your app is ready to be deployed! - -See the section about [deployment](https://facebook.github.io/create-react-app/docs/deployment) for more information. - -### `yarn eject` - -**Note: this is a one-way operation. Once you `eject`, you can’t go back!** - -If you aren’t satisfied with the build tool and configuration choices, you can `eject` at any time. This command will remove the single build dependency from your project. - -Instead, it will copy all the configuration files and the transitive dependencies (webpack, Babel, ESLint, etc) right into your project so you have full control over them. All of the commands except `eject` will still work, but they will point to the copied scripts so you can tweak them. At this point you’re on your own. - -You don’t have to ever use `eject`. The curated feature set is suitable for small and middle deployments, and you shouldn’t feel obligated to use this feature. However we understand that this tool wouldn’t be useful if you couldn’t customize it when you are ready for it. - -## Learn More - -You can learn more in the [Create React App documentation](https://facebook.github.io/create-react-app/docs/getting-started). - -To learn React, check out the [React documentation](https://reactjs.org/). diff --git a/quickwit/quickwit-ui/biome.json b/quickwit/quickwit-ui/biome.json new file mode 100644 index 00000000000..5bc8eb9994a --- /dev/null +++ b/quickwit/quickwit-ui/biome.json @@ -0,0 +1,26 @@ +{ + "$schema": "./node_modules/@biomejs/biome/configuration_schema.json", + "formatter": { + "enabled": true, + "indentStyle": "space", + "includes": ["**", "!build/**"] + }, + "linter": { + "enabled": true, + "rules": { + "recommended": true, + "style": "off", + "complexity": "off", + "correctness": { + "useExhaustiveDependencies": "off" + }, + "suspicious": { + "noTsIgnore": "off", + "useIterableCallbackReturn": "off", + "noExplicitAny": "off", + "noArrayIndexKey": "off" + } + }, + "includes": ["**", "!build/**"] + } +} diff --git a/quickwit/quickwit-ui/cypress.config.js b/quickwit/quickwit-ui/cypress.config.js deleted file mode 100644 index 00266a27db5..00000000000 --- a/quickwit/quickwit-ui/cypress.config.js +++ /dev/null @@ -1,12 +0,0 @@ -const { defineConfig } = require("cypress"); - -module.exports = defineConfig({ - video: false, - screenshotOnRunFailure: false, - - e2e: { - setupNodeEvents(on, config) { - // implement node event listeners here - }, - }, -}); diff --git a/quickwit/quickwit-ui/cypress/e2e/homepage.spec.cy.js b/quickwit/quickwit-ui/cypress/e2e/homepage.spec.cy.js deleted file mode 100644 index cc31c6fef27..00000000000 --- a/quickwit/quickwit-ui/cypress/e2e/homepage.spec.cy.js +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -describe('Home navigation', () => { - it('Should display sidebar links', () => { - cy.visit('http://127.0.0.1:7280/ui'); - cy.get('a') - .should('be.visible') - .should('contain.text', 'Query editor') - .should('contain.text', 'Indexes') - .should('contain.text', 'Cluster'); - }); - it('Should navigate to cluster state', () => { - cy.visit('http://127.0.0.1:7280/ui'); - cy.get('a').contains('Cluster').click(); - cy.get('p').should('contain.text', 'Cluster'); - cy.get('span').should('contain.text', 'cluster_id'); - }); - it('Should display otel logs index page', () => { - cy.visit('http://127.0.0.1:7280/ui/indexes/otel-logs-v0_7'); - cy.get('a') - .should('be.visible') - .should('contain.text', 'Indexes') - }); -}) diff --git a/quickwit/quickwit-ui/cypress/fixtures/example.json b/quickwit/quickwit-ui/cypress/fixtures/example.json deleted file mode 100644 index 02e4254378e..00000000000 --- a/quickwit/quickwit-ui/cypress/fixtures/example.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "Using fixtures to represent data", - "email": "hello@cypress.io", - "body": "Fixtures are a great way to mock data for responses to routes" -} diff --git a/quickwit/quickwit-ui/cypress/plugins/index.js b/quickwit/quickwit-ui/cypress/plugins/index.js deleted file mode 100644 index 59b2bab6e4e..00000000000 --- a/quickwit/quickwit-ui/cypress/plugins/index.js +++ /dev/null @@ -1,22 +0,0 @@ -/// -// *********************************************************** -// This example plugins/index.js can be used to load plugins -// -// You can change the location of this file or turn off loading -// the plugins file with the 'pluginsFile' configuration option. -// -// You can read more here: -// https://on.cypress.io/plugins-guide -// *********************************************************** - -// This function is called when a project is opened or re-opened (e.g. due to -// the project's config changing) - -/** - * @type {Cypress.PluginConfig} - */ -// eslint-disable-next-line no-unused-vars -module.exports = (on, config) => { - // `on` is used to hook into various events Cypress emits - // `config` is the resolved Cypress config -} diff --git a/quickwit/quickwit-ui/cypress/support/commands.js b/quickwit/quickwit-ui/cypress/support/commands.js deleted file mode 100644 index 119ab03f7cd..00000000000 --- a/quickwit/quickwit-ui/cypress/support/commands.js +++ /dev/null @@ -1,25 +0,0 @@ -// *********************************************** -// This example commands.js shows you how to -// create various custom commands and overwrite -// existing commands. -// -// For more comprehensive examples of custom -// commands please read more here: -// https://on.cypress.io/custom-commands -// *********************************************** -// -// -// -- This is a parent command -- -// Cypress.Commands.add('login', (email, password) => { ... }) -// -// -// -- This is a child command -- -// Cypress.Commands.add('drag', { prevSubject: 'element'}, (subject, options) => { ... }) -// -// -// -- This is a dual command -- -// Cypress.Commands.add('dismiss', { prevSubject: 'optional'}, (subject, options) => { ... }) -// -// -// -- This will overwrite an existing command -- -// Cypress.Commands.overwrite('visit', (originalFn, url, options) => { ... }) diff --git a/quickwit/quickwit-ui/cypress/support/e2e.js b/quickwit/quickwit-ui/cypress/support/e2e.js deleted file mode 100644 index 0e7290a13d9..00000000000 --- a/quickwit/quickwit-ui/cypress/support/e2e.js +++ /dev/null @@ -1,20 +0,0 @@ -// *********************************************************** -// This example support/e2e.js is processed and -// loaded automatically before your test files. -// -// This is a great place to put global configuration and -// behavior that modifies Cypress. -// -// You can change the location of this file or turn off -// automatically serving support files with the -// 'supportFile' configuration option. -// -// You can read more here: -// https://on.cypress.io/configuration -// *********************************************************** - -// Import commands.js using ES2015 syntax: -import './commands' - -// Alternatively you can use CommonJS syntax: -// require('./commands') \ No newline at end of file diff --git a/quickwit/quickwit-ui/cypress/support/index.js b/quickwit/quickwit-ui/cypress/support/index.js deleted file mode 100644 index 239169708c4..00000000000 --- a/quickwit/quickwit-ui/cypress/support/index.js +++ /dev/null @@ -1,30 +0,0 @@ -// *********************************************************** -// This example support/index.js is processed and -// loaded automatically before your test files. -// -// This is a great place to put global configuration and -// behavior that modifies Cypress. -// -// You can change the location of this file or turn off -// automatically serving support files with the -// 'supportFile' configuration option. -// -// You can read more here: -// https://on.cypress.io/configuration -// *********************************************************** - -// Import commands.js using ES2015 syntax: -import './commands' - -// Alternatively you can use CommonJS syntax: -// require('./commands') - -Cypress.on('uncaught:exception', (err, runnable) => { - // we expect a 3rd party library error with message 'list not defined' - // and don't want to fail the test so we return false - if (err.message.includes('monaco-editor')) { - return false - } - // we still want to ensure there are no other unexpected - // errors, so we let them fail the test -}) \ No newline at end of file diff --git a/quickwit/quickwit-ui/e2e/homepage.spec.ts b/quickwit/quickwit-ui/e2e/homepage.spec.ts new file mode 100644 index 00000000000..73b86b64f06 --- /dev/null +++ b/quickwit/quickwit-ui/e2e/homepage.spec.ts @@ -0,0 +1,40 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { expect, test } from "@playwright/test"; + +test.describe("Home navigation", () => { + test("Should display sidebar links", async ({ page }) => { + await page.goto("/ui"); + await expect(page.locator("a")).toContainText([ + "Query editor", + "Indexes", + "Cluster", + ]); + }); + + test("Should navigate to cluster state", async ({ page }) => { + await page.goto("/ui"); + await page.getByRole("link", { name: "Cluster" }).click(); + await expect(page.getByLabel("breadcrumb")).toContainText("Cluster"); + await expect(page.getByText("cluster_id")).toBeVisible(); + }); + + test("Should display otel logs index page", async ({ page }) => { + await page.goto("/ui/indexes/otel-logs-v0_7"); + await expect( + page.getByLabel("breadcrumb").getByRole("link", { name: "Indexes" }), + ).toBeVisible(); + }); +}); diff --git a/quickwit/quickwit-ui/index.html b/quickwit/quickwit-ui/index.html new file mode 100644 index 00000000000..bfeffb32ea7 --- /dev/null +++ b/quickwit/quickwit-ui/index.html @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + Quickwit UI + + + +

+ + + diff --git a/quickwit/quickwit-ui/jest.config.js b/quickwit/quickwit-ui/jest.config.js new file mode 100644 index 00000000000..1ed3ed949da --- /dev/null +++ b/quickwit/quickwit-ui/jest.config.js @@ -0,0 +1,38 @@ +module.exports = { + setupFiles: [ + "react-app-polyfill/jsdom", // polyfill jsdom api (such as fetch) + "/jest/setup.js", // polyfill textEncode + ], + + setupFilesAfterEnv: ["@testing-library/jest-dom"], + + testEnvironment: "jsdom", + + transform: { + // transform js file (typescript and es6 import) + "^.+\\.(js|jsx|mjs|cjs|ts|tsx)$": [ + "babel-jest", + { + presets: [["babel-preset-react-app", { runtime: "automatic" }]], + plugins: [ + [ + "@dr.pogodin/babel-plugin-transform-assets", + { extensions: ["svg", "woff2"] }, + ], + ], + babelrc: false, + configFile: false, + }, + ], + }, + + moduleNameMapper: { + "@monaco-editor/react": "/mocks/monacoMock.js", + "swagger-ui-react": "/mocks/swaggerUIMock.js", + "@mui/x-charts": "/mocks/x-charts.js", + }, + + testPathIgnorePatterns: ["/node_modules/", "/e2e/"], + + resetMocks: true, +}; diff --git a/quickwit/quickwit-ui/jest/setup.js b/quickwit/quickwit-ui/jest/setup.js new file mode 100644 index 00000000000..6667bc7e8a3 --- /dev/null +++ b/quickwit/quickwit-ui/jest/setup.js @@ -0,0 +1 @@ +global.TextEncoder = require("util").TextEncoder; diff --git a/quickwit/quickwit-ui/mocks/monacoMock.js b/quickwit/quickwit-ui/mocks/monacoMock.js index 728c7c1c205..4c11a2f0ccf 100644 --- a/quickwit/quickwit-ui/mocks/monacoMock.js +++ b/quickwit/quickwit-ui/mocks/monacoMock.js @@ -12,11 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -import * as React from 'react'; - // Mock MonocoEditor as the current jest setup does not work when Monaco JS files // are loaded. -export default function MonacoEditor(props) { +export const Editor = (props) => { return
{props.value}
; -} - +}; diff --git a/quickwit/quickwit-ui/mocks/swaggerUIMock.js b/quickwit/quickwit-ui/mocks/swaggerUIMock.js index 77db2a196a2..3ccd93b6c0b 100644 --- a/quickwit/quickwit-ui/mocks/swaggerUIMock.js +++ b/quickwit/quickwit-ui/mocks/swaggerUIMock.js @@ -12,11 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -import * as React from 'react'; - // Mock SwaggerUI as the current jest setup does not work when Monaco JS files // are loaded. export default function SwaggerUI(props) { return
{props.url}
; } - diff --git a/quickwit/quickwit-ui/mocks/x-charts.js b/quickwit/quickwit-ui/mocks/x-charts.js index 4a777a1ce84..4ee4fca0e43 100644 --- a/quickwit/quickwit-ui/mocks/x-charts.js +++ b/quickwit/quickwit-ui/mocks/x-charts.js @@ -1 +1 @@ -export const LineChart = ({ children }) => children; +export const LineChart = ({ children }) => children; diff --git a/quickwit/quickwit-ui/package.json b/quickwit/quickwit-ui/package.json index 89312471769..1f6b30061b5 100644 --- a/quickwit/quickwit-ui/package.json +++ b/quickwit/quickwit-ui/package.json @@ -3,85 +3,64 @@ "version": "0.8.0", "license": "Apache-2.0", "private": true, - "proxy": "http://127.0.0.1:7280", - "homepage": "/ui/", + "packageManager": "yarn@1.22.22", "dependencies": { - "@emotion/react": "^11.11.0", - "@emotion/styled": "^11.11.0", - "@mui/icons-material": "^5.11.0", - "@mui/lab": "^5.0.0-alpha.114", - "@mui/material": "^5.11.2", - "@mui/system": "^5.11.2", - "@mui/x-charts": "^7.3.2", - "@mui/x-date-pickers": "^5.0.12", - "@types/jest": "^29.5.6", - "@types/node": "^20.8.7", - "@types/react": "^17.0.20", - "@types/react-dom": "^17.0.9", - "@types/swagger-ui-react": "^4.18.2", - "@typescript-eslint/eslint-plugin": "^6.8.0", - "@typescript-eslint/parser": "^6.8.0", - "dayjs": "^1.11.7", - "monaco-editor": "^0.34.1", - "monaco-editor-webpack-plugin": "^7.0.1", - "prettier": "2.8.1", - "react": "^17.0.2", - "react-app-rewired": "^2.2.1", - "react-dom": "^17.0.2", - "react-monaco-editor": "^0.50.1", - "react-number-format": "^4.9.3", - "react-router-dom": "6", - "react-scripts": "~5.0.1", - "styled-components": "^5.3.6", - "styled-icons": "^10.47.0", - "swagger-ui-react": "^5.9.0", - "typescript": "^4.4.2", - "web-vitals": "^2.1.0" + "@babel/core": "7.29.0", + "@babel/runtime": "7.28.6", + "@biomejs/biome": "2.4.4", + "@dr.pogodin/babel-plugin-transform-assets": "1.2.6", + "@emotion/react": "11.14.0", + "@emotion/styled": "11.14.1", + "@monaco-editor/react": "4.7.0", + "@mui/icons-material": "7.3.8", + "@mui/lab": "7.0.1-beta.22", + "@mui/material": "7.3.8", + "@mui/system": "7.3.8", + "@mui/x-charts": "8.27.0", + "@mui/x-date-pickers": "8.27.2", + "@testing-library/dom": "10.4.1", + "@testing-library/jest-dom": "6.9.1", + "@testing-library/react": "16.3.2", + "@testing-library/user-event": "14.6.1", + "@types/jest": "30.0.0", + "@types/node": "24.10.9", + "@types/react": "19.2.14", + "@types/react-dom": "19.2.3", + "@types/swagger-ui-react": "5.18.0", + "babel-jest": "30.2.0", + "babel-preset-react-app": "10.1.0", + "dayjs": "1.11.19", + "jest": "30.2.0", + "jest-environment-jsdom": "30.2.0", + "monaco-editor": "0.55.1", + "react": "19.2.4", + "react-app-polyfill": "3.0.0", + "react-dom": "19.2.4", + "react-number-format": "5.4.4", + "react-router": "7.13.1", + "styled-components": "6.1.19", + "styled-icons": "10.47.1", + "swagger-ui-react": "5.32.0", + "typescript": "5.9.3", + "vite": "7.3.1" }, "resolutions": { - "@types/react": "17.0.20", - "@types/react-dom": "17.0.9" + "@types/react": "19.2.14", + "@types/react-dom": "19.2.3", + "dompurify": "3.3.1", + "glob": "11.1.0" }, "scripts": { - "start": "react-app-rewired start", - "build": "react-app-rewired build", - "test": "react-app-rewired test", - "eject": "react-app-rewired eject", + "start": "vite", + "build": "vite build --outDir build", + "test": "jest", "postbuild": "cp .gitignore_for_build_directory build/.gitignore", - "lint": "eslint . --ext .ts", - "e2e-test": "cypress run" - }, - "eslintConfig": { - "extends": [ - "react-app", - "react-app/jest" - ] - }, - "browserslist": { - "production": [ - ">0.2%", - "not dead", - "not op_mini all" - ], - "development": [ - "last 1 chrome version", - "last 1 firefox version", - "last 1 safari version" - ] + "lint": "biome check", + "format": "biome check --write", + "type": "tsc", + "e2e-test": "playwright test" }, "devDependencies": { - "@testing-library/jest-dom": "^6.1.4", - "@testing-library/react": "^12.0.0", - "@testing-library/user-event": "^14.5.1", - "cypress": "13.3.2", - "jest": "^29.7.0", - "ts-jest": "^29.1.1" - }, - "jest": { - "moduleNameMapper": { - "monaco-editor": "/mocks/monacoMock.js", - "swagger-ui-react": "/mocks/swaggerUIMock.js", - "@mui/x-charts": "/mocks/x-charts.js" - } + "@playwright/test": "^1.58.2" } } diff --git a/quickwit/quickwit-ui/config-overrides.js b/quickwit/quickwit-ui/playwright.config.ts similarity index 72% rename from quickwit/quickwit-ui/config-overrides.js rename to quickwit/quickwit-ui/playwright.config.ts index 0c838793447..56d41855388 100644 --- a/quickwit/quickwit-ui/config-overrides.js +++ b/quickwit/quickwit-ui/playwright.config.ts @@ -12,12 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -const MonacoWebpackPlugin = require('monaco-editor-webpack-plugin'); - -module.exports = function override(config, env) { - config.plugins.push(new MonacoWebpackPlugin({ - languages: ['json'] - })); - return config; -} +import { defineConfig } from "@playwright/test"; +export default defineConfig({ + testDir: "./e2e", + use: { + baseURL: "http://127.0.0.1:7280/ui", + browserName: "chromium", + video: "off", + screenshot: "off", + }, +}); diff --git a/quickwit/quickwit-ui/public/index.html b/quickwit/quickwit-ui/public/index.html deleted file mode 100644 index 82a199561d7..00000000000 --- a/quickwit/quickwit-ui/public/index.html +++ /dev/null @@ -1,50 +0,0 @@ - - - - - - - - - - - - - - - Quickwit UI - - - -
- - - diff --git a/quickwit/quickwit-ui/src/assets/img/logo.svg b/quickwit/quickwit-ui/src/assets/img/quickwit-logo-monochrome.svg similarity index 100% rename from quickwit/quickwit-ui/src/assets/img/logo.svg rename to quickwit/quickwit-ui/src/assets/img/quickwit-logo-monochrome.svg diff --git a/quickwit/quickwit-ui/src/assets/img/quickwit-logo.svg b/quickwit/quickwit-ui/src/assets/img/quickwit-logo-with-title.svg similarity index 100% rename from quickwit/quickwit-ui/src/assets/img/quickwit-logo.svg rename to quickwit/quickwit-ui/src/assets/img/quickwit-logo-with-title.svg diff --git a/quickwit/quickwit-ui/src/components/ApiUrlFooter.tsx b/quickwit/quickwit-ui/src/components/ApiUrlFooter.tsx index eaa979bef05..e0d10a45763 100644 --- a/quickwit/quickwit-ui/src/components/ApiUrlFooter.tsx +++ b/quickwit/quickwit-ui/src/components/ApiUrlFooter.tsx @@ -12,43 +12,57 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { Box, styled, Typography, Button } from '@mui/material'; -import ContentCopyIcon from '@mui/icons-material/ContentCopy'; -import { QUICKWIT_LIGHT_GREY } from '../utils/theme'; +import ContentCopyIcon from "@mui/icons-material/ContentCopy"; +import { Box, Button, styled, Typography } from "@mui/material"; +import { QUICKWIT_LIGHT_GREY } from "../utils/theme"; const Footer = styled(Box)` -display: flex; -height: 25px; -padding: 0px 5px; -position: absolute; -bottom: 0px; -font-size: 0.90em; -background-color: ${QUICKWIT_LIGHT_GREY}; -opacity: 0.7; -` + display: flex; + height: 25px; + padding: 0px 5px; + position: absolute; + bottom: 0px; + font-size: 0.9em; + background-color: ${QUICKWIT_LIGHT_GREY}; + opacity: 0.7; +`; export default function ApiUrlFooter(url: string) { const urlMaxLength = 80; - const origin = process.env.NODE_ENV === 'development' ? 'http://localhost:7280' : window.location.origin; + const origin = + // @ts-ignore + process.env.NODE_ENV === "development" + ? "http://localhost:7280" + : window.location.origin; const completeUrl = `${origin}/${url}`; const isTooLong = completeUrl.length > urlMaxLength; // TODO show generated aggregation - return
- - API URL: - - -
+ return ( +
+ + API URL: + + +
+ ); } diff --git a/quickwit/quickwit-ui/src/components/IndexSideBar.tsx b/quickwit/quickwit-ui/src/components/IndexSideBar.tsx index 69cbb294c97..d50de353805 100644 --- a/quickwit/quickwit-ui/src/components/IndexSideBar.tsx +++ b/quickwit/quickwit-ui/src/components/IndexSideBar.tsx @@ -12,22 +12,33 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { Autocomplete, Box, Chip, CircularProgress, IconButton, List, ListItem, ListItemText, TextField, Typography } from '@mui/material'; -import React, { useEffect, useMemo, useState } from 'react'; -import styled from '@emotion/styled'; -import { FieldMapping, getAllFields, IndexMetadata } from '../utils/models'; -import { ChevronRight, KeyboardArrowDown } from '@mui/icons-material'; -import Tooltip from '@mui/material/Tooltip'; -import { Client } from '../services/client'; +import styled from "@emotion/styled"; +import { ChevronRight, KeyboardArrowDown } from "@mui/icons-material"; +import { + Autocomplete, + Box, + Chip, + CircularProgress, + IconButton, + List, + ListItem, + ListItemText, + TextField, + Typography, +} from "@mui/material"; +import Tooltip from "@mui/material/Tooltip"; +import React, { useEffect, useMemo, useState } from "react"; +import { Client } from "../services/client"; +import { FieldMapping, getAllFields, IndexMetadata } from "../utils/models"; -const IndexBarWrapper = styled('div')({ - display: 'flex', - height: '100%', - flex: '0 0 260px', - maxWidth: '260px', - flexDirection: 'column', - borderRight: '1px solid rgba(0, 0, 0, 0.12)', - overflow: 'auto', +const IndexBarWrapper = styled("div")({ + display: "flex", + height: "100%", + flex: "0 0 260px", + maxWidth: "260px", + flexDirection: "column", + borderRight: "1px solid rgba(0, 0, 0, 0.12)", + overflow: "auto", }); function IndexAutocomplete(props: IndexMetadataProps) { @@ -53,7 +64,7 @@ function IndexAutocomplete(props: IndexMetadataProps) { (error) => { console.log("Index autocomplete error", error); setLoading(false); - } + }, ); }, [quickwitClient, open]); @@ -66,7 +77,7 @@ function IndexAutocomplete(props: IndexMetadataProps) { }, [open, props.indexMetadata, options.length]); useEffect(() => { - setValue(props.indexMetadata); + setValue(props.indexMetadata); }, [props.indexMetadata]); return ( @@ -78,7 +89,10 @@ function IndexAutocomplete(props: IndexMetadataProps) { onChange={(_, updatedValue) => { setValue(updatedValue); - if (updatedValue == null || updatedValue.index_config.index_id == null) { + if ( + updatedValue == null || + updatedValue.index_config.index_id == null + ) { props.onIndexMetadataUpdate(null); } else { props.onIndexMetadataUpdate(updatedValue); @@ -91,7 +105,9 @@ function IndexAutocomplete(props: IndexMetadataProps) { setOpen(false); setLoading(false); }} - isOptionEqualToValue={(option, value) => option.index_config.index_id === value.index_config.index_id} + isOptionEqualToValue={(option, value) => + option.index_config.index_id === value.index_config.index_id + } getOptionLabel={(option) => option.index_config.index_id} options={options} noOptionsText="No indexes." @@ -99,12 +115,14 @@ function IndexAutocomplete(props: IndexMetadataProps) { renderInput={(params) => ( - {showLoading ? : null} + {showLoading ? ( + + ) : null} {params.InputProps.endAdornment} ), @@ -116,14 +134,13 @@ function IndexAutocomplete(props: IndexMetadataProps) { } export interface IndexMetadataProps { - indexMetadata: null | IndexMetadata, + indexMetadata: null | IndexMetadata; onIndexMetadataUpdate(indexMetadata: IndexMetadata | null): void; } function fieldTypeLabel(fieldMapping: FieldMapping): string { if (fieldMapping.type[0] !== undefined) { return fieldMapping.type[0].toUpperCase(); - } else { return ""; } @@ -131,41 +148,64 @@ function fieldTypeLabel(fieldMapping: FieldMapping): string { export function IndexSideBar(props: IndexMetadataProps) { const [open, setOpen] = useState(true); - const fields = (props.indexMetadata === null) ? [] : getAllFields(props.indexMetadata.index_config.doc_mapping.field_mappings); + const fields = + props.indexMetadata === null + ? [] + : getAllFields( + props.indexMetadata.index_config.doc_mapping.field_mappings, + ); return ( - - + + Index ID - + - + setOpen(!open)} - > - {open ? : } + aria-label="expand row" + size="small" + onClick={() => setOpen(!open)} + > + {open ? : } Fields - { open && - { fields.map(function(field) { - return - } - sx={{paddingLeft: '10px'}} - > - - - - - - })} - - } + {open && ( + + {fields.map(function (field) { + return ( + + } + sx={{ paddingLeft: "10px" }} + > + + + + + + ); + })} + + )} ); diff --git a/quickwit/quickwit-ui/src/components/IndexSummary.tsx b/quickwit/quickwit-ui/src/components/IndexSummary.tsx index 61657b712e8..7be3b8b01ee 100644 --- a/quickwit/quickwit-ui/src/components/IndexSummary.tsx +++ b/quickwit/quickwit-ui/src/components/IndexSummary.tsx @@ -13,28 +13,31 @@ // limitations under the License. import styled from "@emotion/styled"; -import { Paper } from "@mui/material"; +import { Alert, Paper } from "@mui/material"; import dayjs from "dayjs"; -import utc from "dayjs/plugin/utc" +import utc from "dayjs/plugin/utc"; import { FC, ReactNode } from "react"; -import NumberFormat from "react-number-format"; +import { NumericFormat } from "react-number-format"; import { Index } from "../utils/models"; + dayjs.extend(utc); const ItemContainer = styled.div` -padding: 10px; -display: flex; -flex-direction: column; -` + padding: 10px; + display: flex; + flex-direction: column; +`; const Row = styled.div` -padding: 5px; -display: flex; -flex-direction: row; -&:nth-of-type(odd){ background: rgba(0,0,0,0.05) } -` + padding: 5px; + display: flex; + flex-direction: row; + &:nth-of-type(odd) { + background: rgba(0, 0, 0, 0.05); + } +`; const RowKey = styled.div` -width: 350px; -` + width: 350px; +`; const IndexRow: FC<{ title: string; children: ReactNode }> = ({ title, children, @@ -47,41 +50,55 @@ const IndexRow: FC<{ title: string; children: ReactNode }> = ({ export function IndexSummary({ index }: { index: Index }) { const all_splits = index.splits; - const published_splits = all_splits.filter(split => split.split_state == "Published"); - const num_of_staged_splits = all_splits.filter(split => split.split_state == "Staged").length; - const num_of_marked_for_delete_splits = all_splits.filter(split => split.split_state == "MarkedForDeletion").length; + const published_splits = all_splits.filter( + (split) => split.split_state === "Published", + ); + const num_of_staged_splits = all_splits.filter( + (split) => split.split_state === "Staged", + ).length; + const num_of_marked_for_delete_splits = all_splits.filter( + (split) => split.split_state === "MarkedForDeletion", + ).length; const total_num_docs = published_splits - .map(split => split.num_docs) + .map((split) => split.num_docs) .reduce((sum, current) => sum + current, 0); const total_num_bytes = published_splits - .map(split => { - return split.footer_offsets.end + .map((split) => { + return split.footer_offsets.end; }) .reduce((sum, current) => sum + current, 0); const total_uncompressed_num_bytes = published_splits - .map(split => { - return split.uncompressed_docs_size_in_bytes + .map((split) => { + return split.uncompressed_docs_size_in_bytes; }) .reduce((sum, current) => sum + current, 0); return ( - + + {index.split_limit_reached && ( + + Split limit reached. Only the first 10,000 splits were retrieved. + The actual total may be higher. Statistics shown are incomplete. + + )} {dayjs .unix(index.metadata.create_timestamp) .utc() .format("YYYY/MM/DD HH:mm")} - {index.metadata.index_config.index_uri} + + {index.metadata.index_config.index_uri} + - - - {published_splits.length} + + {published_splits.length} + - - {num_of_staged_splits} - {num_of_marked_for_delete_splits} + + {num_of_staged_splits} + + + {num_of_marked_for_delete_splits} + - ) + ); } diff --git a/quickwit/quickwit-ui/src/components/IndexesTable.tsx b/quickwit/quickwit-ui/src/components/IndexesTable.tsx index e8618c7f1ef..83251a8b2fa 100644 --- a/quickwit/quickwit-ui/src/components/IndexesTable.tsx +++ b/quickwit/quickwit-ui/src/components/IndexesTable.tsx @@ -12,19 +12,29 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { Paper, Table, TableBody, TableCell, TableContainer, TableHead, TableRow } from "@mui/material"; -import dayjs from 'dayjs'; -import utc from "dayjs/plugin/utc" +import { + Paper, + Table, + TableBody, + TableCell, + TableContainer, + TableHead, + TableRow, +} from "@mui/material"; +import dayjs from "dayjs"; +import utc from "dayjs/plugin/utc"; +import { useNavigate } from "react-router"; import { IndexMetadata } from "../utils/models"; -import { useNavigate } from "react-router-dom"; -dayjs.extend(utc); +dayjs.extend(utc); -const IndexesTable = ({ indexesMetadata }: Readonly<{indexesMetadata: IndexMetadata[]}>) => { +const IndexesTable = ({ + indexesMetadata, +}: Readonly<{ indexesMetadata: IndexMetadata[] }>) => { const navigate = useNavigate(); - const handleClick = function(indexId: string) { + const handleClick = (indexId: string) => { navigate(`/indexes/${indexId}`); - } + }; return ( @@ -41,16 +51,28 @@ const IndexesTable = ({ indexesMetadata }: Readonly<{indexesMetadata: IndexMetad {indexesMetadata.map((indexMetadata) => ( handleClick(indexMetadata.index_config.index_id)} > {indexMetadata.index_config.index_id} - {indexMetadata.index_config.index_uri} - { dayjs.unix(indexMetadata.create_timestamp).utc().format("YYYY/MM/DD HH:mm") } - { indexMetadata.sources?.length || 'None'} + + {indexMetadata.index_config.index_uri} + + + {dayjs + .unix(indexMetadata.create_timestamp) + .utc() + .format("YYYY/MM/DD HH:mm")} + + + {indexMetadata.sources?.length || "None"} + ))} diff --git a/quickwit/quickwit-ui/src/components/JsonEditor.tsx b/quickwit/quickwit-ui/src/components/JsonEditor.tsx index a5ba510f5ee..a3c88d44bd8 100644 --- a/quickwit/quickwit-ui/src/components/JsonEditor.tsx +++ b/quickwit/quickwit-ui/src/components/JsonEditor.tsx @@ -12,47 +12,54 @@ // See the License for the specific language governing permissions and // limitations under the License. -import MonacoEditor from 'react-monaco-editor'; +import { BeforeMount, Editor, OnMount } from "@monaco-editor/react"; import { useCallback } from "react"; import { EDITOR_THEME } from "../utils/theme"; -export function JsonEditor({content, resizeOnMount}: {content: unknown, resizeOnMount: boolean}) { +export function JsonEditor({ + content, + resizeOnMount, +}: { + content: unknown; + resizeOnMount: boolean; +}) { // Setting editor height based on lines height and count to stretch and fit its content. - const onMount = useCallback((editor) => { - if (!resizeOnMount) { - return; - } - const editorElement = editor.getDomNode(); + const onMount: OnMount = useCallback( + (editor) => { + if (!resizeOnMount) { + return; + } + const editorElement = editor.getDomNode(); - if (!editorElement) { - return; - } + if (!editorElement) { + return; + } - // Weirdly enough, we have to wait a few ms to get the right height - // from `editor.getContentHeight()`. If not, we sometimes end up with - // a height > 7000px... and I don't know why. - setTimeout(() => { - const height = Math.min(800, editor.getContentHeight()); - editorElement.style.height = `${height}px`; - editor.layout(); - }, 10); + // Weirdly enough, we have to wait a few ms to get the right height + // from `editor.getContentHeight()`. If not, we sometimes end up with + // a height > 7000px... and I don't know why. + setTimeout(() => { + const height = Math.min(800, editor.getContentHeight()); + editorElement.style.height = `${height}px`; + editor.layout(); + }, 10); + }, + [resizeOnMount], + ); - }, [resizeOnMount]); - - /* eslint-disable @typescript-eslint/no-explicit-any */ - function beforeMount(monaco: any) { - monaco.editor.defineTheme('quickwit-light', EDITOR_THEME); - } + const beforeMount: BeforeMount = (monaco) => { + monaco.editor.defineTheme("quickwit-light", EDITOR_THEME); + }; return ( - - ) + ); } diff --git a/quickwit/quickwit-ui/src/components/LayoutUtils.tsx b/quickwit/quickwit-ui/src/components/LayoutUtils.tsx index c1ccddb70aa..b193c760cbf 100644 --- a/quickwit/quickwit-ui/src/components/LayoutUtils.tsx +++ b/quickwit/quickwit-ui/src/components/LayoutUtils.tsx @@ -14,7 +14,7 @@ import { Box, Breadcrumbs, styled } from "@mui/material"; -export const APP_BAR_HEIGHT_PX = '48px'; +export const APP_BAR_HEIGHT_PX = "48px"; export const ViewUnderAppBarBox = styled(Box)` display: flex; flex-direction: column; @@ -31,4 +31,4 @@ padding: 16px 24px; `; export const QBreadcrumbs = styled(Breadcrumbs)` padding-bottom: 8px; -` +`; diff --git a/quickwit/quickwit-ui/src/components/Loader.tsx b/quickwit/quickwit-ui/src/components/Loader.tsx index 6a06ed0de6c..b1b1687c012 100644 --- a/quickwit/quickwit-ui/src/components/Loader.tsx +++ b/quickwit/quickwit-ui/src/components/Loader.tsx @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { Box, styled, keyframes } from '@mui/material'; -import { ReactComponent as Logo } from '../assets/img/logo.svg'; +import { Box, keyframes, styled } from "@mui/material"; +import loadinIcongUrl from "../assets/img/quickwit-logo-monochrome.svg"; const spin = keyframes` from { @@ -22,22 +22,28 @@ from { to { transform: rotate(360deg); } -` +`; -const SpinningLogo = styled(Logo)` -height: 10vmin; -pointer-events: none; -fill: #CBD1DD; -animation: ${spin} infinite 5s linear; -` +const LoadingIcon = (props: React.ComponentProps<"img">) => ( + loading icon +); + +const SpinningLoadingIcon = styled(LoadingIcon)` + height: 10vmin; + pointer-events: none; + fill: #cbd1dd; + animation: ${spin} infinite 5s linear; +`; export default function Loader() { - return - - + + + ); } diff --git a/quickwit/quickwit-ui/src/components/QueryActionBar.tsx b/quickwit/quickwit-ui/src/components/QueryActionBar.tsx index 791525526b5..f118a01eb24 100644 --- a/quickwit/quickwit-ui/src/components/QueryActionBar.tsx +++ b/quickwit/quickwit-ui/src/components/QueryActionBar.tsx @@ -12,57 +12,63 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { Box, Button, Tabs, Tab } from "@mui/material"; -import { TimeRangeSelect } from './TimeRangeSelect'; -import PlayArrowIcon from '@mui/icons-material/PlayArrow'; +import PlayArrowIcon from "@mui/icons-material/PlayArrow"; +import { Box, Button, Tab, Tabs } from "@mui/material"; import { SearchComponentProps } from "../utils/SearchComponentProps"; +import { TimeRangeSelect } from "./TimeRangeSelect"; export function QueryEditorActionBar(props: SearchComponentProps) { - const timestamp_field_name = props.index?.metadata.index_config.doc_mapping.timestamp_field; + const timestamp_field_name = + props.index?.metadata.index_config.doc_mapping.timestamp_field; const shouldDisplayTimeRangeSelect = timestamp_field_name ?? false; const handleChange = (_event: React.SyntheticEvent, newTab: number) => { - const updatedSearchRequest = {...props.searchRequest, aggregation: newTab != 0}; + const updatedSearchRequest = { + ...props.searchRequest, + aggregation: newTab !== 0, + }; props.onSearchRequestUpdate(updatedSearchRequest); - props.runSearch(updatedSearchRequest) + props.runSearch(updatedSearchRequest); }; return ( - - + + - - - - + + + + - - - { shouldDisplayTimeRangeSelect && + {shouldDisplayTimeRangeSelect && ( + { + props.runSearch({ ...props.searchRequest, ...timeRange }); }} - onUpdate={ - (timeRange)=>{ - props.runSearch({...props.searchRequest, ...timeRange}); - } - } disabled={props.queryRunning || !props.searchRequest.indexId} - /> - } + /> + )} - ) + ); } diff --git a/quickwit/quickwit-ui/src/components/QueryEditor/AggregationEditor.tsx b/quickwit/quickwit-ui/src/components/QueryEditor/AggregationEditor.tsx index 47301efd919..28d68e4cdc6 100644 --- a/quickwit/quickwit-ui/src/components/QueryEditor/AggregationEditor.tsx +++ b/quickwit/quickwit-ui/src/components/QueryEditor/AggregationEditor.tsx @@ -12,14 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { useRef, useEffect, useState } from 'react'; -import { SearchComponentProps } from '../../utils/SearchComponentProps'; -import { TermAgg, HistogramAgg } from '../../utils/models'; -import { Box } from '@mui/material'; -import MenuItem from '@mui/material/MenuItem'; -import FormControl from '@mui/material/FormControl'; -import Select, { SelectChangeEvent } from '@mui/material/Select'; -import TextField from '@mui/material/TextField'; +import { Box } from "@mui/material"; +import FormControl from "@mui/material/FormControl"; +import MenuItem from "@mui/material/MenuItem"; +import Select, { SelectChangeEvent } from "@mui/material/Select"; +import TextField from "@mui/material/TextField"; +import { useEffect, useRef, useState } from "react"; +import { HistogramAgg, TermAgg } from "../../utils/models"; +import { SearchComponentProps } from "../../utils/SearchComponentProps"; export function AggregationEditor(props: SearchComponentProps) { return ( @@ -29,15 +29,17 @@ export function AggregationEditor(props: SearchComponentProps) { onSearchRequestUpdate={props.onSearchRequestUpdate} runSearch={props.runSearch} index={props.index} - queryRunning={props.queryRunning} /> + queryRunning={props.queryRunning} + /> + queryRunning={props.queryRunning} + /> - ) + ); } export function MetricKind(props: SearchComponentProps) { @@ -46,9 +48,16 @@ export function MetricKind(props: SearchComponentProps) { const handleTypeChange = (event: SelectChangeEvent) => { const value = event.target.value; - const updatedMetric = value != "count" ? {...metricRef.current!, type: value} : null; - const updatedAggregation = {...props.searchRequest.aggregationConfig, metric: updatedMetric}; - const updatedSearchRequest = {...props.searchRequest, aggregationConfig: updatedAggregation}; + const updatedMetric = + value !== "count" ? { ...metricRef.current!, type: value } : null; + const updatedAggregation = { + ...props.searchRequest.aggregationConfig, + metric: updatedMetric, + }; + const updatedSearchRequest = { + ...props.searchRequest, + aggregationConfig: updatedAggregation, + }; props.onSearchRequestUpdate(updatedSearchRequest); metricRef.current = updatedMetric; }; @@ -58,20 +67,26 @@ export function MetricKind(props: SearchComponentProps) { if (metricRef.current == null) { return; } - const updatedMetric = {...metricRef.current!, field: value}; - const updatedAggregation = {...props.searchRequest.aggregationConfig, metric: updatedMetric}; - const updatedSearchRequest = {...props.searchRequest, aggregationConfig: updatedAggregation}; + const updatedMetric = { ...metricRef.current!, field: value }; + const updatedAggregation = { + ...props.searchRequest.aggregationConfig, + metric: updatedMetric, + }; + const updatedSearchRequest = { + ...props.searchRequest, + aggregationConfig: updatedAggregation, + }; props.onSearchRequestUpdate(updatedSearchRequest); metricRef.current = updatedMetric; }; return ( - + handleHistogramChange(pos, e)} - sx={{ "marginLeft": "10px", "minHeight": "44px" }} + sx={{ marginLeft: "10px", minHeight: "44px" }} > 10 seconds 1 minute @@ -252,13 +313,14 @@ export function AggregationKind(props: SearchComponentProps) { ); } if (isTerm(agg)) { - return (<> + return ( + <> handleTermFieldChange(pos, e)} - sx={{ "marginLeft": "10px" }} + sx={{ marginLeft: "10px" }} /> @@ -268,40 +330,44 @@ export function AggregationKind(props: SearchComponentProps) { type="number" onChange={(e) => handleTermCountChange(pos, e)} value={agg.term.size} - sx={{ "marginLeft": "10px" }} + sx={{ marginLeft: "10px" }} /> - ) + + ); } - return (null); - } + return null; + }; return ( <> - + {drawAdditional(0, aggregations)} - - + + {drawAdditional(1, aggregations)} - ) + ); } diff --git a/quickwit/quickwit-ui/src/components/QueryEditor/QueryEditor.tsx b/quickwit/quickwit-ui/src/components/QueryEditor/QueryEditor.tsx index 6f8209efa56..2916f180f88 100644 --- a/quickwit/quickwit-ui/src/components/QueryEditor/QueryEditor.tsx +++ b/quickwit/quickwit-ui/src/components/QueryEditor/QueryEditor.tsx @@ -12,36 +12,42 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { useEffect, useRef, useState } from 'react'; -import MonacoEditor from 'react-monaco-editor'; -import * as monacoEditor from 'monaco-editor/esm/vs/editor/editor.api'; -import { LANGUAGE_CONFIG, LanguageFeatures, createIndexCompletionProvider } from './config'; -import { SearchComponentProps } from '../../utils/SearchComponentProps'; -import { EDITOR_THEME } from '../../utils/theme'; -import { Box } from '@mui/material'; +import { Editor } from "@monaco-editor/react"; +import { Box } from "@mui/material"; +import * as monacoEditor from "monaco-editor/esm/vs/editor/editor.api"; +import React, { useEffect, useRef, useState } from "react"; +import { SearchComponentProps } from "../../utils/SearchComponentProps"; +import { EDITOR_THEME } from "../../utils/theme"; +import { + createIndexCompletionProvider, + LANGUAGE_CONFIG, + LanguageFeatures, +} from "./config"; -const QUICKWIT_EDITOR_THEME_ID = 'quickwit-light'; +const QUICKWIT_EDITOR_THEME_ID = "quickwit-light"; function getLanguageId(indexId: string | null): string { if (indexId === null) { - return ''; + return ""; } return `${indexId}-query-language`; } export function QueryEditor(props: SearchComponentProps) { const monacoRef = useRef(null); - const [languageId, setLanguageId] = useState(''); + const [languageId, setLanguageId] = useState(""); const runSearchRef = useRef(props.runSearch); const searchRequestRef = useRef(props.searchRequest); - const defaultValue = props.searchRequest.query === null ? `// Select an index and type your query. Example: field_name:"phrase query"` : props.searchRequest.query; + const defaultValue = + props.searchRequest.query === null + ? `// Select an index and type your query. Example: field_name:"phrase query"` + : props.searchRequest.query; let resize: () => void; - /* eslint-disable @typescript-eslint/no-explicit-any */ function handleEditorDidMount(editor: any, monaco: any) { monacoRef.current = monaco; editor.addAction({ - id: 'SEARCH', + id: "SEARCH", label: "Run search", keybindings: [ monaco.KeyCode.F9, @@ -50,27 +56,44 @@ export function QueryEditor(props: SearchComponentProps) { run: () => { runSearchRef.current(searchRequestRef.current); }, - }) + }); resize = () => { - editor.layout({width: Math.max(window.innerWidth - (260+180+2*24), 200), height: 84}); - } - window.addEventListener('resize', resize); + editor.layout({ + width: Math.max(window.innerWidth - (260 + 180 + 2 * 24), 200), + height: 84, + }); + }; + window.addEventListener("resize", resize); } - function handleEditorWillUnmount() { - window.removeEventListener('resize', resize); - } + React.useEffect(() => { + return () => window.removeEventListener("resize", resize); + }); useEffect(() => { const updatedLanguageId = getLanguageId(props.searchRequest.indexId); - if (monacoRef.current !== null && updatedLanguageId !== '' && props.index !== null) { + if ( + monacoRef.current !== null && + updatedLanguageId !== "" && + props.index !== null + ) { const monaco = monacoRef.current; - if (!monaco.languages.getLanguages().some(({ id }: {id :string }) => id === updatedLanguageId)) { - console.log('register language', updatedLanguageId); - monaco.languages.register({'id': updatedLanguageId}); - monaco.languages.setMonarchTokensProvider(updatedLanguageId, LanguageFeatures()) + if ( + !monaco.languages + .getLanguages() + .some(({ id }: { id: string }) => id === updatedLanguageId) + ) { + console.log("register language", updatedLanguageId); + monaco.languages.register({ id: updatedLanguageId }); + monaco.languages.setMonarchTokensProvider( + updatedLanguageId, + LanguageFeatures(), + ); if (props.index != null) { - monaco.languages.registerCompletionItemProvider(updatedLanguageId, createIndexCompletionProvider(props.index.metadata)); + monaco.languages.registerCompletionItemProvider( + updatedLanguageId, + createIndexCompletionProvider(props.index.metadata), + ); monaco.languages.setLanguageConfiguration( updatedLanguageId, LANGUAGE_CONFIG, @@ -88,7 +111,9 @@ export function QueryEditor(props: SearchComponentProps) { }, [monacoRef, props.runSearch]); function handleEditorChange(value: any) { - const updatedSearchRequest = Object.assign({}, props.searchRequest, {query: value}); + const updatedSearchRequest = Object.assign({}, props.searchRequest, { + query: value, + }); searchRequestRef.current = updatedSearchRequest; props.onSearchRequestUpdate(updatedSearchRequest); } @@ -98,16 +123,15 @@ export function QueryEditor(props: SearchComponentProps) { } return ( - - + ); diff --git a/quickwit/quickwit-ui/src/components/QueryEditor/config.ts b/quickwit/quickwit-ui/src/components/QueryEditor/config.ts index 38353e69222..a7b3b39914f 100644 --- a/quickwit/quickwit-ui/src/components/QueryEditor/config.ts +++ b/quickwit/quickwit-ui/src/components/QueryEditor/config.ts @@ -19,15 +19,15 @@ export enum CompletionItemKind { Operator = 11, } -const BRACES: [string, string] = ['{', '}']; -const BRACKETS: [string, string] = ['[', ']']; -const PARENTHESES: [string, string] = ['(', ')']; +const BRACES: [string, string] = ["{", "}"]; +const BRACKETS: [string, string] = ["[", "]"]; +const PARENTHESES: [string, string] = ["(", ")"]; export const LANGUAGE_CONFIG = { comments: { lineComment: "//", }, - brackets: [ BRACES, BRACKETS, PARENTHESES ], + brackets: [BRACES, BRACKETS, PARENTHESES], autoClosingPairs: [ { open: "{", close: "}" }, { open: "[", close: "]" }, @@ -45,128 +45,140 @@ export const LANGUAGE_CONFIG = { }; // TODO: clean language features as I (fmassot) did not dig into it yet. -/* eslint-disable @typescript-eslint/no-explicit-any */ export function LanguageFeatures(): any { return { defaultToken: "invalid", //wordDefinition: /(-?\d*\.\d\w*)|([^\`\~\!\#\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g, - operators: ['+', '-'], - brackets: [ - { open: "(", close: ")", token: "delimiter.parenthesis" }, - ], - keywords: [ - 'AND', 'OR', - ], - symbols: /[=>](?!@symbols)/, '@brackets'], - [/@symbols/, { cases: { '@operators': 'operator', - '@default' : '' } } ], + [/[{}()[]]/, "@brackets"], + [/[<>](?!@symbols)/, "@brackets"], + [/@symbols/, { cases: { "@operators": "operator", "@default": "" } }], // @ annotations. // As an example, we emit a debugging log message on these tokens. // Note: message are suppressed during the first load -- change some lines to see them. - [/@\s*[a-zA-Z_$][\w$]*/, { token: 'annotation', log: 'annotation token: $0' }], + [ + /@\s*[a-zA-Z_$][\w$]*/, + { token: "annotation", log: "annotation token: $0" }, + ], // numbers - [/\d*\.\d+([eE][-+]?\d+)?/, 'number.float'], - [/0[xX][0-9a-fA-F]+/, 'number.hex'], - [/\d+/, 'number'], + [/\d*\.\d+([eE][-+]?\d+)?/, "number.float"], + [/0[xX][0-9a-fA-F]+/, "number.hex"], + [/\d+/, "number"], // delimiter: after number because of .\d floats - [/[;,.]/, 'delimiter'], + [/[;,.]/, "delimiter"], // strings - [/"([^"\\]|\\.)*$/, 'string.invalid' ], // non-terminated string - [/"/, { token: 'string.quote', bracket: '@open', next: '@string' } ], + [/"([^"\\]|\\.)*$/, "string.invalid"], // non-terminated string + [/"/, { token: "string.quote", bracket: "@open", next: "@string" }], // characters - [/'[^\\']'/, 'string'], - [/(')(@escapes)(')/, ['string','string.escape','string']], - [/'/, 'string.invalid'] + [/'[^\\']'/, "string"], + [/(')(@escapes)(')/, ["string", "string.escape", "string"]], + [/'/, "string.invalid"], ], comment: [ - [/[^/*]+/, 'comment' ], - [/\/\*/, 'comment', '@push' ], // nested comment - ["\\*/", 'comment', '@pop' ], - [/[/*]/, 'comment' ] + [/[^/*]+/, "comment"], + [/\/\*/, "comment", "@push"], // nested comment + ["\\*/", "comment", "@pop"], + [/[/*]/, "comment"], ], string: [ - [/[^\\"]+/, 'string'], - [/@escapes/, 'string.escape'], - [/\\./, 'string.escape.invalid'], - [/"/, { token: 'string.quote', bracket: '@close', next: '@pop' } ] + [/[^\\"]+/, "string"], + [/@escapes/, "string.escape"], + [/\\./, "string.escape.invalid"], + [/"/, { token: "string.quote", bracket: "@close", next: "@pop" }], ], whitespace: [ - [/[ \t\r\n]+/, 'white'], - [/\/\*/, 'comment', '@comment' ], - [/\/\/.*$/, 'comment'], + [/[ \t\r\n]+/, "white"], + [/\/\*/, "comment", "@comment"], + [/\/\/.*$/, "comment"], ], }, }; } export const createIndexCompletionProvider = (indexMetadata: IndexMetadata) => { - const fields = getAllFields(indexMetadata.index_config.doc_mapping.field_mappings); + const fields = getAllFields( + indexMetadata.index_config.doc_mapping.field_mappings, + ); const completionProvider = { provideCompletionItems(model: any, position: any) { - const word = model.getWordUntilPosition(position) + const word = model.getWordUntilPosition(position); const range = { startLineNumber: position.lineNumber, endLineNumber: position.lineNumber, startColumn: word.startColumn, endColumn: word.endColumn, - } + }; // We want to auto complete all fields except timestamp that is handled with `TimeRangeSelect` component. const fieldSuggestions = fields - .filter(field => field.json_path !== indexMetadata.index_config.doc_mapping.timestamp_field) - .map(field => { + .filter( + (field) => + field.json_path !== + indexMetadata.index_config.doc_mapping.timestamp_field, + ) + .map((field) => { return { label: field.json_path, kind: CompletionItemKind.Field, - insertText: field.field_mapping.type == 'json' ? field.json_path + '.' : field.json_path + ':', + insertText: + field.field_mapping.type === "json" + ? field.json_path + "." + : field.json_path + ":", range: range, - } + }; }); return { suggestions: fieldSuggestions.concat([ { - label: 'OR', + label: "OR", kind: CompletionItemKind.Operator, - insertText: 'OR ', + insertText: "OR ", range: range, }, { - label: 'AND', + label: "AND", kind: CompletionItemKind.Operator, - insertText: 'AND ', + insertText: "AND ", range: range, - } + }, ]), - } + }; }, - } - - return completionProvider -} + }; + return completionProvider; +}; export const setErrorMarker = ( monaco: any, @@ -175,7 +187,7 @@ export const setErrorMarker = ( startColumnNumber: number, message: string, ) => { - const model = editor.getModel() + const model = editor.getModel(); if (model) { monaco.editor.setModelMarkers(model, "QuestDBLanguageName", [ @@ -187,6 +199,6 @@ export const setErrorMarker = ( startColumn: startColumnNumber, endColumn: startColumnNumber, }, - ]) + ]); } -} +}; diff --git a/quickwit/quickwit-ui/src/components/ResponseErrorDisplay.tsx b/quickwit/quickwit-ui/src/components/ResponseErrorDisplay.tsx index 183c929b3ac..b43103b6ac3 100644 --- a/quickwit/quickwit-ui/src/components/ResponseErrorDisplay.tsx +++ b/quickwit/quickwit-ui/src/components/ResponseErrorDisplay.tsx @@ -12,30 +12,47 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { Box } from '@mui/material'; -import { ResponseError } from '../utils/models'; -import SentimentVeryDissatisfiedIcon from '@mui/icons-material/SentimentVeryDissatisfied'; +import SentimentVeryDissatisfiedIcon from "@mui/icons-material/SentimentVeryDissatisfied"; +import { Box } from "@mui/material"; +import { ResponseError } from "../utils/models"; function renderMessage(error: ResponseError) { - if (error.message !== null && error.message.includes('No search node available.')) { - return - Your cluster does not contain any search node. You need at least one search node. + if ( + error.message !== null && + error.message.includes("No search node available.") + ) { + return ( + + Your cluster does not contain any search node. You need at least one + search node. + ); } else { - return <> - - {error.status && Status: {error.status}} - - - Error: {error.message} - - + return ( + <> + + {error.status && Status: {error.status}} + + + Error: {error.message} + + + ); } } export default function ErrorResponseDisplay(error: ResponseError) { - return - - {renderMessage(error)} - + return ( + + + {renderMessage(error)} + + ); } diff --git a/quickwit/quickwit-ui/src/components/SearchResult/AggregationResult.tsx b/quickwit/quickwit-ui/src/components/SearchResult/AggregationResult.tsx index 21138dd70e7..b7a4cf8a23a 100644 --- a/quickwit/quickwit-ui/src/components/SearchResult/AggregationResult.tsx +++ b/quickwit/quickwit-ui/src/components/SearchResult/AggregationResult.tsx @@ -12,10 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { SearchResponse, HistogramResult, TermResult, ParsedAggregationResult, extractAggregationResults } from "../../utils/models"; -import { LineChart } from '@mui/x-charts/LineChart'; -import { BarChart } from '@mui/x-charts/BarChart'; -import { CurveType } from '@mui/x-charts/models/seriesType/line'; +import { BarChart } from "@mui/x-charts/BarChart"; +import { LineChart } from "@mui/x-charts/LineChart"; +import { + extractAggregationResults, + HistogramResult, + ParsedAggregationResult, + SearchResponse, + TermResult, +} from "../../utils/models"; function isHistogram(agg: ParsedAggregationResult): agg is HistogramResult { return agg != null && "timestamps" in agg; @@ -25,38 +30,42 @@ function isTerm(agg: ParsedAggregationResult): agg is TermResult { return Array.isArray(agg); } -export function AggregationResult({searchResponse}: {searchResponse: SearchResponse}) { +export function AggregationResult({ + searchResponse, +}: { + searchResponse: SearchResponse; +}) { const result = extractAggregationResults(searchResponse.aggregations); if (isHistogram(result)) { - const xAxis = [{ - data: result.timestamps, - valueFormatter: (date: number) => { - return new Date(date).toISOString() + const xAxis: React.ComponentProps["xAxis"] = [ + { + data: result.timestamps, + valueFormatter: (date: number) => { + return new Date(date).toISOString(); + }, }, - }]; - const series = result.data.map((line) => { - const curve: CurveType = "monotoneX"; - return { - curve, + ]; + const series: React.ComponentProps["series"] = + result.data.map((line) => { + return { + curve: "monotoneX", label: line.name, data: line.value, - }; - }); + }; + }); // we don't customize colors because we would need a full palette. + return ; + } else if (isTerm(result)) { return ( - entry.value), color: "#004BD9A5" }, + ]} + xAxis={[{ data: result.map((entry) => entry.term), scaleType: "band" }]} + margin={{ top: 10, bottom: 30, left: 40, right: 10 }} /> - ) - } else if (isTerm(result)) { - return ( entry.value), color: "#004BD9A5"}]} - xAxis={[{ data: result.map(entry => entry.term), scaleType: 'band' }]} - margin={{ top: 10, bottom: 30, left: 40, right: 10 }} - />) + ); } else { - return (

no result to display

); + return

no result to display

; } } diff --git a/quickwit/quickwit-ui/src/components/SearchResult/ResultTable.tsx b/quickwit/quickwit-ui/src/components/SearchResult/ResultTable.tsx index 7f79037b51c..8e9dff03c0e 100644 --- a/quickwit/quickwit-ui/src/components/SearchResult/ResultTable.tsx +++ b/quickwit/quickwit-ui/src/components/SearchResult/ResultTable.tsx @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { Table, TableBody, TableContainer, Box, styled } from "@mui/material"; -import { Field as Field, getAllFields, Index, SearchResponse} from "../../utils/models"; +import { Box, styled, Table, TableBody, TableContainer } from "@mui/material"; +import { Field, getAllFields, Index, SearchResponse } from "../../utils/models"; import { Row } from "./Row"; const TableBox = styled(Box)` @@ -22,22 +22,24 @@ flex-direction: column; overflow: auto; flex: 1 1 100%; height: 100%; -` +`; -export function ResultTable({searchResponse, index}: {searchResponse: SearchResponse, index: Index}) { +export function ResultTable({ + searchResponse, + index, +}: { + searchResponse: SearchResponse; + index: Index; +}) { const timestampField = getTimestampField(index); return ( - +
- { searchResponse.hits.map((hit, idx) => - - )} + {searchResponse.hits.map((hit, idx) => ( + + ))}
@@ -46,8 +48,13 @@ export function ResultTable({searchResponse, index}: {searchResponse: SearchResp } function getTimestampField(index: Index): Field | null { - const fields = getAllFields(index.metadata.index_config.doc_mapping.field_mappings); - const timestamp_field_name = index.metadata.index_config.doc_mapping.timestamp_field; - const timestamp_field = fields.filter(field => field.field_mapping.name === timestamp_field_name)[0]; + const fields = getAllFields( + index.metadata.index_config.doc_mapping.field_mappings, + ); + const timestamp_field_name = + index.metadata.index_config.doc_mapping.timestamp_field; + const timestamp_field = fields.filter( + (field) => field.field_mapping.name === timestamp_field_name, + )[0]; return timestamp_field ?? null; } diff --git a/quickwit/quickwit-ui/src/components/SearchResult/Row.tsx b/quickwit/quickwit-ui/src/components/SearchResult/Row.tsx index 4df7cf5de99..2a549f53c5e 100644 --- a/quickwit/quickwit-ui/src/components/SearchResult/Row.tsx +++ b/quickwit/quickwit-ui/src/components/SearchResult/Row.tsx @@ -16,10 +16,16 @@ import { KeyboardArrowDown } from "@mui/icons-material"; import ChevronRight from "@mui/icons-material/ChevronRight"; import { Box, IconButton, styled, TableCell, TableRow } from "@mui/material"; import dayjs from "dayjs"; -import relativeTime from "dayjs/plugin/relativeTime" -import utc from "dayjs/plugin/utc" +import relativeTime from "dayjs/plugin/relativeTime"; +import utc from "dayjs/plugin/utc"; import React, { useState } from "react"; -import { DATE_TIME_WITH_SECONDS_FORMAT as DATE_TIME_WITH_MILLISECONDS_FORMAT, DATE_TIME_WITH_SECONDS_FORMAT, Entry, Field, RawDoc } from "../../utils/models"; +import { + DATE_TIME_WITH_SECONDS_FORMAT as DATE_TIME_WITH_MILLISECONDS_FORMAT, + DATE_TIME_WITH_SECONDS_FORMAT, + Entry, + Field, + RawDoc, +} from "../../utils/models"; import { QUICKWIT_INTERMEDIATE_GREY } from "../../utils/theme"; import { JsonEditor } from "../JsonEditor"; @@ -31,91 +37,126 @@ interface RowProps { row: RawDoc; } -const EntryName = styled('dt')` -display: inline; -background-color: ${QUICKWIT_INTERMEDIATE_GREY}; -color: #343741; -padding: 2px 1px 2px 4px; -margin-right: 4px; -word-break: normal; -border-radius: 3px; +const EntryName = styled("dt")` + display: inline; + background-color: ${QUICKWIT_INTERMEDIATE_GREY}; + color: #343741; + padding: 2px 1px 2px 4px; + margin-right: 4px; + word-break: normal; + border-radius: 3px; `; -const EntryValue = styled('dd')` -display: inline; -margin: 0; -padding: 0; -margin-inline-end: 5px; +const EntryValue = styled("dd")` + display: inline; + margin: 0; + padding: 0; + margin-inline-end: 5px; `; function EntryFormatter(entry: Entry) { // Some field can contains objects, stringify them to render them otherwise React will crash. - const value = typeof entry.value === 'object' ? JSON.stringify(entry.value) : entry.value; + const value = + typeof entry.value === "object" ? JSON.stringify(entry.value) : entry.value; return ( <> {entry.key}: {value} - ) + ); } // Display the timestamp value if found in a `TableCell`. function DisplayTimestampValue(row: RawDoc, timestampField: Field | null) { - if (timestampField === null || timestampField.field_mapping.output_format === null) { + if ( + timestampField === null || + timestampField.field_mapping.output_format === null + ) { return <>; } let field_value = row; for (const path_segment of timestampField.path_segments) { - field_value = field_value[path_segment] + field_value = field_value[path_segment]; } if (!field_value) { - return <> + return <>; } - return - - {formatDateTime(field_value, timestampField.field_mapping.output_format)} - - + return ( + + + {formatDateTime( + field_value, + timestampField.field_mapping.output_format, + )} + + + ); } -/* eslint-disable @typescript-eslint/no-explicit-any */ function formatDateTime(field_value: any, timestampOutputFormat: string): any { // A unix timestamp can be in secs/millis/micros/nanos and need to be converted properly. - if (timestampOutputFormat === 'unix_timestamp_secs' && typeof field_value === 'number') { - return dayjs(field_value * 1000).utc().format(DATE_TIME_WITH_SECONDS_FORMAT); - } else if (timestampOutputFormat === 'unix_timestamp_millis' && typeof field_value === 'number') { + if ( + timestampOutputFormat === "unix_timestamp_secs" && + typeof field_value === "number" + ) { + return dayjs(field_value * 1000) + .utc() + .format(DATE_TIME_WITH_SECONDS_FORMAT); + } else if ( + timestampOutputFormat === "unix_timestamp_millis" && + typeof field_value === "number" + ) { return dayjs(field_value).utc().format(DATE_TIME_WITH_MILLISECONDS_FORMAT); - } else if (timestampOutputFormat === 'unix_timestamp_micros' && typeof field_value === 'number') { - return dayjs(field_value / 1000).utc().format(DATE_TIME_WITH_MILLISECONDS_FORMAT); - } else if (timestampOutputFormat === 'unix_timestamp_nanos' && typeof field_value === 'number') { - return dayjs(field_value/ 1000000).utc().format(DATE_TIME_WITH_MILLISECONDS_FORMAT); + } else if ( + timestampOutputFormat === "unix_timestamp_micros" && + typeof field_value === "number" + ) { + return dayjs(field_value / 1000) + .utc() + .format(DATE_TIME_WITH_MILLISECONDS_FORMAT); + } else if ( + timestampOutputFormat === "unix_timestamp_nanos" && + typeof field_value === "number" + ) { + return dayjs(field_value / 1000000) + .utc() + .format(DATE_TIME_WITH_MILLISECONDS_FORMAT); } else { // Other formats are string values and we can just display it as is. return field_value; } } -const BreakWordBox = styled('dl')({ - verticalAlign: 'top', - display: 'inline-block', - color: '#464646', - wordBreak: 'break-all', - wordWrap: 'break-word', +const BreakWordBox = styled("dl")({ + verticalAlign: "top", + display: "inline-block", + color: "#464646", + wordBreak: "break-all", + wordWrap: "break-word", margin: 1, - overflow: 'hidden', - lineHeight: '1.8em', + overflow: "hidden", + lineHeight: "1.8em", }); export function Row(props: RowProps) { const [open, setOpen] = useState(false); const entries: Entry[] = []; for (const [key, value] of Object.entries(props.row)) { - entries.push({key: key, value: value}); + entries.push({ key: key, value: value }); } return ( <> - + {DisplayTimestampValue(props.row, props.timestampField)} - - {!open && - { entries.map((entry) => {EntryFormatter(entry)}) } + + {!open && ( + + {entries.map((entry) => ( + + {EntryFormatter(entry)} + + ))} - } - {open && - - } + )} + {open && } diff --git a/quickwit/quickwit-ui/src/components/SearchResult/SearchResult.tsx b/quickwit/quickwit-ui/src/components/SearchResult/SearchResult.tsx index 7002787f94c..9a66545796f 100644 --- a/quickwit/quickwit-ui/src/components/SearchResult/SearchResult.tsx +++ b/quickwit/quickwit-ui/src/components/SearchResult/SearchResult.tsx @@ -12,26 +12,26 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { useMemo } from 'react'; import { Box, Typography } from "@mui/material"; -import NumberFormat from "react-number-format"; +import { useMemo } from "react"; +import { NumericFormat } from "react-number-format"; import { Index, ResponseError, SearchResponse } from "../../utils/models"; import Loader from "../Loader"; -import { ResultTable } from "./ResultTable"; -import { AggregationResult } from "./AggregationResult"; import ErrorResponseDisplay from "../ResponseErrorDisplay"; +import { AggregationResult } from "./AggregationResult"; +import { ResultTable } from "./ResultTable"; -function HitCount({searchResponse}: {searchResponse: SearchResponse}) { +function HitCount({ searchResponse }: { searchResponse: SearchResponse }) { return ( - {" "} hits found in  - - ) + ); } interface SearchResultProps { @@ -55,7 +55,12 @@ export default function SearchResult(props: SearchResultProps) { if (props.searchResponse == null || props.index == null) { return null; } else if (props.searchResponse.aggregations === undefined) { - return ; + return ( + + ); } else { return ; } @@ -64,7 +69,7 @@ export default function SearchResult(props: SearchResultProps) { if (props.queryRunning) { return ; } - + if (props.searchError !== null) { return ErrorResponseDisplay(props.searchError); } @@ -74,12 +79,35 @@ export default function SearchResult(props: SearchResultProps) { } return ( - - - + + + - + {result} diff --git a/quickwit/quickwit-ui/src/components/SideBar.tsx b/quickwit/quickwit-ui/src/components/SideBar.tsx index e0866fa02f5..933d4cba981 100644 --- a/quickwit/quickwit-ui/src/components/SideBar.tsx +++ b/quickwit/quickwit-ui/src/components/SideBar.tsx @@ -12,23 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -import * as React from 'react'; -import List from '@mui/material/List'; -import ListItem from '@mui/material/ListItem'; -import ListItemIcon from '@mui/material/ListItemIcon'; -import ListItemText from '@mui/material/ListItemText'; import { - Link as RouterLink, - LinkProps as RouterLinkProps, -} from 'react-router-dom'; -import { ListSubheader, styled, Typography } from '@mui/material'; -import { CodeSSlash } from "@styled-icons/remix-line/CodeSSlash" -import { GroupWork } from '@styled-icons/material-outlined/GroupWork'; -import { Database } from '@styled-icons/feather/Database'; -import { Settings } from '@styled-icons/feather/Settings'; -import { useLocalStorage } from '../providers/LocalStorageProvider'; -import { toUrlSearchRequestParams } from '../utils/urls'; -import { APP_BAR_HEIGHT_PX } from './LayoutUtils'; + ListItemButton, + ListSubheader, + styled, + Typography, +} from "@mui/material"; +import List from "@mui/material/List"; +import ListItemIcon from "@mui/material/ListItemIcon"; +import ListItemText from "@mui/material/ListItemText"; +import { Database } from "@styled-icons/feather/Database"; +import { Settings } from "@styled-icons/feather/Settings"; +import { GroupWork } from "@styled-icons/material-outlined/GroupWork"; +import { CodeSSlash } from "@styled-icons/remix-line/CodeSSlash"; +import * as React from "react"; +import { Link as RouterLink, LinkProps as RouterLinkProps } from "react-router"; +import { useLocalStorage } from "../providers/LocalStorageProvider"; +import { toUrlSearchRequestParams } from "../utils/urls"; +import { APP_BAR_HEIGHT_PX } from "./LayoutUtils"; interface ListItemLinkProps { icon?: React.ReactElement; @@ -41,52 +42,77 @@ function ListItemLink(props: ListItemLinkProps) { const renderLink = React.useMemo( () => - React.forwardRef>(function Link( - itemProps, - ref, - ) { - return ; - }), + React.forwardRef>( + function Link(itemProps, ref) { + return ( + // biome-ignore lint/a11y/useValidAriaRole: remove the role + + ); + }, + ), [to], ); return ( - - {icon ? {icon} : null} + + {icon ? ( + {icon} + ) : null} - + ); } -const SideBarWrapper = styled('div')({ - display: 'flex', +const SideBarWrapper = styled("div")({ + display: "flex", marginTop: `${APP_BAR_HEIGHT_PX}`, height: `calc(100% - ${APP_BAR_HEIGHT_PX})`, - flex: '0 0 180px', - flexDirection: 'column', - borderRight: '1px solid rgba(0, 0, 0, 0.12)', + flex: "0 0 180px", + flexDirection: "column", + borderRight: "1px solid rgba(0, 0, 0, 0.12)", }); const SideBar = () => { const lastSearchRequest = useLocalStorage().lastSearchRequest; - let searchUrl = '/search'; + let searchUrl = "/search"; if (lastSearchRequest.indexId || lastSearchRequest.query) { - searchUrl = '/search?' + toUrlSearchRequestParams(lastSearchRequest).toString(); + searchUrl = + "/search?" + toUrlSearchRequestParams(lastSearchRequest).toString(); } return ( - - + + Discover - Query editor} icon={} /> - + Query editor} + icon={} + /> + Admin - Indexes} icon={} /> - Cluster} icon={} /> - Node info} icon={} /> - API } icon={} /> + Indexes} + icon={} + /> + Cluster} + icon={} + /> + Node info} + icon={} + /> + API } + icon={} + /> ); diff --git a/quickwit/quickwit-ui/src/components/TimeRangeSelect.tsx b/quickwit/quickwit-ui/src/components/TimeRangeSelect.tsx index 88c8b88d26f..21b14972b7d 100644 --- a/quickwit/quickwit-ui/src/components/TimeRangeSelect.tsx +++ b/quickwit/quickwit-ui/src/components/TimeRangeSelect.tsx @@ -12,26 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -import React, { useEffect, useMemo, useState } from "react"; +import { AccessTime, ChevronRight, DateRange } from "@mui/icons-material"; import { Box, Button, Divider, List, - ListItem, + ListItemButton, ListItemIcon, ListItemText, Popover, - TextField, - TextFieldProps, } from "@mui/material"; -import { AccessTime, ChevronRight, DateRange } from "@mui/icons-material"; -import { Dayjs, default as dayjs } from 'dayjs'; -import relativeTime from "dayjs/plugin/relativeTime" -import utc from "dayjs/plugin/utc" -import { DateTimePicker } from '@mui/x-date-pickers'; -import { AdapterDayjs, } from '@mui/x-date-pickers/AdapterDayjs'; -import { LocalizationProvider } from '@mui/x-date-pickers'; +import { DateTimePicker, LocalizationProvider } from "@mui/x-date-pickers"; +import { AdapterDayjs } from "@mui/x-date-pickers/AdapterDayjs"; +import { Dayjs, default as dayjs } from "dayjs"; +import relativeTime from "dayjs/plugin/relativeTime"; +import utc from "dayjs/plugin/utc"; +import React, { JSX, useEffect, useMemo, useState } from "react"; import { DATE_TIME_WITH_SECONDS_FORMAT } from "../utils/models"; dayjs.extend(relativeTime); @@ -48,14 +45,14 @@ const TIME_RANGE_CHOICES = [ ]; type TimeRange = { - startTimestamp: number | null, - endTimestamp: number | null, -} + startTimestamp: number | null; + endTimestamp: number | null; +}; export interface TimeRangeSelectProps { timeRange: TimeRange; disabled?: boolean; - onUpdate(newTimeRange:TimeRange): void; + onUpdate(newTimeRange: TimeRange): void; } interface TimeRangeSelectState { @@ -65,8 +62,12 @@ interface TimeRangeSelectState { } export function TimeRangeSelect(props: TimeRangeSelectProps): JSX.Element { - const getInitialState = () => {return {width: 220, anchor: null, customDatesPanelOpen: false}}; - const initialState = useMemo(() => {return getInitialState(); }, []); + const getInitialState = () => { + return { width: 220, anchor: null, customDatesPanelOpen: false }; + }; + const initialState = useMemo(() => { + return getInitialState(); + }, []); const [state, setState] = useState(initialState); const handleOpenClick = (event: React.MouseEvent) => { @@ -83,13 +84,15 @@ export function TimeRangeSelect(props: TimeRangeSelectProps): JSX.Element { useEffect(() => { setState(initialState); - }, [props.disabled, initialState]) + }, [props.disabled, initialState]); const handleClose = () => { setState(initialState); }; - const handleTimeRangeChoiceClick = (secondsBeforeNow: number | string | undefined) => { + const handleTimeRangeChoiceClick = ( + secondsBeforeNow: number | string | undefined, + ) => { if (secondsBeforeNow === undefined) { return; } @@ -97,18 +100,18 @@ export function TimeRangeSelect(props: TimeRangeSelectProps): JSX.Element { secondsBeforeNow = +secondsBeforeNow; setState(initialState); const startTimestamp = Math.trunc(Date.now() / 1000) - secondsBeforeNow; - props.onUpdate({startTimestamp, endTimestamp:null}) + props.onUpdate({ startTimestamp, endTimestamp: null }); }; const handleReset = () => { - props.onUpdate({startTimestamp:null, endTimestamp:null}) + props.onUpdate({ startTimestamp: null, endTimestamp: null }); }; const open = Boolean(state.anchor); const id = open ? "time-range-select-popover" : undefined; return ( - + {TIME_RANGE_CHOICES.map((value, idx) => { - return handleTimeRangeChoiceClick(value[1])} + return ( + handleTimeRangeChoiceClick(value[1])} > - - + + + ); })} - + - - - + + + - - + + - + {state.anchor !== null && state.customDatesPanelOpen && ( - + )} @@ -183,50 +197,75 @@ function CustomDatesPanel(props: TimeRangeSelectProps): JSX.Element { const [endDate, setEndDate] = useState(null); useEffect(() => { - setStartDate(props.timeRange.startTimestamp ? convertTimestampSecsIntoDateUtc(props.timeRange.startTimestamp) : null); - setEndDate(props.timeRange.endTimestamp ? convertTimestampSecsIntoDateUtc(props.timeRange.endTimestamp) : null); + setStartDate( + props.timeRange.startTimestamp + ? convertTimestampSecsIntoDateUtc(props.timeRange.startTimestamp) + : null, + ); + setEndDate( + props.timeRange.endTimestamp + ? convertTimestampSecsIntoDateUtc(props.timeRange.endTimestamp) + : null, + ); }, [props.timeRange.startTimestamp, props.timeRange.endTimestamp]); const handleReset = (event: React.MouseEvent) => { event.preventDefault(); setStartDate(null); - setEndDate(null) - props.onUpdate({startTimestamp:null, endTimestamp:null}); + setEndDate(null); + props.onUpdate({ startTimestamp: null, endTimestamp: null }); }; const handleApply = (event: React.MouseEvent) => { event.preventDefault(); const startTimestamp = startDate ? startDate.valueOf() / 1000 : null; const endTimestamp = endDate ? endDate.valueOf() / 1000 : null; - props.onUpdate({startTimestamp, endTimestamp}); + props.onUpdate({ startTimestamp, endTimestamp }); }; return ( - + { // By default, newValue is a datetime defined on the local time zone and for now we consider // input/output only in UTC. - setStartDate(newValue ? dayjs(newValue.valueOf() + newValue.utcOffset() * 60 * 1000).utc() : null); + setStartDate( + newValue + ? dayjs( + newValue.valueOf() + newValue.utcOffset() * 60 * 1000, + ).utc() + : null, + ); }} - renderInput={(props: TextFieldProps) => } + slotProps={{ textField: { sx: { width: "100%" } } }} /> { // By default, newValue is a datetime defined on the local time zone and for now we consider // input/output only in UTC. - setEndDate(newValue ? dayjs(newValue.valueOf() + newValue.utcOffset() * 60 * 1000).utc() : null); + setEndDate( + newValue + ? dayjs( + newValue.valueOf() + newValue.utcOffset() * 60 * 1000, + ).utc() + : null, + ); }} - renderInput={(props: TextFieldProps) => } + slotProps={{ textField: { sx: { width: "100%" } } }} /> @@ -236,7 +275,7 @@ function CustomDatesPanel(props: TimeRangeSelectProps): JSX.Element { color="primary" onClick={handleReset} disableElevation - style={{marginRight: 10}} + style={{ marginRight: 10 }} > Reset @@ -262,20 +301,39 @@ interface DateTimeRangeLabelProps { function DateTimeRangeLabel(props: DateTimeRangeLabelProps): JSX.Element { function Label() { if (props.startTimestamp !== null && props.endTimestamp !== null) { - return <> - {convertTimestampSecsIntoDateUtc(props.startTimestamp).format(DATE_TIME_WITH_SECONDS_FORMAT)} -{" "} - {convertTimestampSecsIntoDateUtc(props.endTimestamp).format(DATE_TIME_WITH_SECONDS_FORMAT)} - + return ( + <> + {convertTimestampSecsIntoDateUtc(props.startTimestamp).format( + DATE_TIME_WITH_SECONDS_FORMAT, + )}{" "} + -{" "} + {convertTimestampSecsIntoDateUtc(props.endTimestamp).format( + DATE_TIME_WITH_SECONDS_FORMAT, + )} + + ); } else if (props.startTimestamp !== null && props.endTimestamp === null) { - return <>Since {convertTimestampSecsIntoDateUtc(props.startTimestamp).fromNow(true)} + return ( + <> + Since{" "} + {convertTimestampSecsIntoDateUtc(props.startTimestamp).fromNow(true)} + + ); } else if (props.startTimestamp == null && props.endTimestamp != null) { - return <>Before {convertTimestampSecsIntoDateUtc(props.endTimestamp).format(DATE_TIME_WITH_SECONDS_FORMAT)} + return ( + <> + Before{" "} + {convertTimestampSecsIntoDateUtc(props.endTimestamp).format( + DATE_TIME_WITH_SECONDS_FORMAT, + )} + + ); } - return <>No date range + return <>No date range; } return ( - + ); diff --git a/quickwit/quickwit-ui/src/components/TopBar.tsx b/quickwit/quickwit-ui/src/components/TopBar.tsx index 6aa03be421d..1ed6df3b86c 100644 --- a/quickwit/quickwit-ui/src/components/TopBar.tsx +++ b/quickwit/quickwit-ui/src/components/TopBar.tsx @@ -12,21 +12,33 @@ // See the License for the specific language governing permissions and // limitations under the License. -import AppBar from '@mui/material/AppBar'; -import Toolbar from '@mui/material/Toolbar'; -import GitHubIcon from '@mui/icons-material/GitHub'; -import { Box, IconButton, Link, styled, SvgIcon, Tooltip, Typography } from '@mui/material'; -import { Discord } from '@styled-icons/fa-brands/Discord'; -import { ReactComponent as Logo } from '../assets/img/quickwit-logo.svg'; -import { Client } from '../services/client'; -import { useEffect, useMemo, useState } from 'react'; +import GitHubIcon from "@mui/icons-material/GitHub"; +import { + Box, + IconButton, + Link, + SvgIcon, + styled, + Tooltip, + Typography, +} from "@mui/material"; +import AppBar from "@mui/material/AppBar"; +import Toolbar from "@mui/material/Toolbar"; +import { Discord } from "@styled-icons/fa-brands/Discord"; +import { useEffect, useMemo, useState } from "react"; +import quickwitLogoUrl from "../assets/img/quickwit-logo-with-title.svg"; +import { Client } from "../services/client"; -const StyledAppBar = styled(AppBar)(({ theme })=>({ +const Logo = (props: React.ComponentProps<"img">) => ( + quickwit logo +); + +const StyledAppBar = styled(AppBar)(({ theme }) => ({ zIndex: theme.zIndex.drawer + 1, })); // Update the Button's color prop options -declare module '@mui/material/AppBar' { +declare module "@mui/material/AppBar" { interface AppBarPropsColorOverrides { neutral: true; } @@ -37,24 +49,30 @@ const TopBar = () => { const quickwitClient = useMemo(() => new Client(), []); useEffect(() => { - quickwitClient.cluster().then(cluster => { + quickwitClient.cluster().then((cluster) => { setClusterId(cluster.cluster_id); }); - }, []) + }, [quickwitClient]); return ( - - + + - - {clusterId} - + {clusterId} - Docs + Docs diff --git a/quickwit/quickwit-ui/src/index.css b/quickwit/quickwit-ui/src/index.css index d57d07f0dd8..f04f8d80804 100644 --- a/quickwit/quickwit-ui/src/index.css +++ b/quickwit/quickwit-ui/src/index.css @@ -23,5 +23,5 @@ body { } div.swagger-ui div.information-container { - display: none; + display: none; } diff --git a/quickwit/quickwit-ui/src/index.test.js b/quickwit/quickwit-ui/src/index.test.js index 98803c46e93..533802b05dd 100644 --- a/quickwit/quickwit-ui/src/index.test.js +++ b/quickwit/quickwit-ui/src/index.test.js @@ -12,15 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { render, screen } from '@testing-library/react'; -import { BrowserRouter } from 'react-router-dom'; -import App from './views/App'; +import { describe, expect, it } from "@jest/globals"; +import { render, screen } from "@testing-library/react"; +import { BrowserRouter } from "react-router"; +import App from "./views/App"; -describe('App', function () { - it('Should display side bar links', function () { - render(); - expect(screen.getByText(/Discover/)).toBeInTheDocument(); - expect(screen.getByText(/Query editor/)).toBeInTheDocument(); - expect(screen.getByText(/Admin/)).toBeInTheDocument(); +describe("App", () => { + it("Should display side bar links", () => { + render( + + + , + ); + expect(screen.getByText(/Discover/)).toBeInTheDocument(); + expect(screen.getByText(/Query editor/)).toBeInTheDocument(); + expect(screen.getByText(/Admin/)).toBeInTheDocument(); }); }); diff --git a/quickwit/quickwit-ui/src/index.tsx b/quickwit/quickwit-ui/src/index.tsx index 6c8e7f19012..8aeb2cbd60b 100644 --- a/quickwit/quickwit-ui/src/index.tsx +++ b/quickwit/quickwit-ui/src/index.tsx @@ -12,23 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -import React from 'react'; -import ReactDOM from 'react-dom'; -import './index.css'; -import App from './views/App'; -import reportWebVitals from './reportWebVitals'; -import { BrowserRouter } from 'react-router-dom'; +import React from "react"; +import { createRoot } from "react-dom/client"; +import "./index.css"; +import { BrowserRouter } from "react-router"; +import App from "./views/App"; -ReactDOM.render( +const root = createRoot(document.getElementById("root")!); +root.render( - + , - document.getElementById('root') ); - -// If you want to start measuring performance in your app, pass a function -// to log results (for example: reportWebVitals(console.log)) -// or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals -reportWebVitals(); diff --git a/quickwit/quickwit-ui/src/logo.svg b/quickwit/quickwit-ui/src/logo.svg deleted file mode 100644 index 9dfc1c058ce..00000000000 --- a/quickwit/quickwit-ui/src/logo.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/quickwit/quickwit-ui/src/providers/EditorProvider.tsx b/quickwit/quickwit-ui/src/providers/EditorProvider.tsx index 92423c237a0..a17ad09e550 100644 --- a/quickwit/quickwit-ui/src/providers/EditorProvider.tsx +++ b/quickwit/quickwit-ui/src/providers/EditorProvider.tsx @@ -12,24 +12,30 @@ // See the License for the specific language governing permissions and // limitations under the License. -import * as monacoEditor from 'monaco-editor/esm/vs/editor/editor.api'; -import { createContext, MutableRefObject, PropsWithChildren, useContext, useRef } from "react" +import * as monacoEditor from "monaco-editor/esm/vs/editor/editor.api"; +import { + createContext, + MutableRefObject, + PropsWithChildren, + useContext, + useRef, +} from "react"; type ContextProps = { - editorRef: MutableRefObject | null - monacoRef: MutableRefObject | null -} + editorRef: MutableRefObject | null; + monacoRef: MutableRefObject | null; +}; const defaultValues = { editorRef: null, monacoRef: null, -} +}; const EditorContext = createContext(defaultValues); -export const EditorProvider = ({ children }: PropsWithChildren<{}>) => { - const editorRef = useRef(null) - const monacoRef = useRef(null) +export const EditorProvider = ({ children }: PropsWithChildren) => { + const editorRef = useRef(null); + const monacoRef = useRef(null); return ( ) => { > {children} - ) -} + ); +}; export const useEditor = () => { - return useContext(EditorContext) -} + return useContext(EditorContext); +}; diff --git a/quickwit/quickwit-ui/src/providers/LocalStorageProvider.tsx b/quickwit/quickwit-ui/src/providers/LocalStorageProvider.tsx index 8cc65b75305..75c36b75f65 100644 --- a/quickwit/quickwit-ui/src/providers/LocalStorageProvider.tsx +++ b/quickwit/quickwit-ui/src/providers/LocalStorageProvider.tsx @@ -12,8 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { createContext, PropsWithChildren, useContext, useEffect, useState } from "react" -import { EMPTY_SEARCH_REQUEST, SearchRequest } from "../utils/models" +import { + createContext, + PropsWithChildren, + useContext, + useEffect, + useState, +} from "react"; +import { EMPTY_SEARCH_REQUEST, SearchRequest } from "../utils/models"; type Props = Record; @@ -25,7 +31,7 @@ type ContextProps = { const defaultValues = { lastSearchRequest: EMPTY_SEARCH_REQUEST, updateLastSearchRequest: () => undefined, -} +}; function parseSearchRequest(value: string | null): SearchRequest { if (value === null) { @@ -39,17 +45,23 @@ export const LocalStorageContext = createContext(defaultValues); export const LocalStorageProvider = ({ children, }: PropsWithChildren) => { - const [lastSearchRequest, setLastSearchRequest] = useState(EMPTY_SEARCH_REQUEST); + const [lastSearchRequest, setLastSearchRequest] = + useState(EMPTY_SEARCH_REQUEST); useEffect(() => { - if (localStorage.getItem('lastSearchRequest') !== null) { - const lastSearchRequest = parseSearchRequest(localStorage.getItem('lastSearchRequest')); + if (localStorage.getItem("lastSearchRequest") !== null) { + const lastSearchRequest = parseSearchRequest( + localStorage.getItem("lastSearchRequest"), + ); setLastSearchRequest(lastSearchRequest); } }, []); useEffect(() => { - localStorage.setItem('lastSearchRequest', JSON.stringify(lastSearchRequest)); + localStorage.setItem( + "lastSearchRequest", + JSON.stringify(lastSearchRequest), + ); }, [lastSearchRequest]); function updateLastSearchRequest(searchRequest: SearchRequest) { @@ -61,13 +73,13 @@ export const LocalStorageProvider = ({ value={{ lastSearchRequest, updateLastSearchRequest, - }} + }} > {children} - ) -} + ); +}; export const useLocalStorage = () => { - return useContext(LocalStorageContext) -} + return useContext(LocalStorageContext); +}; diff --git a/quickwit/quickwit-ui/src/react-app-env.d.ts b/quickwit/quickwit-ui/src/react-app-env.d.ts deleted file mode 100644 index d0639830017..00000000000 --- a/quickwit/quickwit-ui/src/react-app-env.d.ts +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/// diff --git a/quickwit/quickwit-ui/src/services/client.test.ts b/quickwit/quickwit-ui/src/services/client.test.ts index b393aabc335..eb2552fc755 100644 --- a/quickwit/quickwit-ui/src/services/client.test.ts +++ b/quickwit/quickwit-ui/src/services/client.test.ts @@ -12,39 +12,44 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { SearchRequest } from '../utils/models'; -import { Client } from './client'; +import { describe, expect, it, jest } from "@jest/globals"; +import { SearchRequest } from "../utils/models"; +import { Client } from "./client"; -describe('Client unit test', () => { - it('Should construct correct search URL', async () => { - // Mocking the fetch function to simulate network requests - const mockFetch = jest.fn(() => Promise.resolve({ ok: true, json: () => Promise.resolve({}) })); - (global as any).fetch = mockFetch; // eslint-disable-line @typescript-eslint/no-explicit-any +describe("Client unit test", () => { + it("Should construct correct search URL", async () => { + // Mocking the fetch function to simulate network requests + const mockFetch = jest.fn((_url: string, _options?: unknown) => + Promise.resolve({ ok: true, json: () => Promise.resolve({}) }), + ); + (global as any).fetch = mockFetch; - const searchRequest: SearchRequest = { - indexId: 'my-new-fresh-index-id', - query: 'severity_error:ERROR', - startTimestamp: 100, - endTimestamp: 200, - maxHits: 20, - sortByField: { - field_name: 'timestamp', - order: 'Desc', - }, - aggregation: false, - aggregationConfig: { - metric: null, - term: null, - histogram: null, - }, - }; + const searchRequest: SearchRequest = { + indexId: "my-new-fresh-index-id", + query: "severity_error:ERROR", + startTimestamp: 100, + endTimestamp: 200, + maxHits: 20, + sortByField: { + field_name: "timestamp", + order: "Desc", + }, + aggregation: false, + aggregationConfig: { + metric: null, + term: null, + histogram: null, + }, + }; - const client = new Client(); - expect(client.buildSearchBody(searchRequest, null)).toBe('{"query":"severity_error:ERROR","max_hits":20,"start_timestamp":100,"end_timestamp":200,"sort_by_field":"+timestamp"}'); + const client = new Client(); + expect(client.buildSearchBody(searchRequest, null)).toBe( + '{"query":"severity_error:ERROR","max_hits":20,"start_timestamp":100,"end_timestamp":200,"sort_by_field":"+timestamp"}', + ); - await client.search(searchRequest, null); - const expectedUrl = `${client.apiRoot()}my-new-fresh-index-id/search`; - expect(mockFetch).toHaveBeenCalledTimes(1); - expect(mockFetch).toHaveBeenCalledWith(expectedUrl, expect.any(Object)); - }); + await client.search(searchRequest, null); + const expectedUrl = `${client.apiRoot()}my-new-fresh-index-id/search`; + expect(mockFetch).toHaveBeenCalledTimes(1); + expect(mockFetch).toHaveBeenCalledWith(expectedUrl, expect.any(Object)); + }); }); diff --git a/quickwit/quickwit-ui/src/services/client.ts b/quickwit/quickwit-ui/src/services/client.ts index 6e8e57e8388..95baaceed99 100644 --- a/quickwit/quickwit-ui/src/services/client.ts +++ b/quickwit/quickwit-ui/src/services/client.ts @@ -12,17 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { Cluster, Index, IndexMetadata, QuickwitBuildInfo, SearchRequest, SearchResponse, SplitMetadata } from "../utils/models"; +import { + Cluster, + Index, + IndexMetadata, + QuickwitBuildInfo, + SearchRequest, + SearchResponse, + SplitMetadata, +} from "../utils/models"; import { serializeSortByField } from "../utils/urls"; export class Client { - private readonly _host: string + private readonly _host: string; constructor(host?: string) { if (!host) { - this._host = window.location.origin + this._host = window.location.origin; } else { - this._host = host + this._host = host; } } @@ -30,10 +38,13 @@ export class Client { return this._host + "/api/v1/"; } - async search(request: SearchRequest, timestamp_field: string | null): Promise { + async search( + request: SearchRequest, + timestamp_field: string | null, + ): Promise { // TODO: improve validation of request. if (request.indexId === null || request.indexId === undefined) { - throw Error("Search request must have and index id.") + throw Error("Search request must have and index id."); } const url = `${this.apiRoot()}${request.indexId}/search`; const body = this.buildSearchBody(request, timestamp_field); @@ -41,16 +52,24 @@ export class Client { } async cluster(): Promise { - return await this.fetch(`${this.apiRoot()}cluster`, this.defaultGetRequestParams()); + return await this.fetch( + `${this.apiRoot()}cluster`, + this.defaultGetRequestParams(), + ); } async buildInfo(): Promise { - return await this.fetch(`${this.apiRoot()}version`, this.defaultGetRequestParams()); + return await this.fetch( + `${this.apiRoot()}version`, + this.defaultGetRequestParams(), + ); } - // eslint-disable-next-line async config(): Promise> { - return await this.fetch(`${this.apiRoot()}config`, this.defaultGetRequestParams()); + return await this.fetch( + `${this.apiRoot()}config`, + this.defaultGetRequestParams(), + ); } // // Index management API @@ -58,34 +77,47 @@ export class Client { async getIndex(indexId: string): Promise { const [metadata, splits] = await Promise.all([ this.getIndexMetadata(indexId), - this.getAllSplits(indexId) + this.getAllSplits(indexId), ]); return { metadata: metadata, - splits: splits - } + splits: splits[0], + split_limit_reached: splits[1], + }; } async getIndexMetadata(indexId: string): Promise { return this.fetch(`${this.apiRoot()}indexes/${indexId}`, {}); } - async getAllSplits(indexId: string): Promise> { + async getAllSplits( + indexId: string, + ): Promise<[Array, boolean]> { // TODO: restrieve all the splits. - const results: {splits: Array} = await this.fetch(`${this.apiRoot()}indexes/${indexId}/splits?limit=10000`, {}); + const results: { splits: Array } = await this.fetch( + `${this.apiRoot()}indexes/${indexId}/splits?limit=10000`, + {}, + ); - return results['splits']; + return [results["splits"], results["splits"].length === 10000]; } async listIndexes(): Promise> { return this.fetch(`${this.apiRoot()}indexes`, {}); } - async fetch(url: string, params: RequestInit, body: string|null = null): Promise { + async fetch( + url: string, + params: RequestInit, + body: string | null = null, + ): Promise { if (body !== null) { params.method = "POST"; params.body = body; - params.headers = {...params.headers, "content-type": "application/json"}; + params.headers = { + ...params.headers, + "content-type": "application/json", + }; } const response = await fetch(url, params); if (response.ok) { @@ -94,7 +126,7 @@ export class Client { const message = await response.text(); return await Promise.reject({ message: message, - status: response.status + status: response.status, }); } @@ -104,11 +136,13 @@ export class Client { headers: { Accept: "application/json" }, mode: "cors", cache: "default", - } + }; } - buildSearchBody(request: SearchRequest, timestamp_field: string | null): string { - /* eslint-disable @typescript-eslint/no-explicit-any */ + buildSearchBody( + request: SearchRequest, + timestamp_field: string | null, + ): string { const body: any = { // TODO: the trim should be done in the backend. query: request.query.trim() || "*", @@ -133,23 +167,25 @@ export class Client { return JSON.stringify(body); } - /* eslint-disable @typescript-eslint/no-explicit-any */ - buildAggregation(request: SearchRequest, timestamp_field: string | null): any { - let aggregation = undefined; + buildAggregation( + request: SearchRequest, + timestamp_field: string | null, + ): any { + let aggregation: any; if (request.aggregationConfig.metric) { const metric = request.aggregationConfig.metric; aggregation = { metric: { [metric.type]: { - field: metric.field - } - } - } + field: metric.field, + }, + }, + }; } if (request.aggregationConfig.histogram && timestamp_field) { const histogram = request.aggregationConfig.histogram; - const interval = histogram.interval - let extended_bounds; + const interval = histogram.interval; + let extended_bounds: any; if (request.startTimestamp && request.endTimestamp) { extended_bounds = { min: request.startTimestamp, @@ -166,9 +202,9 @@ export class Client { fixed_interval: interval, min_doc_count: 0, extended_bounds: extended_bounds, - } - } - } + }, + }, + }; } if (request.aggregationConfig.term) { const term = request.aggregationConfig.term; @@ -179,12 +215,12 @@ export class Client { field: term.field, size: term.size, order: { - _count: "desc" + _count: "desc", }, min_doc_count: 1, - } - } - } + }, + }, + }; } return aggregation; } diff --git a/quickwit/quickwit-ui/src/setupTests.ts b/quickwit/quickwit-ui/src/setupTests.ts deleted file mode 100644 index 5e5df0fd200..00000000000 --- a/quickwit/quickwit-ui/src/setupTests.ts +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// jest-dom adds custom jest matchers for asserting on DOM nodes. -// allows you to do things like: -// expect(element).toHaveTextContent(/react/i) -// learn more: https://github.com/testing-library/jest-dom -import '@testing-library/jest-dom'; diff --git a/quickwit/quickwit-ui/src/utils/models.ts b/quickwit/quickwit-ui/src/utils/models.ts index 2d87598cb42..8abe8acc6e1 100644 --- a/quickwit/quickwit-ui/src/utils/models.ts +++ b/quickwit/quickwit-ui/src/utils/models.ts @@ -12,8 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -/* eslint-disable @typescript-eslint/no-explicit-any */ -export type RawDoc = Record +export type RawDoc = Record; export type FieldMapping = { description: string | null; @@ -25,7 +24,7 @@ export type FieldMapping = { // Specific datetime field attributes. output_format: string | null; field_mappings?: FieldMapping[]; -} +}; export type Field = { // Json path (path segments concatenated as a string with dots between segments). @@ -33,12 +32,12 @@ export type Field = { // Json path of the field. path_segments: string[]; field_mapping: FieldMapping; -} +}; export type Entry = { key: string; value: any; -} +}; export const DATE_TIME_WITH_SECONDS_FORMAT = "YYYY/MM/DD HH:mm:ss"; export const DATE_TIME_WITH_MILLISECONDS_FORMAT = "YYYY/MM/DD HH:mm:ss.SSS"; @@ -47,12 +46,27 @@ export const DATE_TIME_WITH_MILLISECONDS_FORMAT = "YYYY/MM/DD HH:mm:ss.SSS"; export function getAllFields(field_mappings: Array): Field[] { const fields: Field[] = []; for (const field_mapping of field_mappings) { - if (field_mapping.type === 'object' && field_mapping.field_mappings !== undefined) { - for (const child_field_mapping of getAllFields(field_mapping.field_mappings)) { - fields.push({json_path: field_mapping.name + '.' + child_field_mapping.json_path, path_segments: [field_mapping.name].concat(child_field_mapping.path_segments), field_mapping: child_field_mapping.field_mapping}) + if ( + field_mapping.type === "object" && + field_mapping.field_mappings !== undefined + ) { + for (const child_field_mapping of getAllFields( + field_mapping.field_mappings, + )) { + fields.push({ + json_path: field_mapping.name + "." + child_field_mapping.json_path, + path_segments: [field_mapping.name].concat( + child_field_mapping.path_segments, + ), + field_mapping: child_field_mapping.field_mapping, + }); } } else { - fields.push({json_path: field_mapping.name, path_segments: [field_mapping.name], field_mapping: field_mapping}); + fields.push({ + json_path: field_mapping.name, + path_segments: [field_mapping.name], + field_mapping: field_mapping, + }); } } @@ -65,14 +79,14 @@ export type DocMapping = { store: boolean; dynamic_mapping: boolean; timestamp_field: string | null; -} +}; -export type SortOrder = 'Asc' | 'Desc'; +export type SortOrder = "Asc" | "Desc"; export type SortByField = { - field_name: string, - order: SortOrder -} + field_name: string; + order: SortOrder; +}; export type SearchRequest = { indexId: string | null; @@ -83,38 +97,40 @@ export type SearchRequest = { sortByField: SortByField | null; aggregation: boolean; aggregationConfig: Aggregation; -} +}; export type Aggregation = { metric: Metric | null; term: TermAgg | null; histogram: HistogramAgg | null; -} +}; export type Metric = { type: string; field: string; -} +}; export type TermAgg = { field: string; size: number; -} +}; export type HistogramAgg = { interval: string; -} +}; -export type ParsedAggregationResult = TermResult | HistogramResult| null; +export type ParsedAggregationResult = TermResult | HistogramResult | null; -export type TermResult = {term: string, value: number}[]; +export type TermResult = { term: string; value: number }[]; export type HistogramResult = { - timestamps: Date[], - data: {name: string | undefined, value: number[]}[], -} + timestamps: Date[]; + data: { name: string | undefined; value: number[] }[]; +}; -export function extractAggregationResults(aggregation: any): ParsedAggregationResult { +export function extractAggregationResults( + aggregation: any, +): ParsedAggregationResult { const extract_value = (entry: any) => { if ("metric" in entry) { return entry.metric.value || 0; @@ -129,21 +145,23 @@ export function extractAggregationResults(aggregation: any): ParsedAggregationRe // we are in the "simple histogram" case return { timestamps, - data: [{name: undefined, value }] - } + data: [{ name: undefined, value }], + }; } else if ("term_agg" in aggregation) { // we have a term aggregation, but maybe there is an histogram inside const term_buckets = aggregation.term_agg.buckets; - if (term_buckets.length == 0) { + if (term_buckets.length === 0) { return null; } if (term_buckets.length > 0 && "histo_agg" in term_buckets[0]) { // we have a term+histo aggregation const timestamps_set: Set = new Set(); - term_buckets.forEach((bucket: any) => bucket.histo_agg.buckets.forEach( - (entry: any) => timestamps_set.add(entry.key) - )); - const timestamps = [... timestamps_set]; + term_buckets.forEach((bucket: any) => + bucket.histo_agg.buckets.forEach((entry: any) => + timestamps_set.add(entry.key), + ), + ); + const timestamps = [...timestamps_set]; timestamps.sort(); const data = term_buckets.map((bucket: any) => { @@ -151,24 +169,24 @@ export function extractAggregationResults(aggregation: any): ParsedAggregationRe const first_elem_key = histo_buckets[0].key; const last_elem_key = histo_buckets[histo_buckets.length - 1].key; const prefix_len = timestamps.indexOf(first_elem_key); - const suffix_len = timestamps.length - timestamps.indexOf(last_elem_key) - 1; - const value = Array(prefix_len).fill(0).concat( - histo_buckets.map(extract_value), - Array(suffix_len).fill(0), - ); - - return {name: bucket.key, value, } - }) + const suffix_len = + timestamps.length - timestamps.indexOf(last_elem_key) - 1; + const value = Array(prefix_len) + .fill(0) + .concat(histo_buckets.map(extract_value), Array(suffix_len).fill(0)); + + return { name: bucket.key, value }; + }); return { timestamps: timestamps.map((date) => new Date(date)), data, - } + }; } else { return term_buckets.map((bucket: any) => { return { term: bucket.key, value: extract_value(bucket), - } + }; }); } } @@ -177,8 +195,8 @@ export function extractAggregationResults(aggregation: any): ParsedAggregationRe } export const EMPTY_SEARCH_REQUEST: SearchRequest = { - indexId: '', - query: '', + indexId: "", + query: "", startTimestamp: null, endTimestamp: null, maxHits: 100, @@ -189,12 +207,12 @@ export const EMPTY_SEARCH_REQUEST: SearchRequest = { term: null, histogram: null, }, -} +}; export type ResponseError = { status: number | null; message: string | null; -} +}; export type SearchResponse = { num_hits: number; @@ -202,7 +220,7 @@ export type SearchResponse = { elapsed_time_micros: number; errors: Array | undefined; aggregations: any | undefined; -} +}; export type IndexConfig = { version: string; @@ -212,26 +230,26 @@ export type IndexConfig = { indexing_settings: object; search_settings: object; retention: object; -} +}; export type IndexMetadata = { index_config: IndexConfig; checkpoint: object; sources: object[] | undefined; create_timestamp: number; -} +}; export const EMPTY_INDEX_METADATA: IndexMetadata = { index_config: { - version: '', - index_uri: '', - index_id: '', + version: "", + index_uri: "", + index_id: "", doc_mapping: { field_mappings: [], tag_fields: [], store: false, dynamic_mapping: false, - timestamp_field: null + timestamp_field: null, }, indexing_settings: {}, search_settings: {}, @@ -254,62 +272,63 @@ export type SplitMetadata = { tags: string[]; demux_num_ops: number; footer_offsets: Range; -} +}; export type Range = { start: number; end: number; -} +}; export type Index = { metadata: IndexMetadata; splits: SplitMetadata[]; -} + split_limit_reached: boolean; +}; export type Cluster = { - node_id: string, - cluster_id: string, - state: ClusterState, -} + node_id: string; + cluster_id: string; + state: ClusterState; +}; export type ClusterState = { state: ClusterStateSnapshot; live_nodes: any[]; dead_nodes: any[]; -} +}; export type ClusterStateSnapshot = { - seed_addrs: string[], - node_states: Record, -} + seed_addrs: string[]; + node_states: Record; +}; export type NodeState = { - key_values: KeyValues, - max_version: number, -} + key_values: KeyValues; + max_version: number; +}; export type KeyValues = { - available_services: KeyValue, - grpc_address: KeyValue, - heartbeat: KeyValue, -} + available_services: KeyValue; + grpc_address: KeyValue; + heartbeat: KeyValue; +}; export type KeyValue = { - value: any, - version: number, -} + value: any; + version: number; +}; export type QuickwitBuildInfo = { - commit_version_tag: string, - cargo_pkg_version: string, - cargo_build_target: string, - commit_short_hash: string, - commit_date: string, - version: string, -} + commit_version_tag: string; + cargo_pkg_version: string; + cargo_build_target: string; + commit_short_hash: string; + commit_date: string; + version: string; +}; export type NodeId = { - id: string, - grpc_address: string, - self: boolean, -} + id: string; + grpc_address: string; + self: boolean; +}; diff --git a/quickwit/quickwit-ui/src/utils/theme.ts b/quickwit/quickwit-ui/src/utils/theme.ts index cec9f8438fa..25ceb3d725b 100644 --- a/quickwit/quickwit-ui/src/utils/theme.ts +++ b/quickwit/quickwit-ui/src/utils/theme.ts @@ -13,22 +13,21 @@ // limitations under the License. import { createTheme } from "@mui/material"; -import SoehneMonoKraftigWoff2 from "./../assets/fonts/soehne-mono-web-kraftig.woff2"; import SoehneMonoDreiviertelfettWoff2 from "./../assets/fonts/soehne-mono-web-dreiviertelfett.woff2"; -import SoehneHalbfettWoff2 from "./../assets/fonts/soehne-web-halbfett.woff2"; +import SoehneMonoKraftigWoff2 from "./../assets/fonts/soehne-mono-web-kraftig.woff2"; import SoehneBuchWoff2 from "./../assets/fonts/soehne-web-buch.woff2"; +import SoehneHalbfettWoff2 from "./../assets/fonts/soehne-web-halbfett.woff2"; -export const QUICKWIT_BLUE = '#004BD9'; -export const QUICKWIT_RED = '#FF0026'; -export const QUICKWIT_GREEN = '#00D588'; -export const QUICKWIT_GREY = '#CBD1DE'; -export const QUICKWIT_INTERMEDIATE_GREY = 'rgba(203,209,222,0.5)'; -export const QUICKWIT_LIGHT_GREY = '#F8F9FB'; -export const QUICKWIT_BLACK = '#1F232A'; - +export const QUICKWIT_BLUE = "#004BD9"; +export const QUICKWIT_RED = "#FF0026"; +export const QUICKWIT_GREEN = "#00D588"; +export const QUICKWIT_GREY = "#CBD1DE"; +export const QUICKWIT_INTERMEDIATE_GREY = "rgba(203,209,222,0.5)"; +export const QUICKWIT_LIGHT_GREY = "#F8F9FB"; +export const QUICKWIT_BLACK = "#1F232A"; // Update the Typography's var@iant prop options -declare module '@mui/material/Typography' { +declare module "@mui/material/Typography" { interface TypographyPropsVariantOverrides { fontSize: true; poster: true; @@ -36,22 +35,22 @@ declare module '@mui/material/Typography' { } } -declare module '@mui/material/styles' { +declare module "@mui/material/styles" { interface Theme { status: { - danger: React.CSSProperties['color']; + danger: React.CSSProperties["color"]; }; } interface PaletteOptions { - neutral: PaletteOptions['primary']; + neutral: PaletteOptions["primary"]; } interface Palette { - primary: Palette['primary']; - secondary: Palette['secondary']; - text: Palette['text']; - neutral: Palette['primary']; + primary: Palette["primary"]; + secondary: Palette["secondary"]; + text: Palette["text"]; + neutral: Palette["primary"]; } } @@ -59,24 +58,24 @@ export const theme = createTheme({ palette: { primary: { main: "#000000", - contrastText: '#ffffff' + contrastText: "#ffffff", }, secondary: { - main: '#000000', + main: "#000000", }, text: { - primary: '#000000', + primary: "#000000", }, neutral: { - main: '#F8F9FB', - contrastText: '#000000', + main: "#F8F9FB", + contrastText: "#000000", }, }, typography: { fontSize: 12, - fontFamily: 'SoehneMono, Arial', + fontFamily: "SoehneMono, Arial", body1: { - fontSize: '0.8rem', + fontSize: "0.8rem", }, }, components: { @@ -115,19 +114,18 @@ export const theme = createTheme({ }, }); - export const EDITOR_THEME = { - base: 'vs', + base: "vs" as const, inherit: true, rules: [ - { token: 'comment', foreground: '#1F232A', fontStyle: 'italic' }, - { token: 'keyword', foreground: QUICKWIT_BLUE } + { token: "comment", foreground: "#1F232A", fontStyle: "italic" }, + { token: "keyword", foreground: QUICKWIT_BLUE }, ], colors: { - 'editor.comment.foreground': '#CBD1DE', - 'editor.foreground': '#000000', - 'editor.background': QUICKWIT_LIGHT_GREY, - 'editorLineNumber.foreground': 'black', - 'editor.lineHighlightBackground': '#DFE0E1', + "editor.comment.foreground": "#CBD1DE", + "editor.foreground": "#000000", + "editor.background": QUICKWIT_LIGHT_GREY, + "editorLineNumber.foreground": "black", + "editor.lineHighlightBackground": "#DFE0E1", }, }; diff --git a/quickwit/quickwit-ui/src/utils/urls.ts b/quickwit/quickwit-ui/src/utils/urls.ts index cd85b9c77af..1c3d7c9ab14 100644 --- a/quickwit/quickwit-ui/src/utils/urls.ts +++ b/quickwit/quickwit-ui/src/utils/urls.ts @@ -12,28 +12,32 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { SearchRequest, SortByField, SortOrder, Aggregation } from "./models"; +import { Aggregation, SearchRequest, SortByField, SortOrder } from "./models"; export function hasSearchParams(historySearch: string): boolean { const searchParams = new URLSearchParams(historySearch); - return searchParams.has('index_id') || searchParams.has('query') - || searchParams.has('start_timestamp') || searchParams.has('end_timestamp'); + return ( + searchParams.has("index_id") || + searchParams.has("query") || + searchParams.has("start_timestamp") || + searchParams.has("end_timestamp") + ); } export function parseSearchUrl(historySearch: string): SearchRequest { const searchParams = new URLSearchParams(historySearch); const startTimestampString = searchParams.get("start_timestamp"); let startTimestamp = null; - const startTimeStampParsedInt = parseInt(startTimestampString || ""); - if (!isNaN(startTimeStampParsedInt)) { - startTimestamp = startTimeStampParsedInt + const startTimeStampParsedInt = parseInt(startTimestampString || "", 10); + if (!Number.isNaN(startTimeStampParsedInt)) { + startTimestamp = startTimeStampParsedInt; } let endTimestamp = null; const endTimestampString = searchParams.get("end_timestamp"); - const endTimestampParsedInt = parseInt(endTimestampString || ""); - if (!isNaN(endTimestampParsedInt)) { - endTimestamp = endTimestampParsedInt + const endTimestampParsedInt = parseInt(endTimestampString || "", 10); + if (!Number.isNaN(endTimestampParsedInt)) { + endTimestamp = endTimestampParsedInt; } let indexId = null; const indexIdParam = searchParams.get("index_id"); @@ -43,15 +47,15 @@ export function parseSearchUrl(historySearch: string): SearchRequest { let sortByField = null; const sortByFieldParam = searchParams.get("sort_by_field"); if (sortByFieldParam !== null) { - if (sortByFieldParam.startsWith('+')) { - const order: SortOrder = 'Desc'; - sortByField = {field_name: sortByFieldParam.substring(1), order: order}; - } else if (sortByFieldParam.startsWith('-')) { - const order: SortOrder = 'Asc'; - sortByField = {field_name: sortByFieldParam.substring(1), order: order}; + if (sortByFieldParam.startsWith("+")) { + const order: SortOrder = "Desc"; + sortByField = { field_name: sortByFieldParam.substring(1), order: order }; + } else if (sortByFieldParam.startsWith("-")) { + const order: SortOrder = "Asc"; + sortByField = { field_name: sortByFieldParam.substring(1), order: order }; } else { - const order: SortOrder = 'Desc'; - sortByField = {field_name: sortByFieldParam, order: order}; + const order: SortOrder = "Desc"; + sortByField = { field_name: sortByFieldParam, order: order }; } } const aggregationParam = searchParams.get("aggregation"); @@ -68,12 +72,12 @@ export function parseSearchUrl(historySearch: string): SearchRequest { }; } -function parseAggregation(param: string|null): Aggregation { +function parseAggregation(param: string | null): Aggregation { const empty: Aggregation = { metric: null, term: null, histogram: null, - } + }; if (param !== null) { try { const aggregation: Aggregation = JSON.parse(param); @@ -85,9 +89,11 @@ function parseAggregation(param: string|null): Aggregation { return empty; } -export function toUrlSearchRequestParams(request: SearchRequest): URLSearchParams { +export function toUrlSearchRequestParams( + request: SearchRequest, +): URLSearchParams { const params = new URLSearchParams(); - params.append("query", request.query || '*'); + params.append("query", request.query || "*"); // We have to set the index ID in url params as it's not present in the UI path params. // This enables the react app to be able to get index ID from url params // if the user enter directly the UI url. @@ -96,10 +102,7 @@ export function toUrlSearchRequestParams(request: SearchRequest): URLSearchParam params.append("max_hits", request.maxHits.toString()); } if (request.startTimestamp) { - params.append( - "start_timestamp", - request.startTimestamp.toString() - ); + params.append("start_timestamp", request.startTimestamp.toString()); } if (request.endTimestamp) { params.append("end_timestamp", request.endTimestamp.toString()); @@ -108,18 +111,21 @@ export function toUrlSearchRequestParams(request: SearchRequest): URLSearchParam params.append("sort_by_field", serializeSortByField(request.sortByField)); } if (request.aggregation) { - params.append("aggregation", JSON.stringify(request.aggregationConfig, (_, val) => { - if (val == null) { - return undefined; - } else { - return val; - } - })) + params.append( + "aggregation", + JSON.stringify(request.aggregationConfig, (_, val) => { + if (val == null) { + return undefined; + } else { + return val; + } + }), + ); } return params; } export function serializeSortByField(sortByField: SortByField): string { - const order = sortByField.order === 'Desc' ? '+' : '-'; + const order = sortByField.order === "Desc" ? "+" : "-"; return `${order}${sortByField.field_name}`; } diff --git a/quickwit/quickwit-ui/src/views/ApiView.tsx b/quickwit/quickwit-ui/src/views/ApiView.tsx index c87c445a8b6..49cdfe1a15c 100644 --- a/quickwit/quickwit-ui/src/views/ApiView.tsx +++ b/quickwit/quickwit-ui/src/views/ApiView.tsx @@ -12,19 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. -import "swagger-ui-react/swagger-ui.css" -import SwaggerUI from 'swagger-ui-react' -import { ViewUnderAppBarBox, FullBoxContainer } from '../components/LayoutUtils'; +import "swagger-ui-react/swagger-ui.css"; +import SwaggerUI from "swagger-ui-react"; +import { + FullBoxContainer, + ViewUnderAppBarBox, +} from "../components/LayoutUtils"; function ApiView() { - return + return ( + + layout="BaseLayout" + defaultModelsExpandDepth={-1} + url="/openapi.json" + /> + ); } export default ApiView; diff --git a/quickwit/quickwit-ui/src/views/App.tsx b/quickwit/quickwit-ui/src/views/App.tsx index bb073770664..56d26ea5fc8 100644 --- a/quickwit/quickwit-ui/src/views/App.tsx +++ b/quickwit/quickwit-ui/src/views/App.tsx @@ -12,25 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. -import TopBar from '../components/TopBar'; -import { CssBaseline, ThemeProvider } from '@mui/material'; -import SideBar from '../components/SideBar'; -import { Navigate, Route, Routes } from 'react-router-dom'; -import SearchView from './SearchView'; -import IndexesView from './IndexesView'; -import { theme } from '../utils/theme'; -import IndexView from './IndexView'; -import { FullBoxContainer } from '../components/LayoutUtils'; -import { LocalStorageProvider } from '../providers/LocalStorageProvider'; -import ClusterView from './ClusterView'; -import NodeInfoView from './NodeInfoView'; -import ApiView from './ApiView'; +import { CssBaseline, ThemeProvider } from "@mui/material"; +import { Navigate, Route, Routes } from "react-router"; +import { FullBoxContainer } from "../components/LayoutUtils"; +import SideBar from "../components/SideBar"; +import TopBar from "../components/TopBar"; +import { LocalStorageProvider } from "../providers/LocalStorageProvider"; +import { theme } from "../utils/theme"; +import ApiView from "./ApiView"; +import ClusterView from "./ClusterView"; +import IndexesView from "./IndexesView"; +import IndexView from "./IndexView"; +import NodeInfoView from "./NodeInfoView"; +import SearchView from "./SearchView"; function App() { return ( - + diff --git a/quickwit/quickwit-ui/src/views/ClusterView.test.jsx b/quickwit/quickwit-ui/src/views/ClusterView.test.jsx index 332af9d75ad..1fe3b9fb0ad 100644 --- a/quickwit/quickwit-ui/src/views/ClusterView.test.jsx +++ b/quickwit/quickwit-ui/src/views/ClusterView.test.jsx @@ -12,19 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { render, unmountComponentAtNode } from "react-dom"; -import { waitFor } from "@testing-library/react"; -import { screen } from '@testing-library/dom'; -import ClusterView from './ClusterView'; -import { act } from "react-dom/test-utils"; +import { render, screen, waitFor } from "@testing-library/react"; +import { act } from "react"; import { Client } from "../services/client"; +import ClusterView from "./ClusterView"; -jest.mock('../services/client'); -const mockedUsedNavigate = jest.fn(); -jest.mock('react-router-dom', () => ({ - ...jest.requireActual('react-router-dom'), - useNavigate: () => mockedUsedNavigate, -})); +jest.mock("../services/client"); let container = null; beforeEach(() => { @@ -35,43 +28,46 @@ beforeEach(() => { afterEach(() => { // cleanup on exiting - unmountComponentAtNode(container); container.remove(); container = null; }); -test('renders ClusterStateView', async () => { +test("renders ClusterStateView", async () => { const clusterState = { - "state": { - "seed_addrs": [], - "node_states": { - "node-green-uCdq/1656700092": { - "key_values": { - "available_services": { - "value": "searcher", - "version": 3 - }, - "grpc_address": { - "value": "127.0.0.1:7281", - "version": 2 - }, - "heartbeat": { - "value": "24", - "version": 27 - } + state: { + seed_addrs: [], + node_states: { + "node-green-uCdq/1656700092": { + key_values: { + available_services: { + value: "searcher", + version: 3, }, - "max_version": 27 - } - } + grpc_address: { + value: "127.0.0.1:7281", + version: 2, + }, + heartbeat: { + value: "24", + version: 27, + }, + }, + max_version: 27, + }, }, - "live_nodes": [], - "dead_nodes": [] + }, + live_nodes: [], + dead_nodes: [], }; - Client.prototype.cluster.mockImplementation(() => Promise.resolve(clusterState)); + Client.prototype.cluster.mockImplementation(() => + Promise.resolve(clusterState), + ); await act(async () => { render(, container); }); - await waitFor(() => expect(screen.getByText(/node-green-uCdq/)).toBeInTheDocument()); + await waitFor(() => + expect(screen.getByText(/node-green-uCdq/)).toBeInTheDocument(), + ); }); diff --git a/quickwit/quickwit-ui/src/views/ClusterView.tsx b/quickwit/quickwit-ui/src/views/ClusterView.tsx index 48ba6245aaa..f3c6146cf29 100644 --- a/quickwit/quickwit-ui/src/views/ClusterView.tsx +++ b/quickwit/quickwit-ui/src/views/ClusterView.tsx @@ -12,21 +12,26 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { Typography } from '@mui/material'; -import { useEffect, useMemo, useState } from 'react'; -import ApiUrlFooter from '../components/ApiUrlFooter'; -import { JsonEditor } from '../components/JsonEditor'; -import { ViewUnderAppBarBox, FullBoxContainer, QBreadcrumbs } from '../components/LayoutUtils'; -import Loader from '../components/Loader'; -import ErrorResponseDisplay from '../components/ResponseErrorDisplay'; -import { Client } from '../services/client'; -import { Cluster, ResponseError } from '../utils/models'; - +import { Typography } from "@mui/material"; +import { useEffect, useMemo, useState } from "react"; +import ApiUrlFooter from "../components/ApiUrlFooter"; +import { JsonEditor } from "../components/JsonEditor"; +import { + FullBoxContainer, + QBreadcrumbs, + ViewUnderAppBarBox, +} from "../components/LayoutUtils"; +import Loader from "../components/Loader"; +import ErrorResponseDisplay from "../components/ResponseErrorDisplay"; +import { Client } from "../services/client"; +import { Cluster, ResponseError } from "../utils/models"; function ClusterView() { const [loading, setLoading] = useState(false); const [cluster, setCluster] = useState(null); - const [responseError, setResponseError] = useState(null); + const [responseError, setResponseError] = useState( + null, + ); const quickwitClient = useMemo(() => new Client(), []); useEffect(() => { @@ -40,7 +45,7 @@ function ClusterView() { (error) => { setLoading(false); setResponseError(error); - } + }, ); }, [quickwitClient]); @@ -51,8 +56,8 @@ function ClusterView() { if (loading || cluster == null) { return ; } - return - } + return ; + }; return ( @@ -60,11 +65,9 @@ function ClusterView() { Cluster - - { renderResult() } - + {renderResult()} - { ApiUrlFooter('api/v1/cluster') } + {ApiUrlFooter("api/v1/cluster")} ); } diff --git a/quickwit/quickwit-ui/src/views/IndexView.test.jsx b/quickwit/quickwit-ui/src/views/IndexView.test.jsx index a69cca95e7f..bec4905b506 100644 --- a/quickwit/quickwit-ui/src/views/IndexView.test.jsx +++ b/quickwit/quickwit-ui/src/views/IndexView.test.jsx @@ -12,36 +12,36 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { unmountComponentAtNode } from "react-dom"; -import { render, waitFor, screen } from "@testing-library/react"; -import { act } from "react-dom/test-utils"; +import { render, screen, waitFor } from "@testing-library/react"; +import { act } from "react"; +import { BrowserRouter } from "react-router"; import { Client } from "../services/client"; import IndexView from "./IndexView"; -import { BrowserRouter } from "react-router-dom"; -jest.mock('../services/client'); -const mockedUsedNavigate = jest.fn(); -jest.mock('react-router-dom', () => ({ - ...jest.requireActual('react-router-dom'), +jest.mock("../services/client"); +jest.mock("react-router", () => ({ + ...jest.requireActual("react-router"), useParams: () => ({ - indexId: 'my-new-fresh-index-id' - }) + indexId: "my-new-fresh-index-id", + }), })); -test('renders IndexView', async () => { +test("renders IndexView", async () => { const index = { metadata: { index_config: { - index_uri: 'my-new-fresh-index-uri', - } + index_uri: "my-new-fresh-index-uri", + }, }, - splits: [] + splits: [], }; Client.prototype.getIndex.mockImplementation(() => Promise.resolve(index)); await act(async () => { - render( , {wrapper: BrowserRouter}); + render(, { wrapper: BrowserRouter }); }); - await waitFor(() => expect(screen.getByText(/my-new-fresh-index-uri/)).toBeInTheDocument()); + await waitFor(() => + expect(screen.getByText(/my-new-fresh-index-uri/)).toBeInTheDocument(), + ); }); diff --git a/quickwit/quickwit-ui/src/views/IndexView.tsx b/quickwit/quickwit-ui/src/views/IndexView.tsx index 20b658da11b..d7d1646c3d4 100644 --- a/quickwit/quickwit-ui/src/views/IndexView.tsx +++ b/quickwit/quickwit-ui/src/views/IndexView.tsx @@ -12,27 +12,31 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { Box, styled, Typography, Tab } from '@mui/material'; -import Link, { LinkProps } from '@mui/material/Link'; -import React, { useCallback, useEffect, useMemo, useState } from 'react'; -import { Client } from '../services/client'; -import Loader from '../components/Loader'; -import { Link as RouterLink, useParams } from 'react-router-dom'; -import { Index } from '../utils/models'; -import { TabContext, TabList, TabPanel } from '@mui/lab'; -import { IndexSummary } from '../components/IndexSummary'; -import { JsonEditor } from '../components/JsonEditor'; -import { ViewUnderAppBarBox, FullBoxContainer, QBreadcrumbs } from '../components/LayoutUtils'; -import ApiUrlFooter from '../components/ApiUrlFooter'; +import { TabContext, TabList, TabPanel } from "@mui/lab"; +import { Box, styled, Tab, Typography } from "@mui/material"; +import Link, { LinkProps } from "@mui/material/Link"; +import React, { useCallback, useEffect, useMemo, useState } from "react"; +import { Link as RouterLink, useParams } from "react-router"; +import ApiUrlFooter from "../components/ApiUrlFooter"; +import { IndexSummary } from "../components/IndexSummary"; +import { JsonEditor } from "../components/JsonEditor"; +import { + FullBoxContainer, + QBreadcrumbs, + ViewUnderAppBarBox, +} from "../components/LayoutUtils"; +import Loader from "../components/Loader"; +import { Client } from "../services/client"; +import { Index } from "../utils/models"; export type ErrorResult = { error: string; -} +}; const CustomTabPanel = styled(TabPanel)` -padding-left: 0; -padding-right: 0; -height: 100%; + padding-left: 0; + padding-right: 0; + height: 100%; `; // NOTE : https://mui.com/material-ui/react-breadcrumbs/#integration-with-react-router @@ -47,10 +51,10 @@ function LinkRouter(props: LinkRouterProps) { function IndexView() { const { indexId } = useParams(); - const [loading, setLoading] = useState(false) - const [, setLoadingError] = useState(null) - const [tabIndex, setTabIndex] = useState('1'); - const [index, setIndex] = useState() + const [loading, setLoading] = useState(false); + const [, setLoadingError] = useState(null); + const [tabIndex, setTabIndex] = useState("1"); + const [index, setIndex] = useState(); const quickwitClient = useMemo(() => new Client(), []); const handleTabIndexChange = (_: React.SyntheticEvent, newValue: string) => { @@ -71,8 +75,8 @@ function IndexView() { }, (error) => { setLoading(false); - setLoadingError({error: error}); - } + setLoadingError({ error: error }); + }, ); } }, [indexId, quickwitClient]); @@ -82,44 +86,67 @@ function IndexView() { return ; } else { // TODO: remove this css with magic number `48px`. - return - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + return ( + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ); } - } + }; useEffect(() => { fetchIndex(); @@ -134,9 +161,9 @@ function IndexView() { {indexId} - { renderFetchIndexResult() } + {renderFetchIndexResult()} - { ApiUrlFooter('api/v1/indexes/' + indexId) } + {ApiUrlFooter("api/v1/indexes/" + indexId)} ); } diff --git a/quickwit/quickwit-ui/src/views/IndexesView.test.jsx b/quickwit/quickwit-ui/src/views/IndexesView.test.jsx index b0b586de0cc..d64abebedb0 100644 --- a/quickwit/quickwit-ui/src/views/IndexesView.test.jsx +++ b/quickwit/quickwit-ui/src/views/IndexesView.test.jsx @@ -12,16 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { render, unmountComponentAtNode } from "react-dom"; -import {screen} from '@testing-library/dom' -import IndexesView from './IndexesView'; -import { act } from "react-dom/test-utils"; -import {Client} from "../services/client"; +import { render, screen } from "@testing-library/react"; +import { act } from "react"; +import { Client } from "../services/client"; +import IndexesView from "./IndexesView"; -jest.mock('../services/client'); +jest.mock("../services/client"); const mockedUsedNavigate = jest.fn(); -jest.mock('react-router-dom', () => ({ - ...jest.requireActual('react-router-dom'), +jest.mock("react-router", () => ({ + ...jest.requireActual("react-router"), useNavigate: () => mockedUsedNavigate, })); @@ -34,36 +33,39 @@ beforeEach(() => { afterEach(() => { // cleanup on exiting - unmountComponentAtNode(container); container.remove(); container = null; }); -test('renders IndexesView', async () => { - const indexes = [{ - index_config: { - index_id: 'my-new-fresh-index', - index_uri: 'my-uri', - indexing_settings: { - timestamp_field: 'timestamp' - }, - search_settings: {}, - doc_mapping: { - store: false, - field_mappings: [], - tag_fields: [], - dynamic_mapping: false, +test("renders IndexesView", async () => { + const indexes = [ + { + index_config: { + index_id: "my-new-fresh-index", + index_uri: "my-uri", + indexing_settings: { + timestamp_field: "timestamp", + }, + search_settings: {}, + doc_mapping: { + store: false, + field_mappings: [], + tag_fields: [], + dynamic_mapping: false, + }, }, + sources: [], + create_timestamp: 1000, + update_timestamp: 1000, }, - sources: [], - create_timestamp: 1000, - update_timestamp: 1000, - }]; + ]; Client.prototype.listIndexes.mockResolvedValueOnce(() => indexes); await act(async () => { render(, container); }); - expect(screen.getByText(indexes[0].index_config.index_id)).toBeInTheDocument(); + expect( + screen.getByText(indexes[0].index_config.index_id), + ).toBeInTheDocument(); }); diff --git a/quickwit/quickwit-ui/src/views/IndexesView.tsx b/quickwit/quickwit-ui/src/views/IndexesView.tsx index ab637040576..507f5b6291d 100644 --- a/quickwit/quickwit-ui/src/views/IndexesView.tsx +++ b/quickwit/quickwit-ui/src/views/IndexesView.tsx @@ -12,19 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { Box, Typography } from '@mui/material'; -import { useEffect, useMemo, useState } from 'react'; -import IndexesTable from '../components/IndexesTable'; -import { Client } from '../services/client'; -import Loader from '../components/Loader'; -import { IndexMetadata, ResponseError } from '../utils/models'; -import { ViewUnderAppBarBox, FullBoxContainer, QBreadcrumbs } from '../components/LayoutUtils'; -import ApiUrlFooter from '../components/ApiUrlFooter'; -import ErrorResponseDisplay from '../components/ResponseErrorDisplay'; +import { Box, Typography } from "@mui/material"; +import { useEffect, useMemo, useState } from "react"; +import ApiUrlFooter from "../components/ApiUrlFooter"; +import IndexesTable from "../components/IndexesTable"; +import { + FullBoxContainer, + QBreadcrumbs, + ViewUnderAppBarBox, +} from "../components/LayoutUtils"; +import Loader from "../components/Loader"; +import ErrorResponseDisplay from "../components/ResponseErrorDisplay"; +import { Client } from "../services/client"; +import { IndexMetadata, ResponseError } from "../utils/models"; function IndexesView() { const [loading, setLoading] = useState(false); - const [responseError, setResponseError] = useState(null); + const [responseError, setResponseError] = useState( + null, + ); const [indexesMetadata, setIndexesMetadata] = useState(); const quickwitClient = useMemo(() => new Client(), []); @@ -36,14 +42,14 @@ function IndexesView() { return ; } if (indexesMetadata.length > 0) { - return + return ( + + ); } - return - You have no index registered in your metastore. - - } + return You have no index registered in your metastore.; + }; useEffect(() => { setLoading(true); @@ -56,7 +62,7 @@ function IndexesView() { (error) => { setLoading(false); setResponseError(error); - } + }, ); }, [quickwitClient]); @@ -66,9 +72,9 @@ function IndexesView() { Indexes - { renderFetchIndexesResult() } + {renderFetchIndexesResult()} - { ApiUrlFooter('api/v1/indexes') } + {ApiUrlFooter("api/v1/indexes")} ); } diff --git a/quickwit/quickwit-ui/src/views/NodeInfoView.test.jsx b/quickwit/quickwit-ui/src/views/NodeInfoView.test.jsx index b32e981fea3..62f28e1b982 100644 --- a/quickwit/quickwit-ui/src/views/NodeInfoView.test.jsx +++ b/quickwit/quickwit-ui/src/views/NodeInfoView.test.jsx @@ -12,21 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { render, unmountComponentAtNode } from "react-dom"; -import { waitFor } from "@testing-library/react"; -import { screen } from '@testing-library/dom'; -import { act } from "react-dom/test-utils"; +import { render, screen, waitFor } from "@testing-library/react"; +import { act } from "react"; import { Client } from "../services/client"; import NodeInfoView from "./NodeInfoView"; -jest.mock('../services/client'); -const mockedUsedNavigate = jest.fn(); -jest.mock('react-router-dom', () => ({ - ...jest.requireActual('react-router-dom'), - useParams: () => ({ - indexId: 'my-new-fresh-index-id' - }) -})); +jest.mock("../services/client"); let container = null; beforeEach(() => { @@ -37,29 +28,32 @@ beforeEach(() => { afterEach(() => { // cleanup on exiting - unmountComponentAtNode(container); container.remove(); container = null; }); -test('renders NodeInfoView', async () => { +test("renders NodeInfoView", async () => { const cluster = { - cluster_id: 'my cluster id', + cluster_id: "my cluster id", }; Client.prototype.cluster.mockImplementation(() => Promise.resolve(cluster)); const config = { - node_id: 'my-node-id', + node_id: "my-node-id", }; Client.prototype.config.mockImplementation(() => Promise.resolve(config)); const buildInfo = { - version: '0.3.2', + version: "0.3.2", }; - Client.prototype.buildInfo.mockImplementation(() => Promise.resolve(buildInfo)); + Client.prototype.buildInfo.mockImplementation(() => + Promise.resolve(buildInfo), + ); await act(async () => { render(, container); }); - await waitFor(() => expect(screen.getByText(/my-node-id/)).toBeInTheDocument()); + await waitFor(() => + expect(screen.getByText(/my-node-id/)).toBeInTheDocument(), + ); }); diff --git a/quickwit/quickwit-ui/src/views/NodeInfoView.tsx b/quickwit/quickwit-ui/src/views/NodeInfoView.tsx index 76d8ba50b06..8affcf9b0c5 100644 --- a/quickwit/quickwit-ui/src/views/NodeInfoView.tsx +++ b/quickwit/quickwit-ui/src/views/NodeInfoView.tsx @@ -12,35 +12,40 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { TabContext, TabList, TabPanel } from '@mui/lab'; -import { Box, Tab, Typography, styled } from '@mui/material'; -import { useEffect, useMemo, useState } from 'react'; -import ApiUrlFooter from '../components/ApiUrlFooter'; -import { JsonEditor } from '../components/JsonEditor'; -import { ViewUnderAppBarBox, FullBoxContainer, QBreadcrumbs } from '../components/LayoutUtils'; -import Loader from '../components/Loader'; -import { Client } from '../services/client'; -import { QuickwitBuildInfo } from '../utils/models'; +import { TabContext, TabList, TabPanel } from "@mui/lab"; +import { Box, styled, Tab, Typography } from "@mui/material"; +import { useEffect, useMemo, useState } from "react"; +import ApiUrlFooter from "../components/ApiUrlFooter"; +import { JsonEditor } from "../components/JsonEditor"; +import { + FullBoxContainer, + QBreadcrumbs, + ViewUnderAppBarBox, +} from "../components/LayoutUtils"; +import Loader from "../components/Loader"; +import { Client } from "../services/client"; +import { QuickwitBuildInfo } from "../utils/models"; const CustomTabPanel = styled(TabPanel)` -padding-left: 0; -padding-right: 0; -height: 100%; + padding-left: 0; + padding-right: 0; + height: 100%; `; function NodeInfoView() { const [loadingCounter, setLoadingCounter] = useState(2); const [nodeId, setNodeId] = useState(""); - // eslint-disable-next-line - const [nodeConfig, setNodeConfig] = useState>(null); + const [nodeConfig, setNodeConfig] = useState>( + null, + ); const [buildInfo, setBuildInfo] = useState(null); - const [tabIndex, setTabIndex] = useState('1'); + const [tabIndex, setTabIndex] = useState("1"); const quickwitClient = useMemo(() => new Client(), []); const urlByTab: Record = { - '1': 'api/v1/config', - '2': 'api/v1/version', - } + "1": "api/v1/config", + "2": "api/v1/version", + }; const handleTabIndexChange = (_: React.SyntheticEvent, newValue: string) => { setTabIndex(newValue); @@ -52,31 +57,31 @@ function NodeInfoView() { setNodeId(cluster.node_id); }, (error) => { - console.log('Error when fetching cluster info:', error); - } - ) + console.log("Error when fetching cluster info:", error); + }, + ); }); useEffect(() => { setLoadingCounter(2); quickwitClient.buildInfo().then( (fetchedBuildInfo) => { - setLoadingCounter(prevCounter => prevCounter - 1); + setLoadingCounter((prevCounter) => prevCounter - 1); setBuildInfo(fetchedBuildInfo); }, (error) => { - setLoadingCounter(prevCounter => prevCounter - 1); - console.log('Error when fetching build info: ', error); - } + setLoadingCounter((prevCounter) => prevCounter - 1); + console.log("Error when fetching build info: ", error); + }, ); quickwitClient.config().then( (fetchedConfig) => { - setLoadingCounter(prevCounter => prevCounter - 1); + setLoadingCounter((prevCounter) => prevCounter - 1); setNodeConfig(fetchedConfig); }, (error) => { - setLoadingCounter(prevCounter => prevCounter - 1); - console.log('Error when fetching node config: ', error); - } + setLoadingCounter((prevCounter) => prevCounter - 1); + console.log("Error when fetching node config: ", error); + }, ); }, [quickwitClient]); @@ -84,24 +89,26 @@ function NodeInfoView() { if (loadingCounter !== 0) { return ; } else { - return - - - - - - - - - - - - - - - + return ( + + + + + + + + + + + + + + + + + ); } - } + }; return ( @@ -109,9 +116,9 @@ function NodeInfoView() { Node ID: {nodeId} (self) - { renderResult() } + {renderResult()} - { ApiUrlFooter(urlByTab[tabIndex] || '') } + {ApiUrlFooter(urlByTab[tabIndex] || "")} ); } diff --git a/quickwit/quickwit-ui/src/views/SearchView.test.jsx b/quickwit/quickwit-ui/src/views/SearchView.test.jsx index 2d5ddf9d4d0..fe94d4a438b 100644 --- a/quickwit/quickwit-ui/src/views/SearchView.test.jsx +++ b/quickwit/quickwit-ui/src/views/SearchView.test.jsx @@ -12,22 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { render, unmountComponentAtNode } from "react-dom"; -import { waitFor } from "@testing-library/react"; -import { screen } from '@testing-library/dom'; -import { act } from "react-dom/test-utils"; +import { render, screen, waitFor } from "@testing-library/react"; +import { act } from "react"; import { Client } from "../services/client"; import SearchView from "./SearchView"; -jest.mock('../services/client'); +jest.mock("../services/client"); const mockedUsedNavigate = jest.fn(); -jest.mock('react-router-dom', () => ({ - ...jest.requireActual('react-router-dom'), +jest.mock("react-router", () => ({ + ...jest.requireActual("react-router"), useLocation: () => ({ - pathname: '/search', - search: 'index_id=my-new-fresh-index-idmax_hits=10&start_timestamp=1460554590&end_timestamp=1460554592&sort_by_field=-timestamp' + pathname: "/search", + search: + "index_id=my-new-fresh-index-idmax_hits=10&start_timestamp=1460554590&end_timestamp=1460554592&sort_by_field=-timestamp", }), - useNavigate: () => mockedUsedNavigate + useNavigate: () => mockedUsedNavigate, })); let container = null; @@ -39,42 +38,52 @@ beforeEach(() => { afterEach(() => { // cleanup on exiting - unmountComponentAtNode(container); container.remove(); container = null; }); -test('renders SearchView', async () => { +test("renders SearchView", async () => { const index = { metadata: { index_config: { - index_id: 'my-new-fresh-index-id', - index_uri: 'my-new-fresh-index-uri', + index_id: "my-new-fresh-index-id", + index_uri: "my-new-fresh-index-uri", indexing_settings: {}, doc_mapping: { - field_mappings: [{ - name: 'timestamp', - type: 'i64' - }] - } - } + field_mappings: [ + { + name: "timestamp", + type: "i64", + }, + ], + }, + }, }, - splits: [] + splits: [], }; Client.prototype.getIndex.mockImplementation(() => Promise.resolve(index)); - Client.prototype.listIndexes.mockImplementation(() => Promise.resolve([index.metadata])); + Client.prototype.listIndexes.mockImplementation(() => + Promise.resolve([index.metadata]), + ); const searchResponse = { num_hits: 2, - hits: [{body: 'INFO This is an info log'}, {body: 'WARN This is a warn log'}], + hits: [ + { body: "INFO This is an info log" }, + { body: "WARN This is a warn log" }, + ], elapsed_time_micros: 10, - errors: [] - } - Client.prototype.search.mockImplementation(() => Promise.resolve(searchResponse)); + errors: [], + }; + Client.prototype.search.mockImplementation(() => + Promise.resolve(searchResponse), + ); await act(async () => { render(, container); }); - await waitFor(() => expect(screen.getByText(/This is an info log/)).toBeInTheDocument()); + await waitFor(() => + expect(screen.getByText(/This is an info log/)).toBeInTheDocument(), + ); }); diff --git a/quickwit/quickwit-ui/src/views/SearchView.tsx b/quickwit/quickwit-ui/src/views/SearchView.tsx index 070bd4db1f4..c30f253820f 100644 --- a/quickwit/quickwit-ui/src/views/SearchView.tsx +++ b/quickwit/quickwit-ui/src/views/SearchView.tsx @@ -12,26 +12,43 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { useEffect, useMemo, useRef, useState } from 'react'; -import { useLocation, useNavigate } from 'react-router-dom'; -import ApiUrlFooter from '../components/ApiUrlFooter'; -import { IndexSideBar } from '../components/IndexSideBar'; -import { ViewUnderAppBarBox, FullBoxContainer } from '../components/LayoutUtils'; -import { QueryEditorActionBar } from '../components/QueryActionBar'; -import { QueryEditor } from '../components/QueryEditor/QueryEditor'; -import { AggregationEditor } from '../components/QueryEditor/AggregationEditor'; -import SearchResult from '../components/SearchResult/SearchResult'; -import { useLocalStorage } from '../providers/LocalStorageProvider'; -import { Client } from '../services/client'; -import { EMPTY_SEARCH_REQUEST, Index, IndexMetadata, ResponseError, SearchRequest, SearchResponse } from '../utils/models'; -import { hasSearchParams, parseSearchUrl, toUrlSearchRequestParams } from '../utils/urls'; +import { useEffect, useMemo, useRef, useState } from "react"; +import { useLocation, useNavigate } from "react-router"; +import ApiUrlFooter from "../components/ApiUrlFooter"; +import { IndexSideBar } from "../components/IndexSideBar"; +import { + FullBoxContainer, + ViewUnderAppBarBox, +} from "../components/LayoutUtils"; +import { QueryEditorActionBar } from "../components/QueryActionBar"; +import { AggregationEditor } from "../components/QueryEditor/AggregationEditor"; +import { QueryEditor } from "../components/QueryEditor/QueryEditor"; +import SearchResult from "../components/SearchResult/SearchResult"; +import { useLocalStorage } from "../providers/LocalStorageProvider"; +import { Client } from "../services/client"; +import { + EMPTY_SEARCH_REQUEST, + Index, + IndexMetadata, + ResponseError, + SearchRequest, + SearchResponse, +} from "../utils/models"; +import { + hasSearchParams, + parseSearchUrl, + toUrlSearchRequestParams, +} from "../utils/urls"; -function updateSearchRequestWithIndex(index: Index | null, searchRequest: SearchRequest) { +function updateSearchRequestWithIndex( + index: Index | null, + searchRequest: SearchRequest, +) { // If we have a timestamp field, order by desc on the timestamp field. if (index?.metadata.index_config.doc_mapping.timestamp_field) { searchRequest.sortByField = { field_name: index?.metadata.index_config.doc_mapping.timestamp_field, - order: 'Desc' + order: "Desc", }; } else { searchRequest.sortByField = null; @@ -47,11 +64,17 @@ function SearchView() { const location = useLocation(); const navigate = useNavigate(); const [index, setIndex] = useState(null); - const prevIndexIdRef = useRef(); - const [searchResponse, setSearchResponse] = useState(null); + const prevIndexIdRef = useRef(null); + const [searchResponse, setSearchResponse] = useState( + null, + ); const [searchError, setSearchError] = useState(null); const [queryRunning, setQueryRunning] = useState(false); - const [searchRequest, setSearchRequest] = useState(hasSearchParams(location.search) ? parseSearchUrl(location.search) : EMPTY_SEARCH_REQUEST); + const [searchRequest, setSearchRequest] = useState( + hasSearchParams(location.search) + ? parseSearchUrl(location.search) + : EMPTY_SEARCH_REQUEST, + ); const updateLastSearchRequest = useLocalStorage().updateLastSearchRequest; const quickwitClient = useMemo(() => new Client(), []); @@ -60,32 +83,42 @@ function SearchView() { return; } - console.log('Run search...', updatedSearchRequest); + console.log("Run search...", updatedSearchRequest); updateSearchRequestWithIndex(index, updatedSearchRequest); setSearchRequest(updatedSearchRequest); setQueryRunning(true); setSearchError(null); - navigate('/search?' + toUrlSearchRequestParams(updatedSearchRequest).toString()); - const timestamp_field = index?.metadata.index_config.doc_mapping.timestamp_field || null; - quickwitClient.search(updatedSearchRequest, timestamp_field).then((response) => { - updateLastSearchRequest(updatedSearchRequest); - setSearchResponse(response); - setQueryRunning(false); - }, (error) => { - setQueryRunning(false); - setSearchError(error); - console.error('Error when running search request', error); - }); - } + navigate( + "/search?" + toUrlSearchRequestParams(updatedSearchRequest).toString(), + ); + const timestamp_field = + index?.metadata.index_config.doc_mapping.timestamp_field || null; + quickwitClient.search(updatedSearchRequest, timestamp_field).then( + (response) => { + updateLastSearchRequest(updatedSearchRequest); + setSearchResponse(response); + setQueryRunning(false); + }, + (error) => { + setQueryRunning(false); + setSearchError(error); + console.error("Error when running search request", error); + }, + ); + }; const onIndexMetadataUpdate = (indexMetadata: IndexMetadata | null) => { - setSearchRequest(previousRequest => { + setSearchRequest((previousRequest) => { updateSearchRequestWithIndex(index, previousRequest); - return {...previousRequest, indexId: indexMetadata === null ? null : indexMetadata.index_config.index_id}; + return { + ...previousRequest, + indexId: + indexMetadata === null ? null : indexMetadata.index_config.index_id, + }; }); - } + }; const onSearchRequestUpdate = (searchRequest: SearchRequest) => { setSearchRequest(searchRequest); - } + }; useEffect(() => { if (prevIndexIdRef.current !== index?.metadata.index_config.index_id) { setSearchResponse(null); @@ -94,19 +127,26 @@ function SearchView() { if (prevIndexIdRef.current === null) { runSearch(searchRequest); } - prevIndexIdRef.current = index === null ? null : index.metadata.index_config.index_id; + prevIndexIdRef.current = + index === null ? null : index.metadata.index_config.index_id; }, [index]); useEffect(() => { if (!searchRequest.indexId) { return; } - if (index !== null && index.metadata.index_config.index_id === searchRequest.indexId) { + if ( + index !== null && + index.metadata.index_config.index_id === searchRequest.indexId + ) { return; } // If index id is changing, it's better to reset timestamps as the time unit may be different // between indexes. - if (prevIndexIdRef.current !== null && prevIndexIdRef.current !== index?.metadata.index_config.index_id) { + if ( + prevIndexIdRef.current !== null && + prevIndexIdRef.current !== index?.metadata.index_config.index_id + ) { searchRequest.startTimestamp = null; searchRequest.endTimestamp = null; } @@ -118,39 +158,49 @@ function SearchView() { const searchParams = toUrlSearchRequestParams(searchRequest); // `toUrlSearchRequestParams` is used for the UI urls. We need to remove the `indexId` request parameter to generate // the correct API url, this is the only difference. - searchParams.delete('index_id'); + searchParams.delete("index_id"); return ( - - - - - - - - - - { index !== null && ApiUrlFooter(`api/v1/${index?.metadata.index_config.index_id}/search?${searchParams.toString()}`) } + + + + + + + + - + {index !== null && + ApiUrlFooter( + `api/v1/${index?.metadata.index_config.index_id}/search?${searchParams.toString()}`, + )} + + ); } diff --git a/quickwit/quickwit-ui/tsconfig.json b/quickwit/quickwit-ui/tsconfig.json index 294507dc6f1..d5f78d10de0 100644 --- a/quickwit/quickwit-ui/tsconfig.json +++ b/quickwit/quickwit-ui/tsconfig.json @@ -1,11 +1,7 @@ { "compilerOptions": { "target": "ESNext", - "lib": [ - "dom", - "dom.iterable", - "esnext" - ], + "lib": ["dom", "dom.iterable", "esnext"], "allowJs": true, "skipLibCheck": true, "esModuleInterop": true, @@ -23,10 +19,7 @@ "isolatedModules": true, "noEmit": true, "jsx": "react-jsx", + "types": ["vite/client"] }, - "include": [ - "src", "typings", - "tests", - "tests/integration/index.test.ts" - ] + "exclude": ["build"] } diff --git a/quickwit/quickwit-ui/typings/fonts.d.ts b/quickwit/quickwit-ui/typings/fonts.d.ts deleted file mode 100644 index 956c01c4725..00000000000 --- a/quickwit/quickwit-ui/typings/fonts.d.ts +++ /dev/null @@ -1,3 +0,0 @@ -declare module '*.woff'; -declare module '*.woff2'; -declare module '*.svg'; \ No newline at end of file diff --git a/quickwit/quickwit-ui/vite.config.ts b/quickwit/quickwit-ui/vite.config.ts new file mode 100644 index 00000000000..4548ed5f408 --- /dev/null +++ b/quickwit/quickwit-ui/vite.config.ts @@ -0,0 +1,26 @@ +import { UserConfig } from "vite"; + +export default { + base: "/ui", + server: { + proxy: { + "/api": "http://127.0.0.1:7280", + "/openapi.json": "http://127.0.0.1:7280", + }, + port: 3000, + }, + build: { + rollupOptions: { + onwarn(warning, warn) { + // Suppress "use client" directive warnings from material-ui + if ( + warning.code === "MODULE_LEVEL_DIRECTIVE" && + warning.message.includes('"use client"') + ) { + return; + } + warn(warning); + }, + }, + }, +} satisfies UserConfig; diff --git a/quickwit/quickwit-ui/yarn.lock b/quickwit/quickwit-ui/yarn.lock index 76795059630..677ea3267c1 100644 --- a/quickwit/quickwit-ui/yarn.lock +++ b/quickwit/quickwit-ui/yarn.lock @@ -2,10 +2,10 @@ # yarn lockfile v1 -"@adobe/css-tools@^4.3.1": - version "4.3.2" - resolved "https://registry.yarnpkg.com/@adobe/css-tools/-/css-tools-4.3.2.tgz#a6abc715fb6884851fca9dad37fc34739a04fd11" - integrity sha512-DA5a1C0gD/pLOvhv33YMrbf2FK3oUzwNl9oOJqE4XVjuEtt6XIakRcsd7eLiOSPkp1kTRQGICTA8cKra/vFbjw== +"@adobe/css-tools@^4.4.0": + version "4.4.4" + resolved "https://registry.yarnpkg.com/@adobe/css-tools/-/css-tools-4.4.4.tgz#2856c55443d3d461693f32d2b96fb6ea92e1ffa9" + integrity sha512-Elp+iwUx5rN5+Y8xLt5/GRoG20WGoDCQ/1Fb+1LiGtvwbDavuSk0jhD/eZdckHAuzcDzccnkv+rEjyWfRx18gg== "@ampproject/remapping@^2.1.0": version "2.2.0" @@ -15,30 +15,33 @@ "@jridgewell/gen-mapping" "^0.1.0" "@jridgewell/trace-mapping" "^0.3.9" -"@ampproject/remapping@^2.2.0": - version "2.2.1" - resolved "https://registry.yarnpkg.com/@ampproject/remapping/-/remapping-2.2.1.tgz#99e8e11851128b8702cd57c33684f1d0f260b630" - integrity sha512-lFMjJTrFL3j7L9yBxwYfCq2k6qqwHyzuUl/XBnif78PWTJYyL/dfowQHWE3sp6U6ZzqWiiIZnpTMO96zhkjwtg== - dependencies: - "@jridgewell/gen-mapping" "^0.3.0" - "@jridgewell/trace-mapping" "^0.3.9" - -"@apideck/better-ajv-errors@^0.3.1": - version "0.3.6" - resolved "https://registry.yarnpkg.com/@apideck/better-ajv-errors/-/better-ajv-errors-0.3.6.tgz#957d4c28e886a64a8141f7522783be65733ff097" - integrity sha512-P+ZygBLZtkp0qqOAJJVX4oX/sFo5JR3eBWwwuqHHhK0GIgQOKWrAfiAaWX0aArHkRWHMuggFEgAZNxVPwPZYaA== +"@asamuzakjp/css-color@^3.2.0": + version "3.2.0" + resolved "https://registry.yarnpkg.com/@asamuzakjp/css-color/-/css-color-3.2.0.tgz#cc42f5b85c593f79f1fa4f25d2b9b321e61d1794" + integrity sha512-K1A6z8tS3XsmCMM86xoWdn7Fkdn9m6RSVtocUrJYIwZnFVkng/PvkEoWtOWmP+Scc6saYWHWZYbndEEXxl24jw== dependencies: - json-schema "^0.4.0" - jsonpointer "^5.0.0" - leven "^3.1.0" + "@csstools/css-calc" "^2.1.3" + "@csstools/css-color-parser" "^3.0.9" + "@csstools/css-parser-algorithms" "^3.0.4" + "@csstools/css-tokenizer" "^3.0.3" + lru-cache "^10.4.3" -"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.4", "@babel/code-frame@^7.12.13", "@babel/code-frame@^7.16.0", "@babel/code-frame@^7.18.6", "@babel/code-frame@^7.8.3": +"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.18.6": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.18.6.tgz#3b25d38c89600baa2dcc219edfa88a74eb2c427a" integrity sha512-TDCmlK5eOvH+eH7cdAFlNXeVJqWIQ7gW9tY1GJIpUtFb6CmjVyq2VM3u71bOyR8CRihcCgMUYoDNyLXao3+70Q== dependencies: "@babel/highlight" "^7.18.6" +"@babel/code-frame@^7.10.4", "@babel/code-frame@^7.27.1": + version "7.27.1" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.27.1.tgz#200f715e66d52a23b221a9435534a91cc13ad5be" + integrity sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg== + dependencies: + "@babel/helper-validator-identifier" "^7.27.1" + js-tokens "^4.0.0" + picocolors "^1.1.1" + "@babel/code-frame@^7.22.13": version "7.22.13" resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.22.13.tgz#e3c1c099402598483b7a8c46a721d1038803755e" @@ -47,17 +50,52 @@ "@babel/highlight" "^7.22.13" chalk "^2.4.2" +"@babel/code-frame@^7.28.6", "@babel/code-frame@^7.29.0": + version "7.29.0" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.29.0.tgz#7cd7a59f15b3cc0dcd803038f7792712a7d0b15c" + integrity sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw== + dependencies: + "@babel/helper-validator-identifier" "^7.28.5" + js-tokens "^4.0.0" + picocolors "^1.1.1" + "@babel/compat-data@^7.17.7", "@babel/compat-data@^7.20.1", "@babel/compat-data@^7.20.5": version "7.20.10" resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.20.10.tgz#9d92fa81b87542fff50e848ed585b4212c1d34ec" integrity sha512-sEnuDPpOJR/fcafHMjpcpGN5M2jbUGUHwmuWKM/YdPzeEDJg8bgmbcWQFUfE32MQjti1koACvoPVsDe8Uq+idg== -"@babel/compat-data@^7.22.9": - version "7.23.2" - resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.23.2.tgz#6a12ced93455827037bfb5ed8492820d60fc32cc" - integrity sha512-0S9TQMmDHlqAZ2ITT95irXKfxN9bncq8ZCoJhun3nHL/lLUxd2NKBJYoNGWH7S0hz6fRQwWlAWn/ILM0C70KZQ== +"@babel/compat-data@^7.27.2": + version "7.28.5" + resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.28.5.tgz#a8a4962e1567121ac0b3b487f52107443b455c7f" + integrity sha512-6uFXyCayocRbqhZOB+6XcuZbkMNimwfVGFji8CTZnCzOHVGvDqzvitu1re2AU5LROliz7eQPhB8CpAMvnx9EjA== + +"@babel/compat-data@^7.28.6": + version "7.29.0" + resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.29.0.tgz#00d03e8c0ac24dd9be942c5370990cbe1f17d88d" + integrity sha512-T1NCJqT/j9+cn8fvkt7jtwbLBfLC/1y1c7NtCeXFRgzGTsafi68MRv8yzkYSapBnFA6L3U2VSc02ciDzoAJhJg== + +"@babel/core@7.29.0": + version "7.29.0" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.29.0.tgz#5286ad785df7f79d656e88ce86e650d16ca5f322" + integrity sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA== + dependencies: + "@babel/code-frame" "^7.29.0" + "@babel/generator" "^7.29.0" + "@babel/helper-compilation-targets" "^7.28.6" + "@babel/helper-module-transforms" "^7.28.6" + "@babel/helpers" "^7.28.6" + "@babel/parser" "^7.29.0" + "@babel/template" "^7.28.6" + "@babel/traverse" "^7.29.0" + "@babel/types" "^7.29.0" + "@jridgewell/remapping" "^2.3.5" + convert-source-map "^2.0.0" + debug "^4.1.0" + gensync "^1.0.0-beta.2" + json5 "^2.2.3" + semver "^6.3.1" -"@babel/core@^7.1.0", "@babel/core@^7.11.1", "@babel/core@^7.12.3", "@babel/core@^7.16.0", "@babel/core@^7.7.2", "@babel/core@^7.8.0": +"@babel/core@^7.12.3", "@babel/core@^7.16.0": version "7.20.7" resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.20.7.tgz#37072f951bd4d28315445f66e0ec9f6ae0c8c35f" integrity sha512-t1ZjCluspe5DW24bn2Rr1CDb2v9rn/hROtg9a2tmd0+QYf4bsloYfLQzjG4qHPNMhWtKdGC33R5AxGR2Af2cBw== @@ -78,37 +116,28 @@ json5 "^2.2.1" semver "^6.3.0" -"@babel/core@^7.11.6": - version "7.23.2" - resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.23.2.tgz#ed10df0d580fff67c5f3ee70fd22e2e4c90a9f94" - integrity sha512-n7s51eWdaWZ3vGT2tD4T7J6eJs3QoBXydv7vkUM06Bf1cbVD2Kc2UrkzhiQwobfV7NwOnQXYL7UBJ5VPU+RGoQ== - dependencies: - "@ampproject/remapping" "^2.2.0" - "@babel/code-frame" "^7.22.13" - "@babel/generator" "^7.23.0" - "@babel/helper-compilation-targets" "^7.22.15" - "@babel/helper-module-transforms" "^7.23.0" - "@babel/helpers" "^7.23.2" - "@babel/parser" "^7.23.0" - "@babel/template" "^7.22.15" - "@babel/traverse" "^7.23.2" - "@babel/types" "^7.23.0" +"@babel/core@^7.23.9", "@babel/core@^7.27.4": + version "7.28.5" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.28.5.tgz#4c81b35e51e1b734f510c99b07dfbc7bbbb48f7e" + integrity sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw== + dependencies: + "@babel/code-frame" "^7.27.1" + "@babel/generator" "^7.28.5" + "@babel/helper-compilation-targets" "^7.27.2" + "@babel/helper-module-transforms" "^7.28.3" + "@babel/helpers" "^7.28.4" + "@babel/parser" "^7.28.5" + "@babel/template" "^7.27.2" + "@babel/traverse" "^7.28.5" + "@babel/types" "^7.28.5" + "@jridgewell/remapping" "^2.3.5" convert-source-map "^2.0.0" debug "^4.1.0" gensync "^1.0.0-beta.2" json5 "^2.2.3" semver "^6.3.1" -"@babel/eslint-parser@^7.16.3": - version "7.19.1" - resolved "https://registry.yarnpkg.com/@babel/eslint-parser/-/eslint-parser-7.19.1.tgz#4f68f6b0825489e00a24b41b6a1ae35414ecd2f4" - integrity sha512-AqNf2QWt1rtu2/1rLswy6CDP7H9Oh3mMhk177Y67Rg8d7RD9WfOLLv8CGn6tisFvS2htm86yIe1yLF6I1UDaGQ== - dependencies: - "@nicolo-ribaudo/eslint-scope-5-internals" "5.1.1-v1" - eslint-visitor-keys "^2.1.0" - semver "^6.3.0" - -"@babel/generator@^7.20.7", "@babel/generator@^7.7.2": +"@babel/generator@^7.20.7": version "7.20.7" resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.20.7.tgz#f8ef57c8242665c5929fe2e8d82ba75460187b4a" integrity sha512-7wqMOJq8doJMZmP4ApXTzLxSr7+oO2jroJURrVEp6XShrQUObV8Tq/D0NCcoYg2uHqUrjzO0zwBjoYzelxK+sw== @@ -127,13 +156,42 @@ "@jridgewell/trace-mapping" "^0.3.17" jsesc "^2.5.1" -"@babel/helper-annotate-as-pure@^7.16.0", "@babel/helper-annotate-as-pure@^7.18.6": +"@babel/generator@^7.27.5", "@babel/generator@^7.28.5": + version "7.28.5" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.28.5.tgz#712722d5e50f44d07bc7ac9fe84438742dd61298" + integrity sha512-3EwLFhZ38J4VyIP6WNtt2kUdW9dokXA9Cr4IVIFHuCpZ3H8/YFOl5JjZHisrn1fATPBmKKqXzDFvh9fUwHz6CQ== + dependencies: + "@babel/parser" "^7.28.5" + "@babel/types" "^7.28.5" + "@jridgewell/gen-mapping" "^0.3.12" + "@jridgewell/trace-mapping" "^0.3.28" + jsesc "^3.0.2" + +"@babel/generator@^7.29.0": + version "7.29.1" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.29.1.tgz#d09876290111abbb00ef962a7b83a5307fba0d50" + integrity sha512-qsaF+9Qcm2Qv8SRIMMscAvG4O3lJ0F1GuMo5HR/Bp02LopNgnZBC/EkbevHFeGs4ls/oPz9v+Bsmzbkbe+0dUw== + dependencies: + "@babel/parser" "^7.29.0" + "@babel/types" "^7.29.0" + "@jridgewell/gen-mapping" "^0.3.12" + "@jridgewell/trace-mapping" "^0.3.28" + jsesc "^3.0.2" + +"@babel/helper-annotate-as-pure@^7.18.6": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.18.6.tgz#eaa49f6f80d5a33f9a5dd2276e6d6e451be0a6bb" integrity sha512-duORpUiYrEpzKIop6iNbjnwKLAKnJ47csTyRACyEmWj0QdUrm5aqNJGHSSEQSUAvNW0ojX0dOmK9dZduvkfeXA== dependencies: "@babel/types" "^7.18.6" +"@babel/helper-annotate-as-pure@^7.27.3": + version "7.27.3" + resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.27.3.tgz#f31fd86b915fc4daf1f3ac6976c59be7084ed9c5" + integrity sha512-fXSwMQqitTGeHLBC08Eq5yXz2m37E4pJX1qAU1+2cNedz/ifv/bVXft90VeSav5nFO61EcNgwr0aJxbyPaWBPg== + dependencies: + "@babel/types" "^7.27.3" + "@babel/helper-builder-binary-assignment-operator-visitor@^7.18.6": version "7.18.9" resolved "https://registry.yarnpkg.com/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.18.9.tgz#acd4edfd7a566d1d51ea975dff38fd52906981bb" @@ -153,14 +211,25 @@ lru-cache "^5.1.1" semver "^6.3.0" -"@babel/helper-compilation-targets@^7.22.15": - version "7.22.15" - resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.22.15.tgz#0698fc44551a26cf29f18d4662d5bf545a6cfc52" - integrity sha512-y6EEzULok0Qvz8yyLkCvVX+02ic+By2UdOhylwUOvOn9dvYc9mKICJuuU1n1XBI02YWsNsnrY1kc6DVbjcXbtw== +"@babel/helper-compilation-targets@^7.27.2": + version "7.27.2" + resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.27.2.tgz#46a0f6efab808d51d29ce96858dd10ce8732733d" + integrity sha512-2+1thGUUWWjLTYTHZWK1n8Yga0ijBz1XAhUXcKy81rd5g6yh7hGqMp45v7cadSbEHc9G3OTv45SyneRN3ps4DQ== + dependencies: + "@babel/compat-data" "^7.27.2" + "@babel/helper-validator-option" "^7.27.1" + browserslist "^4.24.0" + lru-cache "^5.1.1" + semver "^6.3.1" + +"@babel/helper-compilation-targets@^7.28.6": + version "7.28.6" + resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.28.6.tgz#32c4a3f41f12ed1532179b108a4d746e105c2b25" + integrity sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA== dependencies: - "@babel/compat-data" "^7.22.9" - "@babel/helper-validator-option" "^7.22.15" - browserslist "^4.21.9" + "@babel/compat-data" "^7.28.6" + "@babel/helper-validator-option" "^7.27.1" + browserslist "^4.24.0" lru-cache "^5.1.1" semver "^6.3.1" @@ -177,6 +246,19 @@ "@babel/helper-replace-supers" "^7.20.7" "@babel/helper-split-export-declaration" "^7.18.6" +"@babel/helper-create-class-features-plugin@^7.21.0": + version "7.28.5" + resolved "https://registry.yarnpkg.com/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.28.5.tgz#472d0c28028850968979ad89f173594a6995da46" + integrity sha512-q3WC4JfdODypvxArsJQROfupPBq9+lMwjKq7C33GhbFYJsufD0yd/ziwD+hJucLeWsnFPWZjsU2DNFqBPE7jwQ== + dependencies: + "@babel/helper-annotate-as-pure" "^7.27.3" + "@babel/helper-member-expression-to-functions" "^7.28.5" + "@babel/helper-optimise-call-expression" "^7.27.1" + "@babel/helper-replace-supers" "^7.27.1" + "@babel/helper-skip-transparent-expression-wrappers" "^7.27.1" + "@babel/traverse" "^7.28.5" + semver "^6.3.1" + "@babel/helper-create-regexp-features-plugin@^7.18.6", "@babel/helper-create-regexp-features-plugin@^7.20.5": version "7.20.5" resolved "https://registry.yarnpkg.com/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.20.5.tgz#5ea79b59962a09ec2acf20a963a01ab4d076ccca" @@ -230,6 +312,11 @@ "@babel/template" "^7.22.15" "@babel/types" "^7.23.0" +"@babel/helper-globals@^7.28.0": + version "7.28.0" + resolved "https://registry.yarnpkg.com/@babel/helper-globals/-/helper-globals-7.28.0.tgz#b9430df2aa4e17bc28665eadeae8aa1d985e6674" + integrity sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw== + "@babel/helper-hoist-variables@^7.18.6": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.18.6.tgz#d4d2c8fb4baeaa5c68b99cc8245c56554f926678" @@ -251,19 +338,36 @@ dependencies: "@babel/types" "^7.20.7" -"@babel/helper-module-imports@^7.0.0", "@babel/helper-module-imports@^7.10.4", "@babel/helper-module-imports@^7.16.0", "@babel/helper-module-imports@^7.16.7", "@babel/helper-module-imports@^7.18.6": +"@babel/helper-member-expression-to-functions@^7.27.1", "@babel/helper-member-expression-to-functions@^7.28.5": + version "7.28.5" + resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.28.5.tgz#f3e07a10be37ed7a63461c63e6929575945a6150" + integrity sha512-cwM7SBRZcPCLgl8a7cY0soT1SptSzAlMH39vwiRpOQkJlh53r5hdHwLSCZpQdVLT39sZt+CRpNwYG4Y2v77atg== + dependencies: + "@babel/traverse" "^7.28.5" + "@babel/types" "^7.28.5" + +"@babel/helper-module-imports@^7.16.7", "@babel/helper-module-imports@^7.18.6": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.18.6.tgz#1e3ebdbbd08aad1437b428c50204db13c5a3ca6e" integrity sha512-0NFvs3VkuSYbFi1x2Vd6tKrywq+z/cLeYC/RJNFrIX/30Bf5aiGYbtvGXolEktzJH8o5E5KJ3tT+nkxuuZFVlA== dependencies: "@babel/types" "^7.18.6" -"@babel/helper-module-imports@^7.22.15": - version "7.22.15" - resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.22.15.tgz#16146307acdc40cc00c3b2c647713076464bdbf0" - integrity sha512-0pYVBnDKZO2fnSPCrgM/6WMc7eS20Fbok+0r88fp+YtWVLZrp4CkafFGIp+W0VKw4a22sgebPT99y+FDNMdP4w== +"@babel/helper-module-imports@^7.27.1": + version "7.27.1" + resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.27.1.tgz#7ef769a323e2655e126673bb6d2d6913bbead204" + integrity sha512-0gSFWUPNXNopqtIPQvlD5WgXYI5GY2kP2cCvoT8kczjbfcfuIljTbcWrulD1CIPIX2gt1wghbDy08yE1p+/r3w== dependencies: - "@babel/types" "^7.22.15" + "@babel/traverse" "^7.27.1" + "@babel/types" "^7.27.1" + +"@babel/helper-module-imports@^7.28.6": + version "7.28.6" + resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.28.6.tgz#60632cbd6ffb70b22823187201116762a03e2d5c" + integrity sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw== + dependencies: + "@babel/traverse" "^7.28.6" + "@babel/types" "^7.28.6" "@babel/helper-module-transforms@^7.18.6", "@babel/helper-module-transforms@^7.20.11", "@babel/helper-module-transforms@^7.20.7": version "7.20.11" @@ -279,16 +383,23 @@ "@babel/traverse" "^7.20.10" "@babel/types" "^7.20.7" -"@babel/helper-module-transforms@^7.23.0": - version "7.23.0" - resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.23.0.tgz#3ec246457f6c842c0aee62a01f60739906f7047e" - integrity sha512-WhDWw1tdrlT0gMgUJSlX0IQvoO1eN279zrAUbVB+KpV2c3Tylz8+GnKOLllCS6Z/iZQEyVYxhZVUdPTqs2YYPw== +"@babel/helper-module-transforms@^7.28.3": + version "7.28.3" + resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.28.3.tgz#a2b37d3da3b2344fe085dab234426f2b9a2fa5f6" + integrity sha512-gytXUbs8k2sXS9PnQptz5o0QnpLL51SwASIORY6XaBKF88nsOT0Zw9szLqlSGQDP/4TljBAD5y98p2U1fqkdsw== dependencies: - "@babel/helper-environment-visitor" "^7.22.20" - "@babel/helper-module-imports" "^7.22.15" - "@babel/helper-simple-access" "^7.22.5" - "@babel/helper-split-export-declaration" "^7.22.6" - "@babel/helper-validator-identifier" "^7.22.20" + "@babel/helper-module-imports" "^7.27.1" + "@babel/helper-validator-identifier" "^7.27.1" + "@babel/traverse" "^7.28.3" + +"@babel/helper-module-transforms@^7.28.6": + version "7.28.6" + resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.28.6.tgz#9312d9d9e56edc35aeb6e95c25d4106b50b9eb1e" + integrity sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA== + dependencies: + "@babel/helper-module-imports" "^7.28.6" + "@babel/helper-validator-identifier" "^7.28.5" + "@babel/traverse" "^7.28.6" "@babel/helper-optimise-call-expression@^7.18.6": version "7.18.6" @@ -297,15 +408,22 @@ dependencies: "@babel/types" "^7.18.6" +"@babel/helper-optimise-call-expression@^7.27.1": + version "7.27.1" + resolved "https://registry.yarnpkg.com/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.27.1.tgz#c65221b61a643f3e62705e5dd2b5f115e35f9200" + integrity sha512-URMGH08NzYFhubNSGJrpUEphGKQwMQYBySzat5cAByY1/YgIRkULnIy3tAMeszlL/so2HbeilYloUmSpd7GdVw== + dependencies: + "@babel/types" "^7.27.1" + "@babel/helper-plugin-utils@^7.0.0", "@babel/helper-plugin-utils@^7.10.4", "@babel/helper-plugin-utils@^7.12.13", "@babel/helper-plugin-utils@^7.14.5", "@babel/helper-plugin-utils@^7.16.7", "@babel/helper-plugin-utils@^7.18.6", "@babel/helper-plugin-utils@^7.18.9", "@babel/helper-plugin-utils@^7.19.0", "@babel/helper-plugin-utils@^7.20.2", "@babel/helper-plugin-utils@^7.8.0", "@babel/helper-plugin-utils@^7.8.3": version "7.20.2" resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.20.2.tgz#d1b9000752b18d0877cff85a5c376ce5c3121629" integrity sha512-8RvlJG2mj4huQ4pZ+rU9lqKi9ZKiRmuvGuM2HlWmkmgOhbs6zEAw6IEiJ5cQqGbDzGZOhwuOQNtZMi/ENLjZoQ== -"@babel/helper-plugin-utils@^7.22.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.22.5.tgz#dd7ee3735e8a313b9f7b05a773d892e88e6d7295" - integrity sha512-uLls06UVKgFG9QD4OeFYLEGteMIAa5kpTPcFL28yuCIIzsf6ZyKZMllKVOCZFhiZ5ptnwX4mtKdWCBE/uT4amg== +"@babel/helper-plugin-utils@^7.27.1": + version "7.27.1" + resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.27.1.tgz#ddb2f876534ff8013e6c2b299bf4d39b3c51d44c" + integrity sha512-1gn1Up5YXka3YYAHGKpbideQ5Yjf1tDa9qYcgysz+cNCXukyLl6DjPXhD3VRwSb8c0J9tA4b2+rHEZtc6R0tlw== "@babel/helper-remap-async-to-generator@^7.18.9": version "7.18.9" @@ -329,6 +447,15 @@ "@babel/traverse" "^7.20.7" "@babel/types" "^7.20.7" +"@babel/helper-replace-supers@^7.27.1": + version "7.27.1" + resolved "https://registry.yarnpkg.com/@babel/helper-replace-supers/-/helper-replace-supers-7.27.1.tgz#b1ed2d634ce3bdb730e4b52de30f8cccfd692bc0" + integrity sha512-7EHz6qDZc8RYS5ElPoShMheWvEgERonFCs7IAonWLLUTXW59DP14bCZt89/GKyreYn8g3S83m21FelHKbeDCKA== + dependencies: + "@babel/helper-member-expression-to-functions" "^7.27.1" + "@babel/helper-optimise-call-expression" "^7.27.1" + "@babel/traverse" "^7.27.1" + "@babel/helper-simple-access@^7.20.2": version "7.20.2" resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.20.2.tgz#0ab452687fe0c2cfb1e2b9e0015de07fc2d62dd9" @@ -336,13 +463,6 @@ dependencies: "@babel/types" "^7.20.2" -"@babel/helper-simple-access@^7.22.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.22.5.tgz#4938357dc7d782b80ed6dbb03a0fba3d22b1d5de" - integrity sha512-n0H99E/K+Bika3++WNL17POvo4rKWZ7lZEp1Q+fStVbUi8nxPQEBOlTmCOxW/0JsS56SKKQ+ojAe2pHKJHN35w== - dependencies: - "@babel/types" "^7.22.5" - "@babel/helper-skip-transparent-expression-wrappers@^7.20.0": version "7.20.0" resolved "https://registry.yarnpkg.com/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.20.0.tgz#fbe4c52f60518cab8140d77101f0e63a8a230684" @@ -350,6 +470,14 @@ dependencies: "@babel/types" "^7.20.0" +"@babel/helper-skip-transparent-expression-wrappers@^7.27.1": + version "7.27.1" + resolved "https://registry.yarnpkg.com/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.27.1.tgz#62bb91b3abba8c7f1fec0252d9dbea11b3ee7a56" + integrity sha512-Tub4ZKEXqbPjXgWLl2+3JpQAYBJ8+ikpQ2Ocj/q/r0LwE3UhENh7EUabyHjz2kCEsrRY83ew2DQdHluuiDQFzg== + dependencies: + "@babel/traverse" "^7.27.1" + "@babel/types" "^7.27.1" + "@babel/helper-split-export-declaration@^7.18.6": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.18.6.tgz#7367949bc75b20c6d5a5d4a97bba2824ae8ef075" @@ -374,6 +502,11 @@ resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz#533f36457a25814cf1df6488523ad547d784a99f" integrity sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw== +"@babel/helper-string-parser@^7.27.1": + version "7.27.1" + resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz#54da796097ab19ce67ed9f88b47bb2ec49367687" + integrity sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA== + "@babel/helper-validator-identifier@^7.18.6", "@babel/helper-validator-identifier@^7.19.1": version "7.19.1" resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.19.1.tgz#7eea834cf32901ffdc1a7ee555e2f9c27e249ca2" @@ -384,15 +517,20 @@ resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz#c4ae002c61d2879e724581d96665583dbc1dc0e0" integrity sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A== +"@babel/helper-validator-identifier@^7.27.1", "@babel/helper-validator-identifier@^7.28.5": + version "7.28.5" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz#010b6938fab7cb7df74aa2bbc06aa503b8fe5fb4" + integrity sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q== + "@babel/helper-validator-option@^7.18.6": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.18.6.tgz#bf0d2b5a509b1f336099e4ff36e1a63aa5db4db8" integrity sha512-XO7gESt5ouv/LRJdrVjkShckw6STTaB7l9BrpBaAHDeF5YZT+01PCwmR0SJHnkW6i8OwW/EVWRShfi4j2x+KQw== -"@babel/helper-validator-option@^7.22.15": - version "7.22.15" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.22.15.tgz#694c30dfa1d09a6534cdfcafbe56789d36aba040" - integrity sha512-bMn7RmyFjY/mdECUbgn9eoSY4vqvacUnS9i9vGAGttgFWesO6B4CYWA7XlpbWgBt71iv/hfbPlynohStqnu5hA== +"@babel/helper-validator-option@^7.27.1": + version "7.27.1" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz#fa52f5b1e7db1ab049445b421c4471303897702f" + integrity sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg== "@babel/helper-wrap-function@^7.18.9": version "7.20.5" @@ -413,14 +551,21 @@ "@babel/traverse" "^7.20.7" "@babel/types" "^7.20.7" -"@babel/helpers@^7.23.2": - version "7.23.2" - resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.23.2.tgz#2832549a6e37d484286e15ba36a5330483cac767" - integrity sha512-lzchcp8SjTSVe/fPmLwtWVBFC7+Tbn8LGHDVfDp9JGxpAY5opSaEFgt8UQvrnECWOTdji2mOWMz1rOhkHscmGQ== +"@babel/helpers@^7.28.4": + version "7.28.4" + resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.28.4.tgz#fe07274742e95bdf7cf1443593eeb8926ab63827" + integrity sha512-HFN59MmQXGHVyYadKLVumYsA9dBFun/ldYxipEjzA4196jpLZd8UjEEBLkbEkvfYreDqJhZxYAWFPtrfhNpj4w== dependencies: - "@babel/template" "^7.22.15" - "@babel/traverse" "^7.23.2" - "@babel/types" "^7.23.0" + "@babel/template" "^7.27.2" + "@babel/types" "^7.28.4" + +"@babel/helpers@^7.28.6": + version "7.28.6" + resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.28.6.tgz#fca903a313ae675617936e8998b814c415cbf5d7" + integrity sha512-xOBvwq86HHdB7WUDTfKfT/Vuxh7gElQ+Sfti2Cy6yIWNW05P8iUslOVcZ4/sKbE+/jQaukQAdz/gf3724kYdqw== + dependencies: + "@babel/template" "^7.28.6" + "@babel/types" "^7.28.6" "@babel/highlight@^7.18.6": version "7.18.6" @@ -450,6 +595,20 @@ resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.0.tgz#da950e622420bf96ca0d0f2909cdddac3acd8719" integrity sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw== +"@babel/parser@^7.23.9", "@babel/parser@^7.27.2", "@babel/parser@^7.28.5": + version "7.28.5" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.28.5.tgz#0b0225ee90362f030efd644e8034c99468893b08" + integrity sha512-KKBU1VGYR7ORr3At5HAtUQ+TV3SzRCXmA/8OdDZiLDBIZxVyzXuztPjfLd3BV1PRAQGCMWWSHYhL0F8d5uHBDQ== + dependencies: + "@babel/types" "^7.28.5" + +"@babel/parser@^7.28.6", "@babel/parser@^7.29.0": + version "7.29.0" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.29.0.tgz#669ef345add7d057e92b7ed15f0bac07611831b6" + integrity sha512-IyDgFV5GeDUVX4YdF/3CPULtVGSXXMLh1xVIgdCgxApktqnQV0r7/8Nqthg+8YLGaAtdyIlo2qIdZrbCv4+7ww== + dependencies: + "@babel/types" "^7.29.0" + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@^7.18.6": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.18.6.tgz#da5b8f9a580acdfbe53494dba45ea389fb09a4d2" @@ -588,6 +747,16 @@ "@babel/helper-create-class-features-plugin" "^7.18.6" "@babel/helper-plugin-utils" "^7.18.6" +"@babel/plugin-proposal-private-property-in-object@^7.16.7": + version "7.21.11" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-private-property-in-object/-/plugin-proposal-private-property-in-object-7.21.11.tgz#69d597086b6760c4126525cfa154f34631ff272c" + integrity sha512-0QZ8qP/3RLDVBwBFoWAwCtgcDZJVwA5LUJRZU8x2YFfKNuFq161wK3cuGrALu5yiPu+vzwTAg/sMWVNeWeNyaw== + dependencies: + "@babel/helper-annotate-as-pure" "^7.18.6" + "@babel/helper-create-class-features-plugin" "^7.21.0" + "@babel/helper-plugin-utils" "^7.20.2" + "@babel/plugin-syntax-private-property-in-object" "^7.14.5" + "@babel/plugin-proposal-private-property-in-object@^7.18.6": version "7.20.5" resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-private-property-in-object/-/plugin-proposal-private-property-in-object-7.20.5.tgz#309c7668f2263f1c711aa399b5a9a6291eef6135" @@ -620,7 +789,7 @@ dependencies: "@babel/helper-plugin-utils" "^7.8.0" -"@babel/plugin-syntax-class-properties@^7.12.13", "@babel/plugin-syntax-class-properties@^7.8.3": +"@babel/plugin-syntax-class-properties@^7.12.13": version "7.12.13" resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz#b5c987274c4a3a82b89714796931a6b53544ae10" integrity sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA== @@ -669,7 +838,14 @@ dependencies: "@babel/helper-plugin-utils" "^7.19.0" -"@babel/plugin-syntax-import-meta@^7.8.3": +"@babel/plugin-syntax-import-attributes@^7.24.7": + version "7.27.1" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-import-attributes/-/plugin-syntax-import-attributes-7.27.1.tgz#34c017d54496f9b11b61474e7ea3dfd5563ffe07" + integrity sha512-oFT0FrKHgF53f4vOsZGi2Hh3I35PfSmVs4IBFLFj4dnafP+hIWDLg3VyKmUHfLoLHlyxY4C7DGtmHuJgn+IGww== + dependencies: + "@babel/helper-plugin-utils" "^7.27.1" + +"@babel/plugin-syntax-import-meta@^7.10.4": version "7.10.4" resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz#ee601348c370fa334d2207be158777496521fd51" integrity sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g== @@ -690,14 +866,14 @@ dependencies: "@babel/helper-plugin-utils" "^7.18.6" -"@babel/plugin-syntax-jsx@^7.7.2": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.22.5.tgz#a6b68e84fb76e759fc3b93e901876ffabbe1d918" - integrity sha512-gvyP4hZrgrs/wWMaocvxZ44Hw0b3W8Pe+cMxc8V1ULQ07oh8VNbIRaoD1LRZVTvD+0nieDKjfgKg89sD7rrKrg== +"@babel/plugin-syntax-jsx@^7.27.1": + version "7.27.1" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.27.1.tgz#2f9beb5eff30fa507c5532d107daac7b888fa34c" + integrity sha512-y8YTNIeKoyhGd9O0Jiyzyyqk8gdjnumGTQPsz0xOZOQ2RmkVJeZ1vmmfIvFEKqucBG6axJGBZDE/7iI5suUI/w== dependencies: - "@babel/helper-plugin-utils" "^7.22.5" + "@babel/helper-plugin-utils" "^7.27.1" -"@babel/plugin-syntax-logical-assignment-operators@^7.10.4", "@babel/plugin-syntax-logical-assignment-operators@^7.8.3": +"@babel/plugin-syntax-logical-assignment-operators@^7.10.4": version "7.10.4" resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz#ca91ef46303530448b906652bac2e9fe9941f699" integrity sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig== @@ -711,7 +887,7 @@ dependencies: "@babel/helper-plugin-utils" "^7.8.0" -"@babel/plugin-syntax-numeric-separator@^7.10.4", "@babel/plugin-syntax-numeric-separator@^7.8.3": +"@babel/plugin-syntax-numeric-separator@^7.10.4": version "7.10.4" resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz#b9b070b3e33570cd9fd07ba7fa91c0dd37b9af97" integrity sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug== @@ -746,20 +922,27 @@ dependencies: "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-syntax-top-level-await@^7.14.5", "@babel/plugin-syntax-top-level-await@^7.8.3": +"@babel/plugin-syntax-top-level-await@^7.14.5": version "7.14.5" resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz#c1cfdadc35a646240001f06138247b741c34d94c" integrity sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw== dependencies: "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-syntax-typescript@^7.20.0", "@babel/plugin-syntax-typescript@^7.7.2": +"@babel/plugin-syntax-typescript@^7.20.0": version "7.20.0" resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.20.0.tgz#4e9a0cfc769c85689b77a2e642d24e9f697fc8c7" integrity sha512-rd9TkG+u1CExzS4SM1BlMEhMXwFLKVjOAFFCDx9PbX5ycJWDoWMcwdJH9RhkPu1dOgn5TrxLot/Gx6lWFuAUNQ== dependencies: "@babel/helper-plugin-utils" "^7.19.0" +"@babel/plugin-syntax-typescript@^7.27.1": + version "7.27.1" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.27.1.tgz#5147d29066a793450f220c63fa3a9431b7e6dd18" + integrity sha512-xfYCBMxveHrRMnAWl1ZlPXOZjzkN82THFvLhQhFXFt81Z5HnN+EtUkZhv/zcKpmT3fzmWZB0ywiBrbC3vogbwQ== + dependencies: + "@babel/helper-plugin-utils" "^7.27.1" + "@babel/plugin-transform-arrow-functions@^7.18.6": version "7.20.7" resolved "https://registry.yarnpkg.com/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.20.7.tgz#bea332b0e8b2dab3dafe55a163d8227531ab0551" @@ -953,13 +1136,6 @@ dependencies: "@babel/helper-plugin-utils" "^7.18.6" -"@babel/plugin-transform-react-constant-elements@^7.12.1": - version "7.20.2" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-constant-elements/-/plugin-transform-react-constant-elements-7.20.2.tgz#3f02c784e0b711970d7d8ccc96c4359d64e27ac7" - integrity sha512-KS/G8YI8uwMGKErLFOHS/ekhqdHhpEloxs43NecQHVgo2QuQSyJhGIY1fL8UGl9wy5ItVwwoUL4YxVqsplGq2g== - dependencies: - "@babel/helper-plugin-utils" "^7.20.2" - "@babel/plugin-transform-react-display-name@^7.16.0", "@babel/plugin-transform-react-display-name@^7.18.6": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-display-name/-/plugin-transform-react-display-name-7.18.6.tgz#8b1125f919ef36ebdfff061d664e266c666b9415" @@ -1080,7 +1256,7 @@ "@babel/helper-create-regexp-features-plugin" "^7.18.6" "@babel/helper-plugin-utils" "^7.18.6" -"@babel/preset-env@^7.11.0", "@babel/preset-env@^7.12.1", "@babel/preset-env@^7.16.4": +"@babel/preset-env@^7.16.4": version "7.20.2" resolved "https://registry.yarnpkg.com/@babel/preset-env/-/preset-env-7.20.2.tgz#9b1642aa47bb9f43a86f9630011780dab7f86506" integrity sha512-1G0efQEWR1EHkKvKHqbG+IN/QdgwfByUpM5V5QroDzGV2t3S/WXNQd693cHiHTlCFMpr9B6FkPFXDA2lQcKoDg== @@ -1172,7 +1348,7 @@ "@babel/types" "^7.4.4" esutils "^2.0.2" -"@babel/preset-react@^7.12.5", "@babel/preset-react@^7.16.0": +"@babel/preset-react@^7.16.0": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/preset-react/-/preset-react-7.18.6.tgz#979f76d6277048dc19094c217b507f3ad517dd2d" integrity sha512-zXr6atUmyYdiWRVLOZahakYmOBHtWc2WGCkP8PYTgZi0iJXDY2CN180TdrIW4OGOAdLc7TifzDIvtx6izaRIzg== @@ -1193,52 +1369,19 @@ "@babel/helper-validator-option" "^7.18.6" "@babel/plugin-transform-typescript" "^7.18.6" -"@babel/runtime-corejs3@^7.10.2": - version "7.20.7" - resolved "https://registry.yarnpkg.com/@babel/runtime-corejs3/-/runtime-corejs3-7.20.7.tgz#a1e5ea3d758ba6beb715210142912e3f29981d84" - integrity sha512-jr9lCZ4RbRQmCR28Q8U8Fu49zvFqLxTY9AMOUz+iyMohMoAgpEcVxY+wJNay99oXOpOcCTODkk70NDN2aaJEeg== - dependencies: - core-js-pure "^3.25.1" - regenerator-runtime "^0.13.11" - -"@babel/runtime-corejs3@^7.20.7": - version "7.21.0" - resolved "https://registry.yarnpkg.com/@babel/runtime-corejs3/-/runtime-corejs3-7.21.0.tgz#6e4939d9d9789ff63e2dc58e88f13a3913a24eba" - integrity sha512-TDD4UJzos3JJtM+tHX+w2Uc+KWj7GV+VKKFdMVd2Rx8sdA19hcc3P3AHFYd5LVOw+pYuSd5lICC3gm52B6Rwxw== - dependencies: - core-js-pure "^3.25.1" - regenerator-runtime "^0.13.11" - -"@babel/runtime-corejs3@^7.22.15", "@babel/runtime-corejs3@^7.23.1": - version "7.23.2" - resolved "https://registry.yarnpkg.com/@babel/runtime-corejs3/-/runtime-corejs3-7.23.2.tgz#a5cd9d8b408fb946b2f074b21ea40c04e516795c" - integrity sha512-54cIh74Z1rp4oIjsHjqN+WM4fMyCBYe+LpZ9jWm51CZ1fbH3SkAzQD/3XLoNkjbJ7YEmjobLXyvQrFypRHOrXw== - dependencies: - core-js-pure "^3.30.2" - regenerator-runtime "^0.14.0" - -"@babel/runtime@^7.10.2", "@babel/runtime@^7.11.2", "@babel/runtime@^7.12.5", "@babel/runtime@^7.16.3", "@babel/runtime@^7.18.3", "@babel/runtime@^7.18.9", "@babel/runtime@^7.19.0", "@babel/runtime@^7.20.6", "@babel/runtime@^7.20.7", "@babel/runtime@^7.5.5", "@babel/runtime@^7.8.4", "@babel/runtime@^7.8.7", "@babel/runtime@^7.9.2": - version "7.20.7" - resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.20.7.tgz#fcb41a5a70550e04a7b708037c7c32f7f356d8fd" - integrity sha512-UF0tvkUtxwAgZ5W/KrkHf0Rn0fdnLDU9ScxBrEVNUprE/MzirjK4MJUX1/BVDv00Sv8cljtukVK1aky++X1SjQ== - dependencies: - regenerator-runtime "^0.13.11" - -"@babel/runtime@^7.12.1", "@babel/runtime@^7.3.1": - version "7.21.0" - resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.21.0.tgz#5b55c9d394e5fcf304909a8b00c07dc217b56673" - integrity sha512-xwII0//EObnq89Ji5AKYQaRYiW/nZ3llSv29d49IuxPhKbtJoLP+9QUUZ4nVragQVtaVGeZrpB+ZtG/Pdy/POw== +"@babel/runtime-corejs3@^7.20.7", "@babel/runtime-corejs3@^7.22.15", "@babel/runtime-corejs3@^7.26.10", "@babel/runtime-corejs3@^7.27.1": + version "7.28.6" + resolved "https://registry.yarnpkg.com/@babel/runtime-corejs3/-/runtime-corejs3-7.28.6.tgz#f1e4a0da50f4856ceec1bb51c83f8ed0b202ed12" + integrity sha512-kz2fAQ5UzjV7X7D3ySxmj3vRq89dTpqOZWv76Z6pNPztkwb/0Yj1Mtx1xFrYj6mbIHysxtBot8J4o0JLCblcFw== dependencies: - regenerator-runtime "^0.13.11" + core-js-pure "^3.43.0" -"@babel/runtime@^7.23.9", "@babel/runtime@^7.24.0": - version "7.24.5" - resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.24.5.tgz#230946857c053a36ccc66e1dd03b17dd0c4ed02c" - integrity sha512-Nms86NXrsaeU9vbBJKni6gXiEXZ4CVpYVzEjDH9Sb8vmZ3UljyA1GSOJl/6LGPO8EHLuSF9H+IxNXHPX8QHJ4g== - dependencies: - regenerator-runtime "^0.14.0" +"@babel/runtime@7.28.6", "@babel/runtime@^7.12.5", "@babel/runtime@^7.16.3", "@babel/runtime@^7.18.3", "@babel/runtime@^7.19.0", "@babel/runtime@^7.20.7", "@babel/runtime@^7.28.4", "@babel/runtime@^7.28.6", "@babel/runtime@^7.5.5", "@babel/runtime@^7.8.4", "@babel/runtime@^7.8.7": + version "7.28.6" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.28.6.tgz#d267a43cb1836dc4d182cce93ae75ba954ef6d2b" + integrity sha512-05WQkdpL9COIMz4LjTxGpPNCdlpyimKppYNoJ5Di5EUObifl8t4tuLuUBBZEpoLYOmfvIWrsp9fCl0HoPRVTdA== -"@babel/template@^7.18.10", "@babel/template@^7.20.7", "@babel/template@^7.3.3": +"@babel/template@^7.18.10", "@babel/template@^7.20.7": version "7.20.7" resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.20.7.tgz#a15090c2839a83b02aa996c0b4994005841fd5a8" integrity sha512-8SegXApWe6VoNw0r9JHpSteLKTpTiLZ4rMlGIm9JQ18KiCtyQiAMEazujAHrUS5flrcqYZa75ukev3P6QmUwUw== @@ -1256,7 +1399,25 @@ "@babel/parser" "^7.22.15" "@babel/types" "^7.22.15" -"@babel/traverse@^7.20.10", "@babel/traverse@^7.20.5", "@babel/traverse@^7.20.7", "@babel/traverse@^7.23.2", "@babel/traverse@^7.4.5", "@babel/traverse@^7.7.2": +"@babel/template@^7.27.2": + version "7.27.2" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.27.2.tgz#fa78ceed3c4e7b63ebf6cb39e5852fca45f6809d" + integrity sha512-LPDZ85aEJyYSd18/DkjNh4/y1ntkE5KwUHWTiqgRxruuZL2F1yuHligVHLvcHY2vMHXttKFpJn6LwfI7cw7ODw== + dependencies: + "@babel/code-frame" "^7.27.1" + "@babel/parser" "^7.27.2" + "@babel/types" "^7.27.1" + +"@babel/template@^7.28.6": + version "7.28.6" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.28.6.tgz#0e7e56ecedb78aeef66ce7972b082fce76a23e57" + integrity sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ== + dependencies: + "@babel/code-frame" "^7.28.6" + "@babel/parser" "^7.28.6" + "@babel/types" "^7.28.6" + +"@babel/traverse@^7.20.10", "@babel/traverse@^7.20.5", "@babel/traverse@^7.20.7": version "7.23.2" resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.2.tgz#329c7a06735e144a506bdb2cad0268b7f46f4ad8" integrity sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw== @@ -1272,7 +1433,33 @@ debug "^4.1.0" globals "^11.1.0" -"@babel/types@^7.0.0", "@babel/types@^7.12.6", "@babel/types@^7.18.6", "@babel/types@^7.18.9", "@babel/types@^7.19.0", "@babel/types@^7.20.0", "@babel/types@^7.20.2", "@babel/types@^7.20.5", "@babel/types@^7.20.7", "@babel/types@^7.3.0", "@babel/types@^7.3.3", "@babel/types@^7.4.4": +"@babel/traverse@^7.27.1", "@babel/traverse@^7.28.3", "@babel/traverse@^7.28.5": + version "7.28.5" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.28.5.tgz#450cab9135d21a7a2ca9d2d35aa05c20e68c360b" + integrity sha512-TCCj4t55U90khlYkVV/0TfkJkAkUg3jZFA3Neb7unZT8CPok7iiRfaX0F+WnqWqt7OxhOn0uBKXCw4lbL8W0aQ== + dependencies: + "@babel/code-frame" "^7.27.1" + "@babel/generator" "^7.28.5" + "@babel/helper-globals" "^7.28.0" + "@babel/parser" "^7.28.5" + "@babel/template" "^7.27.2" + "@babel/types" "^7.28.5" + debug "^4.3.1" + +"@babel/traverse@^7.28.6", "@babel/traverse@^7.29.0": + version "7.29.0" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.29.0.tgz#f323d05001440253eead3c9c858adbe00b90310a" + integrity sha512-4HPiQr0X7+waHfyXPZpWPfWL/J7dcN1mx9gL6WdQVMbPnF3+ZhSMs8tCxN7oHddJE9fhNE7+lxdnlyemKfJRuA== + dependencies: + "@babel/code-frame" "^7.29.0" + "@babel/generator" "^7.29.0" + "@babel/helper-globals" "^7.28.0" + "@babel/parser" "^7.29.0" + "@babel/template" "^7.28.6" + "@babel/types" "^7.29.0" + debug "^4.3.1" + +"@babel/types@^7.0.0", "@babel/types@^7.18.6", "@babel/types@^7.18.9", "@babel/types@^7.19.0", "@babel/types@^7.20.0", "@babel/types@^7.20.2", "@babel/types@^7.20.5", "@babel/types@^7.20.7", "@babel/types@^7.3.0", "@babel/types@^7.4.4": version "7.20.7" resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.20.7.tgz#54ec75e252318423fc07fb644dc6a58a64c09b7f" integrity sha512-69OnhBxSSgK0OzTJai4kyPDiKTIe3j+ctaHdIGVbRahTLAT7L3R9oeXHC2aVSuGYt3cVnoAMDmOCgJ2yaiLMvg== @@ -1290,207 +1477,149 @@ "@babel/helper-validator-identifier" "^7.22.20" to-fast-properties "^2.0.0" -"@bcoe/v8-coverage@^0.2.3": - version "0.2.3" - resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" - integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== - -"@braintree/sanitize-url@=6.0.4": - version "6.0.4" - resolved "https://registry.yarnpkg.com/@braintree/sanitize-url/-/sanitize-url-6.0.4.tgz#923ca57e173c6b232bbbb07347b1be982f03e783" - integrity sha512-s3jaWicZd0pkP0jf5ysyHUI/RE7MHos6qlToFcGWXVp+ykHOy77OUMrfbgJ9it2C5bow7OIQwYYaHjk9XlBQ2A== - -"@colors/colors@1.5.0": - version "1.5.0" - resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9" - integrity sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ== - -"@csstools/normalize.css@*": - version "12.0.0" - resolved "https://registry.yarnpkg.com/@csstools/normalize.css/-/normalize.css-12.0.0.tgz#a9583a75c3f150667771f30b60d9f059473e62c4" - integrity sha512-M0qqxAcwCsIVfpFQSlGN5XjXWu8l5JDZN+fPt1LeW5SZexQTgnaEvgXAY+CeygRw0EeppWHi12JxESWiWrB0Sg== - -"@csstools/postcss-cascade-layers@^1.1.1": - version "1.1.1" - resolved "https://registry.yarnpkg.com/@csstools/postcss-cascade-layers/-/postcss-cascade-layers-1.1.1.tgz#8a997edf97d34071dd2e37ea6022447dd9e795ad" - integrity sha512-+KdYrpKC5TgomQr2DlZF4lDEpHcoxnj5IGddYYfBWJAKfj1JtuHUIqMa+E1pJJ+z3kvDViWMqyqPlG4Ja7amQA== - dependencies: - "@csstools/selector-specificity" "^2.0.2" - postcss-selector-parser "^6.0.10" - -"@csstools/postcss-color-function@^1.1.1": - version "1.1.1" - resolved "https://registry.yarnpkg.com/@csstools/postcss-color-function/-/postcss-color-function-1.1.1.tgz#2bd36ab34f82d0497cfacdc9b18d34b5e6f64b6b" - integrity sha512-Bc0f62WmHdtRDjf5f3e2STwRAl89N2CLb+9iAwzrv4L2hncrbDwnQD9PCq0gtAt7pOI2leIV08HIBUd4jxD8cw== - dependencies: - "@csstools/postcss-progressive-custom-properties" "^1.1.0" - postcss-value-parser "^4.2.0" - -"@csstools/postcss-font-format-keywords@^1.0.1": - version "1.0.1" - resolved "https://registry.yarnpkg.com/@csstools/postcss-font-format-keywords/-/postcss-font-format-keywords-1.0.1.tgz#677b34e9e88ae997a67283311657973150e8b16a" - integrity sha512-ZgrlzuUAjXIOc2JueK0X5sZDjCtgimVp/O5CEqTcs5ShWBa6smhWYbS0x5cVc/+rycTDbjjzoP0KTDnUneZGOg== - dependencies: - postcss-value-parser "^4.2.0" - -"@csstools/postcss-hwb-function@^1.0.2": - version "1.0.2" - resolved "https://registry.yarnpkg.com/@csstools/postcss-hwb-function/-/postcss-hwb-function-1.0.2.tgz#ab54a9fce0ac102c754854769962f2422ae8aa8b" - integrity sha512-YHdEru4o3Rsbjmu6vHy4UKOXZD+Rn2zmkAmLRfPet6+Jz4Ojw8cbWxe1n42VaXQhD3CQUXXTooIy8OkVbUcL+w== - dependencies: - postcss-value-parser "^4.2.0" - -"@csstools/postcss-ic-unit@^1.0.1": - version "1.0.1" - resolved "https://registry.yarnpkg.com/@csstools/postcss-ic-unit/-/postcss-ic-unit-1.0.1.tgz#28237d812a124d1a16a5acc5c3832b040b303e58" - integrity sha512-Ot1rcwRAaRHNKC9tAqoqNZhjdYBzKk1POgWfhN4uCOE47ebGcLRqXjKkApVDpjifL6u2/55ekkpnFcp+s/OZUw== - dependencies: - "@csstools/postcss-progressive-custom-properties" "^1.1.0" - postcss-value-parser "^4.2.0" - -"@csstools/postcss-is-pseudo-class@^2.0.7": - version "2.0.7" - resolved "https://registry.yarnpkg.com/@csstools/postcss-is-pseudo-class/-/postcss-is-pseudo-class-2.0.7.tgz#846ae6c0d5a1eaa878fce352c544f9c295509cd1" - integrity sha512-7JPeVVZHd+jxYdULl87lvjgvWldYu+Bc62s9vD/ED6/QTGjy0jy0US/f6BG53sVMTBJ1lzKZFpYmofBN9eaRiA== - dependencies: - "@csstools/selector-specificity" "^2.0.0" - postcss-selector-parser "^6.0.10" - -"@csstools/postcss-nested-calc@^1.0.0": - version "1.0.0" - resolved "https://registry.yarnpkg.com/@csstools/postcss-nested-calc/-/postcss-nested-calc-1.0.0.tgz#d7e9d1d0d3d15cf5ac891b16028af2a1044d0c26" - integrity sha512-JCsQsw1wjYwv1bJmgjKSoZNvf7R6+wuHDAbi5f/7MbFhl2d/+v+TvBTU4BJH3G1X1H87dHl0mh6TfYogbT/dJQ== - dependencies: - postcss-value-parser "^4.2.0" - -"@csstools/postcss-normalize-display-values@^1.0.1": - version "1.0.1" - resolved "https://registry.yarnpkg.com/@csstools/postcss-normalize-display-values/-/postcss-normalize-display-values-1.0.1.tgz#15da54a36e867b3ac5163ee12c1d7f82d4d612c3" - integrity sha512-jcOanIbv55OFKQ3sYeFD/T0Ti7AMXc9nM1hZWu8m/2722gOTxFg7xYu4RDLJLeZmPUVQlGzo4jhzvTUq3x4ZUw== +"@babel/types@^7.27.1", "@babel/types@^7.27.3", "@babel/types@^7.28.4", "@babel/types@^7.28.5": + version "7.28.5" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.28.5.tgz#10fc405f60897c35f07e85493c932c7b5ca0592b" + integrity sha512-qQ5m48eI/MFLQ5PxQj4PFaprjyCTLI37ElWMmNs0K8Lk3dVeOdNpB3ks8jc7yM5CDmVC73eMVk/trk3fgmrUpA== dependencies: - postcss-value-parser "^4.2.0" + "@babel/helper-string-parser" "^7.27.1" + "@babel/helper-validator-identifier" "^7.28.5" -"@csstools/postcss-oklab-function@^1.1.1": - version "1.1.1" - resolved "https://registry.yarnpkg.com/@csstools/postcss-oklab-function/-/postcss-oklab-function-1.1.1.tgz#88cee0fbc8d6df27079ebd2fa016ee261eecf844" - integrity sha512-nJpJgsdA3dA9y5pgyb/UfEzE7W5Ka7u0CX0/HIMVBNWzWemdcTH3XwANECU6anWv/ao4vVNLTMxhiPNZsTK6iA== +"@babel/types@^7.28.6", "@babel/types@^7.29.0": + version "7.29.0" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.29.0.tgz#9f5b1e838c446e72cf3cd4b918152b8c605e37c7" + integrity sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A== dependencies: - "@csstools/postcss-progressive-custom-properties" "^1.1.0" - postcss-value-parser "^4.2.0" + "@babel/helper-string-parser" "^7.27.1" + "@babel/helper-validator-identifier" "^7.28.5" -"@csstools/postcss-progressive-custom-properties@^1.1.0", "@csstools/postcss-progressive-custom-properties@^1.3.0": - version "1.3.0" - resolved "https://registry.yarnpkg.com/@csstools/postcss-progressive-custom-properties/-/postcss-progressive-custom-properties-1.3.0.tgz#542292558384361776b45c85226b9a3a34f276fa" - integrity sha512-ASA9W1aIy5ygskZYuWams4BzafD12ULvSypmaLJT2jvQ8G0M3I8PRQhC0h7mG0Z3LI05+agZjqSR9+K9yaQQjA== - dependencies: - postcss-value-parser "^4.2.0" +"@bcoe/v8-coverage@^0.2.3": + version "0.2.3" + resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" + integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@csstools/postcss-stepped-value-functions@^1.0.1": - version "1.0.1" - resolved "https://registry.yarnpkg.com/@csstools/postcss-stepped-value-functions/-/postcss-stepped-value-functions-1.0.1.tgz#f8772c3681cc2befed695e2b0b1d68e22f08c4f4" - integrity sha512-dz0LNoo3ijpTOQqEJLY8nyaapl6umbmDcgj4AD0lgVQ572b2eqA1iGZYTTWhrcrHztWDDRAX2DGYyw2VBjvCvQ== - dependencies: - postcss-value-parser "^4.2.0" +"@biomejs/biome@2.4.4": + version "2.4.4" + resolved "https://registry.yarnpkg.com/@biomejs/biome/-/biome-2.4.4.tgz#009d1751d061946b320e663e5894af60310f355a" + integrity sha512-tigwWS5KfJf0cABVd52NVaXyAVv4qpUXOWJ1rxFL8xF1RVoeS2q/LK+FHgYoKMclJCuRoCWAPy1IXaN9/mS61Q== + optionalDependencies: + "@biomejs/cli-darwin-arm64" "2.4.4" + "@biomejs/cli-darwin-x64" "2.4.4" + "@biomejs/cli-linux-arm64" "2.4.4" + "@biomejs/cli-linux-arm64-musl" "2.4.4" + "@biomejs/cli-linux-x64" "2.4.4" + "@biomejs/cli-linux-x64-musl" "2.4.4" + "@biomejs/cli-win32-arm64" "2.4.4" + "@biomejs/cli-win32-x64" "2.4.4" + +"@biomejs/cli-darwin-arm64@2.4.4": + version "2.4.4" + resolved "https://registry.yarnpkg.com/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-2.4.4.tgz#a8af5254fcc9dda28a1da26125427df083a13579" + integrity sha512-jZ+Xc6qvD6tTH5jM6eKX44dcbyNqJHssfl2nnwT6vma6B1sj7ZLTGIk6N5QwVBs5xGN52r3trk5fgd3sQ9We9A== + +"@biomejs/cli-darwin-x64@2.4.4": + version "2.4.4" + resolved "https://registry.yarnpkg.com/@biomejs/cli-darwin-x64/-/cli-darwin-x64-2.4.4.tgz#6de17d5bb3afaf3275bc602fb7e61fefe31dab84" + integrity sha512-Dh1a/+W+SUCXhEdL7TiX3ArPTFCQKJTI1mGncZNWfO+6suk+gYA4lNyJcBB+pwvF49uw0pEbUS49BgYOY4hzUg== + +"@biomejs/cli-linux-arm64-musl@2.4.4": + version "2.4.4" + resolved "https://registry.yarnpkg.com/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-2.4.4.tgz#4c55afe87d555782f46e729212acd4e0411b1703" + integrity sha512-+sPAXq3bxmFwhVFJnSwkSF5Rw2ZAJMH3MF6C9IveAEOdSpgajPhoQhbbAK12SehN9j2QrHpk4J/cHsa/HqWaYQ== + +"@biomejs/cli-linux-arm64@2.4.4": + version "2.4.4" + resolved "https://registry.yarnpkg.com/@biomejs/cli-linux-arm64/-/cli-linux-arm64-2.4.4.tgz#2c8848461bd5195d3e887b0c9fc9c1b441c727ec" + integrity sha512-V/NFfbWhsUU6w+m5WYbBenlEAz8eYnSqRMDMAW3K+3v0tYVkNyZn8VU0XPxk/lOqNXLSCCrV7FmV/u3SjCBShg== + +"@biomejs/cli-linux-x64-musl@2.4.4": + version "2.4.4" + resolved "https://registry.yarnpkg.com/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-2.4.4.tgz#f40336d8c1534887c7ef4198b2b4772aca702ee8" + integrity sha512-gGvFTGpOIQDb5CQ2VC0n9Z2UEqlP46c4aNgHmAMytYieTGEcfqhfCFnhs6xjt0S3igE6q5GLuIXtdQt3Izok+g== + +"@biomejs/cli-linux-x64@2.4.4": + version "2.4.4" + resolved "https://registry.yarnpkg.com/@biomejs/cli-linux-x64/-/cli-linux-x64-2.4.4.tgz#397cadb0ec1e76a993d21ee424bcf2cf7976243e" + integrity sha512-R4+ZCDtG9kHArasyBO+UBD6jr/FcFCTH8QkNTOCu0pRJzCWyWC4EtZa2AmUZB5h3e0jD7bRV2KvrENcf8rndBg== + +"@biomejs/cli-win32-arm64@2.4.4": + version "2.4.4" + resolved "https://registry.yarnpkg.com/@biomejs/cli-win32-arm64/-/cli-win32-arm64-2.4.4.tgz#7d767c15ce20ee39bc1430c2057ccd0208cf12fe" + integrity sha512-trzCqM7x+Gn832zZHgr28JoYagQNX4CZkUZhMUac2YxvvyDRLJDrb5m9IA7CaZLlX6lTQmADVfLEKP1et1Ma4Q== + +"@biomejs/cli-win32-x64@2.4.4": + version "2.4.4" + resolved "https://registry.yarnpkg.com/@biomejs/cli-win32-x64/-/cli-win32-x64-2.4.4.tgz#998e87083ff9657a24b4b2f6afcb803a2e820000" + integrity sha512-gnOHKVPFAAPrpoPt2t+Q6FZ7RPry/FDV3GcpU53P3PtLNnQjBmKyN2Vh/JtqXet+H4pme8CC76rScwdjDcT1/A== + +"@csstools/color-helpers@^5.1.0": + version "5.1.0" + resolved "https://registry.yarnpkg.com/@csstools/color-helpers/-/color-helpers-5.1.0.tgz#106c54c808cabfd1ab4c602d8505ee584c2996ef" + integrity sha512-S11EXWJyy0Mz5SYvRmY8nJYTFFd1LCNV+7cXyAgQtOOuzb4EsgfqDufL+9esx72/eLhsRdGZwaldu/h+E4t4BA== -"@csstools/postcss-text-decoration-shorthand@^1.0.0": - version "1.0.0" - resolved "https://registry.yarnpkg.com/@csstools/postcss-text-decoration-shorthand/-/postcss-text-decoration-shorthand-1.0.0.tgz#ea96cfbc87d921eca914d3ad29340d9bcc4c953f" - integrity sha512-c1XwKJ2eMIWrzQenN0XbcfzckOLLJiczqy+YvfGmzoVXd7pT9FfObiSEfzs84bpE/VqfpEuAZ9tCRbZkZxxbdw== - dependencies: - postcss-value-parser "^4.2.0" +"@csstools/css-calc@^2.1.3", "@csstools/css-calc@^2.1.4": + version "2.1.4" + resolved "https://registry.yarnpkg.com/@csstools/css-calc/-/css-calc-2.1.4.tgz#8473f63e2fcd6e459838dd412401d5948f224c65" + integrity sha512-3N8oaj+0juUw/1H3YwmDDJXCgTB1gKU6Hc/bB502u9zR0q2vd786XJH9QfrKIEgFlZmhZiq6epXl4rHqhzsIgQ== -"@csstools/postcss-trigonometric-functions@^1.0.2": - version "1.0.2" - resolved "https://registry.yarnpkg.com/@csstools/postcss-trigonometric-functions/-/postcss-trigonometric-functions-1.0.2.tgz#94d3e4774c36d35dcdc88ce091336cb770d32756" - integrity sha512-woKaLO///4bb+zZC2s80l+7cm07M7268MsyG3M0ActXXEFi6SuhvriQYcb58iiKGbjwwIU7n45iRLEHypB47Og== +"@csstools/css-color-parser@^3.0.9": + version "3.1.0" + resolved "https://registry.yarnpkg.com/@csstools/css-color-parser/-/css-color-parser-3.1.0.tgz#4e386af3a99dd36c46fef013cfe4c1c341eed6f0" + integrity sha512-nbtKwh3a6xNVIp/VRuXV64yTKnb1IjTAEEh3irzS+HkKjAOYLTGNb9pmVNntZ8iVBHcWDA2Dof0QtPgFI1BaTA== dependencies: - postcss-value-parser "^4.2.0" - -"@csstools/postcss-unset-value@^1.0.2": - version "1.0.2" - resolved "https://registry.yarnpkg.com/@csstools/postcss-unset-value/-/postcss-unset-value-1.0.2.tgz#c99bb70e2cdc7312948d1eb41df2412330b81f77" - integrity sha512-c8J4roPBILnelAsdLr4XOAR/GsTm0GJi4XpcfvoWk3U6KiTCqiFYc63KhRMQQX35jYMp4Ao8Ij9+IZRgMfJp1g== - -"@csstools/selector-specificity@^2.0.0", "@csstools/selector-specificity@^2.0.2": - version "2.0.2" - resolved "https://registry.yarnpkg.com/@csstools/selector-specificity/-/selector-specificity-2.0.2.tgz#1bfafe4b7ed0f3e4105837e056e0a89b108ebe36" - integrity sha512-IkpVW/ehM1hWKln4fCA3NzJU8KwD+kIOvPZA4cqxoJHtE21CCzjyp+Kxbu0i5I4tBNOlXPL9mjwnWlL0VEG4Fg== - -"@cypress/request@^3.0.0": - version "3.0.1" - resolved "https://registry.yarnpkg.com/@cypress/request/-/request-3.0.1.tgz#72d7d5425236a2413bd3d8bb66d02d9dc3168960" - integrity sha512-TWivJlJi8ZDx2wGOw1dbLuHJKUYX7bWySw377nlnGOW3hP9/MUKIsEdXT/YngWxVdgNCHRBmFlBipE+5/2ZZlQ== - dependencies: - aws-sign2 "~0.7.0" - aws4 "^1.8.0" - caseless "~0.12.0" - combined-stream "~1.0.6" - extend "~3.0.2" - forever-agent "~0.6.1" - form-data "~2.3.2" - http-signature "~1.3.6" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.19" - performance-now "^2.1.0" - qs "6.10.4" - safe-buffer "^5.1.2" - tough-cookie "^4.1.3" - tunnel-agent "^0.6.0" - uuid "^8.3.2" + "@csstools/color-helpers" "^5.1.0" + "@csstools/css-calc" "^2.1.4" -"@cypress/xvfb@^1.2.4": - version "1.2.4" - resolved "https://registry.yarnpkg.com/@cypress/xvfb/-/xvfb-1.2.4.tgz#2daf42e8275b39f4aa53c14214e557bd14e7748a" - integrity sha512-skbBzPggOVYCbnGgV+0dmBdW/s77ZkAOXIC1knS8NagwDjBrNC1LuXtQJeiN6l+m7lzmHtaoUw/ctJKdqkG57Q== - dependencies: - debug "^3.1.0" - lodash.once "^4.1.1" +"@csstools/css-parser-algorithms@^3.0.4": + version "3.0.5" + resolved "https://registry.yarnpkg.com/@csstools/css-parser-algorithms/-/css-parser-algorithms-3.0.5.tgz#5755370a9a29abaec5515b43c8b3f2cf9c2e3076" + integrity sha512-DaDeUkXZKjdGhgYaHNJTV9pV7Y9B3b644jCLs9Upc3VeNGg6LWARAT6O+Q+/COo+2gg/bM5rhpMAtf70WqfBdQ== -"@date-io/core@^2.15.0", "@date-io/core@^2.16.0": - version "2.16.0" - resolved "https://registry.yarnpkg.com/@date-io/core/-/core-2.16.0.tgz#7871bfc1d9bca9aa35ad444a239505589d0f22f6" - integrity sha512-DYmSzkr+jToahwWrsiRA2/pzMEtz9Bq1euJwoOuYwuwIYXnZFtHajY2E6a1VNVDc9jP8YUXK1BvnZH9mmT19Zg== +"@csstools/css-tokenizer@^3.0.3": + version "3.0.4" + resolved "https://registry.yarnpkg.com/@csstools/css-tokenizer/-/css-tokenizer-3.0.4.tgz#333fedabc3fd1a8e5d0100013731cf19e6a8c5d3" + integrity sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw== -"@date-io/date-fns@^2.15.0": - version "2.16.0" - resolved "https://registry.yarnpkg.com/@date-io/date-fns/-/date-fns-2.16.0.tgz#bd5e09b6ecb47ee55e593fc3a87e7b2caaa3da40" - integrity sha512-bfm5FJjucqlrnQcXDVU5RD+nlGmL3iWgkHTq3uAZWVIuBu6dDmGa3m8a6zo2VQQpu8ambq9H22UyUpn7590joA== +"@dr.pogodin/babel-plugin-transform-assets@1.2.6": + version "1.2.6" + resolved "https://registry.yarnpkg.com/@dr.pogodin/babel-plugin-transform-assets/-/babel-plugin-transform-assets-1.2.6.tgz#fc93014091bab73a4d38912fa144a364f99002dd" + integrity sha512-ZMLKzWr5zCszqOKeEXcDECjGpTozjO+yze+iaH52XlZp83KP5LyE7dVLjjXaJ6LxYe3qWRcCNKl9tr/yxrAdbQ== dependencies: - "@date-io/core" "^2.16.0" + enhanced-resolve "^5.19.0" + loader-utils "^3.3.1" -"@date-io/dayjs@^2.15.0": - version "2.16.0" - resolved "https://registry.yarnpkg.com/@date-io/dayjs/-/dayjs-2.16.0.tgz#0d2c254ad8db1306fdc4b8eda197cb53c9af89dc" - integrity sha512-y5qKyX2j/HG3zMvIxTobYZRGnd1FUW2olZLS0vTj7bEkBQkjd2RO7/FEwDY03Z1geVGlXKnzIATEVBVaGzV4Iw== +"@emnapi/core@^1.4.3": + version "1.7.1" + resolved "https://registry.yarnpkg.com/@emnapi/core/-/core-1.7.1.tgz#3a79a02dbc84f45884a1806ebb98e5746bdfaac4" + integrity sha512-o1uhUASyo921r2XtHYOHy7gdkGLge8ghBEQHMWmyJFoXlpU58kIrhhN3w26lpQb6dspetweapMn2CSNwQ8I4wg== dependencies: - "@date-io/core" "^2.16.0" + "@emnapi/wasi-threads" "1.1.0" + tslib "^2.4.0" -"@date-io/luxon@^2.15.0": - version "2.16.1" - resolved "https://registry.yarnpkg.com/@date-io/luxon/-/luxon-2.16.1.tgz#b08786614cb58831c729a15807753011e4acb966" - integrity sha512-aeYp5K9PSHV28946pC+9UKUi/xMMYoaGelrpDibZSgHu2VWHXrr7zWLEr+pMPThSs5vt8Ei365PO+84pCm37WQ== +"@emnapi/runtime@^1.4.3": + version "1.7.1" + resolved "https://registry.yarnpkg.com/@emnapi/runtime/-/runtime-1.7.1.tgz#a73784e23f5d57287369c808197288b52276b791" + integrity sha512-PVtJr5CmLwYAU9PZDMITZoR5iAOShYREoR45EyyLrbntV50mdePTgUn4AmOw90Ifcj+x2kRjdzr1HP3RrNiHGA== dependencies: - "@date-io/core" "^2.16.0" + tslib "^2.4.0" -"@date-io/moment@^2.15.0": - version "2.16.1" - resolved "https://registry.yarnpkg.com/@date-io/moment/-/moment-2.16.1.tgz#ec6e0daa486871e0e6412036c6f806842a0eeed4" - integrity sha512-JkxldQxUqZBfZtsaCcCMkm/dmytdyq5pS1RxshCQ4fHhsvP5A7gSqPD22QbVXMcJydi3d3v1Y8BQdUKEuGACZQ== +"@emnapi/wasi-threads@1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@emnapi/wasi-threads/-/wasi-threads-1.1.0.tgz#60b2102fddc9ccb78607e4a3cf8403ea69be41bf" + integrity sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ== dependencies: - "@date-io/core" "^2.16.0" + tslib "^2.4.0" -"@emotion/babel-plugin@^11.11.0": - version "11.11.0" - resolved "https://registry.yarnpkg.com/@emotion/babel-plugin/-/babel-plugin-11.11.0.tgz#c2d872b6a7767a9d176d007f5b31f7d504bb5d6c" - integrity sha512-m4HEDZleaaCH+XgDDsPF15Ht6wTLsgDTeR3WYj9Q/k76JtWhrJjcP4+/XlG8LGT/Rol9qUfOIztXeA84ATpqPQ== +"@emotion/babel-plugin@^11.13.5": + version "11.13.5" + resolved "https://registry.yarnpkg.com/@emotion/babel-plugin/-/babel-plugin-11.13.5.tgz#eab8d65dbded74e0ecfd28dc218e75607c4e7bc0" + integrity sha512-pxHCpT2ex+0q+HH91/zsdHkw/lXd468DIN2zvfvLtPKLLMo6gQj7oLObq8PhkrxOZb/gGCq03S3Z7PDhS8pduQ== dependencies: "@babel/helper-module-imports" "^7.16.7" "@babel/runtime" "^7.18.3" - "@emotion/hash" "^0.9.1" - "@emotion/memoize" "^0.8.1" - "@emotion/serialize" "^1.1.2" + "@emotion/hash" "^0.9.2" + "@emotion/memoize" "^0.9.0" + "@emotion/serialize" "^1.3.3" babel-plugin-macros "^3.1.0" convert-source-map "^1.5.0" escape-string-regexp "^4.0.0" @@ -1498,221 +1627,266 @@ source-map "^0.5.7" stylis "4.2.0" -"@emotion/cache@^11.10.5": - version "11.10.5" - resolved "https://registry.yarnpkg.com/@emotion/cache/-/cache-11.10.5.tgz#c142da9351f94e47527ed458f7bbbbe40bb13c12" - integrity sha512-dGYHWyzTdmK+f2+EnIGBpkz1lKc4Zbj2KHd4cX3Wi8/OWr5pKslNjc3yABKH4adRGCvSX4VDC0i04mrrq0aiRA== - dependencies: - "@emotion/memoize" "^0.8.0" - "@emotion/sheet" "^1.2.1" - "@emotion/utils" "^1.2.0" - "@emotion/weak-memoize" "^0.3.0" - stylis "4.1.3" - -"@emotion/cache@^11.11.0": - version "11.11.0" - resolved "https://registry.yarnpkg.com/@emotion/cache/-/cache-11.11.0.tgz#809b33ee6b1cb1a625fef7a45bc568ccd9b8f3ff" - integrity sha512-P34z9ssTCBi3e9EI1ZsWpNHcfY1r09ZO0rZbRO2ob3ZQMnFI35jB536qoXbkdesr5EUhYi22anuEJuyxifaqAQ== +"@emotion/cache@^11.14.0": + version "11.14.0" + resolved "https://registry.yarnpkg.com/@emotion/cache/-/cache-11.14.0.tgz#ee44b26986eeb93c8be82bb92f1f7a9b21b2ed76" + integrity sha512-L/B1lc/TViYk4DcpGxtAVbx0ZyiKM5ktoIyafGkH6zg/tj+mA+NE//aPYKG0k8kCHSHVJrpLpcAlOBEXQ3SavA== dependencies: - "@emotion/memoize" "^0.8.1" - "@emotion/sheet" "^1.2.2" - "@emotion/utils" "^1.2.1" - "@emotion/weak-memoize" "^0.3.1" + "@emotion/memoize" "^0.9.0" + "@emotion/sheet" "^1.4.0" + "@emotion/utils" "^1.4.2" + "@emotion/weak-memoize" "^0.4.0" stylis "4.2.0" -"@emotion/hash@^0.9.1": - version "0.9.1" - resolved "https://registry.yarnpkg.com/@emotion/hash/-/hash-0.9.1.tgz#4ffb0055f7ef676ebc3a5a91fb621393294e2f43" - integrity sha512-gJB6HLm5rYwSLI6PQa+X1t5CFGrv1J1TWG+sOyMCeKz2ojaj6Fnl/rZEspogG+cvqbt4AE/2eIyD2QfLKTBNlQ== - -"@emotion/is-prop-valid@^1.1.0", "@emotion/is-prop-valid@^1.2.0": - version "1.2.0" - resolved "https://registry.yarnpkg.com/@emotion/is-prop-valid/-/is-prop-valid-1.2.0.tgz#7f2d35c97891669f7e276eb71c83376a5dc44c83" - integrity sha512-3aDpDprjM0AwaxGE09bOPkNxHpBd+kA6jty3RnaEXdweX1DF1U3VQpPYb0g1IStAuK7SVQ1cy+bNBBKp4W3Fjg== - dependencies: - "@emotion/memoize" "^0.8.0" +"@emotion/hash@^0.9.2": + version "0.9.2" + resolved "https://registry.yarnpkg.com/@emotion/hash/-/hash-0.9.2.tgz#ff9221b9f58b4dfe61e619a7788734bd63f6898b" + integrity sha512-MyqliTZGuOm3+5ZRSaaBGP3USLw6+EGykkwZns2EPC5g8jJ4z9OrdZY9apkl3+UP9+sdz76YYkwCKP5gh8iY3g== -"@emotion/is-prop-valid@^1.2.1": - version "1.2.1" - resolved "https://registry.yarnpkg.com/@emotion/is-prop-valid/-/is-prop-valid-1.2.1.tgz#23116cf1ed18bfeac910ec6436561ecb1a3885cc" - integrity sha512-61Mf7Ufx4aDxx1xlDeOm8aFFigGHE4z+0sKCa+IHCeZKiyP9RLD0Mmx7m8b9/Cf37f7NAvQOOJAbQQGVr5uERw== +"@emotion/is-prop-valid@1.2.2": + version "1.2.2" + resolved "https://registry.yarnpkg.com/@emotion/is-prop-valid/-/is-prop-valid-1.2.2.tgz#d4175076679c6a26faa92b03bb786f9e52612337" + integrity sha512-uNsoYd37AFmaCdXlg6EYD1KaPOaRWRByMCYzbKUX4+hhMfrxdVSelShywL4JVaAeM/eHUOSprYBQls+/neX3pw== dependencies: "@emotion/memoize" "^0.8.1" -"@emotion/memoize@^0.8.0": - version "0.8.0" - resolved "https://registry.yarnpkg.com/@emotion/memoize/-/memoize-0.8.0.tgz#f580f9beb67176fa57aae70b08ed510e1b18980f" - integrity sha512-G/YwXTkv7Den9mXDO7AhLWkE3q+I92B+VqAE+dYG4NGPaHZGvt3G8Q0p9vmE+sq7rTGphUbAvmQ9YpbfMQGGlA== +"@emotion/is-prop-valid@^1.3.0": + version "1.4.0" + resolved "https://registry.yarnpkg.com/@emotion/is-prop-valid/-/is-prop-valid-1.4.0.tgz#e9ad47adff0b5c94c72db3669ce46de33edf28c0" + integrity sha512-QgD4fyscGcbbKwJmqNvUMSE02OsHUa+lAWKdEUIJKgqe5IwRSKd7+KhibEWdaKwgjLj0DRSHA9biAIqGBk05lw== + dependencies: + "@emotion/memoize" "^0.9.0" "@emotion/memoize@^0.8.1": version "0.8.1" resolved "https://registry.yarnpkg.com/@emotion/memoize/-/memoize-0.8.1.tgz#c1ddb040429c6d21d38cc945fe75c818cfb68e17" integrity sha512-W2P2c/VRW1/1tLox0mVUalvnWXxavmv/Oum2aPsRcoDJuob75FC3Y8FbpfLwUegRcxINtGUMPq0tFCvYNTBXNA== -"@emotion/react@^11.11.0": - version "11.11.1" - resolved "https://registry.yarnpkg.com/@emotion/react/-/react-11.11.1.tgz#b2c36afac95b184f73b08da8c214fdf861fa4157" - integrity sha512-5mlW1DquU5HaxjLkfkGN1GA/fvVGdyHURRiX/0FHl2cfIfRxSOfmxEH5YS43edp0OldZrZ+dkBKbngxcNCdZvA== +"@emotion/memoize@^0.9.0": + version "0.9.0" + resolved "https://registry.yarnpkg.com/@emotion/memoize/-/memoize-0.9.0.tgz#745969d649977776b43fc7648c556aaa462b4102" + integrity sha512-30FAj7/EoJ5mwVPOWhAyCX+FPfMDrVecJAM+Iw9NRoSl4BBAQeqj4cApHHUXOVvIPgLVDsCFoz/hGD+5QQD1GQ== + +"@emotion/react@11.14.0": + version "11.14.0" + resolved "https://registry.yarnpkg.com/@emotion/react/-/react-11.14.0.tgz#cfaae35ebc67dd9ef4ea2e9acc6cd29e157dd05d" + integrity sha512-O000MLDBDdk/EohJPFUqvnp4qnHeYkVP5B0xEG0D/L7cOKP9kefu2DXn8dj74cQfsEzUqh+sr1RzFqiL1o+PpA== dependencies: "@babel/runtime" "^7.18.3" - "@emotion/babel-plugin" "^11.11.0" - "@emotion/cache" "^11.11.0" - "@emotion/serialize" "^1.1.2" - "@emotion/use-insertion-effect-with-fallbacks" "^1.0.1" - "@emotion/utils" "^1.2.1" - "@emotion/weak-memoize" "^0.3.1" + "@emotion/babel-plugin" "^11.13.5" + "@emotion/cache" "^11.14.0" + "@emotion/serialize" "^1.3.3" + "@emotion/use-insertion-effect-with-fallbacks" "^1.2.0" + "@emotion/utils" "^1.4.2" + "@emotion/weak-memoize" "^0.4.0" hoist-non-react-statics "^3.3.1" -"@emotion/serialize@^1.1.2": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@emotion/serialize/-/serialize-1.1.2.tgz#017a6e4c9b8a803bd576ff3d52a0ea6fa5a62b51" - integrity sha512-zR6a/fkFP4EAcCMQtLOhIgpprZOwNmCldtpaISpvz348+DP4Mz8ZoKaGGCQpbzepNIUWbq4w6hNZkwDyKoS+HA== +"@emotion/serialize@^1.3.3": + version "1.3.3" + resolved "https://registry.yarnpkg.com/@emotion/serialize/-/serialize-1.3.3.tgz#d291531005f17d704d0463a032fe679f376509e8" + integrity sha512-EISGqt7sSNWHGI76hC7x1CksiXPahbxEOrC5RjmFRJTqLyEK9/9hZvBbiYn70dw4wuwMKiEMCUlR6ZXTSWQqxA== dependencies: - "@emotion/hash" "^0.9.1" - "@emotion/memoize" "^0.8.1" - "@emotion/unitless" "^0.8.1" - "@emotion/utils" "^1.2.1" + "@emotion/hash" "^0.9.2" + "@emotion/memoize" "^0.9.0" + "@emotion/unitless" "^0.10.0" + "@emotion/utils" "^1.4.2" csstype "^3.0.2" -"@emotion/sheet@^1.2.1": - version "1.2.1" - resolved "https://registry.yarnpkg.com/@emotion/sheet/-/sheet-1.2.1.tgz#0767e0305230e894897cadb6c8df2c51e61a6c2c" - integrity sha512-zxRBwl93sHMsOj4zs+OslQKg/uhF38MB+OMKoCrVuS0nyTkqnau+BM3WGEoOptg9Oz45T/aIGs1qbVAsEFo3nA== - -"@emotion/sheet@^1.2.2": - version "1.2.2" - resolved "https://registry.yarnpkg.com/@emotion/sheet/-/sheet-1.2.2.tgz#d58e788ee27267a14342303e1abb3d508b6d0fec" - integrity sha512-0QBtGvaqtWi+nx6doRwDdBIzhNdZrXUppvTM4dtZZWEGTXL/XE/yJxLMGlDT1Gt+UHH5IX1n+jkXyytE/av7OA== +"@emotion/sheet@^1.4.0": + version "1.4.0" + resolved "https://registry.yarnpkg.com/@emotion/sheet/-/sheet-1.4.0.tgz#c9299c34d248bc26e82563735f78953d2efca83c" + integrity sha512-fTBW9/8r2w3dXWYM4HCB1Rdp8NLibOw2+XELH5m5+AkWiL/KqYX6dc0kKYlaYyKjrQ6ds33MCdMPEwgs2z1rqg== -"@emotion/styled@^11.11.0": - version "11.11.0" - resolved "https://registry.yarnpkg.com/@emotion/styled/-/styled-11.11.0.tgz#26b75e1b5a1b7a629d7c0a8b708fbf5a9cdce346" - integrity sha512-hM5Nnvu9P3midq5aaXj4I+lnSfNi7Pmd4EWk1fOZ3pxookaQTNew6bp4JaCBYM4HVFZF9g7UjJmsUmC2JlxOng== +"@emotion/styled@11.14.1": + version "11.14.1" + resolved "https://registry.yarnpkg.com/@emotion/styled/-/styled-11.14.1.tgz#8c34bed2948e83e1980370305614c20955aacd1c" + integrity sha512-qEEJt42DuToa3gurlH4Qqc1kVpNq8wO8cJtDzU46TjlzWjDlsVyevtYCRijVq3SrHsROS+gVQ8Fnea108GnKzw== dependencies: "@babel/runtime" "^7.18.3" - "@emotion/babel-plugin" "^11.11.0" - "@emotion/is-prop-valid" "^1.2.1" - "@emotion/serialize" "^1.1.2" - "@emotion/use-insertion-effect-with-fallbacks" "^1.0.1" - "@emotion/utils" "^1.2.1" - -"@emotion/stylis@^0.8.4": - version "0.8.5" - resolved "https://registry.yarnpkg.com/@emotion/stylis/-/stylis-0.8.5.tgz#deacb389bd6ee77d1e7fcaccce9e16c5c7e78e04" - integrity sha512-h6KtPihKFn3T9fuIrwvXXUOwlx3rfUvfZIcP5a6rh8Y7zjE3O06hT5Ss4S/YI1AYhuZ1kjaE/5EaOOI2NqSylQ== - -"@emotion/unitless@^0.7.4": - version "0.7.5" - resolved "https://registry.yarnpkg.com/@emotion/unitless/-/unitless-0.7.5.tgz#77211291c1900a700b8a78cfafda3160d76949ed" - integrity sha512-OWORNpfjMsSSUBVrRBVGECkhWcULOAJz9ZW8uK9qgxD+87M7jHRcvh/A96XXNhXTLmKcoYSQtBEX7lHMO7YRwg== - -"@emotion/unitless@^0.8.1": + "@emotion/babel-plugin" "^11.13.5" + "@emotion/is-prop-valid" "^1.3.0" + "@emotion/serialize" "^1.3.3" + "@emotion/use-insertion-effect-with-fallbacks" "^1.2.0" + "@emotion/utils" "^1.4.2" + +"@emotion/unitless@0.8.1": version "0.8.1" resolved "https://registry.yarnpkg.com/@emotion/unitless/-/unitless-0.8.1.tgz#182b5a4704ef8ad91bde93f7a860a88fd92c79a3" integrity sha512-KOEGMu6dmJZtpadb476IsZBclKvILjopjUii3V+7MnXIQCYh8W3NgNcgwo21n9LXZX6EDIKvqfjYxXebDwxKmQ== -"@emotion/use-insertion-effect-with-fallbacks@^1.0.1": - version "1.0.1" - resolved "https://registry.yarnpkg.com/@emotion/use-insertion-effect-with-fallbacks/-/use-insertion-effect-with-fallbacks-1.0.1.tgz#08de79f54eb3406f9daaf77c76e35313da963963" - integrity sha512-jT/qyKZ9rzLErtrjGgdkMBn2OP8wl0G3sQlBb3YPryvKHsjvINUhVaPFfP+fpBcOkmrVOVEEHQFJ7nbj2TH2gw== +"@emotion/unitless@^0.10.0": + version "0.10.0" + resolved "https://registry.yarnpkg.com/@emotion/unitless/-/unitless-0.10.0.tgz#2af2f7c7e5150f497bdabd848ce7b218a27cf745" + integrity sha512-dFoMUuQA20zvtVTuxZww6OHoJYgrzfKM1t52mVySDJnMSEa08ruEvdYQbhvyu6soU+NeLVd3yKfTfT0NeV6qGg== -"@emotion/utils@^1.2.0": +"@emotion/use-insertion-effect-with-fallbacks@^1.2.0": version "1.2.0" - resolved "https://registry.yarnpkg.com/@emotion/utils/-/utils-1.2.0.tgz#9716eaccbc6b5ded2ea5a90d65562609aab0f561" - integrity sha512-sn3WH53Kzpw8oQ5mgMmIzzyAaH2ZqFEbozVVBSYp538E06OSE6ytOp7pRAjNQR+Q/orwqdQYJSe2m3hCOeznkw== - -"@emotion/utils@^1.2.1": - version "1.2.1" - resolved "https://registry.yarnpkg.com/@emotion/utils/-/utils-1.2.1.tgz#bbab58465738d31ae4cb3dbb6fc00a5991f755e4" - integrity sha512-Y2tGf3I+XVnajdItskUCn6LX+VUDmP6lTL4fcqsXAv43dnlbZiuW4MWQW38rW/BVWSE7Q/7+XQocmpnRYILUmg== - -"@emotion/weak-memoize@^0.3.0": - version "0.3.0" - resolved "https://registry.yarnpkg.com/@emotion/weak-memoize/-/weak-memoize-0.3.0.tgz#ea89004119dc42db2e1dba0f97d553f7372f6fcb" - integrity sha512-AHPmaAx+RYfZz0eYu6Gviiagpmiyw98ySSlQvCUhVGDRtDFe4DBS0x1bSjdF3gqUDYOczB+yYvBTtEylYSdRhg== - -"@emotion/weak-memoize@^0.3.1": - version "0.3.1" - resolved "https://registry.yarnpkg.com/@emotion/weak-memoize/-/weak-memoize-0.3.1.tgz#d0fce5d07b0620caa282b5131c297bb60f9d87e6" - integrity sha512-EsBwpc7hBUJWAsNPBmJy4hxWx12v6bshQsldrVmjxJoc3isbxhOrF2IcCpaXxfvq03NwkI7sbsOLXbYuqF/8Ww== + resolved "https://registry.yarnpkg.com/@emotion/use-insertion-effect-with-fallbacks/-/use-insertion-effect-with-fallbacks-1.2.0.tgz#8a8cb77b590e09affb960f4ff1e9a89e532738bf" + integrity sha512-yJMtVdH59sxi/aVJBpk9FQq+OR8ll5GT8oWd57UpeaKEVGab41JWaCFA7FRLoMLloOZF/c/wsPoe+bfGmRKgDg== -"@eslint-community/eslint-utils@^4.4.0": - version "4.4.0" - resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz#a23514e8fb9af1269d5f7788aa556798d61c6b59" - integrity sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA== - dependencies: - eslint-visitor-keys "^3.3.0" - -"@eslint-community/regexpp@^4.5.1": - version "4.9.1" - resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.9.1.tgz#449dfa81a57a1d755b09aa58d826c1262e4283b4" - integrity sha512-Y27x+MBLjXa+0JWDhykM3+JE+il3kHKAEqabfEWq3SDhZjLYb6/BHL/JKFnH3fe207JaXkyDo685Oc2Glt6ifA== - -"@eslint/eslintrc@^1.4.0": - version "1.4.0" - resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-1.4.0.tgz#8ec64e0df3e7a1971ee1ff5158da87389f167a63" - integrity sha512-7yfvXy6MWLgWSFsLhz5yH3iQ52St8cdUY6FoGieKkRDVxuxmrNuUetIuu6cmjNWwniUHiWXjxCr5tTXDrbYS5A== - dependencies: - ajv "^6.12.4" - debug "^4.3.2" - espree "^9.4.0" - globals "^13.19.0" - ignore "^5.2.0" - import-fresh "^3.2.1" - js-yaml "^4.1.0" - minimatch "^3.1.2" - strip-json-comments "^3.1.1" - -"@fastify/busboy@^2.0.0": - version "2.0.0" - resolved "https://registry.yarnpkg.com/@fastify/busboy/-/busboy-2.0.0.tgz#f22824caff3ae506b18207bad4126dbc6ccdb6b8" - integrity sha512-JUFJad5lv7jxj926GPgymrWQxxjPYuJNiNjNMzqT+HiuP6Vl3dk5xzG+8sTX96np0ZAluvaMzPsjhHZ5rNuNQQ== - -"@floating-ui/core@^1.0.0": - version "1.6.1" - resolved "https://registry.yarnpkg.com/@floating-ui/core/-/core-1.6.1.tgz#a4e6fef1b069cda533cbc7a4998c083a37f37573" - integrity sha512-42UH54oPZHPdRHdw6BgoBD6cg/eVTmVrFcgeRDM3jbO7uxSoipVcmcIGFcA5jmOHO5apcyvBhkSKES3fQJnu7A== - dependencies: - "@floating-ui/utils" "^0.2.0" +"@emotion/utils@^1.4.2": + version "1.4.2" + resolved "https://registry.yarnpkg.com/@emotion/utils/-/utils-1.4.2.tgz#6df6c45881fcb1c412d6688a311a98b7f59c1b52" + integrity sha512-3vLclRofFziIa3J2wDh9jjbkUz9qk5Vi3IZ/FSTKViB0k+ef0fPV7dYrUIugbgupYDx7v9ud/SjrtEP8Y4xLoA== -"@floating-ui/dom@^1.0.0": - version "1.6.4" - resolved "https://registry.yarnpkg.com/@floating-ui/dom/-/dom-1.6.4.tgz#3a9d1f3b7ccdab89a4ca05713acc6204b1f67a29" - integrity sha512-0G8R+zOvQsAG1pg2Q99P21jiqxqGBW1iRe/iXHsBRBxnpXKFI8QwbB4x5KmYLggNO5m34IQgOIu9SCRfR/WWiQ== - dependencies: - "@floating-ui/core" "^1.0.0" - "@floating-ui/utils" "^0.2.0" +"@emotion/weak-memoize@^0.4.0": + version "0.4.0" + resolved "https://registry.yarnpkg.com/@emotion/weak-memoize/-/weak-memoize-0.4.0.tgz#5e13fac887f08c44f76b0ccaf3370eb00fec9bb6" + integrity sha512-snKqtPW01tN0ui7yu9rGv69aJXr/a/Ywvl11sUjNtEcRc+ng/mQriFL0wLXMef74iHa/EkftbDzU9F8iFbH+zg== + +"@esbuild/aix-ppc64@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/aix-ppc64/-/aix-ppc64-0.27.3.tgz#815b39267f9bffd3407ea6c376ac32946e24f8d2" + integrity sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg== + +"@esbuild/android-arm64@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.27.3.tgz#19b882408829ad8e12b10aff2840711b2da361e8" + integrity sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg== + +"@esbuild/android-arm@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.27.3.tgz#90be58de27915efa27b767fcbdb37a4470627d7b" + integrity sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA== + +"@esbuild/android-x64@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.27.3.tgz#d7dcc976f16e01a9aaa2f9b938fbec7389f895ac" + integrity sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ== + +"@esbuild/darwin-arm64@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.27.3.tgz#9f6cac72b3a8532298a6a4493ed639a8988e8abd" + integrity sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg== + +"@esbuild/darwin-x64@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.27.3.tgz#ac61d645faa37fd650340f1866b0812e1fb14d6a" + integrity sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg== + +"@esbuild/freebsd-arm64@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.3.tgz#b8625689d73cf1830fe58c39051acdc12474ea1b" + integrity sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w== + +"@esbuild/freebsd-x64@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.27.3.tgz#07be7dd3c9d42fe0eccd2ab9f9ded780bc53bead" + integrity sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA== + +"@esbuild/linux-arm64@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.27.3.tgz#bf31918fe5c798586460d2b3d6c46ed2c01ca0b6" + integrity sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg== + +"@esbuild/linux-arm@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.27.3.tgz#28493ee46abec1dc3f500223cd9f8d2df08f9d11" + integrity sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw== + +"@esbuild/linux-ia32@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.27.3.tgz#750752a8b30b43647402561eea764d0a41d0ee29" + integrity sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg== + +"@esbuild/linux-loong64@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.27.3.tgz#a5a92813a04e71198c50f05adfaf18fc1e95b9ed" + integrity sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA== + +"@esbuild/linux-mips64el@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.27.3.tgz#deb45d7fd2d2161eadf1fbc593637ed766d50bb1" + integrity sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw== + +"@esbuild/linux-ppc64@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.27.3.tgz#6f39ae0b8c4d3d2d61a65b26df79f6e12a1c3d78" + integrity sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA== + +"@esbuild/linux-riscv64@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.27.3.tgz#4c5c19c3916612ec8e3915187030b9df0b955c1d" + integrity sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ== + +"@esbuild/linux-s390x@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.27.3.tgz#9ed17b3198fa08ad5ccaa9e74f6c0aff7ad0156d" + integrity sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw== + +"@esbuild/linux-x64@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.27.3.tgz#12383dcbf71b7cf6513e58b4b08d95a710bf52a5" + integrity sha512-Czi8yzXUWIQYAtL/2y6vogER8pvcsOsk5cpwL4Gk5nJqH5UZiVByIY8Eorm5R13gq+DQKYg0+JyQoytLQas4dA== + +"@esbuild/netbsd-arm64@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.3.tgz#dd0cb2fa543205fcd931df44f4786bfcce6df7d7" + integrity sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA== + +"@esbuild/netbsd-x64@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.27.3.tgz#028ad1807a8e03e155153b2d025b506c3787354b" + integrity sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA== + +"@esbuild/openbsd-arm64@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.3.tgz#e3c16ff3490c9b59b969fffca87f350ffc0e2af5" + integrity sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw== + +"@esbuild/openbsd-x64@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.27.3.tgz#c5a4693fcb03d1cbecbf8b422422468dfc0d2a8b" + integrity sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ== + +"@esbuild/openharmony-arm64@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.3.tgz#082082444f12db564a0775a41e1991c0e125055e" + integrity sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g== + +"@esbuild/sunos-x64@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.27.3.tgz#5ab036c53f929e8405c4e96e865a424160a1b537" + integrity sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA== + +"@esbuild/win32-arm64@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.27.3.tgz#38de700ef4b960a0045370c171794526e589862e" + integrity sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA== + +"@esbuild/win32-ia32@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.27.3.tgz#451b93dc03ec5d4f38619e6cd64d9f9eff06f55c" + integrity sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q== + +"@esbuild/win32-x64@0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.27.3.tgz#0eaf705c941a218a43dba8e09f1df1d6cd2f1f17" + integrity sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA== + +"@isaacs/balanced-match@^4.0.1": + version "4.0.1" + resolved "https://registry.yarnpkg.com/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz#3081dadbc3460661b751e7591d7faea5df39dd29" + integrity sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ== -"@floating-ui/react-dom@^2.0.8": - version "2.0.9" - resolved "https://registry.yarnpkg.com/@floating-ui/react-dom/-/react-dom-2.0.9.tgz#264ba8b061000baa132b5910f0427a6acf7ad7ce" - integrity sha512-q0umO0+LQK4+p6aGyvzASqKbKOJcAHJ7ycE9CuUvfx3s9zTHWmGJTPOIlM/hmSBfUfg/XfY5YhLBLR/LHwShQQ== +"@isaacs/brace-expansion@^5.0.0": + version "5.0.1" + resolved "https://registry.yarnpkg.com/@isaacs/brace-expansion/-/brace-expansion-5.0.1.tgz#0ef5a92d91f2fff2a37646ce54da9e5f599f6eff" + integrity sha512-WMz71T1JS624nWj2n2fnYAuPovhv7EUhk69R6i9dsVyzxt5eM3bjwvgk9L+APE1TRscGysAVMANkB0jh0LQZrQ== dependencies: - "@floating-ui/dom" "^1.0.0" - -"@floating-ui/utils@^0.2.0": - version "0.2.2" - resolved "https://registry.yarnpkg.com/@floating-ui/utils/-/utils-0.2.2.tgz#d8bae93ac8b815b2bd7a98078cf91e2724ef11e5" - integrity sha512-J4yDIIthosAsRZ5CPYP/jQvUAQtlZTTD/4suA08/FEnlxqW3sKS9iAhgsa9VYLZ6vDHn/ixJgIqRQPotoBjxIw== + "@isaacs/balanced-match" "^4.0.1" -"@humanwhocodes/config-array@^0.11.8": - version "0.11.8" - resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.8.tgz#03595ac2075a4dc0f191cc2131de14fbd7d410b9" - integrity sha512-UybHIJzJnR5Qc/MsD9Kr+RpO2h+/P1GhOwdiLPXK5TWk5sgTdu88bTD9UP+CKbPPh5Rni1u0GjAdYQLemG8g+g== +"@isaacs/cliui@^8.0.2": + version "8.0.2" + resolved "https://registry.yarnpkg.com/@isaacs/cliui/-/cliui-8.0.2.tgz#b37667b7bc181c168782259bab42474fbf52b550" + integrity sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA== dependencies: - "@humanwhocodes/object-schema" "^1.2.1" - debug "^4.1.1" - minimatch "^3.0.5" - -"@humanwhocodes/module-importer@^1.0.1": - version "1.0.1" - resolved "https://registry.yarnpkg.com/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz#af5b2691a22b44be847b0ca81641c5fb6ad0172c" - integrity sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA== - -"@humanwhocodes/object-schema@^1.2.1": - version "1.2.1" - resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45" - integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA== + string-width "^5.1.2" + string-width-cjs "npm:string-width@^4.2.0" + strip-ansi "^7.0.1" + strip-ansi-cjs "npm:strip-ansi@^6.0.1" + wrap-ansi "^8.1.0" + wrap-ansi-cjs "npm:wrap-ansi@^7.0.0" "@istanbuljs/load-nyc-config@^1.0.0": version "1.1.0" @@ -1725,412 +1899,243 @@ js-yaml "^3.13.1" resolve-from "^5.0.0" -"@istanbuljs/schema@^0.1.2": +"@istanbuljs/schema@^0.1.2", "@istanbuljs/schema@^0.1.3": version "0.1.3" resolved "https://registry.yarnpkg.com/@istanbuljs/schema/-/schema-0.1.3.tgz#e45e384e4b8ec16bce2fd903af78450f6bf7ec98" integrity sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA== -"@jest/console@^27.5.1": - version "27.5.1" - resolved "https://registry.yarnpkg.com/@jest/console/-/console-27.5.1.tgz#260fe7239602fe5130a94f1aa386eff54b014bba" - integrity sha512-kZ/tNpS3NXn0mlXXXPNuDZnb4c0oZ20r4K5eemM2k30ZC3G0T02nXUvyhf5YdbXWHPEJLc9qGLxEZ216MdL+Zg== - dependencies: - "@jest/types" "^27.5.1" - "@types/node" "*" - chalk "^4.0.0" - jest-message-util "^27.5.1" - jest-util "^27.5.1" - slash "^3.0.0" - -"@jest/console@^28.1.3": - version "28.1.3" - resolved "https://registry.yarnpkg.com/@jest/console/-/console-28.1.3.tgz#2030606ec03a18c31803b8a36382762e447655df" - integrity sha512-QPAkP5EwKdK/bxIr6C1I4Vs0rm2nHiANzj/Z5X2JQkrZo6IqvC4ldZ9K95tF0HdidhA8Bo6egxSzUFPYKcEXLw== - dependencies: - "@jest/types" "^28.1.3" - "@types/node" "*" - chalk "^4.0.0" - jest-message-util "^28.1.3" - jest-util "^28.1.3" - slash "^3.0.0" - -"@jest/console@^29.7.0": - version "29.7.0" - resolved "https://registry.yarnpkg.com/@jest/console/-/console-29.7.0.tgz#cd4822dbdb84529265c5a2bdb529a3c9cc950ffc" - integrity sha512-5Ni4CU7XHQi32IJ398EEP4RrB8eV09sXP2ROqD4bksHrnTree52PsxvX8tpL8LvTZ3pFzXyPbNQReSN41CAhOg== +"@jest/console@30.2.0": + version "30.2.0" + resolved "https://registry.yarnpkg.com/@jest/console/-/console-30.2.0.tgz#c52fcd5b58fdd2e8eb66b2fd8ae56f2f64d05b28" + integrity sha512-+O1ifRjkvYIkBqASKWgLxrpEhQAAE7hY77ALLUufSk5717KfOShg6IbqLmdsLMPdUiFvA2kTs0R7YZy+l0IzZQ== dependencies: - "@jest/types" "^29.6.3" + "@jest/types" "30.2.0" "@types/node" "*" - chalk "^4.0.0" - jest-message-util "^29.7.0" - jest-util "^29.7.0" + chalk "^4.1.2" + jest-message-util "30.2.0" + jest-util "30.2.0" slash "^3.0.0" -"@jest/core@^27.5.1": - version "27.5.1" - resolved "https://registry.yarnpkg.com/@jest/core/-/core-27.5.1.tgz#267ac5f704e09dc52de2922cbf3af9edcd64b626" - integrity sha512-AK6/UTrvQD0Cd24NSqmIA6rKsu0tKIxfiCducZvqxYdmMisOYAsdItspT+fQDQYARPf8XgjAFZi0ogW2agH5nQ== - dependencies: - "@jest/console" "^27.5.1" - "@jest/reporters" "^27.5.1" - "@jest/test-result" "^27.5.1" - "@jest/transform" "^27.5.1" - "@jest/types" "^27.5.1" +"@jest/core@30.2.0": + version "30.2.0" + resolved "https://registry.yarnpkg.com/@jest/core/-/core-30.2.0.tgz#813d59faa5abd5510964a8b3a7b17cc77b775275" + integrity sha512-03W6IhuhjqTlpzh/ojut/pDB2LPRygyWX8ExpgHtQA8H/3K7+1vKmcINx5UzeOX1se6YEsBsOHQ1CRzf3fOwTQ== + dependencies: + "@jest/console" "30.2.0" + "@jest/pattern" "30.0.1" + "@jest/reporters" "30.2.0" + "@jest/test-result" "30.2.0" + "@jest/transform" "30.2.0" + "@jest/types" "30.2.0" "@types/node" "*" - ansi-escapes "^4.2.1" - chalk "^4.0.0" - emittery "^0.8.1" - exit "^0.1.2" - graceful-fs "^4.2.9" - jest-changed-files "^27.5.1" - jest-config "^27.5.1" - jest-haste-map "^27.5.1" - jest-message-util "^27.5.1" - jest-regex-util "^27.5.1" - jest-resolve "^27.5.1" - jest-resolve-dependencies "^27.5.1" - jest-runner "^27.5.1" - jest-runtime "^27.5.1" - jest-snapshot "^27.5.1" - jest-util "^27.5.1" - jest-validate "^27.5.1" - jest-watcher "^27.5.1" - micromatch "^4.0.4" - rimraf "^3.0.0" + ansi-escapes "^4.3.2" + chalk "^4.1.2" + ci-info "^4.2.0" + exit-x "^0.2.2" + graceful-fs "^4.2.11" + jest-changed-files "30.2.0" + jest-config "30.2.0" + jest-haste-map "30.2.0" + jest-message-util "30.2.0" + jest-regex-util "30.0.1" + jest-resolve "30.2.0" + jest-resolve-dependencies "30.2.0" + jest-runner "30.2.0" + jest-runtime "30.2.0" + jest-snapshot "30.2.0" + jest-util "30.2.0" + jest-validate "30.2.0" + jest-watcher "30.2.0" + micromatch "^4.0.8" + pretty-format "30.2.0" slash "^3.0.0" - strip-ansi "^6.0.0" -"@jest/core@^29.7.0": - version "29.7.0" - resolved "https://registry.yarnpkg.com/@jest/core/-/core-29.7.0.tgz#b6cccc239f30ff36609658c5a5e2291757ce448f" - integrity sha512-n7aeXWKMnGtDA48y8TLWJPJmLmmZ642Ceo78cYWEpiD7FzDgmNDV/GCVRorPABdXLJZ/9wzzgZAlHjXjxDHGsg== - dependencies: - "@jest/console" "^29.7.0" - "@jest/reporters" "^29.7.0" - "@jest/test-result" "^29.7.0" - "@jest/transform" "^29.7.0" - "@jest/types" "^29.6.3" - "@types/node" "*" - ansi-escapes "^4.2.1" - chalk "^4.0.0" - ci-info "^3.2.0" - exit "^0.1.2" - graceful-fs "^4.2.9" - jest-changed-files "^29.7.0" - jest-config "^29.7.0" - jest-haste-map "^29.7.0" - jest-message-util "^29.7.0" - jest-regex-util "^29.6.3" - jest-resolve "^29.7.0" - jest-resolve-dependencies "^29.7.0" - jest-runner "^29.7.0" - jest-runtime "^29.7.0" - jest-snapshot "^29.7.0" - jest-util "^29.7.0" - jest-validate "^29.7.0" - jest-watcher "^29.7.0" - micromatch "^4.0.4" - pretty-format "^29.7.0" - slash "^3.0.0" - strip-ansi "^6.0.0" +"@jest/diff-sequences@30.0.1": + version "30.0.1" + resolved "https://registry.yarnpkg.com/@jest/diff-sequences/-/diff-sequences-30.0.1.tgz#0ededeae4d071f5c8ffe3678d15f3a1be09156be" + integrity sha512-n5H8QLDJ47QqbCNn5SuFjCRDrOLEZ0h8vAHCK5RL9Ls7Xa8AQLa/YxAc9UjFqoEDM48muwtBGjtMY5cr0PLDCw== -"@jest/environment@^27.5.1": - version "27.5.1" - resolved "https://registry.yarnpkg.com/@jest/environment/-/environment-27.5.1.tgz#d7425820511fe7158abbecc010140c3fd3be9c74" - integrity sha512-/WQjhPJe3/ghaol/4Bq480JKXV/Rfw8nQdN7f41fM8VDHLcxKXou6QyXAh3EFr9/bVG3x74z1NWDkP87EiY8gA== +"@jest/environment-jsdom-abstract@30.2.0": + version "30.2.0" + resolved "https://registry.yarnpkg.com/@jest/environment-jsdom-abstract/-/environment-jsdom-abstract-30.2.0.tgz#1313f9b3b509c31298c241203161b36622865181" + integrity sha512-kazxw2L9IPuZpQ0mEt9lu9Z98SqR74xcagANmMBU16X0lS23yPc0+S6hGLUz8kVRlomZEs/5S/Zlpqwf5yu6OQ== dependencies: - "@jest/fake-timers" "^27.5.1" - "@jest/types" "^27.5.1" + "@jest/environment" "30.2.0" + "@jest/fake-timers" "30.2.0" + "@jest/types" "30.2.0" + "@types/jsdom" "^21.1.7" "@types/node" "*" - jest-mock "^27.5.1" + jest-mock "30.2.0" + jest-util "30.2.0" -"@jest/environment@^29.7.0": - version "29.7.0" - resolved "https://registry.yarnpkg.com/@jest/environment/-/environment-29.7.0.tgz#24d61f54ff1f786f3cd4073b4b94416383baf2a7" - integrity sha512-aQIfHDq33ExsN4jP1NWGXhxgQ/wixs60gDiKO+XVMd8Mn0NWPWgc34ZQDTb2jKaUWQ7MuwoitXAsN2XVXNMpAw== +"@jest/environment@30.2.0": + version "30.2.0" + resolved "https://registry.yarnpkg.com/@jest/environment/-/environment-30.2.0.tgz#1e673cdb8b93ded707cf6631b8353011460831fa" + integrity sha512-/QPTL7OBJQ5ac09UDRa3EQes4gt1FTEG/8jZ/4v5IVzx+Cv7dLxlVIvfvSVRiiX2drWyXeBjkMSR8hvOWSog5g== dependencies: - "@jest/fake-timers" "^29.7.0" - "@jest/types" "^29.6.3" + "@jest/fake-timers" "30.2.0" + "@jest/types" "30.2.0" "@types/node" "*" - jest-mock "^29.7.0" + jest-mock "30.2.0" -"@jest/expect-utils@^29.7.0": - version "29.7.0" - resolved "https://registry.yarnpkg.com/@jest/expect-utils/-/expect-utils-29.7.0.tgz#023efe5d26a8a70f21677d0a1afc0f0a44e3a1c6" - integrity sha512-GlsNBWiFQFCVi9QVSx7f5AgMeLxe9YCCs5PuP2O2LdjDAA8Jh9eX7lA1Jq/xdXw3Wb3hyvlFNfZIfcRetSzYcA== +"@jest/expect-utils@30.2.0": + version "30.2.0" + resolved "https://registry.yarnpkg.com/@jest/expect-utils/-/expect-utils-30.2.0.tgz#4f95413d4748454fdb17404bf1141827d15e6011" + integrity sha512-1JnRfhqpD8HGpOmQp180Fo9Zt69zNtC+9lR+kT7NVL05tNXIi+QC8Csz7lfidMoVLPD3FnOtcmp0CEFnxExGEA== dependencies: - jest-get-type "^29.6.3" + "@jest/get-type" "30.1.0" -"@jest/expect@^29.7.0": - version "29.7.0" - resolved "https://registry.yarnpkg.com/@jest/expect/-/expect-29.7.0.tgz#76a3edb0cb753b70dfbfe23283510d3d45432bf2" - integrity sha512-8uMeAMycttpva3P1lBHB8VciS9V0XAr3GymPpipdyQXbBcuhkLQOSe8E/p92RyAdToS6ZD1tFkX+CkhoECE0dQ== +"@jest/expect@30.2.0": + version "30.2.0" + resolved "https://registry.yarnpkg.com/@jest/expect/-/expect-30.2.0.tgz#9a5968499bb8add2bbb09136f69f7df5ddbf3185" + integrity sha512-V9yxQK5erfzx99Sf+7LbhBwNWEZ9eZay8qQ9+JSC0TrMR1pMDHLMY+BnVPacWU6Jamrh252/IKo4F1Xn/zfiqA== dependencies: - expect "^29.7.0" - jest-snapshot "^29.7.0" + expect "30.2.0" + jest-snapshot "30.2.0" -"@jest/fake-timers@^27.5.1": - version "27.5.1" - resolved "https://registry.yarnpkg.com/@jest/fake-timers/-/fake-timers-27.5.1.tgz#76979745ce0579c8a94a4678af7a748eda8ada74" - integrity sha512-/aPowoolwa07k7/oM3aASneNeBGCmGQsc3ugN4u6s4C/+s5M64MFo/+djTdiwcbQlRfFElGuDXWzaWj6QgKObQ== +"@jest/fake-timers@30.2.0": + version "30.2.0" + resolved "https://registry.yarnpkg.com/@jest/fake-timers/-/fake-timers-30.2.0.tgz#0941ddc28a339b9819542495b5408622dc9e94ec" + integrity sha512-HI3tRLjRxAbBy0VO8dqqm7Hb2mIa8d5bg/NJkyQcOk7V118ObQML8RC5luTF/Zsg4474a+gDvhce7eTnP4GhYw== dependencies: - "@jest/types" "^27.5.1" - "@sinonjs/fake-timers" "^8.0.1" + "@jest/types" "30.2.0" + "@sinonjs/fake-timers" "^13.0.0" "@types/node" "*" - jest-message-util "^27.5.1" - jest-mock "^27.5.1" - jest-util "^27.5.1" - -"@jest/fake-timers@^29.7.0": - version "29.7.0" - resolved "https://registry.yarnpkg.com/@jest/fake-timers/-/fake-timers-29.7.0.tgz#fd91bf1fffb16d7d0d24a426ab1a47a49881a565" - integrity sha512-q4DH1Ha4TTFPdxLsqDXK1d3+ioSL7yL5oCMJZgDYm6i+6CygW5E5xVr/D1HdsGxjt1ZWSfUAs9OxSB/BNelWrQ== + jest-message-util "30.2.0" + jest-mock "30.2.0" + jest-util "30.2.0" + +"@jest/get-type@30.1.0": + version "30.1.0" + resolved "https://registry.yarnpkg.com/@jest/get-type/-/get-type-30.1.0.tgz#4fcb4dc2ebcf0811be1c04fd1cb79c2dba431cbc" + integrity sha512-eMbZE2hUnx1WV0pmURZY9XoXPkUYjpc55mb0CrhtdWLtzMQPFvu/rZkTLZFTsdaVQa+Tr4eWAteqcUzoawq/uA== + +"@jest/globals@30.2.0": + version "30.2.0" + resolved "https://registry.yarnpkg.com/@jest/globals/-/globals-30.2.0.tgz#2f4b696d5862664b89c4ee2e49ae24d2bb7e0988" + integrity sha512-b63wmnKPaK+6ZZfpYhz9K61oybvbI1aMcIs80++JI1O1rR1vaxHUCNqo3ITu6NU0d4V34yZFoHMn/uoKr/Rwfw== + dependencies: + "@jest/environment" "30.2.0" + "@jest/expect" "30.2.0" + "@jest/types" "30.2.0" + jest-mock "30.2.0" + +"@jest/pattern@30.0.1": + version "30.0.1" + resolved "https://registry.yarnpkg.com/@jest/pattern/-/pattern-30.0.1.tgz#d5304147f49a052900b4b853dedb111d080e199f" + integrity sha512-gWp7NfQW27LaBQz3TITS8L7ZCQ0TLvtmI//4OwlQRx4rnWxcPNIYjxZpDcN4+UlGxgm3jS5QPz8IPTCkb59wZA== dependencies: - "@jest/types" "^29.6.3" - "@sinonjs/fake-timers" "^10.0.2" "@types/node" "*" - jest-message-util "^29.7.0" - jest-mock "^29.7.0" - jest-util "^29.7.0" - -"@jest/globals@^27.5.1": - version "27.5.1" - resolved "https://registry.yarnpkg.com/@jest/globals/-/globals-27.5.1.tgz#7ac06ce57ab966566c7963431cef458434601b2b" - integrity sha512-ZEJNB41OBQQgGzgyInAv0UUfDDj3upmHydjieSxFvTRuZElrx7tXg/uVQ5hYVEwiXs3+aMsAeEc9X7xiSKCm4Q== - dependencies: - "@jest/environment" "^27.5.1" - "@jest/types" "^27.5.1" - expect "^27.5.1" - -"@jest/globals@^29.7.0": - version "29.7.0" - resolved "https://registry.yarnpkg.com/@jest/globals/-/globals-29.7.0.tgz#8d9290f9ec47ff772607fa864ca1d5a2efae1d4d" - integrity sha512-mpiz3dutLbkW2MNFubUGUEVLkTGiqW6yLVTA+JbP6fI6J5iL9Y0Nlg8k95pcF8ctKwCS7WVxteBs29hhfAotzQ== - dependencies: - "@jest/environment" "^29.7.0" - "@jest/expect" "^29.7.0" - "@jest/types" "^29.6.3" - jest-mock "^29.7.0" + jest-regex-util "30.0.1" -"@jest/reporters@^27.5.1": - version "27.5.1" - resolved "https://registry.yarnpkg.com/@jest/reporters/-/reporters-27.5.1.tgz#ceda7be96170b03c923c37987b64015812ffec04" - integrity sha512-cPXh9hWIlVJMQkVk84aIvXuBB4uQQmFqZiacloFuGiP3ah1sbCxCosidXFDfqG8+6fO1oR2dTJTlsOy4VFmUfw== +"@jest/reporters@30.2.0": + version "30.2.0" + resolved "https://registry.yarnpkg.com/@jest/reporters/-/reporters-30.2.0.tgz#a36b28fcbaf0c4595250b108e6f20e363348fd91" + integrity sha512-DRyW6baWPqKMa9CzeiBjHwjd8XeAyco2Vt8XbcLFjiwCOEKOvy82GJ8QQnJE9ofsxCMPjH4MfH8fCWIHHDKpAQ== dependencies: "@bcoe/v8-coverage" "^0.2.3" - "@jest/console" "^27.5.1" - "@jest/test-result" "^27.5.1" - "@jest/transform" "^27.5.1" - "@jest/types" "^27.5.1" + "@jest/console" "30.2.0" + "@jest/test-result" "30.2.0" + "@jest/transform" "30.2.0" + "@jest/types" "30.2.0" + "@jridgewell/trace-mapping" "^0.3.25" "@types/node" "*" - chalk "^4.0.0" - collect-v8-coverage "^1.0.0" - exit "^0.1.2" - glob "^7.1.2" - graceful-fs "^4.2.9" + chalk "^4.1.2" + collect-v8-coverage "^1.0.2" + exit-x "^0.2.2" + glob "^10.3.10" + graceful-fs "^4.2.11" istanbul-lib-coverage "^3.0.0" - istanbul-lib-instrument "^5.1.0" + istanbul-lib-instrument "^6.0.0" istanbul-lib-report "^3.0.0" - istanbul-lib-source-maps "^4.0.0" + istanbul-lib-source-maps "^5.0.0" istanbul-reports "^3.1.3" - jest-haste-map "^27.5.1" - jest-resolve "^27.5.1" - jest-util "^27.5.1" - jest-worker "^27.5.1" + jest-message-util "30.2.0" + jest-util "30.2.0" + jest-worker "30.2.0" slash "^3.0.0" - source-map "^0.6.0" - string-length "^4.0.1" - terminal-link "^2.0.0" - v8-to-istanbul "^8.1.0" + string-length "^4.0.2" + v8-to-istanbul "^9.0.1" -"@jest/reporters@^29.7.0": - version "29.7.0" - resolved "https://registry.yarnpkg.com/@jest/reporters/-/reporters-29.7.0.tgz#04b262ecb3b8faa83b0b3d321623972393e8f4c7" - integrity sha512-DApq0KJbJOEzAFYjHADNNxAE3KbhxQB1y5Kplb5Waqw6zVbuWatSnMjE5gs8FUgEPmNsnZA3NCWl9NG0ia04Pg== +"@jest/schemas@30.0.5": + version "30.0.5" + resolved "https://registry.yarnpkg.com/@jest/schemas/-/schemas-30.0.5.tgz#7bdf69fc5a368a5abdb49fd91036c55225846473" + integrity sha512-DmdYgtezMkh3cpU8/1uyXakv3tJRcmcXxBOcO0tbaozPwpmh4YMsnWrQm9ZmZMfa5ocbxzbFk6O4bDPEc/iAnA== dependencies: - "@bcoe/v8-coverage" "^0.2.3" - "@jest/console" "^29.7.0" - "@jest/test-result" "^29.7.0" - "@jest/transform" "^29.7.0" - "@jest/types" "^29.6.3" - "@jridgewell/trace-mapping" "^0.3.18" - "@types/node" "*" - chalk "^4.0.0" - collect-v8-coverage "^1.0.0" - exit "^0.1.2" - glob "^7.1.3" - graceful-fs "^4.2.9" - istanbul-lib-coverage "^3.0.0" - istanbul-lib-instrument "^6.0.0" - istanbul-lib-report "^3.0.0" - istanbul-lib-source-maps "^4.0.0" - istanbul-reports "^3.1.3" - jest-message-util "^29.7.0" - jest-util "^29.7.0" - jest-worker "^29.7.0" - slash "^3.0.0" - string-length "^4.0.1" - strip-ansi "^6.0.0" - v8-to-istanbul "^9.0.1" + "@sinclair/typebox" "^0.34.0" -"@jest/schemas@^28.1.3": - version "28.1.3" - resolved "https://registry.yarnpkg.com/@jest/schemas/-/schemas-28.1.3.tgz#ad8b86a66f11f33619e3d7e1dcddd7f2d40ff905" - integrity sha512-/l/VWsdt/aBXgjshLWOFyFt3IVdYypu5y2Wn2rOO1un6nkqIn8SLXzgIMYXFyYsRWDyF5EthmKJMIdJvk08grg== +"@jest/snapshot-utils@30.2.0": + version "30.2.0" + resolved "https://registry.yarnpkg.com/@jest/snapshot-utils/-/snapshot-utils-30.2.0.tgz#387858eb90c2f98f67bff327435a532ac5309fbe" + integrity sha512-0aVxM3RH6DaiLcjj/b0KrIBZhSX1373Xci4l3cW5xiUWPctZ59zQ7jj4rqcJQ/Z8JuN/4wX3FpJSa3RssVvCug== dependencies: - "@sinclair/typebox" "^0.24.1" + "@jest/types" "30.2.0" + chalk "^4.1.2" + graceful-fs "^4.2.11" + natural-compare "^1.4.0" -"@jest/schemas@^29.6.3": - version "29.6.3" - resolved "https://registry.yarnpkg.com/@jest/schemas/-/schemas-29.6.3.tgz#430b5ce8a4e0044a7e3819663305a7b3091c8e03" - integrity sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA== +"@jest/source-map@30.0.1": + version "30.0.1" + resolved "https://registry.yarnpkg.com/@jest/source-map/-/source-map-30.0.1.tgz#305ebec50468f13e658b3d5c26f85107a5620aaa" + integrity sha512-MIRWMUUR3sdbP36oyNyhbThLHyJ2eEDClPCiHVbrYAe5g3CHRArIVpBw7cdSB5fr+ofSfIb2Tnsw8iEHL0PYQg== dependencies: - "@sinclair/typebox" "^0.27.8" + "@jridgewell/trace-mapping" "^0.3.25" + callsites "^3.1.0" + graceful-fs "^4.2.11" -"@jest/source-map@^27.5.1": - version "27.5.1" - resolved "https://registry.yarnpkg.com/@jest/source-map/-/source-map-27.5.1.tgz#6608391e465add4205eae073b55e7f279e04e8cf" - integrity sha512-y9NIHUYF3PJRlHk98NdC/N1gl88BL08aQQgu4k4ZopQkCw9t9cV8mtl3TV8b/YCB8XaVTFrmUTAJvjsntDireg== +"@jest/test-result@30.2.0": + version "30.2.0" + resolved "https://registry.yarnpkg.com/@jest/test-result/-/test-result-30.2.0.tgz#9c0124377fb7996cdffb86eda3dbc56eacab363d" + integrity sha512-RF+Z+0CCHkARz5HT9mcQCBulb1wgCP3FBvl9VFokMX27acKphwyQsNuWH3c+ojd1LeWBLoTYoxF0zm6S/66mjg== dependencies: - callsites "^3.0.0" - graceful-fs "^4.2.9" - source-map "^0.6.0" + "@jest/console" "30.2.0" + "@jest/types" "30.2.0" + "@types/istanbul-lib-coverage" "^2.0.6" + collect-v8-coverage "^1.0.2" -"@jest/source-map@^29.6.3": - version "29.6.3" - resolved "https://registry.yarnpkg.com/@jest/source-map/-/source-map-29.6.3.tgz#d90ba772095cf37a34a5eb9413f1b562a08554c4" - integrity sha512-MHjT95QuipcPrpLM+8JMSzFx6eHp5Bm+4XeFDJlwsvVBjmKNiIAvasGK2fxz2WbGRlnvqehFbh07MMa7n3YJnw== +"@jest/test-sequencer@30.2.0": + version "30.2.0" + resolved "https://registry.yarnpkg.com/@jest/test-sequencer/-/test-sequencer-30.2.0.tgz#bf0066bc72e176d58f5dfa7f212b6e7eee44f221" + integrity sha512-wXKgU/lk8fKXMu/l5Hog1R61bL4q5GCdT6OJvdAFz1P+QrpoFuLU68eoKuVc4RbrTtNnTL5FByhWdLgOPSph+Q== dependencies: - "@jridgewell/trace-mapping" "^0.3.18" - callsites "^3.0.0" - graceful-fs "^4.2.9" - -"@jest/test-result@^27.5.1": - version "27.5.1" - resolved "https://registry.yarnpkg.com/@jest/test-result/-/test-result-27.5.1.tgz#56a6585fa80f7cdab72b8c5fc2e871d03832f5bb" - integrity sha512-EW35l2RYFUcUQxFJz5Cv5MTOxlJIQs4I7gxzi2zVU7PJhOwfYq1MdC5nhSmYjX1gmMmLPvB3sIaC+BkcHRBfag== - dependencies: - "@jest/console" "^27.5.1" - "@jest/types" "^27.5.1" - "@types/istanbul-lib-coverage" "^2.0.0" - collect-v8-coverage "^1.0.0" - -"@jest/test-result@^28.1.3": - version "28.1.3" - resolved "https://registry.yarnpkg.com/@jest/test-result/-/test-result-28.1.3.tgz#5eae945fd9f4b8fcfce74d239e6f725b6bf076c5" - integrity sha512-kZAkxnSE+FqE8YjW8gNuoVkkC9I7S1qmenl8sGcDOLropASP+BkcGKwhXoyqQuGOGeYY0y/ixjrd/iERpEXHNg== - dependencies: - "@jest/console" "^28.1.3" - "@jest/types" "^28.1.3" - "@types/istanbul-lib-coverage" "^2.0.0" - collect-v8-coverage "^1.0.0" - -"@jest/test-result@^29.7.0": - version "29.7.0" - resolved "https://registry.yarnpkg.com/@jest/test-result/-/test-result-29.7.0.tgz#8db9a80aa1a097bb2262572686734baed9b1657c" - integrity sha512-Fdx+tv6x1zlkJPcWXmMDAG2HBnaR9XPSd5aDWQVsfrZmLVT3lU1cwyxLgRmXR9yrq4NBoEm9BMsfgFzTQAbJYA== - dependencies: - "@jest/console" "^29.7.0" - "@jest/types" "^29.6.3" - "@types/istanbul-lib-coverage" "^2.0.0" - collect-v8-coverage "^1.0.0" - -"@jest/test-sequencer@^27.5.1": - version "27.5.1" - resolved "https://registry.yarnpkg.com/@jest/test-sequencer/-/test-sequencer-27.5.1.tgz#4057e0e9cea4439e544c6353c6affe58d095745b" - integrity sha512-LCheJF7WB2+9JuCS7VB/EmGIdQuhtqjRNI9A43idHv3E4KltCTsPsLxvdaubFHSYwY/fNjMWjl6vNRhDiN7vpQ== - dependencies: - "@jest/test-result" "^27.5.1" - graceful-fs "^4.2.9" - jest-haste-map "^27.5.1" - jest-runtime "^27.5.1" - -"@jest/test-sequencer@^29.7.0": - version "29.7.0" - resolved "https://registry.yarnpkg.com/@jest/test-sequencer/-/test-sequencer-29.7.0.tgz#6cef977ce1d39834a3aea887a1726628a6f072ce" - integrity sha512-GQwJ5WZVrKnOJuiYiAF52UNUJXgTZx1NHjFSEB0qEMmSZKAkdMoIzw/Cj6x6NF4AvV23AUqDpFzQkN/eYCYTxw== - dependencies: - "@jest/test-result" "^29.7.0" - graceful-fs "^4.2.9" - jest-haste-map "^29.7.0" + "@jest/test-result" "30.2.0" + graceful-fs "^4.2.11" + jest-haste-map "30.2.0" slash "^3.0.0" -"@jest/transform@^27.5.1": - version "27.5.1" - resolved "https://registry.yarnpkg.com/@jest/transform/-/transform-27.5.1.tgz#6c3501dcc00c4c08915f292a600ece5ecfe1f409" - integrity sha512-ipON6WtYgl/1329g5AIJVbUuEh0wZVbdpGwC99Jw4LwuoBNS95MVphU6zOeD9pDkon+LLbFL7lOQRapbB8SCHw== - dependencies: - "@babel/core" "^7.1.0" - "@jest/types" "^27.5.1" - babel-plugin-istanbul "^6.1.1" - chalk "^4.0.0" - convert-source-map "^1.4.0" - fast-json-stable-stringify "^2.0.0" - graceful-fs "^4.2.9" - jest-haste-map "^27.5.1" - jest-regex-util "^27.5.1" - jest-util "^27.5.1" - micromatch "^4.0.4" - pirates "^4.0.4" - slash "^3.0.0" - source-map "^0.6.1" - write-file-atomic "^3.0.0" - -"@jest/transform@^29.7.0": - version "29.7.0" - resolved "https://registry.yarnpkg.com/@jest/transform/-/transform-29.7.0.tgz#df2dd9c346c7d7768b8a06639994640c642e284c" - integrity sha512-ok/BTPFzFKVMwO5eOHRrvnBVHdRy9IrsrW1GpMaQ9MCnilNLXQKmAX8s1YXDFaai9xJpac2ySzV0YeRRECr2Vw== - dependencies: - "@babel/core" "^7.11.6" - "@jest/types" "^29.6.3" - "@jridgewell/trace-mapping" "^0.3.18" - babel-plugin-istanbul "^6.1.1" - chalk "^4.0.0" +"@jest/transform@30.2.0": + version "30.2.0" + resolved "https://registry.yarnpkg.com/@jest/transform/-/transform-30.2.0.tgz#54bef1a4510dcbd58d5d4de4fe2980a63077ef2a" + integrity sha512-XsauDV82o5qXbhalKxD7p4TZYYdwcaEXC77PPD2HixEFF+6YGppjrAAQurTl2ECWcEomHBMMNS9AH3kcCFx8jA== + dependencies: + "@babel/core" "^7.27.4" + "@jest/types" "30.2.0" + "@jridgewell/trace-mapping" "^0.3.25" + babel-plugin-istanbul "^7.0.1" + chalk "^4.1.2" convert-source-map "^2.0.0" fast-json-stable-stringify "^2.1.0" - graceful-fs "^4.2.9" - jest-haste-map "^29.7.0" - jest-regex-util "^29.6.3" - jest-util "^29.7.0" - micromatch "^4.0.4" - pirates "^4.0.4" + graceful-fs "^4.2.11" + jest-haste-map "30.2.0" + jest-regex-util "30.0.1" + jest-util "30.2.0" + micromatch "^4.0.8" + pirates "^4.0.7" slash "^3.0.0" - write-file-atomic "^4.0.2" - -"@jest/types@^27.5.1": - version "27.5.1" - resolved "https://registry.yarnpkg.com/@jest/types/-/types-27.5.1.tgz#3c79ec4a8ba61c170bf937bcf9e98a9df175ec80" - integrity sha512-Cx46iJ9QpwQTjIdq5VJu2QTMMs3QlEjI0x1QbBP5W1+nMzyc2XmimiRR/CbX9TO0cPTeUlxWMOu8mslYsJ8DEw== - dependencies: - "@types/istanbul-lib-coverage" "^2.0.0" - "@types/istanbul-reports" "^3.0.0" - "@types/node" "*" - "@types/yargs" "^16.0.0" - chalk "^4.0.0" - -"@jest/types@^28.1.3": - version "28.1.3" - resolved "https://registry.yarnpkg.com/@jest/types/-/types-28.1.3.tgz#b05de80996ff12512bc5ceb1d208285a7d11748b" - integrity sha512-RyjiyMUZrKz/c+zlMFO1pm70DcIlST8AeWTkoUdZevew44wcNZQHsEVOiCVtgVnlFFD82FPaXycys58cf2muVQ== - dependencies: - "@jest/schemas" "^28.1.3" - "@types/istanbul-lib-coverage" "^2.0.0" - "@types/istanbul-reports" "^3.0.0" - "@types/node" "*" - "@types/yargs" "^17.0.8" - chalk "^4.0.0" + write-file-atomic "^5.0.1" -"@jest/types@^29.6.3": - version "29.6.3" - resolved "https://registry.yarnpkg.com/@jest/types/-/types-29.6.3.tgz#1131f8cf634e7e84c5e77bab12f052af585fba59" - integrity sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw== +"@jest/types@30.2.0": + version "30.2.0" + resolved "https://registry.yarnpkg.com/@jest/types/-/types-30.2.0.tgz#1c678a7924b8f59eafd4c77d56b6d0ba976d62b8" + integrity sha512-H9xg1/sfVvyfU7o3zMfBEjQ1gcsdeTMgqHoYdN79tuLqfTtuu7WckRA1R5whDwOzxaZAeMKTYWqP+WCAi0CHsg== dependencies: - "@jest/schemas" "^29.6.3" - "@types/istanbul-lib-coverage" "^2.0.0" - "@types/istanbul-reports" "^3.0.0" + "@jest/pattern" "30.0.1" + "@jest/schemas" "30.0.5" + "@types/istanbul-lib-coverage" "^2.0.6" + "@types/istanbul-reports" "^3.0.4" "@types/node" "*" - "@types/yargs" "^17.0.8" - chalk "^4.0.0" + "@types/yargs" "^17.0.33" + chalk "^4.1.2" "@jridgewell/gen-mapping@^0.1.0": version "0.1.1" @@ -2140,7 +2145,15 @@ "@jridgewell/set-array" "^1.0.0" "@jridgewell/sourcemap-codec" "^1.4.10" -"@jridgewell/gen-mapping@^0.3.0", "@jridgewell/gen-mapping@^0.3.2": +"@jridgewell/gen-mapping@^0.3.12": + version "0.3.13" + resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz#6342a19f44347518c93e43b1ac69deb3c4656a1f" + integrity sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA== + dependencies: + "@jridgewell/sourcemap-codec" "^1.5.0" + "@jridgewell/trace-mapping" "^0.3.24" + +"@jridgewell/gen-mapping@^0.3.2": version "0.3.2" resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz#c1aedc61e853f2bb9f5dfe6d4442d3b565b253b9" integrity sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A== @@ -2158,6 +2171,14 @@ "@jridgewell/sourcemap-codec" "^1.4.10" "@jridgewell/trace-mapping" "^0.3.24" +"@jridgewell/remapping@^2.3.5": + version "2.3.5" + resolved "https://registry.yarnpkg.com/@jridgewell/remapping/-/remapping-2.3.5.tgz#375c476d1972947851ba1e15ae8f123047445aa1" + integrity sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ== + dependencies: + "@jridgewell/gen-mapping" "^0.3.5" + "@jridgewell/trace-mapping" "^0.3.24" + "@jridgewell/resolve-uri@3.1.0": version "3.1.0" resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz#2203b118c157721addfe69d47b70465463066d78" @@ -2178,22 +2199,6 @@ resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.2.1.tgz#558fb6472ed16a4c850b889530e6b36438c49280" integrity sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A== -"@jridgewell/source-map@^0.3.2": - version "0.3.2" - resolved "https://registry.yarnpkg.com/@jridgewell/source-map/-/source-map-0.3.2.tgz#f45351aaed4527a298512ec72f81040c998580fb" - integrity sha512-m7O9o2uR8k2ObDysZYzdfhb08VuEml5oWGiosa1VdaPZ/A6QyPkAJuwN0Q1lhULOf6B7MtQmHENS743hWtCrgw== - dependencies: - "@jridgewell/gen-mapping" "^0.3.0" - "@jridgewell/trace-mapping" "^0.3.9" - -"@jridgewell/source-map@^0.3.3": - version "0.3.6" - resolved "https://registry.yarnpkg.com/@jridgewell/source-map/-/source-map-0.3.6.tgz#9d71ca886e32502eb9362c9a74a46787c36df81a" - integrity sha512-1ZJTZebgqllO79ue2bm3rIGud/bOe0pP5BjSRCRxxYkEZS8STV7zN84UBbiYu7jy+eCKSnVIUgoWWE/tt+shMQ== - dependencies: - "@jridgewell/gen-mapping" "^0.3.5" - "@jridgewell/trace-mapping" "^0.3.25" - "@jridgewell/sourcemap-codec@1.4.14", "@jridgewell/sourcemap-codec@^1.4.10": version "1.4.14" resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24" @@ -2204,7 +2209,12 @@ resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz#d7c6e6755c78567a951e04ab52ef0fd26de59f32" integrity sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg== -"@jridgewell/trace-mapping@^0.3.12", "@jridgewell/trace-mapping@^0.3.17", "@jridgewell/trace-mapping@^0.3.18": +"@jridgewell/sourcemap-codec@^1.5.0": + version "1.5.5" + resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz#6912b00d2c631c0d15ce1a7ab57cd657f2a8f8ba" + integrity sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og== + +"@jridgewell/trace-mapping@^0.3.12", "@jridgewell/trace-mapping@^0.3.17": version "0.3.20" resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.20.tgz#72e45707cf240fa6b081d0366f8265b0cd10197f" integrity sha512-R8LcPeWZol2zR8mmH3JeKQ6QRCFb7XgUhV9ZlGhHLGyg4wpPiPZNQOOWhFZhxKw8u//yTbNGI42Bx/3paXEQ+Q== @@ -2212,15 +2222,15 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" -"@jridgewell/trace-mapping@^0.3.14", "@jridgewell/trace-mapping@^0.3.9": - version "0.3.17" - resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.17.tgz#793041277af9073b0951a7fe0f0d8c4c98c36985" - integrity sha512-MCNzAp77qzKca9+W/+I0+sEpaUnZoeasnghNeVc41VZCEKaCH73Vq3BZZ/SzWIgrqE4H4ceI+p+b6C0mHf9T4g== +"@jridgewell/trace-mapping@^0.3.23", "@jridgewell/trace-mapping@^0.3.28": + version "0.3.31" + resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz#db15d6781c931f3a251a3dac39501c98a6082fd0" + integrity sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw== dependencies: - "@jridgewell/resolve-uri" "3.1.0" - "@jridgewell/sourcemap-codec" "1.4.14" + "@jridgewell/resolve-uri" "^3.1.0" + "@jridgewell/sourcemap-codec" "^1.4.14" -"@jridgewell/trace-mapping@^0.3.20", "@jridgewell/trace-mapping@^0.3.24", "@jridgewell/trace-mapping@^0.3.25": +"@jridgewell/trace-mapping@^0.3.24", "@jridgewell/trace-mapping@^0.3.25": version "0.3.25" resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz#15f190e98895f3fc23276ee14bc76b675c2e50f0" integrity sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ== @@ -2228,397 +2238,392 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" -"@leichtgewicht/ip-codec@^2.0.1": - version "2.0.4" - resolved "https://registry.yarnpkg.com/@leichtgewicht/ip-codec/-/ip-codec-2.0.4.tgz#b2ac626d6cb9c8718ab459166d4bb405b8ffa78b" - integrity sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A== +"@jridgewell/trace-mapping@^0.3.9": + version "0.3.17" + resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.17.tgz#793041277af9073b0951a7fe0f0d8c4c98c36985" + integrity sha512-MCNzAp77qzKca9+W/+I0+sEpaUnZoeasnghNeVc41VZCEKaCH73Vq3BZZ/SzWIgrqE4H4ceI+p+b6C0mHf9T4g== + dependencies: + "@jridgewell/resolve-uri" "3.1.0" + "@jridgewell/sourcemap-codec" "1.4.14" -"@mui/base@5.0.0-alpha.112": - version "5.0.0-alpha.112" - resolved "https://registry.yarnpkg.com/@mui/base/-/base-5.0.0-alpha.112.tgz#80e815430c5df0316e0a549d34628d54215e05f1" - integrity sha512-KPwb1iYPXsV/P8uu0SNQrj7v7YU6wdN4Eccc2lZQyRDW+f6PJYjHBuFUTYKc408B98Jvs1XbC/z5MN45a2DWrQ== +"@monaco-editor/loader@^1.5.0": + version "1.6.1" + resolved "https://registry.yarnpkg.com/@monaco-editor/loader/-/loader-1.6.1.tgz#c99177d87765abf10de31a0086084e714acfbc0f" + integrity sha512-w3tEnj9HYEC73wtjdpR089AqkUPskFRcdkxsiSFt3SoUc3OHpmu+leP94CXBm4mHfefmhsdfI0ZQu6qJ0wgtPg== dependencies: - "@babel/runtime" "^7.20.7" - "@emotion/is-prop-valid" "^1.2.0" - "@mui/types" "^7.2.3" - "@mui/utils" "^5.11.2" - "@popperjs/core" "^2.11.6" - clsx "^1.2.1" - prop-types "^15.8.1" - react-is "^18.2.0" + state-local "^1.0.6" -"@mui/base@^5.0.0-beta.40": - version "5.0.0-beta.40" - resolved "https://registry.yarnpkg.com/@mui/base/-/base-5.0.0-beta.40.tgz#1f8a782f1fbf3f84a961e954c8176b187de3dae2" - integrity sha512-I/lGHztkCzvwlXpjD2+SNmvNQvB4227xBXhISPjEaJUXGImOQ9f3D2Yj/T3KasSI/h0MLWy74X0J6clhPmsRbQ== +"@monaco-editor/react@4.7.0": + version "4.7.0" + resolved "https://registry.yarnpkg.com/@monaco-editor/react/-/react-4.7.0.tgz#35a1ec01bfe729f38bfc025df7b7bac145602a60" + integrity sha512-cyzXQCtO47ydzxpQtCGSQGOC8Gk3ZUeBXFAxD+CWXYFo5OqZyZUonFl0DwUlTyAfRHntBfw2p3w4s9R6oe1eCA== dependencies: - "@babel/runtime" "^7.23.9" - "@floating-ui/react-dom" "^2.0.8" - "@mui/types" "^7.2.14" - "@mui/utils" "^5.15.14" - "@popperjs/core" "^2.11.8" - clsx "^2.1.0" - prop-types "^15.8.1" + "@monaco-editor/loader" "^1.5.0" -"@mui/core-downloads-tracker@^5.11.2": - version "5.11.2" - resolved "https://registry.yarnpkg.com/@mui/core-downloads-tracker/-/core-downloads-tracker-5.11.2.tgz#0ff8066bee7e9654fd978cb671e9c12c9d2a65b7" - integrity sha512-ztLQELdSSuJFXezng8g5eCzy8mogtzMM8JcfG3HIGgUJ2RlAiBXI2Qe0adKmrJlF4FMat8vTaTeoiRNBZH4t1Q== +"@mui/core-downloads-tracker@^7.3.8": + version "7.3.8" + resolved "https://registry.yarnpkg.com/@mui/core-downloads-tracker/-/core-downloads-tracker-7.3.8.tgz#67949c6b3b4b3ae95dde3a9ddfbb7549f3ca56b1" + integrity sha512-s9UHZo7QJVly7gNArEZkbbsimHqJZhElgBpXIJdehZ4OWXt+CCr0SBDgUCDJnQrqpd1dWK2dLq5rmO4mCBmI3w== -"@mui/icons-material@^5.11.0": - version "5.11.0" - resolved "https://registry.yarnpkg.com/@mui/icons-material/-/icons-material-5.11.0.tgz#9ea6949278b2266d2683866069cd43009eaf6464" - integrity sha512-I2LaOKqO8a0xcLGtIozC9xoXjZAto5G5gh0FYUMAlbsIHNHIjn4Xrw9rvjY20vZonyiGrZNMAlAXYkY6JvhF6A== +"@mui/icons-material@7.3.8": + version "7.3.8" + resolved "https://registry.yarnpkg.com/@mui/icons-material/-/icons-material-7.3.8.tgz#a31d9970be5557b73c83a912f2bbfdacd6319fbc" + integrity sha512-88sWg/UJc1X82OMO+ISR4E3P58I3BjFVg0qkmDu7OWlN8VijneZD3ylFA+ImxuPjMHW3SHosfSJYy1fztoz0fw== dependencies: - "@babel/runtime" "^7.20.6" + "@babel/runtime" "^7.28.6" -"@mui/lab@^5.0.0-alpha.114": - version "5.0.0-alpha.114" - resolved "https://registry.yarnpkg.com/@mui/lab/-/lab-5.0.0-alpha.114.tgz#af4a1893a66bd1b99e8c4f15e51e8db51c094826" - integrity sha512-tChDoLaJ3qcYk37GIwBL1KrCiW0gpmEY//D5z5nHWnO/mzx3axjRJZpBOBeGEvhuoO/Y3QzMz4rhTvqbGNkW0w== +"@mui/lab@7.0.1-beta.22": + version "7.0.1-beta.22" + resolved "https://registry.yarnpkg.com/@mui/lab/-/lab-7.0.1-beta.22.tgz#2c8c81134af99aedcf40553ad2be558b7b8d23b5" + integrity sha512-NHNIg51/CxiJTMTaWCxmqlqYIUfpXbdVjv/OyhSmuAb0z/lIOCMMu9fz0bIbQnZEh6H5L7DhTblynT7xwkXMIQ== dependencies: - "@babel/runtime" "^7.20.7" - "@mui/base" "5.0.0-alpha.112" - "@mui/system" "^5.11.2" - "@mui/types" "^7.2.3" - "@mui/utils" "^5.11.2" - clsx "^1.2.1" + "@babel/runtime" "^7.28.6" + "@mui/system" "^7.3.8" + "@mui/types" "^7.4.11" + "@mui/utils" "^7.3.8" + clsx "^2.1.1" prop-types "^15.8.1" - react-is "^18.2.0" -"@mui/material@^5.11.2": - version "5.11.2" - resolved "https://registry.yarnpkg.com/@mui/material/-/material-5.11.2.tgz#8f0ce18d2cf9da9276f6f19b40a681225098ea09" - integrity sha512-PeraRDsghnDLzejorfe9ps1syxlB8UrGs+UKwg9GGlndv5Tghm+9nwuibrP2TCDC14mlryF+u2WlAOYaPPMwGA== +"@mui/material@7.3.8": + version "7.3.8" + resolved "https://registry.yarnpkg.com/@mui/material/-/material-7.3.8.tgz#230230b5aa6a791558874ecffe9c50ffa0be6817" + integrity sha512-QKd1RhDXE1hf2sQDNayA9ic9jGkEgvZOf0tTkJxlBPG8ns8aS4rS8WwYURw2x5y3739p0HauUXX9WbH7UufFLw== dependencies: - "@babel/runtime" "^7.20.7" - "@mui/base" "5.0.0-alpha.112" - "@mui/core-downloads-tracker" "^5.11.2" - "@mui/system" "^5.11.2" - "@mui/types" "^7.2.3" - "@mui/utils" "^5.11.2" - "@types/react-transition-group" "^4.4.5" - clsx "^1.2.1" - csstype "^3.1.1" + "@babel/runtime" "^7.28.6" + "@mui/core-downloads-tracker" "^7.3.8" + "@mui/system" "^7.3.8" + "@mui/types" "^7.4.11" + "@mui/utils" "^7.3.8" + "@popperjs/core" "^2.11.8" + "@types/react-transition-group" "^4.4.12" + clsx "^2.1.1" + csstype "^3.2.3" prop-types "^15.8.1" - react-is "^18.2.0" + react-is "^19.2.3" react-transition-group "^4.4.5" -"@mui/private-theming@^5.11.2": - version "5.11.2" - resolved "https://registry.yarnpkg.com/@mui/private-theming/-/private-theming-5.11.2.tgz#93eafb317070888a988efa8d6a9ec1f69183a606" - integrity sha512-qZwMaqRFPwlYmqwVKblKBGKtIjJRAj3nsvX93pOmatsXyorW7N/0IPE/swPgz1VwChXhHO75DwBEx8tB+aRMNg== +"@mui/private-theming@^7.3.8": + version "7.3.8" + resolved "https://registry.yarnpkg.com/@mui/private-theming/-/private-theming-7.3.8.tgz#025659d822e92b50586f99bd463dae3857c15feb" + integrity sha512-du5dlPZ9XL3xW2apHoGDXBI+QLtyVJGrXNCfcNYfP/ojkz1RQ0rRV6VG9Rkm1DqEFRG8mjjTL7zmE1Bvn1eR4A== dependencies: - "@babel/runtime" "^7.20.7" - "@mui/utils" "^5.11.2" + "@babel/runtime" "^7.28.6" + "@mui/utils" "^7.3.8" prop-types "^15.8.1" -"@mui/private-theming@^5.15.14": - version "5.15.14" - resolved "https://registry.yarnpkg.com/@mui/private-theming/-/private-theming-5.15.14.tgz#edd9a82948ed01586a01c842eb89f0e3f68970ee" - integrity sha512-UH0EiZckOWcxiXLX3Jbb0K7rC8mxTr9L9l6QhOZxYc4r8FHUkefltV9VDGLrzCaWh30SQiJvAEd7djX3XXY6Xw== +"@mui/styled-engine@^7.3.8": + version "7.3.8" + resolved "https://registry.yarnpkg.com/@mui/styled-engine/-/styled-engine-7.3.8.tgz#0cf761b153061815687488bf4bec5f84ce21e0f9" + integrity sha512-JHAeXQzS0tJ+Fq3C6J4TVDsW+yKhO4uuxuiLaopNStJeQYBIUCXpKYyUCcgXym4AmhbznQnv9RlHywSH6b0FOg== dependencies: - "@babel/runtime" "^7.23.9" - "@mui/utils" "^5.15.14" + "@babel/runtime" "^7.28.6" + "@emotion/cache" "^11.14.0" + "@emotion/serialize" "^1.3.3" + "@emotion/sheet" "^1.4.0" + csstype "^3.2.3" prop-types "^15.8.1" -"@mui/styled-engine@^5.11.0": - version "5.11.0" - resolved "https://registry.yarnpkg.com/@mui/styled-engine/-/styled-engine-5.11.0.tgz#79afb30c612c7807c4b77602cf258526d3997c7b" - integrity sha512-AF06K60Zc58qf0f7X+Y/QjaHaZq16znliLnGc9iVrV/+s8Ln/FCoeNuFvhlCbZZQ5WQcJvcy59zp0nXrklGGPQ== - dependencies: - "@babel/runtime" "^7.20.6" - "@emotion/cache" "^11.10.5" - csstype "^3.1.1" +"@mui/system@7.3.8", "@mui/system@^7.3.8": + version "7.3.8" + resolved "https://registry.yarnpkg.com/@mui/system/-/system-7.3.8.tgz#315ee3f4285f8611f32851d3699a5d5e5ef0b818" + integrity sha512-hoFRj4Zw2Km8DPWZp/nKG+ao5Jw5LSk2m/e4EGc6M3RRwXKEkMSG4TgtfVJg7dS2homRwtdXSMW+iRO0ZJ4+IA== + dependencies: + "@babel/runtime" "^7.28.6" + "@mui/private-theming" "^7.3.8" + "@mui/styled-engine" "^7.3.8" + "@mui/types" "^7.4.11" + "@mui/utils" "^7.3.8" + clsx "^2.1.1" + csstype "^3.2.3" prop-types "^15.8.1" -"@mui/styled-engine@^5.15.14": - version "5.15.14" - resolved "https://registry.yarnpkg.com/@mui/styled-engine/-/styled-engine-5.15.14.tgz#168b154c4327fa4ccc1933a498331d53f61c0de2" - integrity sha512-RILkuVD8gY6PvjZjqnWhz8fu68dVkqhM5+jYWfB5yhlSQKg+2rHkmEwm75XIeAqI3qwOndK6zELK5H6Zxn4NHw== +"@mui/types@^7.4.11": + version "7.4.11" + resolved "https://registry.yarnpkg.com/@mui/types/-/types-7.4.11.tgz#d9c8e028a354a52fe86cb8ffbc306faf4c090608" + integrity sha512-fZ2xO9D08IKOxO2oUBi1nnVKH6oJUD+64cnv4YAaFoC0E5+i1+S5AHbNqqvZlYYsbPEQ6qEVwuBqY3jl5W4G+Q== dependencies: - "@babel/runtime" "^7.23.9" - "@emotion/cache" "^11.11.0" - csstype "^3.1.3" - prop-types "^15.8.1" + "@babel/runtime" "^7.28.6" -"@mui/system@^5.11.2": - version "5.11.2" - resolved "https://registry.yarnpkg.com/@mui/system/-/system-5.11.2.tgz#a5a5865dda0f5f360eed8cdc1ab399a493dbd361" - integrity sha512-PPkYhrcP2MkhscX6SauIl0wPgra0w1LGPtll+hIKc2Z2JbGRSrUCFif93kxejB7I1cAoCay9jWW4mnNhsOqF/g== +"@mui/types@^7.4.8": + version "7.4.8" + resolved "https://registry.yarnpkg.com/@mui/types/-/types-7.4.8.tgz#0c1829353cd7d196be9ac0332a30cdd2792f3558" + integrity sha512-ZNXLBjkPV6ftLCmmRCafak3XmSn8YV0tKE/ZOhzKys7TZXUiE0mZxlH8zKDo6j6TTUaDnuij68gIG+0Ucm7Xhw== dependencies: - "@babel/runtime" "^7.20.7" - "@mui/private-theming" "^5.11.2" - "@mui/styled-engine" "^5.11.0" - "@mui/types" "^7.2.3" - "@mui/utils" "^5.11.2" - clsx "^1.2.1" - csstype "^3.1.1" - prop-types "^15.8.1" - -"@mui/system@^5.15.14": - version "5.15.15" - resolved "https://registry.yarnpkg.com/@mui/system/-/system-5.15.15.tgz#658771b200ce3c4a0f28e58169f02e5e718d1c53" - integrity sha512-aulox6N1dnu5PABsfxVGOZffDVmlxPOVgj56HrUnJE8MCSh8lOvvkd47cebIVQQYAjpwieXQXiDPj5pwM40jTQ== - dependencies: - "@babel/runtime" "^7.23.9" - "@mui/private-theming" "^5.15.14" - "@mui/styled-engine" "^5.15.14" - "@mui/types" "^7.2.14" - "@mui/utils" "^5.15.14" - clsx "^2.1.0" - csstype "^3.1.3" - prop-types "^15.8.1" - -"@mui/types@^7.2.14": - version "7.2.14" - resolved "https://registry.yarnpkg.com/@mui/types/-/types-7.2.14.tgz#8a02ac129b70f3d82f2f9b76ded2c8d48e3fc8c9" - integrity sha512-MZsBZ4q4HfzBsywtXgM1Ksj6HDThtiwmOKUXH1pKYISI9gAVXCNHNpo7TlGoGrBaYWZTdNoirIN7JsQcQUjmQQ== - -"@mui/types@^7.2.3": - version "7.2.3" - resolved "https://registry.yarnpkg.com/@mui/types/-/types-7.2.3.tgz#06faae1c0e2f3a31c86af6f28b3a4a42143670b9" - integrity sha512-tZ+CQggbe9Ol7e/Fs5RcKwg/woU+o8DCtOnccX6KmbBc7YrfqMYEYuaIcXHuhpT880QwNkZZ3wQwvtlDFA2yOw== + "@babel/runtime" "^7.28.4" -"@mui/utils@^5.10.3", "@mui/utils@^5.11.2": - version "5.11.2" - resolved "https://registry.yarnpkg.com/@mui/utils/-/utils-5.11.2.tgz#29764311acb99425159b159b1cb382153ad9be1f" - integrity sha512-AyizuHHlGdAtH5hOOXBW3kriuIwUIKUIgg0P7LzMvzf6jPhoQbENYqY6zJqfoZ7fAWMNNYT8mgN5EftNGzwE2w== +"@mui/utils@^7.3.5": + version "7.3.5" + resolved "https://registry.yarnpkg.com/@mui/utils/-/utils-7.3.5.tgz#77f3e2b83454bbd47877c73b04cd804f490bf028" + integrity sha512-jisvFsEC3sgjUjcPnR4mYfhzjCDIudttSGSbe1o/IXFNu0kZuR+7vqQI0jg8qtcVZBHWrwTfvAZj9MNMumcq1g== dependencies: - "@babel/runtime" "^7.20.7" - "@types/prop-types" "^15.7.5" - "@types/react-is" "^16.7.1 || ^17.0.0" + "@babel/runtime" "^7.28.4" + "@mui/types" "^7.4.8" + "@types/prop-types" "^15.7.15" + clsx "^2.1.1" prop-types "^15.8.1" - react-is "^18.2.0" + react-is "^19.2.0" -"@mui/utils@^5.15.14": - version "5.15.14" - resolved "https://registry.yarnpkg.com/@mui/utils/-/utils-5.15.14.tgz#e414d7efd5db00bfdc875273a40c0a89112ade3a" - integrity sha512-0lF/7Hh/ezDv5X7Pry6enMsbYyGKjADzvHyo3Qrc/SSlTsQ1VkbDMbH0m2t3OR5iIVLwMoxwM7yGd+6FCMtTFA== +"@mui/utils@^7.3.8": + version "7.3.8" + resolved "https://registry.yarnpkg.com/@mui/utils/-/utils-7.3.8.tgz#0d6dff4c623682b603f27d6ff37839b3d16832de" + integrity sha512-kZRcE2620CBGr+XI8YMmwPj6WIPwSF7uMJjvSfqd8zXVvlz0MCJbzRRUGNf8NgflCLthdji2DdS643TeyJ3+nA== dependencies: - "@babel/runtime" "^7.23.9" - "@types/prop-types" "^15.7.11" + "@babel/runtime" "^7.28.6" + "@mui/types" "^7.4.11" + "@types/prop-types" "^15.7.15" + clsx "^2.1.1" prop-types "^15.8.1" - react-is "^18.2.0" - -"@mui/x-charts@^7.3.2": - version "7.3.2" - resolved "https://registry.yarnpkg.com/@mui/x-charts/-/x-charts-7.3.2.tgz#90d711dfbb70cfbbb0e207f10e8532f3420774ed" - integrity sha512-620eMe83R0RDZk0S7ExYFRM2yZLlEOyIZmP5zwwVujZg54OC1fwLsk0tiaFaVZQOBKmeFOtyltDNPxr9e6itJQ== - dependencies: - "@babel/runtime" "^7.24.0" - "@mui/base" "^5.0.0-beta.40" - "@mui/system" "^5.15.14" - "@mui/utils" "^5.15.14" - "@react-spring/rafz" "^9.7.3" - "@react-spring/web" "^9.7.3" - clsx "^2.1.0" + react-is "^19.2.3" + +"@mui/x-charts-vendor@8.26.0": + version "8.26.0" + resolved "https://registry.yarnpkg.com/@mui/x-charts-vendor/-/x-charts-vendor-8.26.0.tgz#58ab81deeb330b832acc535732dd03eb0e6d7d05" + integrity sha512-R//+WSWvsLJRTjTRN90EKX9sgRzAb4HQBvtUA3cTQpkGrmEjmatD4BJAm3IdRdkSagf6yKWF+ypESctyRhbwnA== + dependencies: + "@babel/runtime" "^7.28.4" + "@types/d3-array" "^3.2.2" + "@types/d3-color" "^3.1.3" + "@types/d3-format" "^3.0.4" + "@types/d3-interpolate" "^3.0.4" + "@types/d3-path" "^3.1.1" + "@types/d3-scale" "^4.0.9" + "@types/d3-shape" "^3.1.7" + "@types/d3-time" "^3.0.4" + "@types/d3-time-format" "^4.0.3" + "@types/d3-timer" "^3.0.2" + d3-array "^3.2.4" d3-color "^3.1.0" - d3-delaunay "^6.0.4" + d3-format "^3.1.0" d3-interpolate "^3.0.1" + d3-path "^3.1.0" d3-scale "^4.0.2" d3-shape "^3.2.0" + d3-time "^3.1.0" + d3-time-format "^4.1.0" + d3-timer "^3.0.1" + flatqueue "^3.0.0" + internmap "^2.0.3" + +"@mui/x-charts@8.27.0": + version "8.27.0" + resolved "https://registry.yarnpkg.com/@mui/x-charts/-/x-charts-8.27.0.tgz#8f769c855414cf0ad4151de3fe553fa0c9eaecdd" + integrity sha512-MzP1jeiEkMOPWQfzRNo11iwbTwXcSDP7hKd3s/mSN0U4aINKpyHEQX6RAM8OkWzqK8ijTXYWRPVKaRYTLBx7+A== + dependencies: + "@babel/runtime" "^7.28.4" + "@mui/utils" "^7.3.5" + "@mui/x-charts-vendor" "8.26.0" + "@mui/x-internal-gestures" "0.4.0" + "@mui/x-internals" "8.26.0" + bezier-easing "^2.1.0" + clsx "^2.1.1" + prop-types "^15.8.1" + reselect "^5.1.1" + use-sync-external-store "^1.6.0" + +"@mui/x-date-pickers@8.27.2": + version "8.27.2" + resolved "https://registry.yarnpkg.com/@mui/x-date-pickers/-/x-date-pickers-8.27.2.tgz#5ada1fb3adffff3e0fd0fee7702fba7f770dca68" + integrity sha512-06LFkHFRXJ2O9DMXtWAA3kY0jpbL7XH8iqa8L5cBlN+8bRx/UVLKlZYlhGv06C88jF9kuZWY1bUgrv/EoY/2Ww== + dependencies: + "@babel/runtime" "^7.28.4" + "@mui/utils" "^7.3.5" + "@mui/x-internals" "8.26.0" + "@types/react-transition-group" "^4.4.12" + clsx "^2.1.1" prop-types "^15.8.1" - -"@mui/x-date-pickers@^5.0.12": - version "5.0.12" - resolved "https://registry.yarnpkg.com/@mui/x-date-pickers/-/x-date-pickers-5.0.12.tgz#f80e742aa8045b214f40a72e782fd871fb18ec89" - integrity sha512-HpiPE4nkqlklilKAxUQpf3tecJEDaTIBRRVTGk9GMgpkwli7Cv+5OGn6aSxG5ckMZdiQ/NT1L9kOcm0Bi9Gqjw== - dependencies: - "@babel/runtime" "^7.18.9" - "@date-io/core" "^2.15.0" - "@date-io/date-fns" "^2.15.0" - "@date-io/dayjs" "^2.15.0" - "@date-io/luxon" "^2.15.0" - "@date-io/moment" "^2.15.0" - "@mui/utils" "^5.10.3" - "@types/react-transition-group" "^4.4.5" - clsx "^1.2.1" - prop-types "^15.7.2" react-transition-group "^4.4.5" - rifm "^0.12.1" - -"@nicolo-ribaudo/eslint-scope-5-internals@5.1.1-v1": - version "5.1.1-v1" - resolved "https://registry.yarnpkg.com/@nicolo-ribaudo/eslint-scope-5-internals/-/eslint-scope-5-internals-5.1.1-v1.tgz#dbf733a965ca47b1973177dc0bb6c889edcfb129" - integrity sha512-54/JRvkLIzzDWshCWfuhadfrfZVPiElY8Fcgmg1HroEly/EDSszzhBAsarCux+D/kOslTRquNzuyGSmUSTTHGg== - dependencies: - eslint-scope "5.1.1" - -"@nodelib/fs.scandir@2.1.5": - version "2.1.5" - resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5" - integrity sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g== - dependencies: - "@nodelib/fs.stat" "2.0.5" - run-parallel "^1.1.9" - -"@nodelib/fs.stat@2.0.5", "@nodelib/fs.stat@^2.0.2": - version "2.0.5" - resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz#5bd262af94e9d25bd1e71b05deed44876a222e8b" - integrity sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A== - -"@nodelib/fs.walk@^1.2.3", "@nodelib/fs.walk@^1.2.8": - version "1.2.8" - resolved "https://registry.yarnpkg.com/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz#e95737e8bb6746ddedf69c556953494f196fe69a" - integrity sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg== - dependencies: - "@nodelib/fs.scandir" "2.1.5" - fastq "^1.6.0" - -"@pmmmwh/react-refresh-webpack-plugin@^0.5.3": - version "0.5.10" - resolved "https://registry.yarnpkg.com/@pmmmwh/react-refresh-webpack-plugin/-/react-refresh-webpack-plugin-0.5.10.tgz#2eba163b8e7dbabb4ce3609ab5e32ab63dda3ef8" - integrity sha512-j0Ya0hCFZPd4x40qLzbhGsh9TMtdb+CJQiso+WxLOPNasohq9cc5SNUcwsZaRH6++Xh91Xkm/xHCkuIiIu0LUA== - dependencies: - ansi-html-community "^0.0.8" - common-path-prefix "^3.0.0" - core-js-pure "^3.23.3" - error-stack-parser "^2.0.6" - find-up "^5.0.0" - html-entities "^2.1.0" - loader-utils "^2.0.4" - schema-utils "^3.0.0" - source-map "^0.7.3" - -"@popperjs/core@^2.11.6": - version "2.11.6" - resolved "https://registry.yarnpkg.com/@popperjs/core/-/core-2.11.6.tgz#cee20bd55e68a1720bdab363ecf0c821ded4cd45" - integrity sha512-50/17A98tWUfQ176raKiOGXuYpLyyVMkxxG6oylzL3BPOlA6ADGdK7EYunSa4I064xerltq9TGXs8HmOk5E+vw== - -"@popperjs/core@^2.11.8": - version "2.11.8" - resolved "https://registry.yarnpkg.com/@popperjs/core/-/core-2.11.8.tgz#6b79032e760a0899cd4204710beede972a3a185f" - integrity sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A== - -"@react-spring/animated@~9.7.3": - version "9.7.3" - resolved "https://registry.yarnpkg.com/@react-spring/animated/-/animated-9.7.3.tgz#4211b1a6d48da0ff474a125e93c0f460ff816e0f" - integrity sha512-5CWeNJt9pNgyvuSzQH+uy2pvTg8Y4/OisoscZIR8/ZNLIOI+CatFBhGZpDGTF/OzdNFsAoGk3wiUYTwoJ0YIvw== - dependencies: - "@react-spring/shared" "~9.7.3" - "@react-spring/types" "~9.7.3" -"@react-spring/core@~9.7.3": - version "9.7.3" - resolved "https://registry.yarnpkg.com/@react-spring/core/-/core-9.7.3.tgz#60056bcb397f2c4f371c6c9a5f882db77ae90095" - integrity sha512-IqFdPVf3ZOC1Cx7+M0cXf4odNLxDC+n7IN3MDcVCTIOSBfqEcBebSv+vlY5AhM0zw05PDbjKrNmBpzv/AqpjnQ== +"@mui/x-internal-gestures@0.4.0": + version "0.4.0" + resolved "https://registry.yarnpkg.com/@mui/x-internal-gestures/-/x-internal-gestures-0.4.0.tgz#68d0d23ab794de186e02e6f29116d1fe1d9211c8" + integrity sha512-i0W6v9LoiNY8Yf1goOmaygtz/ncPJGBedhpDfvNg/i8BvzPwJcBaeW4rqPucJfVag9KQ8MSssBBrvYeEnrQmhw== dependencies: - "@react-spring/animated" "~9.7.3" - "@react-spring/shared" "~9.7.3" - "@react-spring/types" "~9.7.3" + "@babel/runtime" "^7.28.4" -"@react-spring/rafz@^9.7.3": - version "9.7.3" - resolved "https://registry.yarnpkg.com/@react-spring/rafz/-/rafz-9.7.3.tgz#d6a9695c581f58a49e047ff7ed5646e0b681396c" - integrity sha512-9vzW1zJPcC4nS3aCV+GgcsK/WLaB520Iyvm55ARHfM5AuyBqycjvh1wbmWmgCyJuX4VPoWigzemq1CaaeRSHhQ== - -"@react-spring/shared@~9.7.3": - version "9.7.3" - resolved "https://registry.yarnpkg.com/@react-spring/shared/-/shared-9.7.3.tgz#4cf29797847c689912aec4e62e34c99a4d5d9e53" - integrity sha512-NEopD+9S5xYyQ0pGtioacLhL2luflh6HACSSDUZOwLHoxA5eku1UPuqcJqjwSD6luKjjLfiLOspxo43FUHKKSA== +"@mui/x-internals@8.26.0": + version "8.26.0" + resolved "https://registry.yarnpkg.com/@mui/x-internals/-/x-internals-8.26.0.tgz#49caacac954c29a1b10425c67418310ceb9c8cfa" + integrity sha512-B9OZau5IQUvIxwpJZhoFJKqRpmWf5r0yMmSXjQuqb5WuqM755EuzWJOenY48denGoENzMLT8hQpA0hRTeU2IPA== dependencies: - "@react-spring/types" "~9.7.3" - -"@react-spring/types@~9.7.3": - version "9.7.3" - resolved "https://registry.yarnpkg.com/@react-spring/types/-/types-9.7.3.tgz#ea78fd447cbc2612c1f5d55852e3c331e8172a0b" - integrity sha512-Kpx/fQ/ZFX31OtlqVEFfgaD1ACzul4NksrvIgYfIFq9JpDHFwQkMVZ10tbo0FU/grje4rcL4EIrjekl3kYwgWw== + "@babel/runtime" "^7.28.4" + "@mui/utils" "^7.3.5" + reselect "^5.1.1" + use-sync-external-store "^1.6.0" -"@react-spring/web@^9.7.3": - version "9.7.3" - resolved "https://registry.yarnpkg.com/@react-spring/web/-/web-9.7.3.tgz#d9f4e17fec259f1d65495a19502ada4f5b57fa3d" - integrity sha512-BXt6BpS9aJL/QdVqEIX9YoUy8CE6TJrU0mNCqSoxdXlIeNcEBWOfIyE6B14ENNsyQKS3wOWkiJfco0tCr/9tUg== +"@napi-rs/wasm-runtime@^0.2.11": + version "0.2.12" + resolved "https://registry.yarnpkg.com/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.12.tgz#3e78a8b96e6c33a6c517e1894efbd5385a7cb6f2" + integrity sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ== dependencies: - "@react-spring/animated" "~9.7.3" - "@react-spring/core" "~9.7.3" - "@react-spring/shared" "~9.7.3" - "@react-spring/types" "~9.7.3" - -"@remix-run/router@1.2.1": - version "1.2.1" - resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.2.1.tgz#812edd4104a15a493dda1ccac0b352270d7a188c" - integrity sha512-XiY0IsyHR+DXYS5vBxpoBe/8veTeoRpMHP+vDosLZxL5bnpetzI0igkxkLZS235ldLzyfkxF+2divEwWHP3vMQ== - -"@rollup/plugin-babel@^5.2.0": - version "5.3.1" - resolved "https://registry.yarnpkg.com/@rollup/plugin-babel/-/plugin-babel-5.3.1.tgz#04bc0608f4aa4b2e4b1aebf284344d0f68fda283" - integrity sha512-WFfdLWU/xVWKeRQnKmIAQULUI7Il0gZnBIH/ZFO069wYIfPu+8zrfp/KMW0atmELoRDq8FbiP3VCss9MhCut7Q== - dependencies: - "@babel/helper-module-imports" "^7.10.4" - "@rollup/pluginutils" "^3.1.0" - -"@rollup/plugin-node-resolve@^11.2.1": - version "11.2.1" - resolved "https://registry.yarnpkg.com/@rollup/plugin-node-resolve/-/plugin-node-resolve-11.2.1.tgz#82aa59397a29cd4e13248b106e6a4a1880362a60" - integrity sha512-yc2n43jcqVyGE2sqV5/YCmocy9ArjVAP/BeXyTtADTBBX6V0e5UMqwO8CdQ0kzjb6zu5P1qMzsScCMRvE9OlVg== - dependencies: - "@rollup/pluginutils" "^3.1.0" - "@types/resolve" "1.17.1" - builtin-modules "^3.1.0" - deepmerge "^4.2.2" - is-module "^1.0.0" - resolve "^1.19.0" + "@emnapi/core" "^1.4.3" + "@emnapi/runtime" "^1.4.3" + "@tybys/wasm-util" "^0.10.0" -"@rollup/plugin-replace@^2.4.1": - version "2.4.2" - resolved "https://registry.yarnpkg.com/@rollup/plugin-replace/-/plugin-replace-2.4.2.tgz#a2d539314fbc77c244858faa523012825068510a" - integrity sha512-IGcu+cydlUMZ5En85jxHH4qj2hta/11BHq95iHEyb2sbgiN0eCdzvUcHw5gt9pBL5lTi4JDYJ1acCoMGpTvEZg== - dependencies: - "@rollup/pluginutils" "^3.1.0" - magic-string "^0.25.7" +"@pkgr/core@^0.2.9": + version "0.2.9" + resolved "https://registry.yarnpkg.com/@pkgr/core/-/core-0.2.9.tgz#d229a7b7f9dac167a156992ef23c7f023653f53b" + integrity sha512-QNqXyfVS2wm9hweSYD2O7F0G06uurj9kZ96TRQE5Y9hU7+tgdZwIkbAKc5Ocy1HxEY2kuDQa6cQ1WRs/O5LFKA== -"@rollup/pluginutils@^3.1.0": - version "3.1.0" - resolved "https://registry.yarnpkg.com/@rollup/pluginutils/-/pluginutils-3.1.0.tgz#706b4524ee6dc8b103b3c995533e5ad680c02b9b" - integrity sha512-GksZ6pr6TpIjHm8h9lSQ8pi8BE9VeubNT0OMJ3B5uZJ8pz73NPiqOtCog/x2/QzM1ENChPKxMDhiQuRHsqc+lg== +"@playwright/test@^1.58.2": + version "1.58.2" + resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.58.2.tgz#b0ad585d2e950d690ef52424967a42f40c6d2cbd" + integrity sha512-akea+6bHYBBfA9uQqSYmlJXn61cTa+jbO87xVLCWbTqbWadRVmhxlXATaOjOgcBaWU4ePo0wB41KMFv3o35IXA== dependencies: - "@types/estree" "0.0.39" - estree-walker "^1.0.1" - picomatch "^2.2.2" + playwright "1.58.2" -"@rushstack/eslint-patch@^1.1.0": - version "1.2.0" - resolved "https://registry.yarnpkg.com/@rushstack/eslint-patch/-/eslint-patch-1.2.0.tgz#8be36a1f66f3265389e90b5f9c9962146758f728" - integrity sha512-sXo/qW2/pAcmT43VoRKOJbDOfV3cYpq3szSVfIThQXNt+E4DfKj361vaAt3c88U5tPUxzEswam7GW48PJqtKAg== - -"@sinclair/typebox@^0.24.1": - version "0.24.51" - resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.24.51.tgz#645f33fe4e02defe26f2f5c0410e1c094eac7f5f" - integrity sha512-1P1OROm/rdubP5aFDSZQILU0vrLCJ4fvHt6EoqHEM+2D/G5MK3bIaymUKLit8Js9gbns5UyJnkP/TZROLw4tUA== +"@popperjs/core@^2.11.8": + version "2.11.8" + resolved "https://registry.yarnpkg.com/@popperjs/core/-/core-2.11.8.tgz#6b79032e760a0899cd4204710beede972a3a185f" + integrity sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A== -"@sinclair/typebox@^0.27.8": - version "0.27.8" - resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.27.8.tgz#6667fac16c436b5434a387a34dedb013198f6e6e" - integrity sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA== +"@rollup/rollup-android-arm-eabi@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.60.0.tgz#7e158ddfc16f78da99c0d5ccbae6cae403ef3284" + integrity sha512-WOhNW9K8bR3kf4zLxbfg6Pxu2ybOUbB2AjMDHSQx86LIF4rH4Ft7vmMwNt0loO0eonglSNy4cpD3MKXXKQu0/A== + +"@rollup/rollup-android-arm64@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.60.0.tgz#49f4ae0e22b6f9ffbcd3818b9a0758fa2d10b1cd" + integrity sha512-u6JHLll5QKRvjciE78bQXDmqRqNs5M/3GVqZeMwvmjaNODJih/WIrJlFVEihvV0MiYFmd+ZyPr9wxOVbPAG2Iw== + +"@rollup/rollup-darwin-arm64@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.60.0.tgz#bb200269069acf5c1c4d79ad142524f77e8b8236" + integrity sha512-qEF7CsKKzSRc20Ciu2Zw1wRrBz4g56F7r/vRwY430UPp/nt1x21Q/fpJ9N5l47WWvJlkNCPJz3QRVw008fi7yA== + +"@rollup/rollup-darwin-x64@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.60.0.tgz#1bf7a92b27ebdd5e0d1d48503c7811160773be1a" + integrity sha512-WADYozJ4QCnXCH4wPB+3FuGmDPoFseVCUrANmA5LWwGmC6FL14BWC7pcq+FstOZv3baGX65tZ378uT6WG8ynTw== + +"@rollup/rollup-freebsd-arm64@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.60.0.tgz#5ccf537b99c5175008444702193ad0b1c36f7f16" + integrity sha512-6b8wGHJlDrGeSE3aH5mGNHBjA0TTkxdoNHik5EkvPHCt351XnigA4pS7Wsj/Eo9Y8RBU6f35cjN9SYmCFBtzxw== + +"@rollup/rollup-freebsd-x64@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.60.0.tgz#1196ecd7bf4e128624ef83cd1f9d785114474a77" + integrity sha512-h25Ga0t4jaylMB8M/JKAyrvvfxGRjnPQIR8lnCayyzEjEOx2EJIlIiMbhpWxDRKGKF8jbNH01NnN663dH638mA== + +"@rollup/rollup-linux-arm-gnueabihf@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.60.0.tgz#cc147633a4af229fee83a737bf2334fbac3dc28e" + integrity sha512-RzeBwv0B3qtVBWtcuABtSuCzToo2IEAIQrcyB/b2zMvBWVbjo8bZDjACUpnaafaxhTw2W+imQbP2BD1usasK4g== + +"@rollup/rollup-linux-arm-musleabihf@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.60.0.tgz#3559f9f060153ea54594a42c3b87a297bedcc26e" + integrity sha512-Sf7zusNI2CIU1HLzuu9Tc5YGAHEZs5Lu7N1ssJG4Tkw6e0MEsN7NdjUDDfGNHy2IU+ENyWT+L2obgWiguWibWQ== + +"@rollup/rollup-linux-arm64-gnu@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.60.0.tgz#e91f887b154123485cfc4b59befe2080fcd8f2df" + integrity sha512-DX2x7CMcrJzsE91q7/O02IJQ5/aLkVtYFryqCjduJhUfGKG6yJV8hxaw8pZa93lLEpPTP/ohdN4wFz7yp/ry9A== + +"@rollup/rollup-linux-arm64-musl@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.60.0.tgz#660752f040df9ba44a24765df698928917c0bf21" + integrity sha512-09EL+yFVbJZlhcQfShpswwRZ0Rg+z/CsSELFCnPt3iK+iqwGsI4zht3secj5vLEs957QvFFXnzAT0FFPIxSrkQ== + +"@rollup/rollup-linux-loong64-gnu@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.60.0.tgz#cb0e939a5fa479ccef264f3f45b31971695f869c" + integrity sha512-i9IcCMPr3EXm8EQg5jnja0Zyc1iFxJjZWlb4wr7U2Wx/GrddOuEafxRdMPRYVaXjgbhvqalp6np07hN1w9kAKw== + +"@rollup/rollup-linux-loong64-musl@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.60.0.tgz#42f86fbc82cd1a81be2d346476dd3231cf5ee442" + integrity sha512-DGzdJK9kyJ+B78MCkWeGnpXJ91tK/iKA6HwHxF4TAlPIY7GXEvMe8hBFRgdrR9Ly4qebR/7gfUs9y2IoaVEyog== + +"@rollup/rollup-linux-ppc64-gnu@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.60.0.tgz#39776a647a789dc95ea049277c5ef8f098df77f9" + integrity sha512-RwpnLsqC8qbS8z1H1AxBA1H6qknR4YpPR9w2XX0vo2Sz10miu57PkNcnHVaZkbqyw/kUWfKMI73jhmfi9BRMUQ== + +"@rollup/rollup-linux-ppc64-musl@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.60.0.tgz#466f20029a8e8b3bb2954c7ddebc9586420cac2c" + integrity sha512-Z8pPf54Ly3aqtdWC3G4rFigZgNvd+qJlOE52fmko3KST9SoGfAdSRCwyoyG05q1HrrAblLbk1/PSIV+80/pxLg== + +"@rollup/rollup-linux-riscv64-gnu@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.60.0.tgz#cff9877c78f12e7aa6246f6902ad913e99edb2b7" + integrity sha512-3a3qQustp3COCGvnP4SvrMHnPQ9d1vzCakQVRTliaz8cIp/wULGjiGpbcqrkv0WrHTEp8bQD/B3HBjzujVWLOA== + +"@rollup/rollup-linux-riscv64-musl@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.60.0.tgz#9a762fb99b5a82a921017f56491b7e892b9fb17d" + integrity sha512-pjZDsVH/1VsghMJ2/kAaxt6dL0psT6ZexQVrijczOf+PeP2BUqTHYejk3l6TlPRydggINOeNRhvpLa0AYpCWSQ== + +"@rollup/rollup-linux-s390x-gnu@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.60.0.tgz#9d25ad8ac7dab681935baf78ac5ea92d14629cdf" + integrity sha512-3ObQs0BhvPgiUVZrN7gqCSvmFuMWvWvsjG5ayJ3Lraqv+2KhOsp+pUbigqbeWqueGIsnn+09HBw27rJ+gYK4VQ== + +"@rollup/rollup-linux-x64-gnu@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.60.0.tgz#5e5139e11819fa38a052368da79422cb4afcf466" + integrity sha512-EtylprDtQPdS5rXvAayrNDYoJhIz1/vzN2fEubo3yLE7tfAw+948dO0g4M0vkTVFhKojnF+n6C8bDNe+gDRdTg== + +"@rollup/rollup-linux-x64-musl@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.60.0.tgz#b6211d46e11b1f945f5504cc794fce839331ed08" + integrity sha512-k09oiRCi/bHU9UVFqD17r3eJR9bn03TyKraCrlz5ULFJGdJGi7VOmm9jl44vOJvRJ6P7WuBi/s2A97LxxHGIdw== + +"@rollup/rollup-openbsd-x64@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.60.0.tgz#e6e09eebaa7012bb9c7331b437a9e992bd94ca35" + integrity sha512-1o/0/pIhozoSaDJoDcec+IVLbnRtQmHwPV730+AOD29lHEEo4F5BEUB24H0OBdhbBBDwIOSuf7vgg0Ywxdfiiw== + +"@rollup/rollup-openharmony-arm64@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.60.0.tgz#f7d99ae857032498e57a5e7259fb7100fd24a87e" + integrity sha512-pESDkos/PDzYwtyzB5p/UoNU/8fJo68vcXM9ZW2V0kjYayj1KaaUfi1NmTUTUpMn4UhU4gTuK8gIaFO4UGuMbA== + +"@rollup/rollup-win32-arm64-msvc@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.60.0.tgz#41e392f5d9f3bf1253fdaf2f6d6f6b1bfc452856" + integrity sha512-hj1wFStD7B1YBeYmvY+lWXZ7ey73YGPcViMShYikqKT1GtstIKQAtfUI6yrzPjAy/O7pO0VLXGmUVWXQMaYgTQ== + +"@rollup/rollup-win32-ia32-msvc@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.60.0.tgz#f41b0490be0e5d3cf459b4dc076a192b532adea9" + integrity sha512-SyaIPFoxmUPlNDq5EHkTbiKzmSEmq/gOYFI/3HHJ8iS/v1mbugVa7dXUzcJGQfoytp9DJFLhHH4U3/eTy2Bq4w== + +"@rollup/rollup-win32-x64-gnu@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.60.0.tgz#0fcf9f1fcb750f0317b13aac3b3231687e6397a5" + integrity sha512-RdcryEfzZr+lAr5kRm2ucN9aVlCCa2QNq4hXelZxb8GG0NJSazq44Z3PCCc8wISRuCVnGs0lQJVX5Vp6fKA+IA== + +"@rollup/rollup-win32-x64-msvc@4.60.0": + version "4.60.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.60.0.tgz#3afdb30405f6d4248df5e72e1ca86c5eab55fab8" + integrity sha512-PrsWNQ8BuE00O3Xsx3ALh2Df8fAj9+cvvX9AIA6o4KpATR98c9mud4XtDWVvsEuyia5U4tVSTKygawyJkjm60w== + +"@scarf/scarf@=1.4.0": + version "1.4.0" + resolved "https://registry.yarnpkg.com/@scarf/scarf/-/scarf-1.4.0.tgz#3bbb984085dbd6d982494538b523be1ce6562972" + integrity sha512-xxeapPiUXdZAE3che6f3xogoJPeZgig6omHEy1rIY5WVsB3H2BHNnZH+gHG6x91SCWyQCzWGsuL2Hh3ClO5/qQ== -"@sinonjs/commons@^1.7.0": - version "1.8.6" - resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-1.8.6.tgz#80c516a4dc264c2a69115e7578d62581ff455ed9" - integrity sha512-Ky+XkAkqPZSm3NLBeUng77EBQl3cmeJhITaGHdYH8kjVB+aun3S4XBRti2zt17mtt0mIUDiNxYeoJm6drVvBJQ== - dependencies: - type-detect "4.0.8" +"@sinclair/typebox@^0.34.0": + version "0.34.41" + resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.34.41.tgz#aa51a6c1946df2c5a11494a2cdb9318e026db16c" + integrity sha512-6gS8pZzSXdyRHTIqoqSVknxolr1kzfy4/CeDnrzsVz8TTIWUbOBr6gnzOmTYJ3eXQNh4IYHIGi5aIL7sOZ2G/g== -"@sinonjs/commons@^3.0.0": - version "3.0.0" - resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-3.0.0.tgz#beb434fe875d965265e04722ccfc21df7f755d72" - integrity sha512-jXBtWAF4vmdNmZgD5FoKsVLv3rPgDnLgPbU84LIJ3otV44vJlDRokVng5v8NFJdCf/da9legHcKaRuZs4L7faA== +"@sinonjs/commons@^3.0.1": + version "3.0.1" + resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-3.0.1.tgz#1029357e44ca901a615585f6d27738dbc89084cd" + integrity sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ== dependencies: type-detect "4.0.8" -"@sinonjs/fake-timers@^10.0.2": - version "10.3.0" - resolved "https://registry.yarnpkg.com/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz#55fdff1ecab9f354019129daf4df0dd4d923ea66" - integrity sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA== - dependencies: - "@sinonjs/commons" "^3.0.0" - -"@sinonjs/fake-timers@^8.0.1": - version "8.1.0" - resolved "https://registry.yarnpkg.com/@sinonjs/fake-timers/-/fake-timers-8.1.0.tgz#3fdc2b6cb58935b21bfb8d1625eb1300484316e7" - integrity sha512-OAPJUAtgeINhh/TAlUID4QTs53Njm7xzddaVlEs/SXwgtiD1tW22zAB/W1wdqfrpmikgaWQ9Fw6Ws+hsiRm5Vg== +"@sinonjs/fake-timers@^13.0.0": + version "13.0.5" + resolved "https://registry.yarnpkg.com/@sinonjs/fake-timers/-/fake-timers-13.0.5.tgz#36b9dbc21ad5546486ea9173d6bea063eb1717d5" + integrity sha512-36/hTbH2uaWuGVERyC6da9YwGWnzUZXuPro/F2LfsdOsLnCojz/iSH8MxUt/FD2S5XBSVPhmArFUXcpCQ2Hkiw== dependencies: - "@sinonjs/commons" "^1.7.0" + "@sinonjs/commons" "^3.0.1" "@styled-icons/bootstrap@10.47.0": version "10.47.0" @@ -2892,7 +2897,14 @@ "@babel/runtime" "^7.19.0" "@styled-icons/styled-icon" "^10.7.0" -"@styled-icons/styled-icon@10.7.0", "@styled-icons/styled-icon@^10.7.0": +"@styled-icons/styled-icon@10.7.1": + version "10.7.1" + resolved "https://registry.yarnpkg.com/@styled-icons/styled-icon/-/styled-icon-10.7.1.tgz#da04b80751e126756f98a5485b3b0dd9b52c9aba" + integrity sha512-WLYaeMTMhMkSxE+v+of+r2ovIk0tceDGfv8iqWHRMxvbm+6zxngVcQ4ELx6Zt/LFxqckmmoAdvo6ehiiYj6I6A== + dependencies: + "@babel/runtime" "^7.20.7" + +"@styled-icons/styled-icon@^10.7.0": version "10.7.0" resolved "https://registry.yarnpkg.com/@styled-icons/styled-icon/-/styled-icon-10.7.0.tgz#d6960e719b8567c8d0d3a87c40fb6f5b4952a228" integrity sha512-SCrhCfRyoY8DY7gUkpz+B0RqUg/n1Zaqrr2+YKmK/AyeNfCcoHuP4R9N4H0p/NA1l7PTU10ZkAWSLi68phnAjw== @@ -2915,498 +2927,592 @@ "@babel/runtime" "^7.19.0" "@styled-icons/styled-icon" "^10.7.0" -"@surma/rollup-plugin-off-main-thread@^2.2.3": - version "2.2.3" - resolved "https://registry.yarnpkg.com/@surma/rollup-plugin-off-main-thread/-/rollup-plugin-off-main-thread-2.2.3.tgz#ee34985952ca21558ab0d952f00298ad2190c053" - integrity sha512-lR8q/9W7hZpMWweNiAKU7NQerBnzQQLvi8qnTDU/fxItPhtZVMbPV3lbCwjhIlNBe9Bbr5V+KHshvWmVSG9cxQ== - dependencies: - ejs "^3.1.6" - json5 "^2.2.0" - magic-string "^0.25.0" - string.prototype.matchall "^4.0.6" - -"@svgr/babel-plugin-add-jsx-attribute@^5.4.0": - version "5.4.0" - resolved "https://registry.yarnpkg.com/@svgr/babel-plugin-add-jsx-attribute/-/babel-plugin-add-jsx-attribute-5.4.0.tgz#81ef61947bb268eb9d50523446f9c638fb355906" - integrity sha512-ZFf2gs/8/6B8PnSofI0inYXr2SDNTDScPXhN7k5EqD4aZ3gi6u+rbmZHVB8IM3wDyx8ntKACZbtXSm7oZGRqVg== - -"@svgr/babel-plugin-remove-jsx-attribute@^5.4.0": - version "5.4.0" - resolved "https://registry.yarnpkg.com/@svgr/babel-plugin-remove-jsx-attribute/-/babel-plugin-remove-jsx-attribute-5.4.0.tgz#6b2c770c95c874654fd5e1d5ef475b78a0a962ef" - integrity sha512-yaS4o2PgUtwLFGTKbsiAy6D0o3ugcUhWK0Z45umJ66EPWunAz9fuFw2gJuje6wqQvQWOTJvIahUwndOXb7QCPg== - -"@svgr/babel-plugin-remove-jsx-empty-expression@^5.0.1": - version "5.0.1" - resolved "https://registry.yarnpkg.com/@svgr/babel-plugin-remove-jsx-empty-expression/-/babel-plugin-remove-jsx-empty-expression-5.0.1.tgz#25621a8915ed7ad70da6cea3d0a6dbc2ea933efd" - integrity sha512-LA72+88A11ND/yFIMzyuLRSMJ+tRKeYKeQ+mR3DcAZ5I4h5CPWN9AHyUzJbWSYp/u2u0xhmgOe0+E41+GjEueA== - -"@svgr/babel-plugin-replace-jsx-attribute-value@^5.0.1": - version "5.0.1" - resolved "https://registry.yarnpkg.com/@svgr/babel-plugin-replace-jsx-attribute-value/-/babel-plugin-replace-jsx-attribute-value-5.0.1.tgz#0b221fc57f9fcd10e91fe219e2cd0dd03145a897" - integrity sha512-PoiE6ZD2Eiy5mK+fjHqwGOS+IXX0wq/YDtNyIgOrc6ejFnxN4b13pRpiIPbtPwHEc+NT2KCjteAcq33/F1Y9KQ== - -"@svgr/babel-plugin-svg-dynamic-title@^5.4.0": - version "5.4.0" - resolved "https://registry.yarnpkg.com/@svgr/babel-plugin-svg-dynamic-title/-/babel-plugin-svg-dynamic-title-5.4.0.tgz#139b546dd0c3186b6e5db4fefc26cb0baea729d7" - integrity sha512-zSOZH8PdZOpuG1ZVx/cLVePB2ibo3WPpqo7gFIjLV9a0QsuQAzJiwwqmuEdTaW2pegyBE17Uu15mOgOcgabQZg== - -"@svgr/babel-plugin-svg-em-dimensions@^5.4.0": - version "5.4.0" - resolved "https://registry.yarnpkg.com/@svgr/babel-plugin-svg-em-dimensions/-/babel-plugin-svg-em-dimensions-5.4.0.tgz#6543f69526632a133ce5cabab965deeaea2234a0" - integrity sha512-cPzDbDA5oT/sPXDCUYoVXEmm3VIoAWAPT6mSPTJNbQaBNUuEKVKyGH93oDY4e42PYHRW67N5alJx/eEol20abw== - -"@svgr/babel-plugin-transform-react-native-svg@^5.4.0": - version "5.4.0" - resolved "https://registry.yarnpkg.com/@svgr/babel-plugin-transform-react-native-svg/-/babel-plugin-transform-react-native-svg-5.4.0.tgz#00bf9a7a73f1cad3948cdab1f8dfb774750f8c80" - integrity sha512-3eYP/SaopZ41GHwXma7Rmxcv9uRslRDTY1estspeB1w1ueZWd/tPlMfEOoccYpEMZU3jD4OU7YitnXcF5hLW2Q== - -"@svgr/babel-plugin-transform-svg-component@^5.5.0": - version "5.5.0" - resolved "https://registry.yarnpkg.com/@svgr/babel-plugin-transform-svg-component/-/babel-plugin-transform-svg-component-5.5.0.tgz#583a5e2a193e214da2f3afeb0b9e8d3250126b4a" - integrity sha512-q4jSH1UUvbrsOtlo/tKcgSeiCHRSBdXoIoqX1pgcKK/aU3JD27wmMKwGtpB8qRYUYoyXvfGxUVKchLuR5pB3rQ== - -"@svgr/babel-preset@^5.5.0": - version "5.5.0" - resolved "https://registry.yarnpkg.com/@svgr/babel-preset/-/babel-preset-5.5.0.tgz#8af54f3e0a8add7b1e2b0fcd5a882c55393df327" - integrity sha512-4FiXBjvQ+z2j7yASeGPEi8VD/5rrGQk4Xrq3EdJmoZgz/tpqChpo5hgXDvmEauwtvOc52q8ghhZK4Oy7qph4ig== - dependencies: - "@svgr/babel-plugin-add-jsx-attribute" "^5.4.0" - "@svgr/babel-plugin-remove-jsx-attribute" "^5.4.0" - "@svgr/babel-plugin-remove-jsx-empty-expression" "^5.0.1" - "@svgr/babel-plugin-replace-jsx-attribute-value" "^5.0.1" - "@svgr/babel-plugin-svg-dynamic-title" "^5.4.0" - "@svgr/babel-plugin-svg-em-dimensions" "^5.4.0" - "@svgr/babel-plugin-transform-react-native-svg" "^5.4.0" - "@svgr/babel-plugin-transform-svg-component" "^5.5.0" - -"@svgr/core@^5.5.0": - version "5.5.0" - resolved "https://registry.yarnpkg.com/@svgr/core/-/core-5.5.0.tgz#82e826b8715d71083120fe8f2492ec7d7874a579" - integrity sha512-q52VOcsJPvV3jO1wkPtzTuKlvX7Y3xIcWRpCMtBF3MrteZJtBfQw/+u0B1BHy5ColpQc1/YVTrPEtSYIMNZlrQ== - dependencies: - "@svgr/plugin-jsx" "^5.5.0" - camelcase "^6.2.0" - cosmiconfig "^7.0.0" - -"@svgr/hast-util-to-babel-ast@^5.5.0": - version "5.5.0" - resolved "https://registry.yarnpkg.com/@svgr/hast-util-to-babel-ast/-/hast-util-to-babel-ast-5.5.0.tgz#5ee52a9c2533f73e63f8f22b779f93cd432a5461" - integrity sha512-cAaR/CAiZRB8GP32N+1jocovUtvlj0+e65TB50/6Lcime+EA49m/8l+P2ko+XPJ4dw3xaPS3jOL4F2X4KWxoeQ== - dependencies: - "@babel/types" "^7.12.6" - -"@svgr/plugin-jsx@^5.5.0": - version "5.5.0" - resolved "https://registry.yarnpkg.com/@svgr/plugin-jsx/-/plugin-jsx-5.5.0.tgz#1aa8cd798a1db7173ac043466d7b52236b369000" - integrity sha512-V/wVh33j12hGh05IDg8GpIUXbjAPnTdPTKuP4VNLggnwaHMPNQNae2pRnyTAILWCQdz5GyMqtO488g7CKM8CBA== - dependencies: - "@babel/core" "^7.12.3" - "@svgr/babel-preset" "^5.5.0" - "@svgr/hast-util-to-babel-ast" "^5.5.0" - svg-parser "^2.0.2" - -"@svgr/plugin-svgo@^5.5.0": - version "5.5.0" - resolved "https://registry.yarnpkg.com/@svgr/plugin-svgo/-/plugin-svgo-5.5.0.tgz#02da55d85320549324e201c7b2e53bf431fcc246" - integrity sha512-r5swKk46GuQl4RrVejVwpeeJaydoxkdwkM1mBKOgJLBUJPGaLci6ylg/IjhrRsREKDkr4kbMWdgOtbXEh0fyLQ== - dependencies: - cosmiconfig "^7.0.0" - deepmerge "^4.2.2" - svgo "^1.2.2" - -"@svgr/webpack@^5.5.0": - version "5.5.0" - resolved "https://registry.yarnpkg.com/@svgr/webpack/-/webpack-5.5.0.tgz#aae858ee579f5fa8ce6c3166ef56c6a1b381b640" - integrity sha512-DOBOK255wfQxguUta2INKkzPj6AIS6iafZYiYmHn6W3pHlycSRRlvWKCfLDG10fXfLWqE3DJHgRUOyJYmARa7g== - dependencies: - "@babel/core" "^7.12.3" - "@babel/plugin-transform-react-constant-elements" "^7.12.1" - "@babel/preset-env" "^7.12.1" - "@babel/preset-react" "^7.12.5" - "@svgr/core" "^5.5.0" - "@svgr/plugin-jsx" "^5.5.0" - "@svgr/plugin-svgo" "^5.5.0" - loader-utils "^2.0.0" - -"@swagger-api/apidom-ast@^0.78.0": - version "0.78.0" - resolved "https://registry.yarnpkg.com/@swagger-api/apidom-ast/-/apidom-ast-0.78.0.tgz#1e7ea0c9774b2236846bf24df18fd531f6f517d0" - integrity sha512-mEXmRmkFlmO6dcBuakFkc2gevN4mC6incPAQE1UciaX4hLuJpiv/5DTH9gVWTR0CWUFw/dXROTD/x6ETV0y03A== - dependencies: - "@babel/runtime-corejs3" "^7.20.7" - "@swagger-api/apidom-error" "^0.78.0" - "@types/ramda" "~0.29.6" - ramda "~0.29.0" - ramda-adjunct "^4.1.1" - stampit "^4.3.2" +"@swagger-api/apidom-ast@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-ast/-/apidom-ast-1.6.0.tgz#c72f345023435e4033a5c67cdc064a51f75cd799" + integrity sha512-ez1KnBdAzoh5a6ijDXzu5nADkVZXlnL1RkLl8n2u2tjiNg9597xxmFdEHLVa31Vxr1yYj0WtYGLA5e2Kp0KNrQ== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-error" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" unraw "^3.0.0" -"@swagger-api/apidom-core@>=0.77.0 <1.0.0", "@swagger-api/apidom-core@^0.78.0": - version "0.78.0" - resolved "https://registry.yarnpkg.com/@swagger-api/apidom-core/-/apidom-core-0.78.0.tgz#752215bade74ee56ec3a0f60fd4eee4d9ac8a951" - integrity sha512-Qx9m+1u6H4Bsa38s73ANtGn8zFGqK0peguM+SFuUR5HirjpoFB8JB7IG5E8+ymUlpWhlU43q9QnJjcaYJw9MTg== +"@swagger-api/apidom-core@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-core/-/apidom-core-1.6.0.tgz#4eb1ff8955ec623d6c1a9e5a5d3fb34714667cb9" + integrity sha512-gA1MVoXe19sjFLKGkWxp5VvSw3Tk0CSChfItJjFeFHpLSGrfm+LlXp37TmNSns53Ky0F7x7TB/5kAX5I/TO4xw== dependencies: - "@babel/runtime-corejs3" "^7.20.7" - "@swagger-api/apidom-ast" "^0.78.0" - "@swagger-api/apidom-error" "^0.78.0" - "@types/ramda" "~0.29.6" + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-ast" "^1.6.0" + "@swagger-api/apidom-error" "^1.6.0" + "@types/ramda" "~0.30.0" minim "~0.23.8" - ramda "~0.29.0" - ramda-adjunct "^4.1.1" - short-unique-id "^5.0.2" - stampit "^4.3.2" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + short-unique-id "^5.3.2" + ts-mixer "^6.0.3" -"@swagger-api/apidom-error@^0.78.0": - version "0.78.0" - resolved "https://registry.yarnpkg.com/@swagger-api/apidom-error/-/apidom-error-0.78.0.tgz#f5402b5d15b44e456db136c6239054c2f30c4756" - integrity sha512-P0enIK3XymxCPHlhGtqc4TU5H+cHf7L0yDFmfjZEcsjDzGDv5A+m5tf429Pr/R+e51DzpT5/xIcPKTnti0gIOw== +"@swagger-api/apidom-error@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-error/-/apidom-error-1.6.0.tgz#e813c5491142bbb2518fed71133b28407e0c846a" + integrity sha512-xp/cQ1xQ/4Vd/hhQfONK7ea9oVc3JUXAYyfRzvDR0lxISly/SyD2jMcqXzHtrylBAnv7V2HSsbC1BWo7ZJDLSQ== dependencies: "@babel/runtime-corejs3" "^7.20.7" - "@types/ramda" "~0.29.6" - ramda "~0.29.0" - ramda-adjunct "^4.0.0" -"@swagger-api/apidom-json-pointer@>=0.77.0 <1.0.0", "@swagger-api/apidom-json-pointer@^0.78.0": - version "0.78.0" - resolved "https://registry.yarnpkg.com/@swagger-api/apidom-json-pointer/-/apidom-json-pointer-0.78.0.tgz#652156a32e28960c60ca71d13cba60a7185f9e19" - integrity sha512-Ly4ZfUGxxbNoHHc9vR814mU96ZLGsjaJflCW0jdZnMVfVv20fDCoDoOOmXat6ajxUbS2YKimgxPvdBth3K/CRQ== +"@swagger-api/apidom-json-pointer@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-json-pointer/-/apidom-json-pointer-1.6.0.tgz#21aad5634d997a7ad8976f1307393ec4d8c1fef0" + integrity sha512-RO6P5Gt64AnthGXKeqIFjQCLVFbAJvLYAb67TkvRQ9US4lNixFtFsYJnhLCC4ymz4dTT1hacG0cmTRGcEHF9ig== dependencies: - "@babel/runtime-corejs3" "^7.20.7" - "@swagger-api/apidom-core" "^0.78.0" - "@swagger-api/apidom-error" "^0.78.0" - "@types/ramda" "~0.29.6" - ramda "~0.29.0" - ramda-adjunct "^4.0.0" + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-error" "^1.6.0" + "@swaggerexpert/json-pointer" "^2.10.1" -"@swagger-api/apidom-ns-api-design-systems@^0.78.0": - version "0.78.0" - resolved "https://registry.yarnpkg.com/@swagger-api/apidom-ns-api-design-systems/-/apidom-ns-api-design-systems-0.78.0.tgz#9345debb65c4b14e28983dc1c8d0637051aa3959" - integrity sha512-WoWE6w1P3qsokG3Qyc5F3xpz+e/WablE0EHGSgiYxk+MQJLqYmz5UhS5LxYGT9d6o9XUs24ykSbKrYWYwkpp4w== - dependencies: - "@babel/runtime-corejs3" "^7.20.7" - "@swagger-api/apidom-core" "^0.78.0" - "@swagger-api/apidom-error" "^0.78.0" - "@swagger-api/apidom-ns-openapi-3-1" "^0.78.0" - "@types/ramda" "~0.29.6" - ramda "~0.29.0" - ramda-adjunct "^4.1.1" - stampit "^4.3.2" - -"@swagger-api/apidom-ns-asyncapi-2@^0.78.0": - version "0.78.0" - resolved "https://registry.yarnpkg.com/@swagger-api/apidom-ns-asyncapi-2/-/apidom-ns-asyncapi-2-0.78.0.tgz#cb40b3fe8d8e18cb842b98166cffab99200350c5" - integrity sha512-QWZohCtXf5UX/I9bnc4MQh16X9jGPdGrByWM93xRvh8X8rIF0BtF9S7lIx028aX3AHYIu4SwYr7JZlqEaZ92Kw== - dependencies: - "@babel/runtime-corejs3" "^7.20.7" - "@swagger-api/apidom-core" "^0.78.0" - "@swagger-api/apidom-ns-json-schema-draft-7" "^0.78.0" - "@types/ramda" "~0.29.6" - ramda "~0.29.0" - ramda-adjunct "^4.1.1" - stampit "^4.3.2" - -"@swagger-api/apidom-ns-json-schema-draft-4@^0.78.0": - version "0.78.0" - resolved "https://registry.yarnpkg.com/@swagger-api/apidom-ns-json-schema-draft-4/-/apidom-ns-json-schema-draft-4-0.78.0.tgz#b2420b49426ba534f57a83abd355a1ed18dec6a7" - integrity sha512-19NR9lTHMOQTIEV4tJq+FlHQAYnjyH+DgI4mmRu6UMFSZjRjutYF7B8lCGogSus9Uwy8YpUk00prLFTld00wgA== - dependencies: - "@babel/runtime-corejs3" "^7.20.7" - "@swagger-api/apidom-ast" "^0.78.0" - "@swagger-api/apidom-core" "^0.78.0" - "@types/ramda" "~0.29.6" - ramda "~0.29.0" - ramda-adjunct "^4.1.1" - stampit "^4.3.2" - -"@swagger-api/apidom-ns-json-schema-draft-6@^0.78.0": - version "0.78.0" - resolved "https://registry.yarnpkg.com/@swagger-api/apidom-ns-json-schema-draft-6/-/apidom-ns-json-schema-draft-6-0.78.0.tgz#2a7bdb97eabd7f0376e3375e4172d497cfd3d072" - integrity sha512-pHyCPU3OWDiPuLepo03rBpi2n+SCH6PZAgguqAB3lDJ2ymitrT2SNpmZ6CcHvPGR9Y7h4/fR5vAypVZfdNr/WQ== - dependencies: - "@babel/runtime-corejs3" "^7.20.7" - "@swagger-api/apidom-core" "^0.78.0" - "@swagger-api/apidom-error" "^0.78.0" - "@swagger-api/apidom-ns-json-schema-draft-4" "^0.78.0" - "@types/ramda" "~0.29.6" - ramda "~0.29.0" - ramda-adjunct "^4.1.1" - stampit "^4.3.2" - -"@swagger-api/apidom-ns-json-schema-draft-7@^0.78.0": - version "0.78.0" - resolved "https://registry.yarnpkg.com/@swagger-api/apidom-ns-json-schema-draft-7/-/apidom-ns-json-schema-draft-7-0.78.0.tgz#54e7b719ce5301ab1a9474cc74ec53a9c2f5bd73" - integrity sha512-ScUiNNAdwnikH3Fo2rUsDmXOjV7zXfQ6CGE+QkY5Wj3t1M6siw2HpDjrBaaCyp6w/bemvogsh280GrzAnxKLIw== - dependencies: - "@babel/runtime-corejs3" "^7.20.7" - "@swagger-api/apidom-core" "^0.78.0" - "@swagger-api/apidom-error" "^0.78.0" - "@swagger-api/apidom-ns-json-schema-draft-6" "^0.78.0" - "@types/ramda" "~0.29.6" - ramda "~0.29.0" - ramda-adjunct "^4.1.1" - stampit "^4.3.2" - -"@swagger-api/apidom-ns-openapi-3-0@^0.78.0": - version "0.78.0" - resolved "https://registry.yarnpkg.com/@swagger-api/apidom-ns-openapi-3-0/-/apidom-ns-openapi-3-0-0.78.0.tgz#c658e0bf6b9b3f5aa19b18cd6d23837a954735a2" - integrity sha512-GRmUOknEzMG37y5sStvjEsk30RLVg5E7iZuougK1rEf+wzzX5XhorSgMx2NQmka5rb814BgzyiqGRmvKQErDBw== - dependencies: - "@babel/runtime-corejs3" "^7.20.7" - "@swagger-api/apidom-core" "^0.78.0" - "@swagger-api/apidom-error" "^0.78.0" - "@swagger-api/apidom-ns-json-schema-draft-4" "^0.78.0" - "@types/ramda" "~0.29.6" - ramda "~0.29.0" - ramda-adjunct "^4.1.1" - stampit "^4.3.2" - -"@swagger-api/apidom-ns-openapi-3-1@>=0.77.0 <1.0.0", "@swagger-api/apidom-ns-openapi-3-1@^0.78.0": - version "0.78.0" - resolved "https://registry.yarnpkg.com/@swagger-api/apidom-ns-openapi-3-1/-/apidom-ns-openapi-3-1-0.78.0.tgz#d96ffe766d848bb19e25cb3390958f06abde1ce8" - integrity sha512-hHpUZLjIiaLK+99cAPiYNV9QzZQxFoMLqBNYo+GQwqizaVOjxQRi5y/hPkfFALqqufZ1L6XWeyjQrtli0ftqBQ== - dependencies: - "@babel/runtime-corejs3" "^7.20.7" - "@swagger-api/apidom-ast" "^0.78.0" - "@swagger-api/apidom-core" "^0.78.0" - "@swagger-api/apidom-ns-openapi-3-0" "^0.78.0" - "@types/ramda" "~0.29.6" - ramda "~0.29.0" - ramda-adjunct "^4.1.1" - stampit "^4.3.2" - -"@swagger-api/apidom-parser-adapter-api-design-systems-json@^0.78.0": - version "0.78.0" - resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-api-design-systems-json/-/apidom-parser-adapter-api-design-systems-json-0.78.0.tgz#245bcb6346a4a9312c241b4a88f25bea79026709" - integrity sha512-g7VlfOrpTzbVV30Ugab0qAJITavLo39apvyFFv2cN2jfuIQa8MlzDP0mZmVtCGQy3IoT4Auns/qWeGcZX0li9w== - dependencies: - "@babel/runtime-corejs3" "^7.20.7" - "@swagger-api/apidom-core" "^0.78.0" - "@swagger-api/apidom-ns-api-design-systems" "^0.78.0" - "@swagger-api/apidom-parser-adapter-json" "^0.78.0" - "@types/ramda" "~0.29.6" - ramda "~0.29.0" - ramda-adjunct "^4.0.0" - -"@swagger-api/apidom-parser-adapter-api-design-systems-yaml@^0.78.0": - version "0.78.0" - resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-api-design-systems-yaml/-/apidom-parser-adapter-api-design-systems-yaml-0.78.0.tgz#6d11fbe85ee6830c6b3172bdde17be8004e2610f" - integrity sha512-ZueYoHOJARRm84ntCggUZLKNwUHz2U0eG9KHIzw75UW43pyvQVbxAE2ELdyP5f8vr51wMuMp6XYRcFOsNi/oeQ== - dependencies: - "@babel/runtime-corejs3" "^7.20.7" - "@swagger-api/apidom-core" "^0.78.0" - "@swagger-api/apidom-ns-api-design-systems" "^0.78.0" - "@swagger-api/apidom-parser-adapter-yaml-1-2" "^0.78.0" - "@types/ramda" "~0.29.6" - ramda "~0.29.0" - ramda-adjunct "^4.0.0" - -"@swagger-api/apidom-parser-adapter-asyncapi-json-2@^0.78.0": - version "0.78.0" - resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-asyncapi-json-2/-/apidom-parser-adapter-asyncapi-json-2-0.78.0.tgz#760d9cbdc26045a0bccac0ee6c804e5ce3a01c91" - integrity sha512-Jm0hbNXWOH2QJIiF+5QgY+ioVSOBqV3WlhTeyrF5kSxHinah16nR1jUkz5tMsSc9sxTZHzWYVLneyBMW3VSHrw== - dependencies: - "@babel/runtime-corejs3" "^7.20.7" - "@swagger-api/apidom-core" "^0.78.0" - "@swagger-api/apidom-ns-asyncapi-2" "^0.78.0" - "@swagger-api/apidom-parser-adapter-json" "^0.78.0" - "@types/ramda" "~0.29.6" - ramda "~0.29.0" - ramda-adjunct "^4.0.0" - -"@swagger-api/apidom-parser-adapter-asyncapi-yaml-2@^0.78.0": - version "0.78.0" - resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-asyncapi-yaml-2/-/apidom-parser-adapter-asyncapi-yaml-2-0.78.0.tgz#f1da4fe140cd7f1ea1c9b7182d1521236309362a" - integrity sha512-zpP8gQBXhrR/t91Z/Jl0nD/cUSzmYjzhE5qWHkfhbGvzaWatiLrNY+CnFS9RcgF4pb2LSqS5cjDVAExBbjdLdQ== - dependencies: - "@babel/runtime-corejs3" "^7.20.7" - "@swagger-api/apidom-core" "^0.78.0" - "@swagger-api/apidom-ns-asyncapi-2" "^0.78.0" - "@swagger-api/apidom-parser-adapter-yaml-1-2" "^0.78.0" - "@types/ramda" "~0.29.6" - ramda "~0.29.0" - ramda-adjunct "^4.0.0" - -"@swagger-api/apidom-parser-adapter-json@^0.78.0": - version "0.78.0" - resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-json/-/apidom-parser-adapter-json-0.78.0.tgz#fa27e907738c71c81edb803f0826f465fde8153f" - integrity sha512-d/8gFj5cc+pnCo7ORGN5dJPGWzTleYkIwGfsyFuLZNjb4KlrOrKlPl0LKQ/t7MSEbVpSStxbgezoUtfdVhGscw== - dependencies: - "@babel/runtime-corejs3" "^7.20.7" - "@swagger-api/apidom-ast" "^0.78.0" - "@swagger-api/apidom-core" "^0.78.0" - "@swagger-api/apidom-error" "^0.78.0" - "@types/ramda" "~0.29.6" - ramda "~0.29.0" - ramda-adjunct "^4.1.1" - stampit "^4.3.2" - tree-sitter "=0.20.4" - tree-sitter-json "=0.20.1" - web-tree-sitter "=0.20.3" - -"@swagger-api/apidom-parser-adapter-openapi-json-3-0@^0.78.0": - version "0.78.0" - resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-openapi-json-3-0/-/apidom-parser-adapter-openapi-json-3-0-0.78.0.tgz#c90ffd87c4ee065a35a87bdaf51935655f165571" - integrity sha512-MjXkPAiEyTZIljzjEgvAmqaZel0jpKBBqdtC8nWH/9C2ugkKHetKMSgYu+5wvFh//ixJZZE7dM1QHEIBoPl9nA== - dependencies: - "@babel/runtime-corejs3" "^7.20.7" - "@swagger-api/apidom-core" "^0.78.0" - "@swagger-api/apidom-ns-openapi-3-0" "^0.78.0" - "@swagger-api/apidom-parser-adapter-json" "^0.78.0" - "@types/ramda" "~0.29.6" - ramda "~0.29.0" - ramda-adjunct "^4.0.0" - -"@swagger-api/apidom-parser-adapter-openapi-json-3-1@^0.78.0": - version "0.78.0" - resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-openapi-json-3-1/-/apidom-parser-adapter-openapi-json-3-1-0.78.0.tgz#dbd6495b838be40120709f9d80ef07a23fdd52fa" - integrity sha512-k+rT6kwu1jAN1lYIP1wVshQdaLu9M+jjCfpvMXXkL/2VpZqq1yP6daFm0ExiHllVUcHWeqRXhubFV3wWkFm6eA== - dependencies: - "@babel/runtime-corejs3" "^7.20.7" - "@swagger-api/apidom-core" "^0.78.0" - "@swagger-api/apidom-ns-openapi-3-1" "^0.78.0" - "@swagger-api/apidom-parser-adapter-json" "^0.78.0" - "@types/ramda" "~0.29.6" - ramda "~0.29.0" - ramda-adjunct "^4.0.0" - -"@swagger-api/apidom-parser-adapter-openapi-yaml-3-0@^0.78.0": - version "0.78.0" - resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-openapi-yaml-3-0/-/apidom-parser-adapter-openapi-yaml-3-0-0.78.0.tgz#77ae78bfea6d69dd4776f7949999f98ec179071c" - integrity sha512-RzcqL0kvUl5G75H4qOFSi9FTaVfBtRnjzEcjd8SOKVLg3JJsCv3vrk68laRm8HXocyWgGstU51UzBqkMStXy4A== - dependencies: - "@babel/runtime-corejs3" "^7.20.7" - "@swagger-api/apidom-core" "^0.78.0" - "@swagger-api/apidom-ns-openapi-3-0" "^0.78.0" - "@swagger-api/apidom-parser-adapter-yaml-1-2" "^0.78.0" - "@types/ramda" "~0.29.6" - ramda "~0.29.0" - ramda-adjunct "^4.0.0" - -"@swagger-api/apidom-parser-adapter-openapi-yaml-3-1@^0.78.0": - version "0.78.0" - resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-openapi-yaml-3-1/-/apidom-parser-adapter-openapi-yaml-3-1-0.78.0.tgz#78d180229fbfe301a8f9d47b23320c28c51a0f95" - integrity sha512-1hB+mcEJd14RJC8lH3yJsoQRDhA8TNNKl3EyQ17eFY0dK29JlluDEbDHIRQpLT1l2jCK/NfqAk2hc37yIwydfw== - dependencies: - "@babel/runtime-corejs3" "^7.20.7" - "@swagger-api/apidom-core" "^0.78.0" - "@swagger-api/apidom-ns-openapi-3-1" "^0.78.0" - "@swagger-api/apidom-parser-adapter-yaml-1-2" "^0.78.0" - "@types/ramda" "~0.29.6" - ramda "~0.29.0" - ramda-adjunct "^4.0.0" - -"@swagger-api/apidom-parser-adapter-yaml-1-2@^0.78.0": - version "0.78.0" - resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-yaml-1-2/-/apidom-parser-adapter-yaml-1-2-0.78.0.tgz#235a2cb3458c103ed36271769e1a7524c502a58f" - integrity sha512-L37X+nRNp+2PyJkAwMdSQjP8tb3xoc6FVk2QXLHogghe1Phrmfaal3TPu2rWJNn7NSBcvSyiTAR7gEIULitugA== +"@swagger-api/apidom-ns-api-design-systems@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-ns-api-design-systems/-/apidom-ns-api-design-systems-1.6.0.tgz#9d1b17c3b660fa310f31e77da8b3fc53b430ad83" + integrity sha512-EYJfQ4JYuUo2J4QiiLnA/8LmM1k7AQcf1XVE+NrIpZ1160GIzqE+W5uOXkhAOImkP2Cb7EZZdE2cFE/tMYxNvw== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-error" "^1.6.0" + "@swagger-api/apidom-ns-openapi-3-1" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + ts-mixer "^6.0.3" + +"@swagger-api/apidom-ns-arazzo-1@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-ns-arazzo-1/-/apidom-ns-arazzo-1-1.6.0.tgz#0eb9ef09046e347df120c74f67462b623637c2a5" + integrity sha512-5rF8PyBiIHh6NfC5Y0WypW11X6hQIWr88EKNOQbBuT/nnzAsOznrUCfQ99FYGLucwdOHaMIBn/b/n4ejGBto/A== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-ns-json-schema-2020-12" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + ts-mixer "^6.0.3" + +"@swagger-api/apidom-ns-asyncapi-2@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-ns-asyncapi-2/-/apidom-ns-asyncapi-2-1.6.0.tgz#c5a45530720429c1bc2513b1d9f7704454d4c176" + integrity sha512-tOodfX+o7lonEAnSAxet7nCayW+EqtKPegT06WXt7Llq1LS9eYZ9YzXdFgIwCm8UzfEpZdVLqtxbdLX9vuUtSg== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-ns-json-schema-draft-7" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + ts-mixer "^6.0.3" + +"@swagger-api/apidom-ns-asyncapi-3@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-ns-asyncapi-3/-/apidom-ns-asyncapi-3-1.6.0.tgz#af66f63137479ffa298f4f4b76a58db690894853" + integrity sha512-lRMvwTdtuPcwJEYLTX/UGtECpHi9UNYeT9rmWMw3LiKZrZzYc2L8q4ipPbpWwH8t7QfsF2u0iggCODU99lXCnw== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-ns-asyncapi-2" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + ts-mixer "^6.0.3" + +"@swagger-api/apidom-ns-json-schema-2019-09@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-ns-json-schema-2019-09/-/apidom-ns-json-schema-2019-09-1.6.0.tgz#439faf74e8ffa1006ae144b00a8dfb89b8cefba9" + integrity sha512-dee1i8wcAFgDEOzTsyoCzQhFLZ2JKzkK5KkRuryabvwS0hG2mKlogToFc8cO2MkkiLSpERm7DREALwSTFVHa0w== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-error" "^1.6.0" + "@swagger-api/apidom-ns-json-schema-draft-7" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + ts-mixer "^6.0.4" + +"@swagger-api/apidom-ns-json-schema-2020-12@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-ns-json-schema-2020-12/-/apidom-ns-json-schema-2020-12-1.6.0.tgz#d2e0e1ef89141c0eea013687aebaf2580e8a6c7c" + integrity sha512-ldTxSnnIXskwpN6yCJkasqs32pJXwoXyad95crKT0xjZZr4fTrcAXXIyzdjBubiY9tK6elSrQGQxinJcV7ivWw== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-error" "^1.6.0" + "@swagger-api/apidom-ns-json-schema-2019-09" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + ts-mixer "^6.0.4" + +"@swagger-api/apidom-ns-json-schema-draft-4@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-ns-json-schema-draft-4/-/apidom-ns-json-schema-draft-4-1.6.0.tgz#57f2f0fb4c4a2b907cd096efc96e0bc9cd3594fc" + integrity sha512-t9HvHwrevEG7usosO6AdXmC8oYqje5nxHpUmODr72tUtCeAeGEGEb9lgqx7fBhjc3BYsRzOL1hX56m1gjEyCog== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-ast" "^1.6.0" + "@swagger-api/apidom-core" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + ts-mixer "^6.0.4" + +"@swagger-api/apidom-ns-json-schema-draft-6@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-ns-json-schema-draft-6/-/apidom-ns-json-schema-draft-6-1.6.0.tgz#6d191a7ebf8c64cb6e29002a4f06ee454b960d64" + integrity sha512-aoyvQWgAOcZGTe5OfJ3r24DvXHHbrkKtAnxTOEdZzV/uOm6/cbuT8m02+aMOqWPxei1naC3ZHW9iHrETtfgV3w== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-error" "^1.6.0" + "@swagger-api/apidom-ns-json-schema-draft-4" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + ts-mixer "^6.0.4" + +"@swagger-api/apidom-ns-json-schema-draft-7@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-ns-json-schema-draft-7/-/apidom-ns-json-schema-draft-7-1.6.0.tgz#5f95e9b055c9c9b1dc00c468a4f88449bdc89865" + integrity sha512-GjmC4+AHQh22fRZOmV+jSYMJTXh243XvdACfIQ//39kQu7gQsimF4PVSY2IgWSvS/I1ukWdPBYmDvOKryBPGrw== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-error" "^1.6.0" + "@swagger-api/apidom-ns-json-schema-draft-6" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + ts-mixer "^6.0.4" + +"@swagger-api/apidom-ns-openapi-2@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-ns-openapi-2/-/apidom-ns-openapi-2-1.6.0.tgz#2a8fe09b6e063ffe90ad75c31023ccbda558e097" + integrity sha512-xbmYzagnB8rO7sYwNGVyxYbNBkjCWnMhlnMrxkPtfQ/2u2ANAmTnCB/S/cMswX5XofiRJbznKAjLDSKBS+mLpQ== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-error" "^1.6.0" + "@swagger-api/apidom-ns-json-schema-draft-4" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + ts-mixer "^6.0.3" + +"@swagger-api/apidom-ns-openapi-3-0@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-ns-openapi-3-0/-/apidom-ns-openapi-3-0-1.6.0.tgz#2f838c8ba216541d835793fd5947e1fed7b87445" + integrity sha512-AOvW7a2H27inepcTBAWaBMjJLrCh5IPWD4nTU+gysULC7IW6gphO8hj3iUuTmFBcGh9be89GBbvv2y/EGAfx9w== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-error" "^1.6.0" + "@swagger-api/apidom-ns-json-schema-draft-4" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + ts-mixer "^6.0.3" + +"@swagger-api/apidom-ns-openapi-3-1@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-ns-openapi-3-1/-/apidom-ns-openapi-3-1-1.6.0.tgz#786698fc0a1c9162782024a66d8be2b4e0e106db" + integrity sha512-jCVypc8503zDSxAQlyV8j1vzwc75VBdWHtE2O0F+q5j9qNtGxw/ekbDkgrydYRaGBl92mf16dtPjtp5LwJD0Hw== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-ast" "^1.6.0" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-json-pointer" "^1.6.0" + "@swagger-api/apidom-ns-json-schema-2020-12" "^1.6.0" + "@swagger-api/apidom-ns-openapi-3-0" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + ts-mixer "^6.0.3" + +"@swagger-api/apidom-ns-openapi-3-2@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-ns-openapi-3-2/-/apidom-ns-openapi-3-2-1.6.0.tgz#0ba997d1321a2af1af36ccd30f335b0c9da24753" + integrity sha512-QcFAUucaPaWiOKOEaaGqSfK3OtjeGJodWZLsuBQ0vrHaHkWyQ7jwsM1DJbc1Y8geOBeD2wIwdrdRjoulmqU1SA== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-ast" "^1.6.0" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-json-pointer" "^1.6.0" + "@swagger-api/apidom-ns-json-schema-2020-12" "^1.6.0" + "@swagger-api/apidom-ns-openapi-3-0" "^1.6.0" + "@swagger-api/apidom-ns-openapi-3-1" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + ts-mixer "^6.0.3" + +"@swagger-api/apidom-parser-adapter-api-design-systems-json@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-api-design-systems-json/-/apidom-parser-adapter-api-design-systems-json-1.6.0.tgz#8495b058dc7962ca2c17b0528aed78dac394f654" + integrity sha512-vz/9k0X/kh6mLm+Fi+LGNk/yyFq28wxI29ZVLW+b7ulcODikv+NaDnyN2n2kLKCvIchPATzAEvqMvVMuuQwWlg== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-ns-api-design-systems" "^1.6.0" + "@swagger-api/apidom-parser-adapter-json" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + +"@swagger-api/apidom-parser-adapter-api-design-systems-yaml@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-api-design-systems-yaml/-/apidom-parser-adapter-api-design-systems-yaml-1.6.0.tgz#a56b618504f0168b3687a42553b5d26860d85adc" + integrity sha512-QAq4H6YzRtysSpvLtlJ8WZ22/1Mht+/iarrUOijxDZQPAGfYeUoIicnCqxkVZYSea85sQl+3kiCCB3nhSH+L0g== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-ns-api-design-systems" "^1.6.0" + "@swagger-api/apidom-parser-adapter-yaml-1-2" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + +"@swagger-api/apidom-parser-adapter-arazzo-json-1@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-arazzo-json-1/-/apidom-parser-adapter-arazzo-json-1-1.6.0.tgz#63790158816fc50d81bcf59631b3536e3af8d5ca" + integrity sha512-syKPG3a9IGRvlGhXIEUzWhwbEuFbj+UwwtqaKu8zu771V+DRtH+wxyOkX54vKAIlApz/FgeUbmlWA1ZtYBlSIQ== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-ns-arazzo-1" "^1.6.0" + "@swagger-api/apidom-parser-adapter-json" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + +"@swagger-api/apidom-parser-adapter-arazzo-yaml-1@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-arazzo-yaml-1/-/apidom-parser-adapter-arazzo-yaml-1-1.6.0.tgz#390a74c9cabaf175f35086075e82579edcefeb40" + integrity sha512-IVVLn+a8Q1iQcQsm4tXiAPghHJuJSB1rhIlDyHe3tSQgt9HOSiVpbnJDpwE/JBxxDxSAkeT6Ovo+fi2T5AmHYg== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-ns-arazzo-1" "^1.6.0" + "@swagger-api/apidom-parser-adapter-yaml-1-2" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + +"@swagger-api/apidom-parser-adapter-asyncapi-json-2@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-asyncapi-json-2/-/apidom-parser-adapter-asyncapi-json-2-1.6.0.tgz#3573506e856350ef0a24c64b517c3461eaa64fa7" + integrity sha512-aSUi22ELTDvdCLA3nIUOehuNBcHSeCqU7S7YNiHP/mwE4Q07pwQrYXijH2PROfCdjlZNNN34m6Ptakd92jliJQ== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-ns-asyncapi-2" "^1.6.0" + "@swagger-api/apidom-parser-adapter-json" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + +"@swagger-api/apidom-parser-adapter-asyncapi-json-3@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-asyncapi-json-3/-/apidom-parser-adapter-asyncapi-json-3-1.6.0.tgz#c18fb9cb2ce69fbe82a4b4fee0bb88bd972956f2" + integrity sha512-Ic53vcFF9zniDyCXOGSwwuAdEBUn5lFEAa0m2i30R36cQFHBCCuvbzbMQjWdr+oML0Aw4XoqOwZCQgkJJICpPA== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-ns-asyncapi-3" "^1.6.0" + "@swagger-api/apidom-parser-adapter-json" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + +"@swagger-api/apidom-parser-adapter-asyncapi-yaml-2@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-asyncapi-yaml-2/-/apidom-parser-adapter-asyncapi-yaml-2-1.6.0.tgz#209e4ceefa2c70ce0bfd957bf0e786474f763fb9" + integrity sha512-d/w7X+T4vT+KPqb+8xUm6n4pbHsGB28jdxE9rNVbxhu6D3owny2uxfglwaFh4fJG6FQMavCwl/QzfB4newdoKQ== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-ns-asyncapi-2" "^1.6.0" + "@swagger-api/apidom-parser-adapter-yaml-1-2" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + +"@swagger-api/apidom-parser-adapter-asyncapi-yaml-3@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-asyncapi-yaml-3/-/apidom-parser-adapter-asyncapi-yaml-3-1.6.0.tgz#37f0c6d4562d93a910c883392f51ba3e2e004e17" + integrity sha512-Wmf0LY59TZxQhqrJU2pcnUikcChVB4IqGPgjtOFLUoqPpz8FSwYbJ/SPnSMSl+QuncxROheSFsgZ6Tupv0sPHw== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-ns-asyncapi-3" "^1.6.0" + "@swagger-api/apidom-parser-adapter-yaml-1-2" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + +"@swagger-api/apidom-parser-adapter-json@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-json/-/apidom-parser-adapter-json-1.6.0.tgz#cb950140c66cb27cc7517f604e50e9c8bb6b2753" + integrity sha512-WdAS+dBAB2t18HuUgSZy5b8JM7uXfn1RlPymJNRMUsrKYCTtPrQ/0q3YfnBjPhtjSSNCp+p1wajxHAFS7cj2VA== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-ast" "^1.6.0" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-error" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + tree-sitter "=0.21.1" + tree-sitter-json "=0.24.8" + web-tree-sitter "=0.24.5" + +"@swagger-api/apidom-parser-adapter-openapi-json-2@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-openapi-json-2/-/apidom-parser-adapter-openapi-json-2-1.6.0.tgz#278eac1ec8fec3b0c7e2cad6cc8f75f636ae8a3c" + integrity sha512-Q36W1FzdVaY7Oh98533dzCUghwb8k3ZMdlnV37V1H13FlUkj3tVZiWaeaCLwIakzQ7XXYaQTOP+VrRhDRjzhUA== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-ns-openapi-2" "^1.6.0" + "@swagger-api/apidom-parser-adapter-json" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + +"@swagger-api/apidom-parser-adapter-openapi-json-3-0@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-openapi-json-3-0/-/apidom-parser-adapter-openapi-json-3-0-1.6.0.tgz#9dfebd27032acef17d1c48820ab6d95db6561910" + integrity sha512-UY+obOLTPHJvnXscdMY9XwZyuqcnBe6cu9TURjJgkO/QpOpPDqqZoRyurKZgRrX0Pv9B1zR3EIzhl01u/jeUaw== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-ns-openapi-3-0" "^1.6.0" + "@swagger-api/apidom-parser-adapter-json" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + +"@swagger-api/apidom-parser-adapter-openapi-json-3-1@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-openapi-json-3-1/-/apidom-parser-adapter-openapi-json-3-1-1.6.0.tgz#2ede833b39831973525f8b45680010e957ead469" + integrity sha512-4ch04/96lYMXQu6odqa6H0aJmV8UefnBJKX1CPuL4qcPSPMFCurcXHGpPHrwMu1p/4Q9H+yRVlYeNQV10xvM0w== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-ns-openapi-3-1" "^1.6.0" + "@swagger-api/apidom-parser-adapter-json" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + +"@swagger-api/apidom-parser-adapter-openapi-json-3-2@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-openapi-json-3-2/-/apidom-parser-adapter-openapi-json-3-2-1.6.0.tgz#e019e0321928affbdb7e3b581da811b6da9b3665" + integrity sha512-fWR2gjMQg00QIimcXQMSVeLnCH/2iuDD/Dx8TzVHmKV/IKlu+TnmIVosdlDfRmOB+4duwU6/yfoA79IEhFeZdw== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-ns-openapi-3-2" "^1.6.0" + "@swagger-api/apidom-parser-adapter-json" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + +"@swagger-api/apidom-parser-adapter-openapi-yaml-2@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-openapi-yaml-2/-/apidom-parser-adapter-openapi-yaml-2-1.6.0.tgz#6ca83d9027e1adf9a2a8e7ef1f0b4599bcf6fbf0" + integrity sha512-dkEh1Rw9uvuIAOTfKjWRX2rLWP+xJ/Eqdkqeo0I0BWFKXX49YcDpHJV4XHpmd5FbsjJ9vBYr0hAmkbl32TtR4g== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-ns-openapi-2" "^1.6.0" + "@swagger-api/apidom-parser-adapter-yaml-1-2" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + +"@swagger-api/apidom-parser-adapter-openapi-yaml-3-0@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-openapi-yaml-3-0/-/apidom-parser-adapter-openapi-yaml-3-0-1.6.0.tgz#bd9ad26ed181153fb69eb73bda9fca9291b4eaed" + integrity sha512-6azq5YonWdzHcO9llK9zn1a+rGxlTz2Uf8p8NWDQnl2AZ56neDLYEL3mNDlrMXAy8dSJIHw+u9VF1OOzdslIHQ== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-ns-openapi-3-0" "^1.6.0" + "@swagger-api/apidom-parser-adapter-yaml-1-2" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + +"@swagger-api/apidom-parser-adapter-openapi-yaml-3-1@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-openapi-yaml-3-1/-/apidom-parser-adapter-openapi-yaml-3-1-1.6.0.tgz#cb9e42504cd38564c7c444b958ccb8656314eac5" + integrity sha512-g2tGCXyIAC0IA6JjA0HVxHWyCovyfAxDQ+pMAJ6qm4PfrZHB+oXKWKZHNNmQaFiKdc/SVdMQq6Up0mXOQs7IOQ== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-ns-openapi-3-1" "^1.6.0" + "@swagger-api/apidom-parser-adapter-yaml-1-2" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + +"@swagger-api/apidom-parser-adapter-openapi-yaml-3-2@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-openapi-yaml-3-2/-/apidom-parser-adapter-openapi-yaml-3-2-1.6.0.tgz#5851ff4cfeae7712ffab022b347e4877859ffc17" + integrity sha512-NGkdG9X5Svi89ZBluNseyUBNdgB9MkbTTNmerVKKOmCCHaVbzIb6UFPXf1MifSFyT+wTeGZk6WZLgRIDsTAZ5Q== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-ns-openapi-3-2" "^1.6.0" + "@swagger-api/apidom-parser-adapter-yaml-1-2" "^1.6.0" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + +"@swagger-api/apidom-parser-adapter-yaml-1-2@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-parser-adapter-yaml-1-2/-/apidom-parser-adapter-yaml-1-2-1.6.0.tgz#3bd4c52755f39c76a159645aadc9f96bec9156cd" + integrity sha512-UwSE5pPUJ+ag7ZCbesgx/SJ8zUD3Sx+2U4AD3/1G1EJ+0gb7FMYgihuOT8ujmBfZVGGm3HMIEIa1w3zha08v2g== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-ast" "^1.6.0" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-error" "^1.6.0" + "@tree-sitter-grammars/tree-sitter-yaml" "=0.7.1" + "@types/ramda" "~0.30.0" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + tree-sitter "=0.22.4" + web-tree-sitter "=0.24.5" + +"@swagger-api/apidom-reference@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@swagger-api/apidom-reference/-/apidom-reference-1.6.0.tgz#1060b54d135b1888850678cd35d862ffb1ac6f77" + integrity sha512-gYTDfWQM1heqrCCrCsZH+EWDyAkIGqEJnSJcVWKngwOkXJKeUwat8p1TOW4q3rkaTT+fBaYbrjTr9SkFtVbdMg== + dependencies: + "@babel/runtime-corejs3" "^7.26.10" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-error" "^1.6.0" + "@types/ramda" "~0.30.0" + axios "^1.12.2" + minimatch "^10.2.1" + ramda "~0.30.0" + ramda-adjunct "^5.0.0" + optionalDependencies: + "@swagger-api/apidom-json-pointer" "^1.6.0" + "@swagger-api/apidom-ns-arazzo-1" "^1.6.0" + "@swagger-api/apidom-ns-asyncapi-2" "^1.6.0" + "@swagger-api/apidom-ns-openapi-2" "^1.6.0" + "@swagger-api/apidom-ns-openapi-3-0" "^1.6.0" + "@swagger-api/apidom-ns-openapi-3-1" "^1.6.0" + "@swagger-api/apidom-ns-openapi-3-2" "^1.6.0" + "@swagger-api/apidom-parser-adapter-api-design-systems-json" "^1.6.0" + "@swagger-api/apidom-parser-adapter-api-design-systems-yaml" "^1.6.0" + "@swagger-api/apidom-parser-adapter-arazzo-json-1" "^1.6.0" + "@swagger-api/apidom-parser-adapter-arazzo-yaml-1" "^1.6.0" + "@swagger-api/apidom-parser-adapter-asyncapi-json-2" "^1.6.0" + "@swagger-api/apidom-parser-adapter-asyncapi-json-3" "^1.6.0" + "@swagger-api/apidom-parser-adapter-asyncapi-yaml-2" "^1.6.0" + "@swagger-api/apidom-parser-adapter-asyncapi-yaml-3" "^1.6.0" + "@swagger-api/apidom-parser-adapter-json" "^1.6.0" + "@swagger-api/apidom-parser-adapter-openapi-json-2" "^1.6.0" + "@swagger-api/apidom-parser-adapter-openapi-json-3-0" "^1.6.0" + "@swagger-api/apidom-parser-adapter-openapi-json-3-1" "^1.6.0" + "@swagger-api/apidom-parser-adapter-openapi-json-3-2" "^1.6.0" + "@swagger-api/apidom-parser-adapter-openapi-yaml-2" "^1.6.0" + "@swagger-api/apidom-parser-adapter-openapi-yaml-3-0" "^1.6.0" + "@swagger-api/apidom-parser-adapter-openapi-yaml-3-1" "^1.6.0" + "@swagger-api/apidom-parser-adapter-openapi-yaml-3-2" "^1.6.0" + "@swagger-api/apidom-parser-adapter-yaml-1-2" "^1.6.0" + +"@swaggerexpert/cookie@^2.0.2": + version "2.0.2" + resolved "https://registry.yarnpkg.com/@swaggerexpert/cookie/-/cookie-2.0.2.tgz#2b1bc2b5082955372539ce3f2b52c35831b32ef8" + integrity sha512-DPI8YJ0Vznk4CT+ekn3rcFNq1uQwvUHZhH6WvTSPD0YKBIlMS9ur2RYKghXuxxOiqOam/i4lHJH4xTIiTgs3Mg== dependencies: - "@babel/runtime-corejs3" "^7.20.7" - "@swagger-api/apidom-ast" "^0.78.0" - "@swagger-api/apidom-core" "^0.78.0" - "@swagger-api/apidom-error" "^0.78.0" - "@types/ramda" "~0.29.6" - ramda "~0.29.0" - ramda-adjunct "^4.1.1" - stampit "^4.3.2" - tree-sitter "=0.20.4" - tree-sitter-yaml "=0.5.0" - web-tree-sitter "=0.20.3" - -"@swagger-api/apidom-reference@>=0.77.0 <1.0.0": - version "0.78.0" - resolved "https://registry.yarnpkg.com/@swagger-api/apidom-reference/-/apidom-reference-0.78.0.tgz#7ae6fc2c09aa1b3cba015b07c5ef56cebc4795f3" - integrity sha512-IiOaMgy+CzpQe5fFwyge4B/lkHQnBhiuNGPgIJELYXJMZle+pN6K/V4muLCG6JjAXllucbCqMpW/KLmPxGAXaw== + apg-lite "^1.0.3" + +"@swaggerexpert/json-pointer@^2.10.1": + version "2.10.2" + resolved "https://registry.yarnpkg.com/@swaggerexpert/json-pointer/-/json-pointer-2.10.2.tgz#3f4f8082f166e77021113a401f1f2c4e4b45d574" + integrity sha512-qMx1nOrzoB+PF+pzb26Q4Tc2sOlrx9Ba2UBNX9hB31Omrq+QoZ2Gly0KLrQWw4Of1AQ4J9lnD+XOdwOdcdXqqw== dependencies: - "@babel/runtime-corejs3" "^7.20.7" - "@swagger-api/apidom-core" "^0.78.0" - "@types/ramda" "~0.29.6" - axios "^1.4.0" - minimatch "^7.4.3" - process "^0.11.10" - ramda "~0.29.0" - ramda-adjunct "^4.1.1" - stampit "^4.3.2" - optionalDependencies: - "@swagger-api/apidom-error" "^0.78.0" - "@swagger-api/apidom-json-pointer" "^0.78.0" - "@swagger-api/apidom-ns-asyncapi-2" "^0.78.0" - "@swagger-api/apidom-ns-openapi-3-0" "^0.78.0" - "@swagger-api/apidom-ns-openapi-3-1" "^0.78.0" - "@swagger-api/apidom-parser-adapter-api-design-systems-json" "^0.78.0" - "@swagger-api/apidom-parser-adapter-api-design-systems-yaml" "^0.78.0" - "@swagger-api/apidom-parser-adapter-asyncapi-json-2" "^0.78.0" - "@swagger-api/apidom-parser-adapter-asyncapi-yaml-2" "^0.78.0" - "@swagger-api/apidom-parser-adapter-json" "^0.78.0" - "@swagger-api/apidom-parser-adapter-openapi-json-3-0" "^0.78.0" - "@swagger-api/apidom-parser-adapter-openapi-json-3-1" "^0.78.0" - "@swagger-api/apidom-parser-adapter-openapi-yaml-3-0" "^0.78.0" - "@swagger-api/apidom-parser-adapter-openapi-yaml-3-1" "^0.78.0" - "@swagger-api/apidom-parser-adapter-yaml-1-2" "^0.78.0" - -"@testing-library/dom@^8.0.0": - version "8.19.1" - resolved "https://registry.yarnpkg.com/@testing-library/dom/-/dom-8.19.1.tgz#0e2dafd281dedb930bb235eac1045470b4129d0e" - integrity sha512-P6iIPyYQ+qH8CvGauAqanhVnjrnRe0IZFSYCeGkSRW9q3u8bdVn2NPI+lasFyVsEQn1J/IFmp5Aax41+dAP9wg== + apg-lite "^1.0.4" + +"@testing-library/dom@10.4.1": + version "10.4.1" + resolved "https://registry.yarnpkg.com/@testing-library/dom/-/dom-10.4.1.tgz#d444f8a889e9a46e9a3b4f3b88e0fcb3efb6cf95" + integrity sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg== dependencies: "@babel/code-frame" "^7.10.4" "@babel/runtime" "^7.12.5" "@types/aria-query" "^5.0.1" - aria-query "^5.0.0" - chalk "^4.1.0" + aria-query "5.3.0" dom-accessibility-api "^0.5.9" - lz-string "^1.4.4" + lz-string "^1.5.0" + picocolors "1.1.1" pretty-format "^27.0.2" -"@testing-library/jest-dom@^6.1.4": - version "6.1.4" - resolved "https://registry.yarnpkg.com/@testing-library/jest-dom/-/jest-dom-6.1.4.tgz#cf0835c33bc5ef00befb9e672b1e3e6a710e30e3" - integrity sha512-wpoYrCYwSZ5/AxcrjLxJmCU6I5QAJXslEeSiMQqaWmP2Kzpd1LvF/qxmAIW2qposULGWq2gw30GgVNFLSc2Jnw== +"@testing-library/jest-dom@6.9.1": + version "6.9.1" + resolved "https://registry.yarnpkg.com/@testing-library/jest-dom/-/jest-dom-6.9.1.tgz#7613a04e146dd2976d24ddf019730d57a89d56c2" + integrity sha512-zIcONa+hVtVSSep9UT3jZ5rizo2BsxgyDYU7WFD5eICBE7no3881HGeb/QkGfsJs6JTkY1aQhT7rIPC7e+0nnA== dependencies: - "@adobe/css-tools" "^4.3.1" - "@babel/runtime" "^7.9.2" + "@adobe/css-tools" "^4.4.0" aria-query "^5.0.0" - chalk "^3.0.0" css.escape "^1.5.1" - dom-accessibility-api "^0.5.6" - lodash "^4.17.15" + dom-accessibility-api "^0.6.3" + picocolors "^1.1.1" redent "^3.0.0" -"@testing-library/react@^12.0.0": - version "12.1.5" - resolved "https://registry.yarnpkg.com/@testing-library/react/-/react-12.1.5.tgz#bb248f72f02a5ac9d949dea07279095fa577963b" - integrity sha512-OfTXCJUFgjd/digLUuPxa0+/3ZxsQmE7ub9kcbW/wi96Bh3o/p5vrETcBGfP17NWPGqeYYl5LTRpwyGoMC4ysg== +"@testing-library/react@16.3.2": + version "16.3.2" + resolved "https://registry.yarnpkg.com/@testing-library/react/-/react-16.3.2.tgz#672883b7acb8e775fc0492d9e9d25e06e89786d0" + integrity sha512-XU5/SytQM+ykqMnAnvB2umaJNIOsLF3PVv//1Ew4CTcpz0/BRyy/af40qqrt7SjKpDdT1saBMc42CUok5gaw+g== dependencies: "@babel/runtime" "^7.12.5" - "@testing-library/dom" "^8.0.0" - "@types/react-dom" "<18.0.0" -"@testing-library/user-event@^14.5.1": - version "14.5.1" - resolved "https://registry.yarnpkg.com/@testing-library/user-event/-/user-event-14.5.1.tgz#27337d72046d5236b32fd977edee3f74c71d332f" - integrity sha512-UCcUKrUYGj7ClomOo2SpNVvx4/fkd/2BbIHDCle8A0ax+P3bU7yJwDBDrS6ZwdTMARWTGODX1hEsCcO+7beJjg== +"@testing-library/user-event@14.6.1": + version "14.6.1" + resolved "https://registry.yarnpkg.com/@testing-library/user-event/-/user-event-14.6.1.tgz#13e09a32d7a8b7060fe38304788ebf4197cd2149" + integrity sha512-vq7fv0rnt+QTXgPxr5Hjc210p6YKq2kmdziLgnsZGgLJ9e6VAShx1pACLuRjd/AS/sr7phAR58OIIpf0LlmQNw== -"@tootallnate/once@1": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82" - integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw== +"@tree-sitter-grammars/tree-sitter-yaml@=0.7.1": + version "0.7.1" + resolved "https://registry.yarnpkg.com/@tree-sitter-grammars/tree-sitter-yaml/-/tree-sitter-yaml-0.7.1.tgz#9fcf9c56c7b4adb19097f869ada29c2d7a62c93d" + integrity sha512-AynBwkIoQCTgjDR33bDUp9Mqq+YTco0is3n5hRApMqG9of/6A4eQsfC1/uSEeHSUyMQSYawcAWamsexnVpIP4Q== + dependencies: + node-addon-api "^8.3.1" + node-gyp-build "^4.8.4" -"@trysound/sax@0.2.0": - version "0.2.0" - resolved "https://registry.yarnpkg.com/@trysound/sax/-/sax-0.2.0.tgz#cccaab758af56761eb7bf37af6f03f326dd798ad" - integrity sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA== +"@tybys/wasm-util@^0.10.0": + version "0.10.1" + resolved "https://registry.yarnpkg.com/@tybys/wasm-util/-/wasm-util-0.10.1.tgz#ecddd3205cf1e2d5274649ff0eedd2991ed7f414" + integrity sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg== + dependencies: + tslib "^2.4.0" "@types/aria-query@^5.0.1": - version "5.0.1" - resolved "https://registry.yarnpkg.com/@types/aria-query/-/aria-query-5.0.1.tgz#3286741fb8f1e1580ac28784add4c7a1d49bdfbc" - integrity sha512-XTIieEY+gvJ39ChLcB4If5zHtPxt3Syj5rgZR+e1ctpmK8NjPf0zFqsz4JpLJT0xla9GFDKjy8Cpu331nrmE1Q== + version "5.0.4" + resolved "https://registry.yarnpkg.com/@types/aria-query/-/aria-query-5.0.4.tgz#1a31c3d378850d2778dabb6374d036dcba4ba708" + integrity sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw== -"@types/babel__core@^7.0.0", "@types/babel__core@^7.1.14": - version "7.1.20" - resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.20.tgz#e168cdd612c92a2d335029ed62ac94c95b362359" - integrity sha512-PVb6Bg2QuscZ30FvOU7z4guG6c926D9YRvOxEaelzndpMsvP+YM74Q/dAFASpg2l6+XLalxSGxcq/lrgYWZtyQ== +"@types/babel__core@^7.20.5": + version "7.20.5" + resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.20.5.tgz#3df15f27ba85319caa07ba08d0721889bb39c017" + integrity sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA== dependencies: - "@babel/parser" "^7.1.0" - "@babel/types" "^7.0.0" + "@babel/parser" "^7.20.7" + "@babel/types" "^7.20.7" "@types/babel__generator" "*" "@types/babel__template" "*" "@types/babel__traverse" "*" @@ -3426,131 +3532,91 @@ "@babel/parser" "^7.1.0" "@babel/types" "^7.0.0" -"@types/babel__traverse@*", "@types/babel__traverse@^7.0.4", "@types/babel__traverse@^7.0.6": +"@types/babel__traverse@*": version "7.18.3" resolved "https://registry.yarnpkg.com/@types/babel__traverse/-/babel__traverse-7.18.3.tgz#dfc508a85781e5698d5b33443416b6268c4b3e8d" integrity sha512-1kbcJ40lLB7MHsj39U4Sh1uTd2E7rLEa79kmDpI6cy+XiXsteB3POdQomoq4FxszMrO3ZYchkhYJw7A2862b3w== dependencies: "@babel/types" "^7.3.0" -"@types/body-parser@*": - version "1.19.2" - resolved "https://registry.yarnpkg.com/@types/body-parser/-/body-parser-1.19.2.tgz#aea2059e28b7658639081347ac4fab3de166e6f0" - integrity sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g== - dependencies: - "@types/connect" "*" - "@types/node" "*" +"@types/d3-array@^3.2.2": + version "3.2.2" + resolved "https://registry.yarnpkg.com/@types/d3-array/-/d3-array-3.2.2.tgz#e02151464d02d4a1b44646d0fcdb93faf88fde8c" + integrity sha512-hOLWVbm7uRza0BYXpIIW5pxfrKe0W+D5lrFiAEYR+pb6w3N2SwSMaJbXdUfSEv+dT4MfHBLtn5js0LAWaO6otw== -"@types/bonjour@^3.5.9": - version "3.5.10" - resolved "https://registry.yarnpkg.com/@types/bonjour/-/bonjour-3.5.10.tgz#0f6aadfe00ea414edc86f5d106357cda9701e275" - integrity sha512-p7ienRMiS41Nu2/igbJxxLDWrSZ0WxM8UQgCeO9KhoVF7cOVFkrKsiDr1EsJIla8vV3oEEjGcz11jc5yimhzZw== - dependencies: - "@types/node" "*" +"@types/d3-color@*", "@types/d3-color@^3.1.3": + version "3.1.3" + resolved "https://registry.yarnpkg.com/@types/d3-color/-/d3-color-3.1.3.tgz#368c961a18de721da8200e80bf3943fb53136af2" + integrity sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A== -"@types/connect-history-api-fallback@^1.3.5": - version "1.3.5" - resolved "https://registry.yarnpkg.com/@types/connect-history-api-fallback/-/connect-history-api-fallback-1.3.5.tgz#d1f7a8a09d0ed5a57aee5ae9c18ab9b803205dae" - integrity sha512-h8QJa8xSb1WD4fpKBDcATDNGXghFj6/3GRWG6dhmRcu0RX1Ubasur2Uvx5aeEwlf0MwblEC2bMzzMQntxnw/Cw== - dependencies: - "@types/express-serve-static-core" "*" - "@types/node" "*" - -"@types/connect@*": - version "3.4.35" - resolved "https://registry.yarnpkg.com/@types/connect/-/connect-3.4.35.tgz#5fcf6ae445e4021d1fc2219a4873cc73a3bb2ad1" - integrity sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ== - dependencies: - "@types/node" "*" +"@types/d3-format@^3.0.4": + version "3.0.4" + resolved "https://registry.yarnpkg.com/@types/d3-format/-/d3-format-3.0.4.tgz#b1e4465644ddb3fdf3a263febb240a6cd616de90" + integrity sha512-fALi2aI6shfg7vM5KiR1wNJnZ7r6UuggVqtDA+xiEdPZQwy/trcQaHnwShLuLdta2rTymCNpxYTiMZX/e09F4g== -"@types/eslint@^7.29.0 || ^8.4.1": - version "8.4.10" - resolved "https://registry.yarnpkg.com/@types/eslint/-/eslint-8.4.10.tgz#19731b9685c19ed1552da7052b6f668ed7eb64bb" - integrity sha512-Sl/HOqN8NKPmhWo2VBEPm0nvHnu2LL3v9vKo8MEq0EtbJ4eVzGPl41VNPvn5E1i5poMk4/XD8UriLHpJvEP/Nw== +"@types/d3-interpolate@^3.0.4": + version "3.0.4" + resolved "https://registry.yarnpkg.com/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz#412b90e84870285f2ff8a846c6eb60344f12a41c" + integrity sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA== dependencies: - "@types/estree" "*" - "@types/json-schema" "*" - -"@types/estree@*": - version "1.0.0" - resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.0.tgz#5fb2e536c1ae9bf35366eed879e827fa59ca41c2" - integrity sha512-WulqXMDUTYAXCjZnk6JtIHPigp55cVtDgDrO2gHRwhyJto21+1zbVCtOYB2L1F9w4qCQ0rOGWBnBe0FNTiEJIQ== + "@types/d3-color" "*" -"@types/estree@0.0.39": - version "0.0.39" - resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.39.tgz#e177e699ee1b8c22d23174caaa7422644389509f" - integrity sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw== - -"@types/estree@^1.0.5": - version "1.0.5" - resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.5.tgz#a6ce3e556e00fd9895dd872dd172ad0d4bd687f4" - integrity sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw== - -"@types/express-serve-static-core@*", "@types/express-serve-static-core@^4.17.31": - version "4.17.31" - resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.17.31.tgz#a1139efeab4e7323834bb0226e62ac019f474b2f" - integrity sha512-DxMhY+NAsTwMMFHBTtJFNp5qiHKJ7TeqOo23zVEM9alT1Ml27Q3xcTH0xwxn7Q0BbMcVEJOs/7aQtUWupUQN3Q== - dependencies: - "@types/node" "*" - "@types/qs" "*" - "@types/range-parser" "*" +"@types/d3-path@*", "@types/d3-path@^3.1.1": + version "3.1.1" + resolved "https://registry.yarnpkg.com/@types/d3-path/-/d3-path-3.1.1.tgz#f632b380c3aca1dba8e34aa049bcd6a4af23df8a" + integrity sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg== -"@types/express@*", "@types/express@^4.17.13": - version "4.17.15" - resolved "https://registry.yarnpkg.com/@types/express/-/express-4.17.15.tgz#9290e983ec8b054b65a5abccb610411953d417ff" - integrity sha512-Yv0k4bXGOH+8a+7bELd2PqHQsuiANB+A8a4gnQrkRWzrkKlb6KHaVvyXhqs04sVW/OWlbPyYxRgYlIXLfrufMQ== +"@types/d3-scale@^4.0.9": + version "4.0.9" + resolved "https://registry.yarnpkg.com/@types/d3-scale/-/d3-scale-4.0.9.tgz#57a2f707242e6fe1de81ad7bfcccaaf606179afb" + integrity sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw== dependencies: - "@types/body-parser" "*" - "@types/express-serve-static-core" "^4.17.31" - "@types/qs" "*" - "@types/serve-static" "*" + "@types/d3-time" "*" -"@types/graceful-fs@^4.1.2": - version "4.1.5" - resolved "https://registry.yarnpkg.com/@types/graceful-fs/-/graceful-fs-4.1.5.tgz#21ffba0d98da4350db64891f92a9e5db3cdb4e15" - integrity sha512-anKkLmZZ+xm4p8JWBf4hElkM4XR+EZeA2M9BAkkTldmcyDY4mbdIJnRghDJH3Ov5ooY7/UAoENtmdMSkaAd7Cw== +"@types/d3-shape@^3.1.7": + version "3.1.7" + resolved "https://registry.yarnpkg.com/@types/d3-shape/-/d3-shape-3.1.7.tgz#2b7b423dc2dfe69c8c93596e673e37443348c555" + integrity sha512-VLvUQ33C+3J+8p+Daf+nYSOsjB4GXp19/S/aGo60m9h1v6XaxjiT82lKVWJCfzhtuZ3yD7i/TPeC/fuKLLOSmg== dependencies: - "@types/node" "*" + "@types/d3-path" "*" -"@types/graceful-fs@^4.1.3": - version "4.1.8" - resolved "https://registry.yarnpkg.com/@types/graceful-fs/-/graceful-fs-4.1.8.tgz#417e461e4dc79d957dc3107f45fe4973b09c2915" - integrity sha512-NhRH7YzWq8WiNKVavKPBmtLYZHxNY19Hh+az28O/phfp68CF45pMFud+ZzJ8ewnxnC5smIdF3dqFeiSUQ5I+pw== - dependencies: - "@types/node" "*" +"@types/d3-time-format@^4.0.3": + version "4.0.3" + resolved "https://registry.yarnpkg.com/@types/d3-time-format/-/d3-time-format-4.0.3.tgz#d6bc1e6b6a7db69cccfbbdd4c34b70632d9e9db2" + integrity sha512-5xg9rC+wWL8kdDj153qZcsJ0FWiFt0J5RB6LYUNZjwSnesfblqrI/bJ1wBdJ8OQfncgbJG5+2F+qfqnqyzYxyg== -"@types/hast@^2.0.0": - version "2.3.4" - resolved "https://registry.yarnpkg.com/@types/hast/-/hast-2.3.4.tgz#8aa5ef92c117d20d974a82bdfb6a648b08c0bafc" - integrity sha512-wLEm0QvaoawEDoTRwzTXp4b4jpwiJDvR5KMnFnVodm3scufTlBOWRD6N1OBf9TZMhjlNsSfcO5V+7AF4+Vy+9g== - dependencies: - "@types/unist" "*" +"@types/d3-time@*", "@types/d3-time@^3.0.4": + version "3.0.4" + resolved "https://registry.yarnpkg.com/@types/d3-time/-/d3-time-3.0.4.tgz#8472feecd639691450dd8000eb33edd444e1323f" + integrity sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g== -"@types/hoist-non-react-statics@^3.3.1": - version "3.3.1" - resolved "https://registry.yarnpkg.com/@types/hoist-non-react-statics/-/hoist-non-react-statics-3.3.1.tgz#1124aafe5118cb591977aeb1ceaaed1070eb039f" - integrity sha512-iMIqiko6ooLrTh1joXodJK5X9xeEALT1kM5G3ZLhD3hszxBdIEd5C75U834D9mLcINgD4OyZf5uQXjkuYydWvA== - dependencies: - "@types/react" "*" - hoist-non-react-statics "^3.3.0" +"@types/d3-timer@^3.0.2": + version "3.0.2" + resolved "https://registry.yarnpkg.com/@types/d3-timer/-/d3-timer-3.0.2.tgz#70bbda77dc23aa727413e22e214afa3f0e852f70" + integrity sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw== -"@types/html-minifier-terser@^6.0.0": - version "6.1.0" - resolved "https://registry.yarnpkg.com/@types/html-minifier-terser/-/html-minifier-terser-6.1.0.tgz#4fc33a00c1d0c16987b1a20cf92d20614c55ac35" - integrity sha512-oh/6byDPnL1zeNXFrDXFLyZjkr1MsBG667IM792caf1L2UPOOMf65NFzjUH/ltyfwjAGfs1rsX1eftK0jC/KIg== +"@types/estree@1.0.8": + version "1.0.8" + resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.8.tgz#958b91c991b1867ced318bedea0e215ee050726e" + integrity sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w== -"@types/http-proxy@^1.17.8": - version "1.17.9" - resolved "https://registry.yarnpkg.com/@types/http-proxy/-/http-proxy-1.17.9.tgz#7f0e7931343761efde1e2bf48c40f02f3f75705a" - integrity sha512-QsbSjA/fSk7xB+UXlCT3wHBy5ai9wOcNDWwZAtud+jXhwOM3l+EYZh8Lng4+/6n8uar0J7xILzqftJdJ/Wdfkw== +"@types/hast@^3.0.0": + version "3.0.4" + resolved "https://registry.yarnpkg.com/@types/hast/-/hast-3.0.4.tgz#1d6b39993b82cea6ad783945b0508c25903e15aa" + integrity sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ== dependencies: - "@types/node" "*" + "@types/unist" "*" -"@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0", "@types/istanbul-lib-coverage@^2.0.1": +"@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.1": version "2.0.4" resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.4.tgz#8467d4b3c087805d63580480890791277ce35c44" integrity sha512-z/QT1XN4K4KYuslS23k62yDIDLwLFkzxOuMplDtObz0+y7VqJCaO2o+SPwHCvLFZh7xazvvoor2tA/hPz9ee7g== +"@types/istanbul-lib-coverage@^2.0.6": + version "2.0.6" + resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz#7739c232a1fee9b4d3ce8985f314c0c6d33549d7" + integrity sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w== + "@types/istanbul-lib-report@*": version "3.0.0" resolved "https://registry.yarnpkg.com/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.0.tgz#c14c24f18ea8190c118ee7562b7ff99a36552686" @@ -3558,720 +3624,249 @@ dependencies: "@types/istanbul-lib-coverage" "*" -"@types/istanbul-reports@^3.0.0": - version "3.0.1" - resolved "https://registry.yarnpkg.com/@types/istanbul-reports/-/istanbul-reports-3.0.1.tgz#9153fe98bba2bd565a63add9436d6f0d7f8468ff" - integrity sha512-c3mAZEuK0lvBp8tmuL74XRKn1+y2dcwOUpH7x4WrF6gk1GIgiluDRgMYQtw2OFcBvAJWlt6ASU3tSqxp0Uu0Aw== +"@types/istanbul-reports@^3.0.4": + version "3.0.4" + resolved "https://registry.yarnpkg.com/@types/istanbul-reports/-/istanbul-reports-3.0.4.tgz#0f03e3d2f670fbdac586e34b433783070cc16f54" + integrity sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ== dependencies: "@types/istanbul-lib-report" "*" -"@types/jest@^29.5.6": - version "29.5.6" - resolved "https://registry.yarnpkg.com/@types/jest/-/jest-29.5.6.tgz#f4cf7ef1b5b0bfc1aa744e41b24d9cc52533130b" - integrity sha512-/t9NnzkOpXb4Nfvg17ieHE6EeSjDS2SGSpNYfoLbUAeL/EOueU/RSdOWFpfQTXBEM7BguYW1XQ0EbM+6RlIh6w== +"@types/jest@30.0.0": + version "30.0.0" + resolved "https://registry.yarnpkg.com/@types/jest/-/jest-30.0.0.tgz#5e85ae568006712e4ad66f25433e9bdac8801f1d" + integrity sha512-XTYugzhuwqWjws0CVz8QpM36+T+Dz5mTEBKhNs/esGLnCIlGdRy+Dq78NRjd7ls7r8BC8ZRMOrKlkO1hU0JOwA== dependencies: - expect "^29.0.0" - pretty-format "^29.0.0" - -"@types/json-schema@*", "@types/json-schema@^7.0.4", "@types/json-schema@^7.0.5", "@types/json-schema@^7.0.8", "@types/json-schema@^7.0.9": - version "7.0.11" - resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.11.tgz#d421b6c527a3037f7c84433fd2c4229e016863d3" - integrity sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ== - -"@types/json-schema@^7.0.12": - version "7.0.14" - resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.14.tgz#74a97a5573980802f32c8e47b663530ab3b6b7d1" - integrity sha512-U3PUjAudAdJBeC2pgN8uTIKgxrb4nlDF3SF0++EldXQvQBGkpFZMSnwQiIoDU77tv45VgNkl/L4ouD+rEomujw== - -"@types/json5@^0.0.29": - version "0.0.29" - resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" - integrity sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ== + expect "^30.0.0" + pretty-format "^30.0.0" -"@types/mime@*": - version "3.0.1" - resolved "https://registry.yarnpkg.com/@types/mime/-/mime-3.0.1.tgz#5f8f2bca0a5863cb69bc0b0acd88c96cb1d4ae10" - integrity sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA== +"@types/jsdom@^21.1.7": + version "21.1.7" + resolved "https://registry.yarnpkg.com/@types/jsdom/-/jsdom-21.1.7.tgz#9edcb09e0b07ce876e7833922d3274149c898cfa" + integrity sha512-yOriVnggzrnQ3a9OKOCxaVuSug3w3/SbOj5i7VwXWZEyUNl3bLF9V3MfxGbZKuwqJOQyRfqXyROBB1CoZLFWzA== + dependencies: + "@types/node" "*" + "@types/tough-cookie" "*" + parse5 "^7.0.0" "@types/node@*": version "18.11.18" resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f" integrity sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA== -"@types/node@^18.17.5": - version "18.18.6" - resolved "https://registry.yarnpkg.com/@types/node/-/node-18.18.6.tgz#26da694f75cdb057750f49d099da5e3f3824cb3e" - integrity sha512-wf3Vz+jCmOQ2HV1YUJuCWdL64adYxumkrxtc+H1VUQlnQI04+5HtH+qZCOE21lBE7gIrt+CwX2Wv8Acrw5Ak6w== - -"@types/node@^20.8.7": - version "20.8.7" - resolved "https://registry.yarnpkg.com/@types/node/-/node-20.8.7.tgz#ad23827850843de973096edfc5abc9e922492a25" - integrity sha512-21TKHHh3eUHIi2MloeptJWALuCu5H7HQTdTrWIFReA8ad+aggoX+lRes3ex7/FtpC+sVUpFMQ+QTfYr74mruiQ== +"@types/node@24.10.9": + version "24.10.9" + resolved "https://registry.yarnpkg.com/@types/node/-/node-24.10.9.tgz#1aeb5142e4a92957489cac12b07f9c7fe26057d0" + integrity sha512-ne4A0IpG3+2ETuREInjPNhUGis1SFjv1d5asp8MzEAGtOZeTeHVDOYqOgqfhvseqg/iXty2hjBf1zAOb7RNiNw== dependencies: - undici-types "~5.25.1" + undici-types "~7.16.0" "@types/parse-json@^4.0.0": version "4.0.0" resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0" integrity sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA== -"@types/prettier@^2.1.5": - version "2.7.2" - resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.7.2.tgz#6c2324641cc4ba050a8c710b2b251b377581fbf0" - integrity sha512-KufADq8uQqo1pYKVIYzfKbJfBAc0sOeXqGbFaSpv8MRmC/zXgowNZmFcbngndGk922QDmOASEXUZCaY48gs4cg== - -"@types/prop-types@*", "@types/prop-types@^15.7.5": - version "15.7.5" - resolved "https://registry.yarnpkg.com/@types/prop-types/-/prop-types-15.7.5.tgz#5f19d2b85a98e9558036f6a3cacc8819420f05cf" - integrity sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w== - -"@types/prop-types@^15.7.11": - version "15.7.12" - resolved "https://registry.yarnpkg.com/@types/prop-types/-/prop-types-15.7.12.tgz#12bb1e2be27293c1406acb6af1c3f3a1481d98c6" - integrity sha512-5zvhXYtRNRluoE/jAp4GVsSduVUzNWKkOZrCDBWYtE7biZywwdC2AcEzg+cSMLFRfVgeAFqpfNabiPjxFddV1Q== - -"@types/q@^1.5.1": - version "1.5.5" - resolved "https://registry.yarnpkg.com/@types/q/-/q-1.5.5.tgz#75a2a8e7d8ab4b230414505d92335d1dcb53a6df" - integrity sha512-L28j2FcJfSZOnL1WBjDYp2vUHCeIFlyYI/53EwD/rKUBQ7MtUUfbQWiyKJGpcnv4/WgrhWsFKrcPstcAt/J0tQ== - -"@types/qs@*": - version "6.9.7" - resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb" - integrity sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw== - -"@types/ramda@~0.29.6": - version "0.29.7" - resolved "https://registry.yarnpkg.com/@types/ramda/-/ramda-0.29.7.tgz#e78747e4d6b498b335b841200b5fa03287a9c60c" - integrity sha512-IUl6U95qwlQtVvZkSX4ODj08oJVtPyWMFRtPVNqhxc2rt+Bh7lCzTrGMYMZ7dmRKcAjtot3xrPnYGwsjdt8gzQ== - dependencies: - types-ramda "^0.29.5" - -"@types/range-parser@*": - version "1.2.4" - resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc" - integrity sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw== - -"@types/react-dom@17.0.9", "@types/react-dom@<18.0.0": - version "17.0.9" - resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-17.0.9.tgz#441a981da9d7be117042e1a6fd3dac4b30f55add" - integrity sha512-wIvGxLfgpVDSAMH5utdL9Ngm5Owu0VsGmldro3ORLXV8CShrL8awVj06NuEXFQ5xyaYfdca7Sgbk/50Ri1GdPg== - dependencies: - "@types/react" "*" - -"@types/react-dom@^17.0.9": - version "17.0.18" - resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-17.0.18.tgz#8f7af38f5d9b42f79162eea7492e5a1caff70dc2" - integrity sha512-rLVtIfbwyur2iFKykP2w0pl/1unw26b5td16d5xMgp7/yjTHomkyxPYChFoCr/FtEX1lN9wY6lFj1qvKdS5kDw== - dependencies: - "@types/react" "^17" - -"@types/react-is@^16.7.1 || ^17.0.0": - version "17.0.3" - resolved "https://registry.yarnpkg.com/@types/react-is/-/react-is-17.0.3.tgz#2d855ba575f2fc8d17ef9861f084acc4b90a137a" - integrity sha512-aBTIWg1emtu95bLTLx0cpkxwGW3ueZv71nE2YFBpL8k/z5czEW8yYpOo8Dp+UUAFAtKwNaOsh/ioSeQnWlZcfw== - dependencies: - "@types/react" "*" - -"@types/react-transition-group@^4.4.5": - version "4.4.5" - resolved "https://registry.yarnpkg.com/@types/react-transition-group/-/react-transition-group-4.4.5.tgz#aae20dcf773c5aa275d5b9f7cdbca638abc5e416" - integrity sha512-juKD/eiSM3/xZYzjuzH6ZwpP+/lejltmiS3QEzV/vmb/Q8+HfDmxu+Baga8UEMGBqV88Nbg4l2hY/K2DkyaLLA== - dependencies: - "@types/react" "*" - -"@types/react@*", "@types/react@17.0.20", "@types/react@^17": - version "17.0.20" - resolved "https://registry.yarnpkg.com/@types/react/-/react-17.0.20.tgz#a4284b184d47975c71658cd69e759b6bd37c3b8c" - integrity sha512-wWZrPlihslrPpcKyCSlmIlruakxr57/buQN1RjlIeaaTWDLtJkTtRW429MoQJergvVKc4IWBpRhWw7YNh/7GVA== - dependencies: - "@types/prop-types" "*" - "@types/scheduler" "*" - csstype "^3.0.2" +"@types/prismjs@^1.0.0": + version "1.26.5" + resolved "https://registry.yarnpkg.com/@types/prismjs/-/prismjs-1.26.5.tgz#72499abbb4c4ec9982446509d2f14fb8483869d6" + integrity sha512-AUZTa7hQ2KY5L7AmtSiqxlhWxb4ina0yd8hNbl4TWuqnv/pFP0nDMb3YrfSBf4hJVGLh2YEIBfKaBW/9UEl6IQ== -"@types/react@^17.0.20": - version "17.0.65" - resolved "https://registry.yarnpkg.com/@types/react/-/react-17.0.65.tgz#95f6a2ab61145ffb69129d07982d047f9e0870cd" - integrity sha512-oxur785xZYHvnI7TRS61dXbkIhDPnGfsXKv0cNXR/0ml4SipRIFpSMzA7HMEfOywFwJ5AOnPrXYTEiTRUQeGlQ== - dependencies: - "@types/prop-types" "*" - "@types/scheduler" "*" - csstype "^3.0.2" +"@types/prop-types@^15.7.15": + version "15.7.15" + resolved "https://registry.yarnpkg.com/@types/prop-types/-/prop-types-15.7.15.tgz#e6e5a86d602beaca71ce5163fadf5f95d70931c7" + integrity sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw== -"@types/resolve@1.17.1": - version "1.17.1" - resolved "https://registry.yarnpkg.com/@types/resolve/-/resolve-1.17.1.tgz#3afd6ad8967c77e4376c598a82ddd58f46ec45d6" - integrity sha512-yy7HuzQhj0dhGpD8RLXSZWEkLsV9ibvxvi6EiJ3bkqLAO1RGo0WbkWQiwpRlSFymTJRz0d3k5LM3kkx8ArDbLw== +"@types/ramda@~0.30.0": + version "0.30.2" + resolved "https://registry.yarnpkg.com/@types/ramda/-/ramda-0.30.2.tgz#70661b20c1bb969589a551b7134ae75008ffbfb6" + integrity sha512-PyzHvjCalm2BRYjAU6nIB3TprYwMNOUY/7P/N8bSzp9W/yM2YrtGtAnnVtaCNSeOZ8DzKyFDvaqQs7LnWwwmBA== dependencies: - "@types/node" "*" + types-ramda "^0.30.1" -"@types/retry@0.12.0": - version "0.12.0" - resolved "https://registry.yarnpkg.com/@types/retry/-/retry-0.12.0.tgz#2b35eccfcee7d38cd72ad99232fbd58bffb3c84d" - integrity sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA== - -"@types/scheduler@*": - version "0.16.2" - resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.2.tgz#1a62f89525723dde24ba1b01b092bf5df8ad4d39" - integrity sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew== - -"@types/semver@^7.3.12": - version "7.3.13" - resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.3.13.tgz#da4bfd73f49bd541d28920ab0e2bf0ee80f71c91" - integrity sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw== - -"@types/semver@^7.5.0": - version "7.5.4" - resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.4.tgz#0a41252ad431c473158b22f9bfb9a63df7541cff" - integrity sha512-MMzuxN3GdFwskAnb6fz0orFvhfqi752yjaXylr0Rp4oDg5H0Zn1IuyRhDVvYOwAXoJirx2xuS16I3WjxnAIHiQ== +"@types/react-dom@19.2.3": + version "19.2.3" + resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-19.2.3.tgz#c1e305d15a52a3e508d54dca770d202cb63abf2c" + integrity sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ== -"@types/serve-index@^1.9.1": - version "1.9.1" - resolved "https://registry.yarnpkg.com/@types/serve-index/-/serve-index-1.9.1.tgz#1b5e85370a192c01ec6cec4735cf2917337a6278" - integrity sha512-d/Hs3nWDxNL2xAczmOVZNj92YZCS6RGxfBPjKzuu/XirCgXdpKEb88dYNbrYGint6IVWLNP+yonwVAuRC0T2Dg== - dependencies: - "@types/express" "*" +"@types/react-transition-group@^4.4.12": + version "4.4.12" + resolved "https://registry.yarnpkg.com/@types/react-transition-group/-/react-transition-group-4.4.12.tgz#b5d76568485b02a307238270bfe96cb51ee2a044" + integrity sha512-8TV6R3h2j7a91c+1DXdJi3Syo69zzIZbz7Lg5tORM5LEJG7X/E6a1V3drRyBRZq7/utz7A+c4OgYLiLcYGHG6w== -"@types/serve-static@*", "@types/serve-static@^1.13.10": - version "1.15.0" - resolved "https://registry.yarnpkg.com/@types/serve-static/-/serve-static-1.15.0.tgz#c7930ff61afb334e121a9da780aac0d9b8f34155" - integrity sha512-z5xyF6uh8CbjAu9760KDKsH2FcDxZ2tFCsA4HIMWE6IkiYMXfVoa+4f9KX+FN0ZLsaMw1WNG2ETLA6N+/YA+cg== +"@types/react@*", "@types/react@19.2.14": + version "19.2.14" + resolved "https://registry.yarnpkg.com/@types/react/-/react-19.2.14.tgz#39604929b5e3957e3a6fa0001dafb17c7af70bad" + integrity sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w== dependencies: - "@types/mime" "*" - "@types/node" "*" - -"@types/sinonjs__fake-timers@8.1.1": - version "8.1.1" - resolved "https://registry.yarnpkg.com/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-8.1.1.tgz#b49c2c70150141a15e0fa7e79cf1f92a72934ce3" - integrity sha512-0kSuKjAS0TrGLJ0M/+8MaFkGsQhZpB6pxOmvS3K8FYI72K//YmdfoW9X2qPsAKh1mkwxGD5zib9s1FIFed6E8g== + csstype "^3.2.2" -"@types/sizzle@^2.3.2": - version "2.3.3" - resolved "https://registry.yarnpkg.com/@types/sizzle/-/sizzle-2.3.3.tgz#ff5e2f1902969d305225a047c8a0fd5c915cebef" - integrity sha512-JYM8x9EGF163bEyhdJBpR2QX1R5naCJHC8ucJylJ3w9/CVBaskdQ8WqBf8MmQrd1kRvp/a4TS8HJ+bxzR7ZJYQ== - -"@types/sockjs@^0.3.33": - version "0.3.33" - resolved "https://registry.yarnpkg.com/@types/sockjs/-/sockjs-0.3.33.tgz#570d3a0b99ac995360e3136fd6045113b1bd236f" - integrity sha512-f0KEEe05NvUnat+boPTZ0dgaLZ4SfSouXUgv5noUiefG2ajgKjmETo9ZJyuqsl7dfl2aHlLJUiki6B4ZYldiiw== - dependencies: - "@types/node" "*" +"@types/stack-utils@^2.0.3": + version "2.0.3" + resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.3.tgz#6209321eb2c1712a7e7466422b8cb1fc0d9dd5d8" + integrity sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw== -"@types/stack-utils@^2.0.0": - version "2.0.1" - resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.1.tgz#20f18294f797f2209b5f65c8e3b5c8e8261d127c" - integrity sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw== +"@types/stylis@4.2.5": + version "4.2.5" + resolved "https://registry.yarnpkg.com/@types/stylis/-/stylis-4.2.5.tgz#1daa6456f40959d06157698a653a9ab0a70281df" + integrity sha512-1Xve+NMN7FWjY14vLoY5tL3BVEQ/n42YLwaqJIPYhotZ9uBHt87VceMwWQpzmdEt2TNXIorIFG+YeCUUW7RInw== -"@types/swagger-ui-react@^4.18.2": - version "4.18.2" - resolved "https://registry.yarnpkg.com/@types/swagger-ui-react/-/swagger-ui-react-4.18.2.tgz#009a7469b48457b349e0ec7501b21f11c0fc4ca0" - integrity sha512-uLiHyNGFdgQINNSq3hgca+TNnNMS7/X59M02EbmDfARgvh7KcB6wU+FVfkVY13qE4Zk9opKvjGqygwjjP7+u6w== +"@types/swagger-ui-react@5.18.0": + version "5.18.0" + resolved "https://registry.yarnpkg.com/@types/swagger-ui-react/-/swagger-ui-react-5.18.0.tgz#dd9617deed6a4080d755a93a004f118606bc448c" + integrity sha512-c2M9adVG7t28t1pq19K9Jt20VLQf0P/fwJwnfcmsVVsdkwCWhRmbKDu+tIs0/NGwJ/7GY8lBx+iKZxuDI5gDbw== dependencies: "@types/react" "*" -"@types/trusted-types@^2.0.2": - version "2.0.2" - resolved "https://registry.yarnpkg.com/@types/trusted-types/-/trusted-types-2.0.2.tgz#fc25ad9943bcac11cceb8168db4f275e0e72e756" - integrity sha512-F5DIZ36YVLE+PN+Zwws4kJogq47hNgX3Nx6WyDJ3kcplxyke3XIzB8uK5n/Lpm1HBsbGzd6nmGehL8cPekP+Tg== +"@types/tough-cookie@*": + version "4.0.5" + resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.5.tgz#cb6e2a691b70cb177c6e3ae9c1d2e8b2ea8cd304" + integrity sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA== + +"@types/trusted-types@^2.0.7": + version "2.0.7" + resolved "https://registry.yarnpkg.com/@types/trusted-types/-/trusted-types-2.0.7.tgz#baccb07a970b91707df3a3e8ba6896c57ead2d11" + integrity sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw== "@types/unist@*": version "2.0.6" resolved "https://registry.yarnpkg.com/@types/unist/-/unist-2.0.6.tgz#250a7b16c3b91f672a24552ec64678eeb1d3a08d" integrity sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ== -"@types/use-sync-external-store@^0.0.3": - version "0.0.3" - resolved "https://registry.yarnpkg.com/@types/use-sync-external-store/-/use-sync-external-store-0.0.3.tgz#b6725d5f4af24ace33b36fafd295136e75509f43" - integrity sha512-EwmlvuaxPNej9+T4v5AuBPJa2x2UOJVdjCtDHgcDqitUeOtjnJKJ+apYjVcAoBEMjKW1VVFGZLUb5+qqa09XFA== +"@types/unist@^2.0.0": + version "2.0.11" + resolved "https://registry.yarnpkg.com/@types/unist/-/unist-2.0.11.tgz#11af57b127e32487774841f7a4e54eab166d03c4" + integrity sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA== -"@types/ws@^8.5.1": - version "8.5.4" - resolved "https://registry.yarnpkg.com/@types/ws/-/ws-8.5.4.tgz#bb10e36116d6e570dd943735f86c933c1587b8a5" - integrity sha512-zdQDHKUgcX/zBc4GrwsE/7dVdAD8JR4EuiAXiiUhhfyIJXXb2+PrGshFyeXWQPMmmZ2XxgaqclgpIC7eTXc1mg== - dependencies: - "@types/node" "*" +"@types/use-sync-external-store@^0.0.6": + version "0.0.6" + resolved "https://registry.yarnpkg.com/@types/use-sync-external-store/-/use-sync-external-store-0.0.6.tgz#60be8d21baab8c305132eb9cb912ed497852aadc" + integrity sha512-zFDAD+tlpf2r4asuHEj0XH6pY6i0g5NeAHPn+15wk3BV6JA69eERFXC1gyGThDkVa1zCyKr5jox1+2LbV/AMLg== "@types/yargs-parser@*": version "21.0.0" resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-21.0.0.tgz#0c60e537fa790f5f9472ed2776c2b71ec117351b" integrity sha512-iO9ZQHkZxHn4mSakYV0vFHAVDyEOIJQrV2uZ06HxEPcx+mt8swXoZHIbaaJ2crJYFfErySgktuTZ3BeLz+XmFA== -"@types/yargs@^16.0.0": - version "16.0.4" - resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-16.0.4.tgz#26aad98dd2c2a38e421086ea9ad42b9e51642977" - integrity sha512-T8Yc9wt/5LbJyCaLiHPReJa0kApcIgJ7Bn735GjItUfh08Z1pJvu8QZqb9s+mMvKV6WUQRV7K2R46YbjMXTTJw== +"@types/yargs@^17.0.33": + version "17.0.34" + resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-17.0.34.tgz#1c2f9635b71d5401827373a01ce2e8a7670ea839" + integrity sha512-KExbHVa92aJpw9WDQvzBaGVE2/Pz+pLZQloT2hjL8IqsZnV62rlPOYvNnLmf/L2dyllfVUOVBj64M0z/46eR2A== dependencies: "@types/yargs-parser" "*" -"@types/yargs@^17.0.8": - version "17.0.18" - resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-17.0.18.tgz#466225ab4fbabb9aa711f5b406796daf1374a5b7" - integrity sha512-eIJR1UER6ur3EpKM3d+2Pgd+ET+k6Kn9B4ZItX0oPjjVI5PrfaRjKyLT5UYendDpLuoiJMNJvovLQbEXqhsPaw== - dependencies: - "@types/yargs-parser" "*" - -"@types/yauzl@^2.9.1": - version "2.10.0" - resolved "https://registry.yarnpkg.com/@types/yauzl/-/yauzl-2.10.0.tgz#b3248295276cf8c6f153ebe6a9aba0c988cb2599" - integrity sha512-Cn6WYCm0tXv8p6k+A8PvbDG763EDpBoTzHdA+Q/MF6H3sapGjCm9NzoaJncJS9tUKSuCoDs9XHxYYsQDgxR6kw== - dependencies: - "@types/node" "*" - -"@typescript-eslint/eslint-plugin@^5.5.0": - version "5.47.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.47.1.tgz#50cc5085578a7fa22cd46a0806c2e5eae858af02" - integrity sha512-r4RZ2Jl9kcQN7K/dcOT+J7NAimbiis4sSM9spvWimsBvDegMhKLA5vri2jG19PmIPbDjPeWzfUPQ2hjEzA4Nmg== - dependencies: - "@typescript-eslint/scope-manager" "5.47.1" - "@typescript-eslint/type-utils" "5.47.1" - "@typescript-eslint/utils" "5.47.1" - debug "^4.3.4" - ignore "^5.2.0" - natural-compare-lite "^1.4.0" - regexpp "^3.2.0" - semver "^7.3.7" - tsutils "^3.21.0" - -"@typescript-eslint/eslint-plugin@^6.8.0": - version "6.8.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.8.0.tgz#06abe4265e7c82f20ade2dcc0e3403c32d4f148b" - integrity sha512-GosF4238Tkes2SHPQ1i8f6rMtG6zlKwMEB0abqSJ3Npvos+doIlc/ATG+vX1G9coDF3Ex78zM3heXHLyWEwLUw== - dependencies: - "@eslint-community/regexpp" "^4.5.1" - "@typescript-eslint/scope-manager" "6.8.0" - "@typescript-eslint/type-utils" "6.8.0" - "@typescript-eslint/utils" "6.8.0" - "@typescript-eslint/visitor-keys" "6.8.0" - debug "^4.3.4" - graphemer "^1.4.0" - ignore "^5.2.4" - natural-compare "^1.4.0" - semver "^7.5.4" - ts-api-utils "^1.0.1" - -"@typescript-eslint/experimental-utils@^5.0.0": - version "5.47.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/experimental-utils/-/experimental-utils-5.47.1.tgz#32f83e1ad1483d3772461f602679fb5b4516baa1" - integrity sha512-zWHo/VbqAiRvhXP5byQqW7rGQtdanajHnItGqtmv8JaIi58zMPnmGZ1bW/drXIjU1fuOyfTVoDkNS7aEWGDSLg== - dependencies: - "@typescript-eslint/utils" "5.47.1" - -"@typescript-eslint/parser@^5.5.0": - version "5.47.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-5.47.1.tgz#c4bf16f8c3c7608ce4bf8ff804b677fc899f173f" - integrity sha512-9Vb+KIv29r6GPu4EboWOnQM7T+UjpjXvjCPhNORlgm40a9Ia9bvaPJswvtae1gip2QEeVeGh6YquqAzEgoRAlw== - dependencies: - "@typescript-eslint/scope-manager" "5.47.1" - "@typescript-eslint/types" "5.47.1" - "@typescript-eslint/typescript-estree" "5.47.1" - debug "^4.3.4" - -"@typescript-eslint/parser@^6.8.0": - version "6.8.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-6.8.0.tgz#bb2a969d583db242f1ee64467542f8b05c2e28cb" - integrity sha512-5tNs6Bw0j6BdWuP8Fx+VH4G9fEPDxnVI7yH1IAPkQH5RUtvKwRoqdecAPdQXv4rSOADAaz1LFBZvZG7VbXivSg== - dependencies: - "@typescript-eslint/scope-manager" "6.8.0" - "@typescript-eslint/types" "6.8.0" - "@typescript-eslint/typescript-estree" "6.8.0" - "@typescript-eslint/visitor-keys" "6.8.0" - debug "^4.3.4" - -"@typescript-eslint/scope-manager@5.47.1": - version "5.47.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-5.47.1.tgz#0d302b3c2f20ab24e4787bf3f5a0d8c449b823bd" - integrity sha512-9hsFDsgUwrdOoW1D97Ewog7DYSHaq4WKuNs0LHF9RiCmqB0Z+XRR4Pf7u7u9z/8CciHuJ6yxNws1XznI3ddjEw== - dependencies: - "@typescript-eslint/types" "5.47.1" - "@typescript-eslint/visitor-keys" "5.47.1" - -"@typescript-eslint/scope-manager@6.8.0": - version "6.8.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-6.8.0.tgz#5cac7977385cde068ab30686889dd59879811efd" - integrity sha512-xe0HNBVwCph7rak+ZHcFD6A+q50SMsFwcmfdjs9Kz4qDh5hWhaPhFjRs/SODEhroBI5Ruyvyz9LfwUJ624O40g== - dependencies: - "@typescript-eslint/types" "6.8.0" - "@typescript-eslint/visitor-keys" "6.8.0" - -"@typescript-eslint/type-utils@5.47.1": - version "5.47.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-5.47.1.tgz#aee13314f840ab336c1adb49a300856fd16d04ce" - integrity sha512-/UKOeo8ee80A7/GJA427oIrBi/Gd4osk/3auBUg4Rn9EahFpevVV1mUK8hjyQD5lHPqX397x6CwOk5WGh1E/1w== - dependencies: - "@typescript-eslint/typescript-estree" "5.47.1" - "@typescript-eslint/utils" "5.47.1" - debug "^4.3.4" - tsutils "^3.21.0" - -"@typescript-eslint/type-utils@6.8.0": - version "6.8.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-6.8.0.tgz#50365e44918ca0fd159844b5d6ea96789731e11f" - integrity sha512-RYOJdlkTJIXW7GSldUIHqc/Hkto8E+fZN96dMIFhuTJcQwdRoGN2rEWA8U6oXbLo0qufH7NPElUb+MceHtz54g== - dependencies: - "@typescript-eslint/typescript-estree" "6.8.0" - "@typescript-eslint/utils" "6.8.0" - debug "^4.3.4" - ts-api-utils "^1.0.1" - -"@typescript-eslint/types@5.47.1": - version "5.47.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-5.47.1.tgz#459f07428aec5a8c4113706293c2ae876741ac8e" - integrity sha512-CmALY9YWXEpwuu6377ybJBZdtSAnzXLSQcxLSqSQSbC7VfpMu/HLVdrnVJj7ycI138EHqocW02LPJErE35cE9A== - -"@typescript-eslint/types@6.8.0": - version "6.8.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-6.8.0.tgz#1ab5d4fe1d613e3f65f6684026ade6b94f7e3ded" - integrity sha512-p5qOxSum7W3k+llc7owEStXlGmSl8FcGvhYt8Vjy7FqEnmkCVlM3P57XQEGj58oqaBWDQXbJDZxwUWMS/EAPNQ== - -"@typescript-eslint/typescript-estree@5.47.1": - version "5.47.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-5.47.1.tgz#b9d8441308aca53df7f69b2c67a887b82c9ed418" - integrity sha512-4+ZhFSuISAvRi2xUszEj0xXbNTHceV9GbH9S8oAD2a/F9SW57aJNQVOCxG8GPfSWH/X4eOPdMEU2jYVuWKEpWA== - dependencies: - "@typescript-eslint/types" "5.47.1" - "@typescript-eslint/visitor-keys" "5.47.1" - debug "^4.3.4" - globby "^11.1.0" - is-glob "^4.0.3" - semver "^7.3.7" - tsutils "^3.21.0" - -"@typescript-eslint/typescript-estree@6.8.0": - version "6.8.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-6.8.0.tgz#9565f15e0cd12f55cf5aa0dfb130a6cb0d436ba1" - integrity sha512-ISgV0lQ8XgW+mvv5My/+iTUdRmGspducmQcDw5JxznasXNnZn3SKNrTRuMsEXv+V/O+Lw9AGcQCfVaOPCAk/Zg== - dependencies: - "@typescript-eslint/types" "6.8.0" - "@typescript-eslint/visitor-keys" "6.8.0" - debug "^4.3.4" - globby "^11.1.0" - is-glob "^4.0.3" - semver "^7.5.4" - ts-api-utils "^1.0.1" - -"@typescript-eslint/utils@5.47.1", "@typescript-eslint/utils@^5.13.0": - version "5.47.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-5.47.1.tgz#595f25ac06e9ee28c339fd43c709402820b13d7b" - integrity sha512-l90SdwqfmkuIVaREZ2ykEfCezepCLxzWMo5gVfcJsJCaT4jHT+QjgSkYhs5BMQmWqE9k3AtIfk4g211z/sTMVw== - dependencies: - "@types/json-schema" "^7.0.9" - "@types/semver" "^7.3.12" - "@typescript-eslint/scope-manager" "5.47.1" - "@typescript-eslint/types" "5.47.1" - "@typescript-eslint/typescript-estree" "5.47.1" - eslint-scope "^5.1.1" - eslint-utils "^3.0.0" - semver "^7.3.7" - -"@typescript-eslint/utils@6.8.0": - version "6.8.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-6.8.0.tgz#d42939c2074c6b59844d0982ce26a51d136c4029" - integrity sha512-dKs1itdE2qFG4jr0dlYLQVppqTE+Itt7GmIf/vX6CSvsW+3ov8PbWauVKyyfNngokhIO9sKZeRGCUo1+N7U98Q== - dependencies: - "@eslint-community/eslint-utils" "^4.4.0" - "@types/json-schema" "^7.0.12" - "@types/semver" "^7.5.0" - "@typescript-eslint/scope-manager" "6.8.0" - "@typescript-eslint/types" "6.8.0" - "@typescript-eslint/typescript-estree" "6.8.0" - semver "^7.5.4" - -"@typescript-eslint/visitor-keys@5.47.1": - version "5.47.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-5.47.1.tgz#d35c2da544dbb685db9c5b5b85adac0a1d74d1f2" - integrity sha512-rF3pmut2JCCjh6BLRhNKdYjULMb1brvoaiWDlHfLNVgmnZ0sBVJrs3SyaKE1XoDDnJuAx/hDQryHYmPUuNq0ig== - dependencies: - "@typescript-eslint/types" "5.47.1" - eslint-visitor-keys "^3.3.0" - -"@typescript-eslint/visitor-keys@6.8.0": - version "6.8.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-6.8.0.tgz#cffebed56ae99c45eba901c378a6447b06be58b8" - integrity sha512-oqAnbA7c+pgOhW2OhGvxm0t1BULX5peQI/rLsNDpGM78EebV3C9IGbX5HNZabuZ6UQrYveCLjKo8Iy/lLlBkkg== - dependencies: - "@typescript-eslint/types" "6.8.0" - eslint-visitor-keys "^3.4.1" - -"@webassemblyjs/ast@1.12.1", "@webassemblyjs/ast@^1.12.1": - version "1.12.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/ast/-/ast-1.12.1.tgz#bb16a0e8b1914f979f45864c23819cc3e3f0d4bb" - integrity sha512-EKfMUOPRRUTy5UII4qJDGPpqfwjOmZ5jeGFwid9mnoqIFK+e0vqoi1qH56JpmZSzEL53jKnNzScdmftJyG5xWg== - dependencies: - "@webassemblyjs/helper-numbers" "1.11.6" - "@webassemblyjs/helper-wasm-bytecode" "1.11.6" - -"@webassemblyjs/floating-point-hex-parser@1.11.6": - version "1.11.6" - resolved "https://registry.yarnpkg.com/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.6.tgz#dacbcb95aff135c8260f77fa3b4c5fea600a6431" - integrity sha512-ejAj9hfRJ2XMsNHk/v6Fu2dGS+i4UaXBXGemOfQ/JfQ6mdQg/WXtwleQRLLS4OvfDhv8rYnVwH27YJLMyYsxhw== - -"@webassemblyjs/helper-api-error@1.11.6": - version "1.11.6" - resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.6.tgz#6132f68c4acd59dcd141c44b18cbebbd9f2fa768" - integrity sha512-o0YkoP4pVu4rN8aTJgAyj9hC2Sv5UlkzCHhxqWj8butaLvnpdc2jOwh4ewE6CX0txSfLn/UYaV/pheS2Txg//Q== - -"@webassemblyjs/helper-buffer@1.12.1": - version "1.12.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-buffer/-/helper-buffer-1.12.1.tgz#6df20d272ea5439bf20ab3492b7fb70e9bfcb3f6" - integrity sha512-nzJwQw99DNDKr9BVCOZcLuJJUlqkJh+kVzVl6Fmq/tI5ZtEyWT1KZMyOXltXLZJmDtvLCDgwsyrkohEtopTXCw== - -"@webassemblyjs/helper-numbers@1.11.6": - version "1.11.6" - resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.6.tgz#cbce5e7e0c1bd32cf4905ae444ef64cea919f1b5" - integrity sha512-vUIhZ8LZoIWHBohiEObxVm6hwP034jwmc9kuq5GdHZH0wiLVLIPcMCdpJzG4C11cHoQ25TFIQj9kaVADVX7N3g== - dependencies: - "@webassemblyjs/floating-point-hex-parser" "1.11.6" - "@webassemblyjs/helper-api-error" "1.11.6" - "@xtuc/long" "4.2.2" - -"@webassemblyjs/helper-wasm-bytecode@1.11.6": - version "1.11.6" - resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.6.tgz#bb2ebdb3b83aa26d9baad4c46d4315283acd51e9" - integrity sha512-sFFHKwcmBprO9e7Icf0+gddyWYDViL8bpPjJJl0WHxCdETktXdmtWLGVzoHbqUcY4Be1LkNfwTmXOJUFZYSJdA== - -"@webassemblyjs/helper-wasm-section@1.12.1": - version "1.12.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.12.1.tgz#3da623233ae1a60409b509a52ade9bc22a37f7bf" - integrity sha512-Jif4vfB6FJlUlSbgEMHUyk1j234GTNG9dBJ4XJdOySoj518Xj0oGsNi59cUQF4RRMS9ouBUxDDdyBVfPTypa5g== - dependencies: - "@webassemblyjs/ast" "1.12.1" - "@webassemblyjs/helper-buffer" "1.12.1" - "@webassemblyjs/helper-wasm-bytecode" "1.11.6" - "@webassemblyjs/wasm-gen" "1.12.1" - -"@webassemblyjs/ieee754@1.11.6": - version "1.11.6" - resolved "https://registry.yarnpkg.com/@webassemblyjs/ieee754/-/ieee754-1.11.6.tgz#bb665c91d0b14fffceb0e38298c329af043c6e3a" - integrity sha512-LM4p2csPNvbij6U1f19v6WR56QZ8JcHg3QIJTlSwzFcmx6WSORicYj6I63f9yU1kEUtrpG+kjkiIAkevHpDXrg== - dependencies: - "@xtuc/ieee754" "^1.2.0" - -"@webassemblyjs/leb128@1.11.6": - version "1.11.6" - resolved "https://registry.yarnpkg.com/@webassemblyjs/leb128/-/leb128-1.11.6.tgz#70e60e5e82f9ac81118bc25381a0b283893240d7" - integrity sha512-m7a0FhE67DQXgouf1tbN5XQcdWoNgaAuoULHIfGFIEVKA6tu/edls6XnIlkmS6FrXAquJRPni3ZZKjw6FSPjPQ== - dependencies: - "@xtuc/long" "4.2.2" - -"@webassemblyjs/utf8@1.11.6": - version "1.11.6" - resolved "https://registry.yarnpkg.com/@webassemblyjs/utf8/-/utf8-1.11.6.tgz#90f8bc34c561595fe156603be7253cdbcd0fab5a" - integrity sha512-vtXf2wTQ3+up9Zsg8sa2yWiQpzSsMyXj0qViVP6xKGCUT8p8YJ6HqI7l5eCnWx1T/FYdsv07HQs2wTFbbof/RA== - -"@webassemblyjs/wasm-edit@^1.12.1": - version "1.12.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-edit/-/wasm-edit-1.12.1.tgz#9f9f3ff52a14c980939be0ef9d5df9ebc678ae3b" - integrity sha512-1DuwbVvADvS5mGnXbE+c9NfA8QRcZ6iKquqjjmR10k6o+zzsRVesil54DKexiowcFCPdr/Q0qaMgB01+SQ1u6g== - dependencies: - "@webassemblyjs/ast" "1.12.1" - "@webassemblyjs/helper-buffer" "1.12.1" - "@webassemblyjs/helper-wasm-bytecode" "1.11.6" - "@webassemblyjs/helper-wasm-section" "1.12.1" - "@webassemblyjs/wasm-gen" "1.12.1" - "@webassemblyjs/wasm-opt" "1.12.1" - "@webassemblyjs/wasm-parser" "1.12.1" - "@webassemblyjs/wast-printer" "1.12.1" - -"@webassemblyjs/wasm-gen@1.12.1": - version "1.12.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-gen/-/wasm-gen-1.12.1.tgz#a6520601da1b5700448273666a71ad0a45d78547" - integrity sha512-TDq4Ojh9fcohAw6OIMXqiIcTq5KUXTGRkVxbSo1hQnSy6lAM5GSdfwWeSxpAo0YzgsgF182E/U0mDNhuA0tW7w== - dependencies: - "@webassemblyjs/ast" "1.12.1" - "@webassemblyjs/helper-wasm-bytecode" "1.11.6" - "@webassemblyjs/ieee754" "1.11.6" - "@webassemblyjs/leb128" "1.11.6" - "@webassemblyjs/utf8" "1.11.6" - -"@webassemblyjs/wasm-opt@1.12.1": - version "1.12.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-opt/-/wasm-opt-1.12.1.tgz#9e6e81475dfcfb62dab574ac2dda38226c232bc5" - integrity sha512-Jg99j/2gG2iaz3hijw857AVYekZe2SAskcqlWIZXjji5WStnOpVoat3gQfT/Q5tb2djnCjBtMocY/Su1GfxPBg== - dependencies: - "@webassemblyjs/ast" "1.12.1" - "@webassemblyjs/helper-buffer" "1.12.1" - "@webassemblyjs/wasm-gen" "1.12.1" - "@webassemblyjs/wasm-parser" "1.12.1" - -"@webassemblyjs/wasm-parser@1.12.1", "@webassemblyjs/wasm-parser@^1.12.1": - version "1.12.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-parser/-/wasm-parser-1.12.1.tgz#c47acb90e6f083391e3fa61d113650eea1e95937" - integrity sha512-xikIi7c2FHXysxXe3COrVUPSheuBtpcfhbpFj4gmu7KRLYOzANztwUU0IbsqvMqzuNK2+glRGWCEqZo1WCLyAQ== - dependencies: - "@webassemblyjs/ast" "1.12.1" - "@webassemblyjs/helper-api-error" "1.11.6" - "@webassemblyjs/helper-wasm-bytecode" "1.11.6" - "@webassemblyjs/ieee754" "1.11.6" - "@webassemblyjs/leb128" "1.11.6" - "@webassemblyjs/utf8" "1.11.6" - -"@webassemblyjs/wast-printer@1.12.1": - version "1.12.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wast-printer/-/wast-printer-1.12.1.tgz#bcecf661d7d1abdaf989d8341a4833e33e2b31ac" - integrity sha512-+X4WAlOisVWQMikjbcvY2e0rwPsKQ9F688lksZhBcPycBBuii3O7m8FACbDMWDojpAqvjIncrG8J0XHKyQfVeA== - dependencies: - "@webassemblyjs/ast" "1.12.1" - "@xtuc/long" "4.2.2" - -"@xtuc/ieee754@^1.2.0": - version "1.2.0" - resolved "https://registry.yarnpkg.com/@xtuc/ieee754/-/ieee754-1.2.0.tgz#eef014a3145ae477a1cbc00cd1e552336dceb790" - integrity sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA== - -"@xtuc/long@4.2.2": - version "4.2.2" - resolved "https://registry.yarnpkg.com/@xtuc/long/-/long-4.2.2.tgz#d291c6a4e97989b5c61d9acf396ae4fe133a718d" - integrity sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ== - -"@yarnpkg/lockfile@^1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz#e77a97fbd345b76d83245edcd17d393b1b41fb31" - integrity sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ== - -abab@^2.0.3, abab@^2.0.5: - version "2.0.6" - resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.6.tgz#41b80f2c871d19686216b82309231cfd3cb3d291" - integrity sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA== - -accepts@~1.3.4, accepts@~1.3.5, accepts@~1.3.8: - version "1.3.8" - resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.8.tgz#0bf0be125b67014adcb0b0921e62db7bffe16b2e" - integrity sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw== - dependencies: - mime-types "~2.1.34" - negotiator "0.6.3" - -acorn-globals@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/acorn-globals/-/acorn-globals-6.0.0.tgz#46cdd39f0f8ff08a876619b55f5ac8a6dc770b45" - integrity sha512-ZQl7LOWaF5ePqqcX4hLuv/bLXYQNfNWw2c0/yX/TsPRKamzHcTGQnlCjHT3TsmkOUVEPS3crCxiPfdzE/Trlhg== - dependencies: - acorn "^7.1.1" - acorn-walk "^7.1.1" - -acorn-import-attributes@^1.9.5: - version "1.9.5" - resolved "https://registry.yarnpkg.com/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz#7eb1557b1ba05ef18b5ed0ec67591bfab04688ef" - integrity sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ== - -acorn-jsx@^5.3.2: - version "5.3.2" - resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937" - integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ== - -acorn-node@^1.8.2: - version "1.8.2" - resolved "https://registry.yarnpkg.com/acorn-node/-/acorn-node-1.8.2.tgz#114c95d64539e53dede23de8b9d96df7c7ae2af8" - integrity sha512-8mt+fslDufLYntIoPAaIMUe/lrbrehIiwmR3t2k9LljIzoigEPF27eLk2hy8zSGzmR/ogr7zbRKINMo1u0yh5A== - dependencies: - acorn "^7.0.0" - acorn-walk "^7.0.0" - xtend "^4.0.2" - -acorn-walk@^7.0.0, acorn-walk@^7.1.1: - version "7.2.0" - resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.2.0.tgz#0de889a601203909b0fbe07b8938dc21d2e967bc" - integrity sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA== - -acorn@^7.0.0, acorn@^7.1.1: - version "7.4.1" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" - integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== - -acorn@^8.2.4, acorn@^8.5.0, acorn@^8.7.1, acorn@^8.8.0: - version "8.8.1" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.1.tgz#0a3f9cbecc4ec3bea6f0a80b66ae8dd2da250b73" - integrity sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA== - -acorn@^8.8.2: - version "8.12.1" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.12.1.tgz#71616bdccbe25e27a54439e0046e89ca76df2248" - integrity sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg== - -address@^1.0.1, address@^1.1.2: - version "1.2.2" - resolved "https://registry.yarnpkg.com/address/-/address-1.2.2.tgz#2b5248dac5485a6390532c6a517fda2e3faac89e" - integrity sha512-4B/qKCfeE/ODUaAUpSwfzazo5x29WD4r3vXiWsB7I2mSDAihwEqKO+g8GELZUQSSAo5e1XTYh3ZVfLyxBc12nA== - -adjust-sourcemap-loader@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/adjust-sourcemap-loader/-/adjust-sourcemap-loader-4.0.0.tgz#fc4a0fd080f7d10471f30a7320f25560ade28c99" - integrity sha512-OXwN5b9pCUXNQHJpwwD2qP40byEmSgzj8B4ydSN0uMNYWiFmJ6x6KwUllMmfk8Rwu/HJDFR7U8ubsWBoN0Xp0A== - dependencies: - loader-utils "^2.0.0" - regex-parser "^2.2.11" - -agent-base@6: - version "6.0.2" - resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77" - integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ== - dependencies: - debug "4" - -aggregate-error@^3.0.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-3.1.0.tgz#92670ff50f5359bdb7a3e0d40d0ec30c5737687a" - integrity sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA== - dependencies: - clean-stack "^2.0.0" - indent-string "^4.0.0" - -ajv-formats@^2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/ajv-formats/-/ajv-formats-2.1.1.tgz#6e669400659eb74973bbf2e33327180a0996b520" - integrity sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA== - dependencies: - ajv "^8.0.0" - -ajv-keywords@^3.4.1, ajv-keywords@^3.5.2: - version "3.5.2" - resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.5.2.tgz#31f29da5ab6e00d1c2d329acf7b5929614d5014d" - integrity sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ== - -ajv-keywords@^5.0.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-5.1.0.tgz#69d4d385a4733cdbeab44964a1170a88f87f0e16" - integrity sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw== - dependencies: - fast-deep-equal "^3.1.3" - -ajv@^6.10.0, ajv@^6.12.2, ajv@^6.12.4, ajv@^6.12.5: - version "6.12.6" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" - integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== - dependencies: - fast-deep-equal "^3.1.1" - fast-json-stable-stringify "^2.0.0" - json-schema-traverse "^0.4.1" - uri-js "^4.2.2" - -ajv@^8.0.0, ajv@^8.6.0, ajv@^8.8.0: - version "8.11.2" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.11.2.tgz#aecb20b50607acf2569b6382167b65a96008bb78" - integrity sha512-E4bfmKAhGiSTvMfL1Myyycaub+cUEU2/IvpylXkUu7CHBkBj1f/ikdzbD7YQ6FKUbixDxeYvB/xY4fvyroDlQg== - dependencies: - fast-deep-equal "^3.1.1" - json-schema-traverse "^1.0.0" - require-from-string "^2.0.2" - uri-js "^4.2.2" - -ansi-colors@^4.1.1: - version "4.1.3" - resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-4.1.3.tgz#37611340eb2243e70cc604cad35d63270d48781b" - integrity sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw== - -ansi-escapes@^4.2.1, ansi-escapes@^4.3.0, ansi-escapes@^4.3.1: +"@ungap/structured-clone@^1.3.0": + version "1.3.0" + resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.3.0.tgz#d06bbb384ebcf6c505fde1c3d0ed4ddffe0aaff8" + integrity sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g== + +"@unrs/resolver-binding-android-arm-eabi@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@unrs/resolver-binding-android-arm-eabi/-/resolver-binding-android-arm-eabi-1.11.1.tgz#9f5b04503088e6a354295e8ea8fe3cb99e43af81" + integrity sha512-ppLRUgHVaGRWUx0R0Ut06Mjo9gBaBkg3v/8AxusGLhsIotbBLuRk51rAzqLC8gq6NyyAojEXglNjzf6R948DNw== + +"@unrs/resolver-binding-android-arm64@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@unrs/resolver-binding-android-arm64/-/resolver-binding-android-arm64-1.11.1.tgz#7414885431bd7178b989aedc4d25cccb3865bc9f" + integrity sha512-lCxkVtb4wp1v+EoN+HjIG9cIIzPkX5OtM03pQYkG+U5O/wL53LC4QbIeazgiKqluGeVEeBlZahHalCaBvU1a2g== + +"@unrs/resolver-binding-darwin-arm64@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@unrs/resolver-binding-darwin-arm64/-/resolver-binding-darwin-arm64-1.11.1.tgz#b4a8556f42171fb9c9f7bac8235045e82aa0cbdf" + integrity sha512-gPVA1UjRu1Y/IsB/dQEsp2V1pm44Of6+LWvbLc9SDk1c2KhhDRDBUkQCYVWe6f26uJb3fOK8saWMgtX8IrMk3g== + +"@unrs/resolver-binding-darwin-x64@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@unrs/resolver-binding-darwin-x64/-/resolver-binding-darwin-x64-1.11.1.tgz#fd4d81257b13f4d1a083890a6a17c00de571f0dc" + integrity sha512-cFzP7rWKd3lZaCsDze07QX1SC24lO8mPty9vdP+YVa3MGdVgPmFc59317b2ioXtgCMKGiCLxJ4HQs62oz6GfRQ== + +"@unrs/resolver-binding-freebsd-x64@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@unrs/resolver-binding-freebsd-x64/-/resolver-binding-freebsd-x64-1.11.1.tgz#d2513084d0f37c407757e22f32bd924a78cfd99b" + integrity sha512-fqtGgak3zX4DCB6PFpsH5+Kmt/8CIi4Bry4rb1ho6Av2QHTREM+47y282Uqiu3ZRF5IQioJQ5qWRV6jduA+iGw== + +"@unrs/resolver-binding-linux-arm-gnueabihf@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@unrs/resolver-binding-linux-arm-gnueabihf/-/resolver-binding-linux-arm-gnueabihf-1.11.1.tgz#844d2605d057488d77fab09705f2866b86164e0a" + integrity sha512-u92mvlcYtp9MRKmP+ZvMmtPN34+/3lMHlyMj7wXJDeXxuM0Vgzz0+PPJNsro1m3IZPYChIkn944wW8TYgGKFHw== + +"@unrs/resolver-binding-linux-arm-musleabihf@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@unrs/resolver-binding-linux-arm-musleabihf/-/resolver-binding-linux-arm-musleabihf-1.11.1.tgz#204892995cefb6bd1d017d52d097193bc61ddad3" + integrity sha512-cINaoY2z7LVCrfHkIcmvj7osTOtm6VVT16b5oQdS4beibX2SYBwgYLmqhBjA1t51CarSaBuX5YNsWLjsqfW5Cw== + +"@unrs/resolver-binding-linux-arm64-gnu@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@unrs/resolver-binding-linux-arm64-gnu/-/resolver-binding-linux-arm64-gnu-1.11.1.tgz#023eb0c3aac46066a10be7a3f362e7b34f3bdf9d" + integrity sha512-34gw7PjDGB9JgePJEmhEqBhWvCiiWCuXsL9hYphDF7crW7UgI05gyBAi6MF58uGcMOiOqSJ2ybEeCvHcq0BCmQ== + +"@unrs/resolver-binding-linux-arm64-musl@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@unrs/resolver-binding-linux-arm64-musl/-/resolver-binding-linux-arm64-musl-1.11.1.tgz#9e6f9abb06424e3140a60ac996139786f5d99be0" + integrity sha512-RyMIx6Uf53hhOtJDIamSbTskA99sPHS96wxVE/bJtePJJtpdKGXO1wY90oRdXuYOGOTuqjT8ACccMc4K6QmT3w== + +"@unrs/resolver-binding-linux-ppc64-gnu@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@unrs/resolver-binding-linux-ppc64-gnu/-/resolver-binding-linux-ppc64-gnu-1.11.1.tgz#b111417f17c9d1b02efbec8e08398f0c5527bb44" + integrity sha512-D8Vae74A4/a+mZH0FbOkFJL9DSK2R6TFPC9M+jCWYia/q2einCubX10pecpDiTmkJVUH+y8K3BZClycD8nCShA== + +"@unrs/resolver-binding-linux-riscv64-gnu@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@unrs/resolver-binding-linux-riscv64-gnu/-/resolver-binding-linux-riscv64-gnu-1.11.1.tgz#92ffbf02748af3e99873945c9a8a5ead01d508a9" + integrity sha512-frxL4OrzOWVVsOc96+V3aqTIQl1O2TjgExV4EKgRY09AJ9leZpEg8Ak9phadbuX0BA4k8U5qtvMSQQGGmaJqcQ== + +"@unrs/resolver-binding-linux-riscv64-musl@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@unrs/resolver-binding-linux-riscv64-musl/-/resolver-binding-linux-riscv64-musl-1.11.1.tgz#0bec6f1258fc390e6b305e9ff44256cb207de165" + integrity sha512-mJ5vuDaIZ+l/acv01sHoXfpnyrNKOk/3aDoEdLO/Xtn9HuZlDD6jKxHlkN8ZhWyLJsRBxfv9GYM2utQ1SChKew== + +"@unrs/resolver-binding-linux-s390x-gnu@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@unrs/resolver-binding-linux-s390x-gnu/-/resolver-binding-linux-s390x-gnu-1.11.1.tgz#577843a084c5952f5906770633ccfb89dac9bc94" + integrity sha512-kELo8ebBVtb9sA7rMe1Cph4QHreByhaZ2QEADd9NzIQsYNQpt9UkM9iqr2lhGr5afh885d/cB5QeTXSbZHTYPg== + +"@unrs/resolver-binding-linux-x64-gnu@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@unrs/resolver-binding-linux-x64-gnu/-/resolver-binding-linux-x64-gnu-1.11.1.tgz#36fb318eebdd690f6da32ac5e0499a76fa881935" + integrity sha512-C3ZAHugKgovV5YvAMsxhq0gtXuwESUKc5MhEtjBpLoHPLYM+iuwSj3lflFwK3DPm68660rZ7G8BMcwSro7hD5w== + +"@unrs/resolver-binding-linux-x64-musl@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@unrs/resolver-binding-linux-x64-musl/-/resolver-binding-linux-x64-musl-1.11.1.tgz#bfb9af75f783f98f6a22c4244214efe4df1853d6" + integrity sha512-rV0YSoyhK2nZ4vEswT/QwqzqQXw5I6CjoaYMOX0TqBlWhojUf8P94mvI7nuJTeaCkkds3QE4+zS8Ko+GdXuZtA== + +"@unrs/resolver-binding-wasm32-wasi@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@unrs/resolver-binding-wasm32-wasi/-/resolver-binding-wasm32-wasi-1.11.1.tgz#752c359dd875684b27429500d88226d7cc72f71d" + integrity sha512-5u4RkfxJm+Ng7IWgkzi3qrFOvLvQYnPBmjmZQ8+szTK/b31fQCnleNl1GgEt7nIsZRIf5PLhPwT0WM+q45x/UQ== + dependencies: + "@napi-rs/wasm-runtime" "^0.2.11" + +"@unrs/resolver-binding-win32-arm64-msvc@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@unrs/resolver-binding-win32-arm64-msvc/-/resolver-binding-win32-arm64-msvc-1.11.1.tgz#ce5735e600e4c2fbb409cd051b3b7da4a399af35" + integrity sha512-nRcz5Il4ln0kMhfL8S3hLkxI85BXs3o8EYoattsJNdsX4YUU89iOkVn7g0VHSRxFuVMdM4Q1jEpIId1Ihim/Uw== + +"@unrs/resolver-binding-win32-ia32-msvc@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@unrs/resolver-binding-win32-ia32-msvc/-/resolver-binding-win32-ia32-msvc-1.11.1.tgz#72fc57bc7c64ec5c3de0d64ee0d1810317bc60a6" + integrity sha512-DCEI6t5i1NmAZp6pFonpD5m7i6aFrpofcp4LA2i8IIq60Jyo28hamKBxNrZcyOwVOZkgsRp9O2sXWBWP8MnvIQ== + +"@unrs/resolver-binding-win32-x64-msvc@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@unrs/resolver-binding-win32-x64-msvc/-/resolver-binding-win32-x64-msvc-1.11.1.tgz#538b1e103bf8d9864e7b85cc96fa8d6fb6c40777" + integrity sha512-lrW200hZdbfRtztbygyaq/6jP6AKE8qQN2KvPcJ+x7wiD038YtnYtZ82IMNJ69GJibV7bwL3y9FgK+5w/pYt6g== + +agent-base@^7.1.0, agent-base@^7.1.2: + version "7.1.4" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-7.1.4.tgz#e3cd76d4c548ee895d3c3fd8dc1f6c5b9032e7a8" + integrity sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ== + +ansi-escapes@^4.3.2: version "4.3.2" resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-4.3.2.tgz#6b2291d1db7d98b6521d5f1efa42d0f3a9feb65e" integrity sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ== dependencies: type-fest "^0.21.3" -ansi-html-community@^0.0.8: - version "0.0.8" - resolved "https://registry.yarnpkg.com/ansi-html-community/-/ansi-html-community-0.0.8.tgz#69fbc4d6ccbe383f9736934ae34c3f8290f1bf41" - integrity sha512-1APHAyr3+PCamwNw3bXCPp4HFLONZt/yIH0sZp0/469KWNTEy+qN5jQ3GVX6DMZ1UXAi34yVwtTeaG/HpBuuzw== - ansi-regex@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" @@ -4296,12 +3891,17 @@ ansi-styles@^4.0.0, ansi-styles@^4.1.0: dependencies: color-convert "^2.0.1" -ansi-styles@^5.0.0: +ansi-styles@^5.0.0, ansi-styles@^5.2.0: version "5.2.0" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-5.2.0.tgz#07449690ad45777d1924ac2abb2fc8895dba836b" integrity sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA== -anymatch@^3.0.3, anymatch@~3.1.2: +ansi-styles@^6.1.0: + version "6.2.3" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-6.2.3.tgz#c044d5dcc521a076413472597a1acb1f103c4041" + integrity sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg== + +anymatch@^3.1.3: version "3.1.3" resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.3.tgz#790c58b19ba1720a84205b57c618d5ad8524973e" integrity sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw== @@ -4309,15 +3909,10 @@ anymatch@^3.0.3, anymatch@~3.1.2: normalize-path "^3.0.0" picomatch "^2.0.4" -arch@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/arch/-/arch-2.2.0.tgz#1bc47818f305764f23ab3306b0bfc086c5a29d11" - integrity sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ== - -arg@^5.0.2: - version "5.0.2" - resolved "https://registry.yarnpkg.com/arg/-/arg-5.0.2.tgz#c81433cc427c92c4dcf4865142dbca6f15acd59c" - integrity sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg== +apg-lite@^1.0.3, apg-lite@^1.0.4: + version "1.0.5" + resolved "https://registry.yarnpkg.com/apg-lite/-/apg-lite-1.0.5.tgz#17aee0e8452cb4ce7c7018dc54e53046cb31878b" + integrity sha512-SlI+nLMQDzCZfS39ihzjGp3JNBQfJXyMi6cg9tkLOCPVErgFsUIAEdO9IezR7kbP5Xd0ozcPNQBkf9TO5cHgWw== argparse@^1.0.10, argparse@^1.0.7: version "1.0.10" @@ -4331,13 +3926,12 @@ argparse@^2.0.1: resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== -aria-query@^4.2.2: - version "4.2.2" - resolved "https://registry.yarnpkg.com/aria-query/-/aria-query-4.2.2.tgz#0d2ca6c9aceb56b8977e9fed6aed7e15bbd2f83b" - integrity sha512-o/HelwhuKpTj/frsOsbNLNgnNGVIFsVP/SW2BSF14gVl7kAfMOJ6/8wUAUvG1R1NHKrfG+2sHZTu0yauT1qBrA== +aria-query@5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/aria-query/-/aria-query-5.3.0.tgz#650c569e41ad90b51b3d7df5e5eed1c7549c103e" + integrity sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A== dependencies: - "@babel/runtime" "^7.10.2" - "@babel/runtime-corejs3" "^7.10.2" + dequal "^2.0.3" aria-query@^5.0.0: version "5.1.3" @@ -4346,116 +3940,16 @@ aria-query@^5.0.0: dependencies: deep-equal "^2.0.5" -array-flatten@1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2" - integrity sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg== - -array-flatten@^2.1.2: - version "2.1.2" - resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-2.1.2.tgz#24ef80a28c1a893617e2149b0c6d0d788293b099" - integrity sha512-hNfzcOV8W4NdualtqBFPyVO+54DSJuZGY9qT4pRroB6S9e3iiido2ISIC5h9R2sPJ8H3FHCIiEnsv1lPXO3KtQ== - -array-includes@^3.1.4, array-includes@^3.1.5, array-includes@^3.1.6: - version "3.1.6" - resolved "https://registry.yarnpkg.com/array-includes/-/array-includes-3.1.6.tgz#9e9e720e194f198266ba9e18c29e6a9b0e4b225f" - integrity sha512-sgTbLvL6cNnw24FnbaDyjmvddQ2ML8arZsgaJhoABMoplz/4QRhtrYS+alr1BUM1Bwp6dhx8vVCBSLG+StwOFw== - dependencies: - call-bind "^1.0.2" - define-properties "^1.1.4" - es-abstract "^1.20.4" - get-intrinsic "^1.1.3" - is-string "^1.0.7" - -array-union@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d" - integrity sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw== - -array.prototype.flat@^1.2.5: - version "1.3.1" - resolved "https://registry.yarnpkg.com/array.prototype.flat/-/array.prototype.flat-1.3.1.tgz#ffc6576a7ca3efc2f46a143b9d1dda9b4b3cf5e2" - integrity sha512-roTU0KWIOmJ4DRLmwKd19Otg0/mT3qPNt0Qb3GWW8iObuZXxrjB/pzn0R3hqpRSWg4HCwqx+0vwOnWnvlOyeIA== - dependencies: - call-bind "^1.0.2" - define-properties "^1.1.4" - es-abstract "^1.20.4" - es-shim-unscopables "^1.0.0" - -array.prototype.flatmap@^1.3.1: - version "1.3.1" - resolved "https://registry.yarnpkg.com/array.prototype.flatmap/-/array.prototype.flatmap-1.3.1.tgz#1aae7903c2100433cb8261cd4ed310aab5c4a183" - integrity sha512-8UGn9O1FDVvMNB0UlLv4voxRMze7+FpHyF5mSMRjWHUMlpoDViniy05870VlxhfgTnLbpuwTzvD76MTtWxB/mQ== - dependencies: - call-bind "^1.0.2" - define-properties "^1.1.4" - es-abstract "^1.20.4" - es-shim-unscopables "^1.0.0" - -array.prototype.reduce@^1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/array.prototype.reduce/-/array.prototype.reduce-1.0.5.tgz#6b20b0daa9d9734dd6bc7ea66b5bbce395471eac" - integrity sha512-kDdugMl7id9COE8R7MHF5jWk7Dqt/fs4Pv+JXoICnYwqpjjjbUurz6w5fT5IG6brLdJhv6/VoHB0H7oyIBXd+Q== - dependencies: - call-bind "^1.0.2" - define-properties "^1.1.4" - es-abstract "^1.20.4" - es-array-method-boxes-properly "^1.0.0" - is-string "^1.0.7" - -array.prototype.tosorted@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/array.prototype.tosorted/-/array.prototype.tosorted-1.1.1.tgz#ccf44738aa2b5ac56578ffda97c03fd3e23dd532" - integrity sha512-pZYPXPRl2PqWcsUs6LOMn+1f1532nEoPTYowBtqLwAW+W8vSVhkIGnmOX1t/UQjD6YGI0vcD2B1U7ZFGQH9jnQ== - dependencies: - call-bind "^1.0.2" - define-properties "^1.1.4" - es-abstract "^1.20.4" - es-shim-unscopables "^1.0.0" - get-intrinsic "^1.1.3" - asap@~2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46" integrity sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA== -asn1@~0.2.3: - version "0.2.6" - resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.6.tgz#0d3a7bb6e64e02a90c0303b31f292868ea09a08d" - integrity sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ== - dependencies: - safer-buffer "~2.1.0" - -assert-plus@1.0.0, assert-plus@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525" - integrity sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw== - -ast-types-flow@^0.0.7: - version "0.0.7" - resolved "https://registry.yarnpkg.com/ast-types-flow/-/ast-types-flow-0.0.7.tgz#f70b735c6bca1a5c9c22d982c3e39e7feba3bdad" - integrity sha512-eBvWn1lvIApYMhzQMsu9ciLfkBY499mFZlNqG+/9WR7PVlroQw0vG30cOQQbaKz3sCEc44TAOu2ykzqXSNnwag== - -astral-regex@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-2.0.0.tgz#483143c567aeed4785759c0865786dc77d7d2e31" - integrity sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ== - -async@^3.2.0, async@^3.2.3: - version "3.2.4" - resolved "https://registry.yarnpkg.com/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c" - integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ== - asynckit@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== -at-least-node@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/at-least-node/-/at-least-node-1.0.0.tgz#602cd4b46e844ad4effc92a8011a3c46e0238dc2" - integrity sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg== - autolinker@^3.11.0: version "3.16.2" resolved "https://registry.yarnpkg.com/autolinker/-/autolinker-3.16.2.tgz#6bb4f32432fc111b65659336863e653973bfbcc9" @@ -4463,119 +3957,57 @@ autolinker@^3.11.0: dependencies: tslib "^2.3.0" -autoprefixer@^10.4.13: - version "10.4.13" - resolved "https://registry.yarnpkg.com/autoprefixer/-/autoprefixer-10.4.13.tgz#b5136b59930209a321e9fa3dca2e7c4d223e83a8" - integrity sha512-49vKpMqcZYsJjwotvt4+h/BCjJVnhGwcLpDt5xkcaOG3eLrG/HUYLagrihYsQ+qrIBgIzX1Rw7a6L8I/ZA1Atg== - dependencies: - browserslist "^4.21.4" - caniuse-lite "^1.0.30001426" - fraction.js "^4.2.0" - normalize-range "^0.1.2" - picocolors "^1.0.0" - postcss-value-parser "^4.2.0" - available-typed-arrays@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz#92f95616501069d07d10edb2fc37d3e1c65123b7" integrity sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw== -aws-sign2@~0.7.0: - version "0.7.0" - resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" - integrity sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA== - -aws4@^1.8.0: - version "1.11.0" - resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59" - integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== - -axe-core@^4.4.3: - version "4.6.1" - resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.6.1.tgz#79cccdee3e3ab61a8f42c458d4123a6768e6fbce" - integrity sha512-lCZN5XRuOnpG4bpMq8v0khrWtUOn+i8lZSb6wHZH56ZfbIEv6XwJV84AAueh9/zi7qPVJ/E4yz6fmsiyOmXR4w== +available-typed-arrays@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz#a5cc375d6a03c2efc87a553f3e0b1522def14846" + integrity sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ== + dependencies: + possible-typed-array-names "^1.0.0" -axios@^1.4.0: - version "1.7.7" - resolved "https://registry.yarnpkg.com/axios/-/axios-1.7.7.tgz#2f554296f9892a72ac8d8e4c5b79c14a91d0a47f" - integrity sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q== +axios@^1.12.2: + version "1.13.6" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.13.6.tgz#c3f92da917dc209a15dd29936d20d5089b6b6c98" + integrity sha512-ChTCHMouEe2kn713WHbQGcuYrr6fXTBiu460OTwWrWob16g1bXn4vtz07Ope7ewMozJAnEquLk5lWQWtBig9DQ== dependencies: - follow-redirects "^1.15.6" - form-data "^4.0.0" + follow-redirects "^1.15.11" + form-data "^4.0.5" proxy-from-env "^1.1.0" -axobject-query@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/axobject-query/-/axobject-query-2.2.0.tgz#943d47e10c0b704aa42275e20edf3722648989be" - integrity sha512-Td525n+iPOOyUQIeBfcASuG6uJsDOITl7Mds5gFyerkWiX7qhUTdYUBlSgNMyVqtSJqwpt1kXGLdUt6SykLMRA== - -babel-jest@^27.4.2, babel-jest@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-27.5.1.tgz#a1bf8d61928edfefd21da27eb86a695bfd691444" - integrity sha512-cdQ5dXjGRd0IBRATiQ4mZGlGlRE8kJpjPOixdNRdT+m3UcNqmYWN6rK6nvtXYfY3D76cb8s/O1Ss8ea24PIwcg== - dependencies: - "@jest/transform" "^27.5.1" - "@jest/types" "^27.5.1" - "@types/babel__core" "^7.1.14" - babel-plugin-istanbul "^6.1.1" - babel-preset-jest "^27.5.1" - chalk "^4.0.0" - graceful-fs "^4.2.9" - slash "^3.0.0" - -babel-jest@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-29.7.0.tgz#f4369919225b684c56085998ac63dbd05be020d5" - integrity sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg== - dependencies: - "@jest/transform" "^29.7.0" - "@types/babel__core" "^7.1.14" - babel-plugin-istanbul "^6.1.1" - babel-preset-jest "^29.6.3" - chalk "^4.0.0" - graceful-fs "^4.2.9" - slash "^3.0.0" - -babel-loader@^8.2.3: - version "8.3.0" - resolved "https://registry.yarnpkg.com/babel-loader/-/babel-loader-8.3.0.tgz#124936e841ba4fe8176786d6ff28add1f134d6a8" - integrity sha512-H8SvsMF+m9t15HNLMipppzkC+Y2Yq+v3SonZyU70RBL/h1gxPkH08Ot8pEE9Z4Kd+czyWJClmFS8qzIP9OZ04Q== +babel-jest@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-30.2.0.tgz#fd44a1ec9552be35ead881f7381faa7d8f3b95ac" + integrity sha512-0YiBEOxWqKkSQWL9nNGGEgndoeL0ZpWrbLMNL5u/Kaxrli3Eaxlt3ZtIDktEvXt4L/R9r3ODr2zKwGM/2BjxVw== dependencies: - find-cache-dir "^3.3.1" - loader-utils "^2.0.0" - make-dir "^3.1.0" - schema-utils "^2.6.5" + "@jest/transform" "30.2.0" + "@types/babel__core" "^7.20.5" + babel-plugin-istanbul "^7.0.1" + babel-preset-jest "30.2.0" + chalk "^4.1.2" + graceful-fs "^4.2.11" + slash "^3.0.0" -babel-plugin-istanbul@^6.1.1: - version "6.1.1" - resolved "https://registry.yarnpkg.com/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz#fa88ec59232fd9b4e36dbbc540a8ec9a9b47da73" - integrity sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA== +babel-plugin-istanbul@^7.0.1: + version "7.0.1" + resolved "https://registry.yarnpkg.com/babel-plugin-istanbul/-/babel-plugin-istanbul-7.0.1.tgz#d8b518c8ea199364cf84ccc82de89740236daf92" + integrity sha512-D8Z6Qm8jCvVXtIRkBnqNHX0zJ37rQcFJ9u8WOS6tkYOsRdHBzypCstaxWiu5ZIlqQtviRYbgnRLSoCEvjqcqbA== dependencies: "@babel/helper-plugin-utils" "^7.0.0" "@istanbuljs/load-nyc-config" "^1.0.0" - "@istanbuljs/schema" "^0.1.2" - istanbul-lib-instrument "^5.0.4" + "@istanbuljs/schema" "^0.1.3" + istanbul-lib-instrument "^6.0.2" test-exclude "^6.0.0" -babel-plugin-jest-hoist@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-27.5.1.tgz#9be98ecf28c331eb9f5df9c72d6f89deb8181c2e" - integrity sha512-50wCwD5EMNW4aRpOwtqzyZHIewTYNxLA4nhB+09d8BIssfNfzBRhkBIHiaPv1Si226TQSvp8gxAJm2iY2qs2hQ== - dependencies: - "@babel/template" "^7.3.3" - "@babel/types" "^7.3.3" - "@types/babel__core" "^7.0.0" - "@types/babel__traverse" "^7.0.6" - -babel-plugin-jest-hoist@^29.6.3: - version "29.6.3" - resolved "https://registry.yarnpkg.com/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-29.6.3.tgz#aadbe943464182a8922c3c927c3067ff40d24626" - integrity sha512-ESAc/RJvGTFEzRwOTT4+lNDk/GNHMkKbNzsvT0qKRfDyyYTskxB5rnU2njIDYVxXCBHHEI1c0YwHob3WaYujOg== +babel-plugin-jest-hoist@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-30.2.0.tgz#94c250d36b43f95900f3a219241e0f4648191ce2" + integrity sha512-ftzhzSGMUnOzcCXd6WHdBGMyuwy15Wnn0iyyWGKgBDLxf9/s5ABuraCSpBX2uG0jUg4rqJnxsLc5+oYBqoxVaA== dependencies: - "@babel/template" "^7.3.3" - "@babel/types" "^7.3.3" - "@types/babel__core" "^7.1.14" - "@types/babel__traverse" "^7.0.6" + "@types/babel__core" "^7.20.5" babel-plugin-macros@^3.1.0: version "3.1.0" @@ -4586,11 +4018,6 @@ babel-plugin-macros@^3.1.0: cosmiconfig "^7.0.0" resolve "^1.19.0" -babel-plugin-named-asset-import@^0.3.8: - version "0.3.8" - resolved "https://registry.yarnpkg.com/babel-plugin-named-asset-import/-/babel-plugin-named-asset-import-0.3.8.tgz#6b7fa43c59229685368683c28bc9734f24524cc2" - integrity sha512-WXiAc++qo7XcJ1ZnTYGtLxmBCVbddAml3CEXgWaBzNzLNoxtQ8AiGEFDMOhot9XjTCQbvP5E77Fj9Gk924f00Q== - babel-plugin-polyfill-corejs2@^0.3.3: version "0.3.3" resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-corejs2/-/babel-plugin-polyfill-corejs2-0.3.3.tgz#5d1bd3836d0a19e1b84bbf2d9640ccb6f951c122" @@ -4615,65 +4042,44 @@ babel-plugin-polyfill-regenerator@^0.4.1: dependencies: "@babel/helper-define-polyfill-provider" "^0.3.3" -"babel-plugin-styled-components@>= 1.12.0": - version "2.0.7" - resolved "https://registry.yarnpkg.com/babel-plugin-styled-components/-/babel-plugin-styled-components-2.0.7.tgz#c81ef34b713f9da2b7d3f5550df0d1e19e798086" - integrity sha512-i7YhvPgVqRKfoQ66toiZ06jPNA3p6ierpfUuEWxNF+fV27Uv5gxBkf8KZLHUCc1nFA9j6+80pYoIpqCeyW3/bA== - dependencies: - "@babel/helper-annotate-as-pure" "^7.16.0" - "@babel/helper-module-imports" "^7.16.0" - babel-plugin-syntax-jsx "^6.18.0" - lodash "^4.17.11" - picomatch "^2.3.0" - -babel-plugin-syntax-jsx@^6.18.0: - version "6.18.0" - resolved "https://registry.yarnpkg.com/babel-plugin-syntax-jsx/-/babel-plugin-syntax-jsx-6.18.0.tgz#0af32a9a6e13ca7a3fd5069e62d7b0f58d0d8946" - integrity sha512-qrPaCSo9c8RHNRHIotaufGbuOBN8rtdC4QrrFFc43vyWCCz7Kl7GL1PGaXtMGQZUXrkCjNEgxDfmAuAabr/rlw== - babel-plugin-transform-react-remove-prop-types@^0.4.24: version "0.4.24" resolved "https://registry.yarnpkg.com/babel-plugin-transform-react-remove-prop-types/-/babel-plugin-transform-react-remove-prop-types-0.4.24.tgz#f2edaf9b4c6a5fbe5c1d678bfb531078c1555f3a" integrity sha512-eqj0hVcJUR57/Ug2zE1Yswsw4LhuqqHhD+8v120T1cl3kjg76QwtyBrdIk4WVwK+lAhBJVYCd/v+4nc4y+8JsA== -babel-preset-current-node-syntax@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.0.1.tgz#b4399239b89b2a011f9ddbe3e4f401fc40cff73b" - integrity sha512-M7LQ0bxarkxQoN+vz5aJPsLBn77n8QgTFmo8WK0/44auK2xlCXrYcUxHFxgU7qW5Yzw/CjmLRK2uJzaCd7LvqQ== +babel-preset-current-node-syntax@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.2.0.tgz#20730d6cdc7dda5d89401cab10ac6a32067acde6" + integrity sha512-E/VlAEzRrsLEb2+dv8yp3bo4scof3l9nR4lrld+Iy5NyVqgVYUJnDAmunkhPMisRI32Qc4iRiz425d8vM++2fg== dependencies: "@babel/plugin-syntax-async-generators" "^7.8.4" "@babel/plugin-syntax-bigint" "^7.8.3" - "@babel/plugin-syntax-class-properties" "^7.8.3" - "@babel/plugin-syntax-import-meta" "^7.8.3" + "@babel/plugin-syntax-class-properties" "^7.12.13" + "@babel/plugin-syntax-class-static-block" "^7.14.5" + "@babel/plugin-syntax-import-attributes" "^7.24.7" + "@babel/plugin-syntax-import-meta" "^7.10.4" "@babel/plugin-syntax-json-strings" "^7.8.3" - "@babel/plugin-syntax-logical-assignment-operators" "^7.8.3" + "@babel/plugin-syntax-logical-assignment-operators" "^7.10.4" "@babel/plugin-syntax-nullish-coalescing-operator" "^7.8.3" - "@babel/plugin-syntax-numeric-separator" "^7.8.3" + "@babel/plugin-syntax-numeric-separator" "^7.10.4" "@babel/plugin-syntax-object-rest-spread" "^7.8.3" "@babel/plugin-syntax-optional-catch-binding" "^7.8.3" "@babel/plugin-syntax-optional-chaining" "^7.8.3" - "@babel/plugin-syntax-top-level-await" "^7.8.3" - -babel-preset-jest@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/babel-preset-jest/-/babel-preset-jest-27.5.1.tgz#91f10f58034cb7989cb4f962b69fa6eef6a6bc81" - integrity sha512-Nptf2FzlPCWYuJg41HBqXVT8ym6bXOevuCTbhxlUpjwtysGaIWFvDEjp4y+G7fl13FgOdjs7P/DmErqH7da0Ag== - dependencies: - babel-plugin-jest-hoist "^27.5.1" - babel-preset-current-node-syntax "^1.0.0" + "@babel/plugin-syntax-private-property-in-object" "^7.14.5" + "@babel/plugin-syntax-top-level-await" "^7.14.5" -babel-preset-jest@^29.6.3: - version "29.6.3" - resolved "https://registry.yarnpkg.com/babel-preset-jest/-/babel-preset-jest-29.6.3.tgz#fa05fa510e7d493896d7b0dd2033601c840f171c" - integrity sha512-0B3bhxR6snWXJZtR/RliHTDPRgn1sNHOR0yVtq/IiQFyuOVjFS+wuio/R4gSNkyYmKmJB4wGZv2NZanmKmTnNA== +babel-preset-jest@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/babel-preset-jest/-/babel-preset-jest-30.2.0.tgz#04717843e561347781d6d7f69c81e6bcc3ed11ce" + integrity sha512-US4Z3NOieAQumwFnYdUWKvUKh8+YSnS/gB3t6YBiz0bskpu7Pine8pPCheNxlPEW4wnUkma2a94YuW2q3guvCQ== dependencies: - babel-plugin-jest-hoist "^29.6.3" - babel-preset-current-node-syntax "^1.0.0" + babel-plugin-jest-hoist "30.2.0" + babel-preset-current-node-syntax "^1.2.0" -babel-preset-react-app@^10.0.1: - version "10.0.1" - resolved "https://registry.yarnpkg.com/babel-preset-react-app/-/babel-preset-react-app-10.0.1.tgz#ed6005a20a24f2c88521809fa9aea99903751584" - integrity sha512-b0D9IZ1WhhCWkrTXyFuIIgqGzSkRIH5D5AmB0bXbzYAB1OBAwHcUeyWW2LorutLWF5btNo/N7r/cIdmvvKJlYg== +babel-preset-react-app@10.1.0: + version "10.1.0" + resolved "https://registry.yarnpkg.com/babel-preset-react-app/-/babel-preset-react-app-10.1.0.tgz#e367f223f6c27878e6cc28471d0d506a9ab9f96c" + integrity sha512-f9B1xMdnkCIqe+2dHrJsoQFRz7reChaAHE/65SdaykPklQqhme2WaC08oD3is77x9ff98/9EazAKFDZv5rFEQg== dependencies: "@babel/core" "^7.16.0" "@babel/plugin-proposal-class-properties" "^7.16.0" @@ -4682,6 +4088,7 @@ babel-preset-react-app@^10.0.1: "@babel/plugin-proposal-numeric-separator" "^7.16.0" "@babel/plugin-proposal-optional-chaining" "^7.16.0" "@babel/plugin-proposal-private-methods" "^7.16.0" + "@babel/plugin-proposal-private-property-in-object" "^7.16.7" "@babel/plugin-transform-flow-strip-types" "^7.16.0" "@babel/plugin-transform-react-display-name" "^7.16.0" "@babel/plugin-transform-runtime" "^7.16.4" @@ -4697,94 +4104,25 @@ balanced-match@^1.0.0: resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== +balanced-match@^4.0.2: + version "4.0.4" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-4.0.4.tgz#bfb10662feed8196a2c62e7c68e17720c274179a" + integrity sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA== + base64-js@^1.3.1, base64-js@^1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== -batch@0.6.1: - version "0.6.1" - resolved "https://registry.yarnpkg.com/batch/-/batch-0.6.1.tgz#dc34314f4e679318093fc760272525f94bf25c16" - integrity sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw== - -bcrypt-pbkdf@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e" - integrity sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w== - dependencies: - tweetnacl "^0.14.3" - -bfj@^7.0.2: - version "7.0.2" - resolved "https://registry.yarnpkg.com/bfj/-/bfj-7.0.2.tgz#1988ce76f3add9ac2913fd8ba47aad9e651bfbb2" - integrity sha512-+e/UqUzwmzJamNF50tBV6tZPTORow7gQ96iFow+8b562OdMpEK0BcJEq2OSPEDmAbSMBQ7PKZ87ubFkgxpYWgw== - dependencies: - bluebird "^3.5.5" - check-types "^11.1.1" - hoopy "^0.1.4" - tryer "^1.0.1" - -big.js@^5.2.2: - version "5.2.2" - resolved "https://registry.yarnpkg.com/big.js/-/big.js-5.2.2.tgz#65f0af382f578bcdc742bd9c281e9cb2d7768328" - integrity sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ== - -binary-extensions@^2.0.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d" - integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA== - -bl@^4.0.3: - version "4.1.0" - resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a" - integrity sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w== - dependencies: - buffer "^5.5.0" - inherits "^2.0.4" - readable-stream "^3.4.0" +baseline-browser-mapping@^2.8.25: + version "2.8.28" + resolved "https://registry.yarnpkg.com/baseline-browser-mapping/-/baseline-browser-mapping-2.8.28.tgz#9ef511f5a7c19d74a94cafcbf951608398e9bdb3" + integrity sha512-gYjt7OIqdM0PcttNYP2aVrr2G0bMALkBaoehD4BuRGjAOtipg0b6wHg1yNL+s5zSnLZZrGHOw4IrND8CD+3oIQ== -blob-util@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/blob-util/-/blob-util-2.0.2.tgz#3b4e3c281111bb7f11128518006cdc60b403a1eb" - integrity sha512-T7JQa+zsXXEa6/8ZhHcQEW1UFfVM49Ts65uBkFL6fz2QmrElqmbajIDJvuA0tEhRe5eIjpV9ZF+0RfZR9voJFQ== - -bluebird@^3.5.5, bluebird@^3.7.2: - version "3.7.2" - resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" - integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== - -body-parser@1.20.3: - version "1.20.3" - resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.3.tgz#1953431221c6fb5cd63c4b36d53fab0928e548c6" - integrity sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g== - dependencies: - bytes "3.1.2" - content-type "~1.0.5" - debug "2.6.9" - depd "2.0.0" - destroy "1.2.0" - http-errors "2.0.0" - iconv-lite "0.4.24" - on-finished "2.4.1" - qs "6.13.0" - raw-body "2.5.2" - type-is "~1.6.18" - unpipe "1.0.0" - -bonjour-service@^1.0.11: - version "1.0.14" - resolved "https://registry.yarnpkg.com/bonjour-service/-/bonjour-service-1.0.14.tgz#c346f5bc84e87802d08f8d5a60b93f758e514ee7" - integrity sha512-HIMbgLnk1Vqvs6B4Wq5ep7mxvj9sGz5d1JJyDNSGNIdA/w2MCz6GTjWTdjqOJV1bEPj+6IkxDvWNFKEBxNt4kQ== - dependencies: - array-flatten "^2.1.2" - dns-equal "^1.0.0" - fast-deep-equal "^3.1.3" - multicast-dns "^7.2.5" - -boolbase@^1.0.0, boolbase@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e" - integrity sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww== +bezier-easing@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/bezier-easing/-/bezier-easing-2.1.0.tgz#c04dfe8b926d6ecaca1813d69ff179b7c2025d86" + integrity sha512-gbIqZ/eslnUFC1tjEvtz0sgx+xTK20wDnYMIA27VA04R7w6xxXQPZDbibjA9DTWZRA2CXtwHykkVzlCaAJAZig== brace-expansion@^1.1.7: version "1.1.11" @@ -4794,26 +4132,21 @@ brace-expansion@^1.1.7: balanced-match "^1.0.0" concat-map "0.0.1" -brace-expansion@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae" - integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA== +brace-expansion@^5.0.2: + version "5.0.4" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-5.0.4.tgz#614daaecd0a688f660bbbc909a8748c3d80d4336" + integrity sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg== dependencies: - balanced-match "^1.0.0" + balanced-match "^4.0.2" -braces@^3.0.3, braces@~3.0.2: +braces@^3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789" integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA== dependencies: fill-range "^7.1.1" -browser-process-hrtime@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz#3c9b4b7d782c8121e56f10106d84c0d0ffc94626" - integrity sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow== - -browserslist@^4.0.0, browserslist@^4.16.6, browserslist@^4.18.1, browserslist@^4.21.3, browserslist@^4.21.4: +browserslist@^4.21.3, browserslist@^4.21.4: version "4.21.4" resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.21.4.tgz#e7496bbc67b9e39dd0f98565feccdcb0d4ff6987" integrity sha512-CBHJJdDmgjl3daYjN5Cp5kbTf1mUhZoS+beLklHIvkOWscs83YAhLlF3Wsh/lciQYAcbBJgTOD44VtG31ZM4Hw== @@ -4823,32 +4156,16 @@ browserslist@^4.0.0, browserslist@^4.16.6, browserslist@^4.18.1, browserslist@^4 node-releases "^2.0.6" update-browserslist-db "^1.0.9" -browserslist@^4.21.10: - version "4.23.3" - resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.23.3.tgz#debb029d3c93ebc97ffbc8d9cbb03403e227c800" - integrity sha512-btwCFJVjI4YWDNfau8RhZ+B1Q/VLoUITrm3RlP6y1tYGWIOa+InuYiRGXUBXo8nA1qKmHMyLB/iVQg5TT4eFoA== - dependencies: - caniuse-lite "^1.0.30001646" - electron-to-chromium "^1.5.4" - node-releases "^2.0.18" - update-browserslist-db "^1.1.0" - -browserslist@^4.21.9: - version "4.22.1" - resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.22.1.tgz#ba91958d1a59b87dab6fed8dfbcb3da5e2e9c619" - integrity sha512-FEVc202+2iuClEhZhrWy6ZiAcRLvNMyYcxZ8raemul1DYVOVdFsbqckWLdsixQZCpJlwe77Z3UTalE7jsjnKfQ== +browserslist@^4.24.0: + version "4.28.0" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.28.0.tgz#9cefece0a386a17a3cd3d22ebf67b9deca1b5929" + integrity sha512-tbydkR/CxfMwelN0vwdP/pLkDwyAASZ+VfWm4EOwlB6SWhx1sYnWLqo8N5j0rAzPfzfRaxt0mM/4wPU/Su84RQ== dependencies: - caniuse-lite "^1.0.30001541" - electron-to-chromium "^1.4.535" - node-releases "^2.0.13" - update-browserslist-db "^1.0.13" - -bs-logger@0.x: - version "0.2.6" - resolved "https://registry.yarnpkg.com/bs-logger/-/bs-logger-0.2.6.tgz#eb7d365307a72cf974cc6cda76b68354ad336bd8" - integrity sha512-pd8DCoxmbgc7hyPKOvxtqNcjYoOsABPQdcCUjGp3d42VR2CX1ORhk2A87oqqu5R1kk+76nsxZupkmyd+MVtCog== - dependencies: - fast-json-stable-stringify "2.x" + baseline-browser-mapping "^2.8.25" + caniuse-lite "^1.0.30001754" + electron-to-chromium "^1.5.249" + node-releases "^2.0.27" + update-browserslist-db "^1.1.4" bser@2.1.1: version "2.1.1" @@ -4857,43 +4174,26 @@ bser@2.1.1: dependencies: node-int64 "^0.4.0" -buffer-crc32@~0.2.3: - version "0.2.13" - resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" - integrity sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ== - buffer-from@^1.0.0: version "1.1.2" resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== -buffer@^5.5.0, buffer@^5.6.0: - version "5.7.1" - resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" - integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ== +buffer@^6.0.3: + version "6.0.3" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-6.0.3.tgz#2ace578459cc8fbe2a70aaa8f52ee63b6a74c6c6" + integrity sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA== dependencies: base64-js "^1.3.1" - ieee754 "^1.1.13" - -builtin-modules@^3.1.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-3.3.0.tgz#cae62812b89801e9656336e46223e030386be7b6" - integrity sha512-zhaCDicdLuWN5UbN5IMnFqNMhNfo919sH85y2/ea+5Yg9TsTkeZxpL+JLbp6cgYFS4sRLp3YV4S6yDuqVWHYOw== - -bytes@3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.0.0.tgz#d32815404d689699f85a4ea4fa8755dd13a96048" - integrity sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw== - -bytes@3.1.2: - version "3.1.2" - resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5" - integrity sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg== + ieee754 "^1.2.1" -cachedir@^2.3.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/cachedir/-/cachedir-2.3.0.tgz#0c75892a052198f0b21c7c1804d8331edfcae0e8" - integrity sha512-A+Fezp4zxnit6FanDmv9EqXNAi3vt9DWp51/71UEhXukb7QUuvtv9344h91dyAxuTLoSYJFU299qzR3tzwPAhw== +call-bind-apply-helpers@^1.0.0, call-bind-apply-helpers@^1.0.1, call-bind-apply-helpers@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz#4b5428c222be985d79c3d82657479dbe0b59b2d6" + integrity sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ== + dependencies: + es-errors "^1.3.0" + function-bind "^1.1.2" call-bind@^1.0.0, call-bind@^1.0.2: version "1.0.2" @@ -4903,41 +4203,35 @@ call-bind@^1.0.0, call-bind@^1.0.2: function-bind "^1.1.1" get-intrinsic "^1.0.2" -call-bind@^1.0.7: - version "1.0.7" - resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.7.tgz#06016599c40c56498c18769d2730be242b6fa3b9" - integrity sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w== +call-bind@^1.0.8: + version "1.0.8" + resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.8.tgz#0736a9660f537e3388826f440d5ec45f744eaa4c" + integrity sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww== dependencies: + call-bind-apply-helpers "^1.0.0" es-define-property "^1.0.0" - es-errors "^1.3.0" - function-bind "^1.1.2" get-intrinsic "^1.2.4" - set-function-length "^1.2.1" + set-function-length "^1.2.2" -callsites@^3.0.0: +call-bound@^1.0.3, call-bound@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/call-bound/-/call-bound-1.0.4.tgz#238de935d2a2a692928c538c7ccfa91067fd062a" + integrity sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg== + dependencies: + call-bind-apply-helpers "^1.0.2" + get-intrinsic "^1.3.0" + +callsites@^3.0.0, callsites@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== -camel-case@^4.1.2: - version "4.1.2" - resolved "https://registry.yarnpkg.com/camel-case/-/camel-case-4.1.2.tgz#9728072a954f805228225a6deea6b38461e1bd5a" - integrity sha512-gxGWBrTT1JuMx6R+o5PTXMmUnhnVzLQ9SNutD4YqKtI6ap897t3tKECYla6gCWEkplXnlNybEkZg9GEGxKFCgw== - dependencies: - pascal-case "^3.1.2" - tslib "^2.0.3" - -camelcase-css@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/camelcase-css/-/camelcase-css-2.0.1.tgz#ee978f6947914cc30c6b44741b6ed1df7f043fd5" - integrity sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA== - camelcase@^5.3.1: version "5.3.1" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320" integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg== -camelcase@^6.2.0, camelcase@^6.2.1: +camelcase@^6.3.0: version "6.3.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a" integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== @@ -4947,37 +4241,17 @@ camelize@^1.0.0: resolved "https://registry.yarnpkg.com/camelize/-/camelize-1.0.1.tgz#89b7e16884056331a35d6b5ad064332c91daa6c3" integrity sha512-dU+Tx2fsypxTgtLoE36npi3UqcjSSMNYfkqgmoEhtZrraP5VWq0K7FkWVTYa8eMPtnU/G2txVsfdCJTn9uzpuQ== -caniuse-api@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/caniuse-api/-/caniuse-api-3.0.0.tgz#5e4d90e2274961d46291997df599e3ed008ee4c0" - integrity sha512-bsTwuIg/BZZK/vreVTYYbSWoe2F+71P7K5QGEX+pT250DZbfU1MQ5prOKpPR+LL6uWKK3KMwMCAS74QB3Um1uw== - dependencies: - browserslist "^4.0.0" - caniuse-lite "^1.0.0" - lodash.memoize "^4.1.2" - lodash.uniq "^4.5.0" - -caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001400, caniuse-lite@^1.0.30001426, caniuse-lite@^1.0.30001541: +caniuse-lite@^1.0.30001400: version "1.0.30001636" resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001636.tgz" integrity sha512-bMg2vmr8XBsbL6Lr0UHXy/21m84FTxDLWn2FSqMd5PrlbMxwJlQnC2YWYxVgp66PZE+BBNF2jYQUBKCo1FDeZg== -caniuse-lite@^1.0.30001646: - version "1.0.30001653" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001653.tgz#b8af452f8f33b1c77f122780a4aecebea0caca56" - integrity sha512-XGWQVB8wFQ2+9NZwZ10GxTYC5hk0Fa+q8cSkr0tgvMhYhMHP/QC+WTgrePMDBWiWc/pV+1ik82Al20XOK25Gcw== - -case-sensitive-paths-webpack-plugin@^2.4.0: - version "2.4.0" - resolved "https://registry.yarnpkg.com/case-sensitive-paths-webpack-plugin/-/case-sensitive-paths-webpack-plugin-2.4.0.tgz#db64066c6422eed2e08cc14b986ca43796dbc6d4" - integrity sha512-roIFONhcxog0JSSWbvVAh3OocukmSgpqOH6YpMkCvav/ySIV3JKg4Dc8vYtQjYi/UxpNE36r/9v+VqTQqgkYmw== +caniuse-lite@^1.0.30001754: + version "1.0.30001754" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001754.tgz#7758299d9a72cce4e6b038788a15b12b44002759" + integrity sha512-x6OeBXueoAceOmotzx3PO4Zpt4rzpeIFsSr6AAePTZxSkXiYDUmpypEl7e2+8NCd9bD7bXjqyef8CJYPC1jfxg== -caseless@~0.12.0: - version "0.12.0" - resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" - integrity sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw== - -chalk@^2.0.0, chalk@^2.4.1, chalk@^2.4.2: +chalk@^2.0.0, chalk@^2.4.2: version "2.4.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== @@ -4986,15 +4260,7 @@ chalk@^2.0.0, chalk@^2.4.1, chalk@^2.4.2: escape-string-regexp "^1.0.5" supports-color "^5.3.0" -chalk@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-3.0.0.tgz#3f73c2bf526591f574cc492c51e2456349f844e4" - integrity sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg== - dependencies: - ansi-styles "^4.1.0" - supports-color "^7.1.0" - -chalk@^4.0.0, chalk@^4.0.2, chalk@^4.1.0, chalk@^4.1.2: +chalk@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== @@ -5007,125 +4273,35 @@ char-regex@^1.0.2: resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf" integrity sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw== -char-regex@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-2.0.1.tgz#6dafdb25f9d3349914079f010ba8d0e6ff9cd01e" - integrity sha512-oSvEeo6ZUD7NepqAat3RqoucZ5SeqLJgOvVIwkafu6IP3V0pO38s/ypdVUmDDK6qIIHNlYHJAKX9E7R7HoKElw== - -character-entities-legacy@^1.0.0: - version "1.1.4" - resolved "https://registry.yarnpkg.com/character-entities-legacy/-/character-entities-legacy-1.1.4.tgz#94bc1845dce70a5bb9d2ecc748725661293d8fc1" - integrity sha512-3Xnr+7ZFS1uxeiUDvV02wQ+QDbc55o97tIV5zHScSPJpcLm/r0DFPcoY3tYRp+VZukxuMeKgXYmsXQHO05zQeA== - -character-entities@^1.0.0: - version "1.2.4" - resolved "https://registry.yarnpkg.com/character-entities/-/character-entities-1.2.4.tgz#e12c3939b7eaf4e5b15e7ad4c5e28e1d48c5b16b" - integrity sha512-iBMyeEHxfVnIakwOuDXpVkc54HijNgCyQB2w0VfGQThle6NXn50zU6V/u+LDhxHcDUPojn6Kpga3PTAD8W1bQw== - -character-reference-invalid@^1.0.0: - version "1.1.4" - resolved "https://registry.yarnpkg.com/character-reference-invalid/-/character-reference-invalid-1.1.4.tgz#083329cda0eae272ab3dbbf37e9a382c13af1560" - integrity sha512-mKKUkUbhPpQlCOfIuZkvSEgktjPFIsZKRRbC6KWVEMvlzblj3i3asQv5ODsrwt0N3pHAEvjP8KTQPHkp0+6jOg== - -check-more-types@^2.24.0: - version "2.24.0" - resolved "https://registry.yarnpkg.com/check-more-types/-/check-more-types-2.24.0.tgz#1420ffb10fd444dcfc79b43891bbfffd32a84600" - integrity sha512-Pj779qHxV2tuapviy1bSZNEL1maXr13bPYpsvSDB68HlYcYuhlDrmGd63i0JHMCLKzc7rUSNIrpdJlhVlNwrxA== - -check-types@^11.1.1: - version "11.2.2" - resolved "https://registry.yarnpkg.com/check-types/-/check-types-11.2.2.tgz#7afc0b6a860d686885062f2dba888ba5710335b4" - integrity sha512-HBiYvXvn9Z70Z88XKjz3AEKd4HJhBXsa3j7xFnITAzoS8+q6eIGi8qDB8FKPBAjtuxjI/zFpwuiCb8oDtKOYrA== - -chokidar@^3.4.2, chokidar@^3.5.3: - version "3.5.3" - resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.3.tgz#1cf37c8707b932bd1af1ae22c0432e2acd1903bd" - integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw== - dependencies: - anymatch "~3.1.2" - braces "~3.0.2" - glob-parent "~5.1.2" - is-binary-path "~2.1.0" - is-glob "~4.0.1" - normalize-path "~3.0.0" - readdirp "~3.6.0" - optionalDependencies: - fsevents "~2.3.2" - -chownr@^1.1.1: - version "1.1.4" - resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" - integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg== - -chrome-trace-event@^1.0.2: - version "1.0.3" - resolved "https://registry.yarnpkg.com/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz#1015eced4741e15d06664a957dbbf50d041e26ac" - integrity sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg== - -ci-info@^3.2.0: - version "3.7.0" - resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.7.0.tgz#6d01b3696c59915b6ce057e4aa4adfc2fa25f5ef" - integrity sha512-2CpRNYmImPx+RXKLq6jko/L07phmS9I02TyqkcNU20GCF/GgaWvc58hPtjxDX8lPpkdwc9sNh72V9k00S7ezog== - -ci-info@^3.7.0: - version "3.9.0" - resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.9.0.tgz#4279a62028a7b1f262f3473fc9605f5e218c59b4" - integrity sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ== - -cjs-module-lexer@^1.0.0: - version "1.2.2" - resolved "https://registry.yarnpkg.com/cjs-module-lexer/-/cjs-module-lexer-1.2.2.tgz#9f84ba3244a512f3a54e5277e8eef4c489864e40" - integrity sha512-cOU9usZw8/dXIXKtwa8pM0OTJQuJkxMN6w30csNRUerHfeQ5R6U3kkU/FtJeIf3M202OHfY2U8ccInBG7/xogA== - -classnames@^2.3.1: - version "2.3.2" - resolved "https://registry.yarnpkg.com/classnames/-/classnames-2.3.2.tgz#351d813bf0137fcc6a76a16b88208d2560a0d924" - integrity sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw== - -clean-css@^5.2.2: - version "5.3.1" - resolved "https://registry.yarnpkg.com/clean-css/-/clean-css-5.3.1.tgz#d0610b0b90d125196a2894d35366f734e5d7aa32" - integrity sha512-lCr8OHhiWCTw4v8POJovCoh4T7I9U11yVsPjMWWnnMmp9ZowCxyad1Pathle/9HjaDp+fdQKjO9fQydE6RHTZg== - dependencies: - source-map "~0.6.0" +character-entities-legacy@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz#76bc83a90738901d7bc223a9e93759fdd560125b" + integrity sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ== -clean-stack@^2.0.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b" - integrity sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A== +character-entities@^2.0.0: + version "2.0.2" + resolved "https://registry.yarnpkg.com/character-entities/-/character-entities-2.0.2.tgz#2d09c2e72cd9523076ccb21157dff66ad43fcc22" + integrity sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ== -cli-cursor@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-3.1.0.tgz#264305a7ae490d1d03bf0c9ba7c925d1753af307" - integrity sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw== - dependencies: - restore-cursor "^3.1.0" +character-reference-invalid@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz#85c66b041e43b47210faf401278abf808ac45cb9" + integrity sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw== -cli-table3@~0.6.1: - version "0.6.3" - resolved "https://registry.yarnpkg.com/cli-table3/-/cli-table3-0.6.3.tgz#61ab765aac156b52f222954ffc607a6f01dbeeb2" - integrity sha512-w5Jac5SykAeZJKntOxJCrm63Eg5/4dhMWIcuTbo9rpE+brgaSZo0RuNJZeOyMgsUdhDeojvgyQLmjI+K50ZGyg== - dependencies: - string-width "^4.2.0" - optionalDependencies: - "@colors/colors" "1.5.0" +ci-info@^4.2.0: + version "4.3.1" + resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-4.3.1.tgz#355ad571920810b5623e11d40232f443f16f1daa" + integrity sha512-Wdy2Igu8OcBpI2pZePZ5oWjPC38tmDVx5WKUXKwlLYkA0ozo85sLsLvkBbBn/sZaSCMFOGZJ14fvW9t5/d7kdA== -cli-truncate@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/cli-truncate/-/cli-truncate-2.1.0.tgz#c39e28bf05edcde5be3b98992a22deed5a2b93c7" - integrity sha512-n8fOixwDD6b/ObinzTrp1ZKFzbgvKZvuz/TvejnLn1aQfC6r52XEx85FmuC+3HI+JM7coBRXUvNqEU2PHVrHpg== - dependencies: - slice-ansi "^3.0.0" - string-width "^4.2.0" +cjs-module-lexer@^2.1.0: + version "2.1.1" + resolved "https://registry.yarnpkg.com/cjs-module-lexer/-/cjs-module-lexer-2.1.1.tgz#bff23b0609cc9afa428bd35f1918f7d03b448562" + integrity sha512-+CmxIZ/L2vNcEfvNtLdU0ZQ6mbq3FZnwAP2PPTiKP+1QOoKwlKlPgb8UKV0Dds7QVaMnHm+FwSft2VB0s/SLjQ== -cliui@^7.0.2: - version "7.0.4" - resolved "https://registry.yarnpkg.com/cliui/-/cliui-7.0.4.tgz#a0265ee655476fc807aea9df3df8df7783808b4f" - integrity sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ== - dependencies: - string-width "^4.2.0" - strip-ansi "^6.0.0" - wrap-ansi "^7.0.0" +classnames@^2.5.1: + version "2.5.1" + resolved "https://registry.yarnpkg.com/classnames/-/classnames-2.5.1.tgz#ba774c614be0f016da105c858e7159eae8e7687b" + integrity sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow== cliui@^8.0.1: version "8.0.1" @@ -5136,12 +4312,7 @@ cliui@^8.0.1: strip-ansi "^6.0.1" wrap-ansi "^7.0.0" -clsx@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/clsx/-/clsx-1.2.1.tgz#0ddc4a20a549b59c93a4116bb26f5294ca17dc12" - integrity sha512-EcR6r5a8bj6pu3ycsa/E/cKVGuTgZJZdsyUYHOksG/UHIiKfjxzRxYJpyVBwYaQeOvghal9fcc4PidlgzugAQg== - -clsx@^2.1.0: +clsx@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/clsx/-/clsx-2.1.1.tgz#eed397c9fd8bd882bfb18deab7102049a2f32999" integrity sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA== @@ -5151,19 +4322,10 @@ co@^4.6.0: resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" integrity sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ== -coa@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/coa/-/coa-2.0.2.tgz#43f6c21151b4ef2bf57187db0d73de229e3e7ec3" - integrity sha512-q5/jG+YQnSy4nRTV4F7lPepBJZ8qBNJJDBuJdoejDyLXgmL7IEo+Le2JDZudFTFt7mrCqIRaSjws4ygRCTCAXA== - dependencies: - "@types/q" "^1.5.1" - chalk "^2.4.1" - q "^1.1.2" - -collect-v8-coverage@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/collect-v8-coverage/-/collect-v8-coverage-1.0.1.tgz#cc2c8e94fc18bbdffe64d6534570c8a673b27f59" - integrity sha512-iBPtljfCNcTKNAto0KEtDfZ3qzjJvqE3aTGZsbhjSBlorqpXJlaWWtPO35D+ZImoC3KWejX64o+yPGxhWSTzfg== +collect-v8-coverage@^1.0.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/collect-v8-coverage/-/collect-v8-coverage-1.0.3.tgz#cc1f01eb8d02298cbc9a437c74c70ab4e5210b80" + integrity sha512-1L5aqIkwPfiodaMgQunkF1zRhNqifHBmtbbbxcr6yVxxBnliw4TDOW6NxpO8DJLgJ16OT+Y4ztZqP6p/FtXnAw== color-convert@^1.9.0: version "1.9.3" @@ -5184,121 +4346,29 @@ color-name@1.1.3: resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" integrity sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw== -color-name@^1.1.4, color-name@~1.1.4: +color-name@~1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== -colord@^2.9.1: - version "2.9.3" - resolved "https://registry.yarnpkg.com/colord/-/colord-2.9.3.tgz#4f8ce919de456f1d5c1c368c307fe20f3e59fb43" - integrity sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw== - -colorette@^2.0.10, colorette@^2.0.16: - version "2.0.19" - resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.19.tgz#cdf044f47ad41a0f4b56b3a0d5b4e6e1a2d5a798" - integrity sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ== - -combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6: +combined-stream@^1.0.8: version "1.0.8" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== dependencies: delayed-stream "~1.0.0" -comma-separated-tokens@^1.0.0: - version "1.0.8" - resolved "https://registry.yarnpkg.com/comma-separated-tokens/-/comma-separated-tokens-1.0.8.tgz#632b80b6117867a158f1080ad498b2fbe7e3f5ea" - integrity sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw== - -commander@^2.20.0: - version "2.20.3" - resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" - integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== - -commander@^6.2.1: - version "6.2.1" - resolved "https://registry.yarnpkg.com/commander/-/commander-6.2.1.tgz#0792eb682dfbc325999bb2b84fddddba110ac73c" - integrity sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA== - -commander@^7.2.0: - version "7.2.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7" - integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw== - -commander@^8.3.0: - version "8.3.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-8.3.0.tgz#4837ea1b2da67b9c616a67afbb0fafee567bca66" - integrity sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww== - -common-path-prefix@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/common-path-prefix/-/common-path-prefix-3.0.0.tgz#7d007a7e07c58c4b4d5f433131a19141b29f11e0" - integrity sha512-QE33hToZseCH3jS0qN96O/bSh3kaw/h+Tq7ngyY9eWDUnTlTNUyqfqvCXioLe5Na5jFsL78ra/wuBU4iuEgd4w== - -common-tags@^1.8.0: - version "1.8.2" - resolved "https://registry.yarnpkg.com/common-tags/-/common-tags-1.8.2.tgz#94ebb3c076d26032745fd54face7f688ef5ac9c6" - integrity sha512-gk/Z852D2Wtb//0I+kRFNKKE9dIIVirjoqPoA1wJU+XePVXZfGeBpk45+A1rKO4Q43prqWBNY/MiIeRLbPWUaA== - -commondir@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b" - integrity sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg== - -compressible@~2.0.16: - version "2.0.18" - resolved "https://registry.yarnpkg.com/compressible/-/compressible-2.0.18.tgz#af53cca6b070d4c3c0750fbd77286a6d7cc46fba" - integrity sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg== - dependencies: - mime-db ">= 1.43.0 < 2" - -compression@^1.7.4: - version "1.7.4" - resolved "https://registry.yarnpkg.com/compression/-/compression-1.7.4.tgz#95523eff170ca57c29a0ca41e6fe131f41e5bb8f" - integrity sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ== - dependencies: - accepts "~1.3.5" - bytes "3.0.0" - compressible "~2.0.16" - debug "2.6.9" - on-headers "~1.0.2" - safe-buffer "5.1.2" - vary "~1.1.2" +comma-separated-tokens@^2.0.0: + version "2.0.3" + resolved "https://registry.yarnpkg.com/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz#4e89c9458acb61bc8fef19f4529973b2392839ee" + integrity sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg== concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg== -confusing-browser-globals@^1.0.11: - version "1.0.11" - resolved "https://registry.yarnpkg.com/confusing-browser-globals/-/confusing-browser-globals-1.0.11.tgz#ae40e9b57cdd3915408a2805ebd3a5585608dc81" - integrity sha512-JsPKdmh8ZkmnHxDk55FZ1TqVLvEQTvoByJZRN9jzI0UjxK/QgAmsphz7PGtqgPieQZ/CQcHWXCR7ATDNhGe+YA== - -connect-history-api-fallback@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/connect-history-api-fallback/-/connect-history-api-fallback-2.0.0.tgz#647264845251a0daf25b97ce87834cace0f5f1c8" - integrity sha512-U73+6lQFmfiNPrYbXqr6kZ1i1wiRqXnp2nhMsINseWXO8lDau0LGEffJ8kQi4EjLZympVgRdvqjAgiZ1tgzDDA== - -content-disposition@0.5.4: - version "0.5.4" - resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.4.tgz#8b82b4efac82512a02bb0b1dcec9d2c5e8eb5bfe" - integrity sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ== - dependencies: - safe-buffer "5.2.1" - -content-type@~1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b" - integrity sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA== - -content-type@~1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.5.tgz#8b773162656d1d1086784c8f23a54ce6d73d7918" - integrity sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA== - -convert-source-map@^1.4.0, convert-source-map@^1.5.0, convert-source-map@^1.6.0, convert-source-map@^1.7.0: +convert-source-map@^1.5.0, convert-source-map@^1.7.0: version "1.9.0" resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.9.0.tgz#7faae62353fb4213366d0ca98358d22e8368b05f" integrity sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A== @@ -5308,20 +4378,10 @@ convert-source-map@^2.0.0: resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-2.0.0.tgz#4b560f649fc4e918dd0ab75cf4961e8bc882d82a" integrity sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg== -cookie-signature@1.0.6: - version "1.0.6" - resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c" - integrity sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ== - -cookie@0.6.0: - version "0.6.0" - resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.6.0.tgz#2798b04b071b0ecbff0dbb62a505a8efa4e19051" - integrity sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw== - -cookie@~0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.5.0.tgz#d1f5d71adec6558c58f389987c366aa47e994f8b" - integrity sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw== +cookie@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/cookie/-/cookie-1.0.2.tgz#27360701532116bd3f1f9416929d176afe1e4610" + integrity sha512-9Kr/j4O16ISv8zBBhJoi4bXOYNTkFLOqSL3UDB0njXxCXNezjeyVrJyGOWtgfs/q2km1gwBcfH8q1yEGoMYunA== copy-to-clipboard@^3.3.1: version "3.3.3" @@ -5337,42 +4397,16 @@ core-js-compat@^3.25.1: dependencies: browserslist "^4.21.4" -core-js-pure@^3.23.3, core-js-pure@^3.25.1: - version "3.27.1" - resolved "https://registry.yarnpkg.com/core-js-pure/-/core-js-pure-3.27.1.tgz#ede4a6b8440585c7190062757069c01d37a19dca" - integrity sha512-BS2NHgwwUppfeoqOXqi08mUqS5FiZpuRuJJpKsaME7kJz0xxuk0xkhDdfMIlP/zLa80krBqss1LtD7f889heAw== - -core-js-pure@^3.30.2: - version "3.33.1" - resolved "https://registry.yarnpkg.com/core-js-pure/-/core-js-pure-3.33.1.tgz#7f27dd239da8eb97dbea30120071be8e5565cb0e" - integrity sha512-wCXGbLjnsP10PlK/thHSQlOLlLKNEkaWbTzVvHHZ79fZNeN1gUmw2gBlpItxPv/pvqldevEXFh/d5stdNvl6EQ== +core-js-pure@^3.43.0: + version "3.47.0" + resolved "https://registry.yarnpkg.com/core-js-pure/-/core-js-pure-3.47.0.tgz#1104df8a3b6eb9189fcc559b5a65b90f66e7e887" + integrity sha512-BcxeDbzUrRnXGYIVAGFtcGQVNpFcUhVjr6W7F8XktvQW2iJP9e66GP6xdKotCRFlrxBvNIBrhwKteRXqMV86Nw== core-js@^3.19.2: version "3.27.1" resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.27.1.tgz#23cc909b315a6bb4e418bf40a52758af2103ba46" integrity sha512-GutwJLBChfGCpwwhbYoqfv03LAfmiz7e7D/BNxzeMxwQf10GRSzqiOjx7AmtEk+heiD/JWmBuyBPgFtx0Sg1ww== -core-util-is@1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" - integrity sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ== - -core-util-is@~1.0.0: - version "1.0.3" - resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" - integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== - -cosmiconfig@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-6.0.0.tgz#da4fee853c52f6b1e6935f41c1a2fc50bd4a9982" - integrity sha512-xb3ZL6+L8b9JLLCx3ZdoZy4+2ECphCMo2PwqgP1tlfVq6M6YReyzBJtvWWtbDSpNr9hn96pkCiZqUcFEc+54Qg== - dependencies: - "@types/parse-json" "^4.0.0" - import-fresh "^3.1.0" - parse-json "^5.0.0" - path-type "^4.0.0" - yaml "^1.7.2" - cosmiconfig@^7.0.0: version "7.1.0" resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-7.1.0.tgz#1443b9afa596b670082ea46cbd8f6a62b84635f6" @@ -5384,20 +4418,7 @@ cosmiconfig@^7.0.0: path-type "^4.0.0" yaml "^1.10.0" -create-jest@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/create-jest/-/create-jest-29.7.0.tgz#a355c5b3cb1e1af02ba177fe7afd7feee49a5320" - integrity sha512-Adz2bdH0Vq3F53KEMJOoftQFutWCukm6J24wbPWRO4k1kMY7gS7ds/uoJkNuV8wDCtWWnuwGcJwpWcih+zEW1Q== - dependencies: - "@jest/types" "^29.6.3" - chalk "^4.0.0" - exit "^0.1.2" - graceful-fs "^4.2.9" - jest-config "^29.7.0" - jest-util "^29.7.0" - prompts "^2.0.1" - -cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3: +cross-spawn@^7.0.3, cross-spawn@^7.0.6: version "7.0.6" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f" integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA== @@ -5406,275 +4427,49 @@ cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3: shebang-command "^2.0.0" which "^2.0.1" -crypto-random-string@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-2.0.0.tgz#ef2a7a966ec11083388369baa02ebead229b30d5" - integrity sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA== - -css-blank-pseudo@^3.0.3: - version "3.0.3" - resolved "https://registry.yarnpkg.com/css-blank-pseudo/-/css-blank-pseudo-3.0.3.tgz#36523b01c12a25d812df343a32c322d2a2324561" - integrity sha512-VS90XWtsHGqoM0t4KpH053c4ehxZ2E6HtGI7x68YFV0pTo/QmkV/YFA+NnlvK8guxZVNWGQhVNJGC39Q8XF4OQ== - dependencies: - postcss-selector-parser "^6.0.9" - css-color-keywords@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/css-color-keywords/-/css-color-keywords-1.0.0.tgz#fea2616dc676b2962686b3af8dbdbe180b244e05" integrity sha512-FyyrDHZKEjXDpNJYvVsV960FiqQyXc/LlYmsxl2BcdMb2WPx0OGRVgTg55rPSyLSNMqP52R9r8geSp7apN3Ofg== -css-declaration-sorter@^6.3.1: - version "6.3.1" - resolved "https://registry.yarnpkg.com/css-declaration-sorter/-/css-declaration-sorter-6.3.1.tgz#be5e1d71b7a992433fb1c542c7a1b835e45682ec" - integrity sha512-fBffmak0bPAnyqc/HO8C3n2sHrp9wcqQz6ES9koRF2/mLOVAx9zIQ3Y7R29sYCteTPqMCwns4WYQoCX91Xl3+w== - -css-has-pseudo@^3.0.4: - version "3.0.4" - resolved "https://registry.yarnpkg.com/css-has-pseudo/-/css-has-pseudo-3.0.4.tgz#57f6be91ca242d5c9020ee3e51bbb5b89fc7af73" - integrity sha512-Vse0xpR1K9MNlp2j5w1pgWIJtm1a8qS0JwS9goFYcImjlHEmywP9VUF05aGBXzGpDJF86QXk4L0ypBmwPhGArw== - dependencies: - postcss-selector-parser "^6.0.9" - -css-loader@^6.5.1: - version "6.7.3" - resolved "https://registry.yarnpkg.com/css-loader/-/css-loader-6.7.3.tgz#1e8799f3ccc5874fdd55461af51137fcc5befbcd" - integrity sha512-qhOH1KlBMnZP8FzRO6YCH9UHXQhVMcEGLyNdb7Hv2cpcmJbW0YrddO+tG1ab5nT41KpHIYGsbeHqxB9xPu1pKQ== - dependencies: - icss-utils "^5.1.0" - postcss "^8.4.19" - postcss-modules-extract-imports "^3.0.0" - postcss-modules-local-by-default "^4.0.0" - postcss-modules-scope "^3.0.0" - postcss-modules-values "^4.0.0" - postcss-value-parser "^4.2.0" - semver "^7.3.8" - -css-minimizer-webpack-plugin@^3.2.0: - version "3.4.1" - resolved "https://registry.yarnpkg.com/css-minimizer-webpack-plugin/-/css-minimizer-webpack-plugin-3.4.1.tgz#ab78f781ced9181992fe7b6e4f3422e76429878f" - integrity sha512-1u6D71zeIfgngN2XNRJefc/hY7Ybsxd74Jm4qngIXyUEk7fss3VUzuHxLAq/R8NAba4QU9OUSaMZlbpRc7bM4Q== - dependencies: - cssnano "^5.0.6" - jest-worker "^27.0.2" - postcss "^8.3.5" - schema-utils "^4.0.0" - serialize-javascript "^6.0.0" - source-map "^0.6.1" - -css-prefers-color-scheme@^6.0.3: - version "6.0.3" - resolved "https://registry.yarnpkg.com/css-prefers-color-scheme/-/css-prefers-color-scheme-6.0.3.tgz#ca8a22e5992c10a5b9d315155e7caee625903349" - integrity sha512-4BqMbZksRkJQx2zAjrokiGMd07RqOa2IxIrrN10lyBe9xhn9DEvjUK79J6jkeiv9D9hQFXKb6g1jwU62jziJZA== - -css-select-base-adapter@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/css-select-base-adapter/-/css-select-base-adapter-0.1.1.tgz#3b2ff4972cc362ab88561507a95408a1432135d7" - integrity sha512-jQVeeRG70QI08vSTwf1jHxp74JoZsr2XSgETae8/xC8ovSnL2WF87GTLO86Sbwdt2lK4Umg4HnnwMO4YF3Ce7w== - -css-select@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/css-select/-/css-select-2.1.0.tgz#6a34653356635934a81baca68d0255432105dbef" - integrity sha512-Dqk7LQKpwLoH3VovzZnkzegqNSuAziQyNZUcrdDM401iY+R5NkGBXGmtO05/yaXQziALuPogeG0b7UAgjnTJTQ== - dependencies: - boolbase "^1.0.0" - css-what "^3.2.1" - domutils "^1.7.0" - nth-check "^1.0.2" - -css-select@^4.1.3: - version "4.3.0" - resolved "https://registry.yarnpkg.com/css-select/-/css-select-4.3.0.tgz#db7129b2846662fd8628cfc496abb2b59e41529b" - integrity sha512-wPpOYtnsVontu2mODhA19JrqWxNsfdatRKd64kmpRbQgh1KtItko5sTnEpPdpSaJszTOhEMlF/RPz28qj4HqhQ== - dependencies: - boolbase "^1.0.0" - css-what "^6.0.1" - domhandler "^4.3.1" - domutils "^2.8.0" - nth-check "^2.0.1" - -css-to-react-native@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/css-to-react-native/-/css-to-react-native-3.0.0.tgz#62dbe678072a824a689bcfee011fc96e02a7d756" - integrity sha512-Ro1yETZA813eoyUp2GDBhG2j+YggidUmzO1/v9eYBKR2EHVEniE2MI/NqpTQ954BMpTPZFsGNPm46qFB9dpaPQ== +css-to-react-native@3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/css-to-react-native/-/css-to-react-native-3.2.0.tgz#cdd8099f71024e149e4f6fe17a7d46ecd55f1e32" + integrity sha512-e8RKaLXMOFii+02mOlqwjbD00KSEKqblnpO9e++1aXS1fPQOpS1YoqdVHBqPjHNoxeF2mimzVqawm2KCbEdtHQ== dependencies: camelize "^1.0.0" css-color-keywords "^1.0.0" postcss-value-parser "^4.0.2" -css-tree@1.0.0-alpha.37: - version "1.0.0-alpha.37" - resolved "https://registry.yarnpkg.com/css-tree/-/css-tree-1.0.0-alpha.37.tgz#98bebd62c4c1d9f960ec340cf9f7522e30709a22" - integrity sha512-DMxWJg0rnz7UgxKT0Q1HU/L9BeJI0M6ksor0OgqOnF+aRCDWg/N2641HmVyU9KVIu0OVVWOb2IpC9A+BJRnejg== - dependencies: - mdn-data "2.0.4" - source-map "^0.6.1" - -css-tree@^1.1.2, css-tree@^1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/css-tree/-/css-tree-1.1.3.tgz#eb4870fb6fd7707327ec95c2ff2ab09b5e8db91d" - integrity sha512-tRpdppF7TRazZrjJ6v3stzv93qxRcSsFmW6cX0Zm2NVKpxE1WV1HblnghVv9TreireHkqI/VDEsfolRF1p6y7Q== - dependencies: - mdn-data "2.0.14" - source-map "^0.6.1" - -css-what@^3.2.1: - version "3.4.2" - resolved "https://registry.yarnpkg.com/css-what/-/css-what-3.4.2.tgz#ea7026fcb01777edbde52124e21f327e7ae950e4" - integrity sha512-ACUm3L0/jiZTqfzRM3Hi9Q8eZqd6IK37mMWPLz9PJxkLWllYeRf+EHUSHYEtFop2Eqytaq1FizFVh7XfBnXCDQ== - -css-what@^6.0.1: - version "6.1.0" - resolved "https://registry.yarnpkg.com/css-what/-/css-what-6.1.0.tgz#fb5effcf76f1ddea2c81bdfaa4de44e79bac70f4" - integrity sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw== - css.escape@1.5.1, css.escape@^1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/css.escape/-/css.escape-1.5.1.tgz#42e27d4fa04ae32f931a4b4d4191fa9cddee97cb" integrity sha512-YUifsXXuknHlUsmlgyY0PKzgPOr7/FjCePfHNt0jxm83wHZi44VDMQ7/fGNkjY3/jV1MC+1CmZbaHzugyeRtpg== -cssdb@^7.1.0: - version "7.2.0" - resolved "https://registry.yarnpkg.com/cssdb/-/cssdb-7.2.0.tgz#f44bd4abc430f0ff7f4c64b8a1fb857a753f77a8" - integrity sha512-JYlIsE7eKHSi0UNuCyo96YuIDFqvhGgHw4Ck6lsN+DP0Tp8M64UTDT2trGbkMDqnCoEjks7CkS0XcjU0rkvBdg== - -cssesc@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/cssesc/-/cssesc-3.0.0.tgz#37741919903b868565e1c09ea747445cd18983ee" - integrity sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg== - -cssnano-preset-default@^5.2.13: - version "5.2.13" - resolved "https://registry.yarnpkg.com/cssnano-preset-default/-/cssnano-preset-default-5.2.13.tgz#e7353b0c57975d1bdd97ac96e68e5c1b8c68e990" - integrity sha512-PX7sQ4Pb+UtOWuz8A1d+Rbi+WimBIxJTRyBdgGp1J75VU0r/HFQeLnMYgHiCAp6AR4rqrc7Y4R+1Rjk3KJz6DQ== - dependencies: - css-declaration-sorter "^6.3.1" - cssnano-utils "^3.1.0" - postcss-calc "^8.2.3" - postcss-colormin "^5.3.0" - postcss-convert-values "^5.1.3" - postcss-discard-comments "^5.1.2" - postcss-discard-duplicates "^5.1.0" - postcss-discard-empty "^5.1.1" - postcss-discard-overridden "^5.1.0" - postcss-merge-longhand "^5.1.7" - postcss-merge-rules "^5.1.3" - postcss-minify-font-values "^5.1.0" - postcss-minify-gradients "^5.1.1" - postcss-minify-params "^5.1.4" - postcss-minify-selectors "^5.2.1" - postcss-normalize-charset "^5.1.0" - postcss-normalize-display-values "^5.1.0" - postcss-normalize-positions "^5.1.1" - postcss-normalize-repeat-style "^5.1.1" - postcss-normalize-string "^5.1.0" - postcss-normalize-timing-functions "^5.1.0" - postcss-normalize-unicode "^5.1.1" - postcss-normalize-url "^5.1.0" - postcss-normalize-whitespace "^5.1.1" - postcss-ordered-values "^5.1.3" - postcss-reduce-initial "^5.1.1" - postcss-reduce-transforms "^5.1.0" - postcss-svgo "^5.1.0" - postcss-unique-selectors "^5.1.1" - -cssnano-utils@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/cssnano-utils/-/cssnano-utils-3.1.0.tgz#95684d08c91511edfc70d2636338ca37ef3a6861" - integrity sha512-JQNR19/YZhz4psLX/rQ9M83e3z2Wf/HdJbryzte4a3NSuafyp9w/I4U+hx5C2S9g41qlstH7DEWnZaaj83OuEA== - -cssnano@^5.0.6: - version "5.1.14" - resolved "https://registry.yarnpkg.com/cssnano/-/cssnano-5.1.14.tgz#07b0af6da73641276fe5a6d45757702ebae2eb05" - integrity sha512-Oou7ihiTocbKqi0J1bB+TRJIQX5RMR3JghA8hcWSw9mjBLQ5Y3RWqEDoYG3sRNlAbCIXpqMoZGbq5KDR3vdzgw== - dependencies: - cssnano-preset-default "^5.2.13" - lilconfig "^2.0.3" - yaml "^1.10.2" - -csso@^4.0.2, csso@^4.2.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/csso/-/csso-4.2.0.tgz#ea3a561346e8dc9f546d6febedd50187cf389529" - integrity sha512-wvlcdIbf6pwKEk7vHj8/Bkc0B4ylXZruLvOgs9doS5eOsOpuodOV2zJChSpkp+pRpYQLQMeF04nr3Z68Sta9jA== +cssstyle@^4.2.1: + version "4.6.0" + resolved "https://registry.yarnpkg.com/cssstyle/-/cssstyle-4.6.0.tgz#ea18007024e3167f4f105315f3ec2d982bf48ed9" + integrity sha512-2z+rWdzbbSZv6/rhtvzvqeZQHrBaqgogqt85sqFNbabZOuFbCVFb8kPeEtZjiKkbrm395irpNKiYeFeLiQnFPg== dependencies: - css-tree "^1.1.2" + "@asamuzakjp/css-color" "^3.2.0" + rrweb-cssom "^0.8.0" -cssom@^0.4.4: - version "0.4.4" - resolved "https://registry.yarnpkg.com/cssom/-/cssom-0.4.4.tgz#5a66cf93d2d0b661d80bf6a44fb65f5c2e4e0a10" - integrity sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw== - -cssom@~0.3.6: - version "0.3.8" - resolved "https://registry.yarnpkg.com/cssom/-/cssom-0.3.8.tgz#9f1276f5b2b463f2114d3f2c75250af8c1a36f4a" - integrity sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg== - -cssstyle@^2.3.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/cssstyle/-/cssstyle-2.3.0.tgz#ff665a0ddbdc31864b09647f34163443d90b0852" - integrity sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A== - dependencies: - cssom "~0.3.6" +csstype@3.1.3: + version "3.1.3" + resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.3.tgz#d80ff294d114fb0e6ac500fbf85b60137d7eff81" + integrity sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw== -csstype@^3.0.2, csstype@^3.1.1: +csstype@^3.0.2: version "3.1.1" resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.1.tgz#841b532c45c758ee546a11d5bd7b7b473c8c30b9" integrity sha512-DJR/VvkAvSZW9bTouZue2sSxDwdTN92uHjqeKVm+0dAqdfNykRzQ95tay8aXMBAAPpUiq4Qcug2L7neoRh2Egw== -csstype@^3.1.3: - version "3.1.3" - resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.3.tgz#d80ff294d114fb0e6ac500fbf85b60137d7eff81" - integrity sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw== - -cypress@13.3.2: - version "13.3.2" - resolved "https://registry.yarnpkg.com/cypress/-/cypress-13.3.2.tgz#b4baa64ce37d7874f6bdd8efbc28a9c722c0686f" - integrity sha512-ArLmZObcLC+xxCp7zJZZbhby9FUf5CueLej9dUM4+5j37FTS4iMSgHxQLDu01PydFUvDXcNoIVRCYrHHxD7Ybg== - dependencies: - "@cypress/request" "^3.0.0" - "@cypress/xvfb" "^1.2.4" - "@types/node" "^18.17.5" - "@types/sinonjs__fake-timers" "8.1.1" - "@types/sizzle" "^2.3.2" - arch "^2.2.0" - blob-util "^2.0.2" - bluebird "^3.7.2" - buffer "^5.6.0" - cachedir "^2.3.0" - chalk "^4.1.0" - check-more-types "^2.24.0" - cli-cursor "^3.1.0" - cli-table3 "~0.6.1" - commander "^6.2.1" - common-tags "^1.8.0" - dayjs "^1.10.4" - debug "^4.3.4" - enquirer "^2.3.6" - eventemitter2 "6.4.7" - execa "4.1.0" - executable "^4.1.1" - extract-zip "2.0.1" - figures "^3.2.0" - fs-extra "^9.1.0" - getos "^3.2.1" - is-ci "^3.0.0" - is-installed-globally "~0.4.0" - lazy-ass "^1.6.0" - listr2 "^3.8.3" - lodash "^4.17.21" - log-symbols "^4.0.0" - minimist "^1.2.8" - ospath "^1.2.2" - pretty-bytes "^5.6.0" - process "^0.11.10" - proxy-from-env "1.0.0" - request-progress "^3.0.0" - semver "^7.5.3" - supports-color "^8.1.1" - tmp "~0.2.1" - untildify "^4.0.0" - yauzl "^2.10.0" +csstype@^3.2.2, csstype@^3.2.3: + version "3.2.3" + resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.2.3.tgz#ec48c0f3e993e50648c86da559e2610995cf989a" + integrity sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ== -"d3-array@2 - 3", "d3-array@2.10.0 - 3": +"d3-array@2 - 3", "d3-array@2.10.0 - 3", d3-array@^3.2.4: version "3.2.4" resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-3.2.4.tgz#15fec33b237f97ac5d7c986dc77da273a8ed0bb5" integrity sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg== @@ -5686,18 +4481,16 @@ cypress@13.3.2: resolved "https://registry.yarnpkg.com/d3-color/-/d3-color-3.1.0.tgz#395b2833dfac71507f12ac2f7af23bf819de24e2" integrity sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA== -d3-delaunay@^6.0.4: - version "6.0.4" - resolved "https://registry.yarnpkg.com/d3-delaunay/-/d3-delaunay-6.0.4.tgz#98169038733a0a5babbeda55054f795bb9e4a58b" - integrity sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A== - dependencies: - delaunator "5" - "d3-format@1 - 3": version "3.1.0" resolved "https://registry.yarnpkg.com/d3-format/-/d3-format-3.1.0.tgz#9260e23a28ea5cb109e93b21a06e24e2ebd55641" integrity sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA== +d3-format@^3.1.0: + version "3.1.2" + resolved "https://registry.yarnpkg.com/d3-format/-/d3-format-3.1.2.tgz#01fdb46b58beb1f55b10b42ad70b6e344d5eb2ae" + integrity sha512-AJDdYOdnyRDV5b6ArilzCPPwc1ejkHcoyFarqlPqT7zRYjhavcT3uSrqcMvsgh2CgoPbK3RCwyHaVyxYcP2Arg== + "d3-interpolate@1.2.0 - 3", d3-interpolate@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/d3-interpolate/-/d3-interpolate-3.0.1.tgz#3c47aa5b32c5b3dfb56ef3fd4342078a632b400d" @@ -5728,88 +4521,68 @@ d3-shape@^3.2.0: dependencies: d3-path "^3.1.0" -"d3-time-format@2 - 4": +"d3-time-format@2 - 4", d3-time-format@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/d3-time-format/-/d3-time-format-4.1.0.tgz#7ab5257a5041d11ecb4fe70a5c7d16a195bb408a" integrity sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg== dependencies: d3-time "1 - 3" -"d3-time@1 - 3", "d3-time@2.1.1 - 3": +"d3-time@1 - 3", "d3-time@2.1.1 - 3", d3-time@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/d3-time/-/d3-time-3.1.0.tgz#9310db56e992e3c0175e1ef385e545e48a9bb5c7" integrity sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q== dependencies: d3-array "2 - 3" -damerau-levenshtein@^1.0.8: - version "1.0.8" - resolved "https://registry.yarnpkg.com/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz#b43d286ccbd36bc5b2f7ed41caf2d0aba1f8a6e7" - integrity sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA== - -dashdash@^1.12.0: - version "1.14.1" - resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" - integrity sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g== - dependencies: - assert-plus "^1.0.0" +d3-timer@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/d3-timer/-/d3-timer-3.0.1.tgz#6284d2a2708285b1abb7e201eda4380af35e63b0" + integrity sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA== -data-urls@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/data-urls/-/data-urls-2.0.0.tgz#156485a72963a970f5d5821aaf642bef2bf2db9b" - integrity sha512-X5eWTSXO/BJmpdIKCRuKUgSCgAN0OwliVK3yPKbwIWU1Tdw5BRajxlzMidvh+gwko9AfQ9zIj52pzF91Q3YAvQ== +data-urls@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/data-urls/-/data-urls-5.0.0.tgz#2f76906bce1824429ffecb6920f45a0b30f00dde" + integrity sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg== dependencies: - abab "^2.0.3" - whatwg-mimetype "^2.3.0" - whatwg-url "^8.0.0" + whatwg-mimetype "^4.0.0" + whatwg-url "^14.0.0" -dayjs@^1.10.4, dayjs@^1.11.7: - version "1.11.7" - resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.7.tgz#4b296922642f70999544d1144a2c25730fce63e2" - integrity sha512-+Yw9U6YO5TQohxLcIkrXBeY73WP3ejHWVvx8XCk3gxvQDCTEmS48ZrSZCKciI7Bhl/uCMyxYtE9UqRILmFphkQ== - -debug@2.6.9, debug@^2.6.0, debug@^2.6.9: - version "2.6.9" - resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" - integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== - dependencies: - ms "2.0.0" +dayjs@1.11.19: + version "1.11.19" + resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.19.tgz#15dc98e854bb43917f12021806af897c58ae2938" + integrity sha512-t5EcLVS6QPBNqM2z8fakk/NKel+Xzshgt8FFKAn+qwlD1pzZWxh0nVCrvFK7ZDb6XucZeF9z8C7CBWTRIVApAw== -debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.2, debug@^4.3.4: +debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.4: version "4.3.4" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== dependencies: ms "2.1.2" -debug@^3.1.0, debug@^3.2.7: - version "3.2.7" - resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" - integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== +debug@^4.3.1: + version "4.4.3" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.4.3.tgz#c6ae432d9bd9662582fce08709b038c58e9e3d6a" + integrity sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA== dependencies: - ms "^2.1.1" + ms "^2.1.3" -decimal.js@^10.2.1: - version "10.4.3" - resolved "https://registry.yarnpkg.com/decimal.js/-/decimal.js-10.4.3.tgz#1044092884d245d1b7f65725fa4ad4c6f781cc23" - integrity sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA== +decimal.js@^10.5.0: + version "10.6.0" + resolved "https://registry.yarnpkg.com/decimal.js/-/decimal.js-10.6.0.tgz#e649a43e3ab953a72192ff5983865e509f37ed9a" + integrity sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg== -decompress-response@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-6.0.0.tgz#ca387612ddb7e104bd16d85aab00d5ecf09c66fc" - integrity sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ== +decode-named-character-reference@^1.0.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/decode-named-character-reference/-/decode-named-character-reference-1.2.0.tgz#25c32ae6dd5e21889549d40f676030e9514cc0ed" + integrity sha512-c6fcElNV6ShtZXmsgNgFFV5tVX2PaV4g+MOAkb8eXHvn6sryJBrZa9r0zV6+dtTyoCKxtDy5tyQ5ZwQuidtd+Q== dependencies: - mimic-response "^3.1.0" - -dedent@^0.7.0: - version "0.7.0" - resolved "https://registry.yarnpkg.com/dedent/-/dedent-0.7.0.tgz#2495ddbaf6eb874abb0e1be9df22d2e5a544326c" - integrity sha512-Q6fKUPqnAHAyhiUgFU7BUzLiv0kd8saH9al7tnu5Q/okj6dnupxyTgFIBjVzJATdfIAm9NAsvXNzjaKa+bxVyA== + character-entities "^2.0.0" -dedent@^1.0.0: - version "1.5.1" - resolved "https://registry.yarnpkg.com/dedent/-/dedent-1.5.1.tgz#4f3fc94c8b711e9bb2800d185cd6ad20f2a90aff" - integrity sha512-+LxW+KLWxu3HW3M2w2ympwtqPrqYRzU8fqi6Fhd18fBALe15blJPI/I4+UHveMVG6lJqB4JNd4UG0S5cnVHwIg== +dedent@^1.6.0: + version "1.7.0" + resolved "https://registry.yarnpkg.com/dedent/-/dedent-1.7.0.tgz#c1f9445335f0175a96587be245a282ff451446ca" + integrity sha512-HGFtf8yhuhGhqO07SV79tRp+br4MnbdjeVxotpn1QBl30pcLLCQjX5b2295ll0fv8RKDKsmWYrl05usHM9CewQ== deep-equal@^2.0.5: version "2.1.0" @@ -5832,33 +4605,21 @@ deep-equal@^2.0.5: which-collection "^1.0.1" which-typed-array "^1.1.8" -deep-extend@0.6.0, deep-extend@^0.6.0: +deep-extend@0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.6.0.tgz#c4fa7c95404a17a9c3e8ca7e1537312b736330ac" integrity sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA== -deep-is@^0.1.3, deep-is@~0.1.3: - version "0.1.4" - resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.4.tgz#a6f2dce612fadd2ef1f519b73551f17e85199831" - integrity sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ== - -deepmerge@^4.2.2: - version "4.2.2" - resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.2.2.tgz#44d2ea3679b8f4d4ffba33f03d865fc1e7bf4955" - integrity sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg== +deepmerge@^4.3.1: + version "4.3.1" + resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.3.1.tgz#44b5f2147cd3b00d4b56137685966f26fd25dd4a" + integrity sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A== deepmerge@~4.3.0: version "4.3.0" resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.3.0.tgz#65491893ec47756d44719ae520e0e2609233b59b" integrity sha512-z2wJZXrmeHdvYJp/Ux55wIjqo81G5Bp4c+oELTW+7ar6SogWHajt5a9gO3s3IDaGSAXjDk0vlQKN3rms8ab3og== -default-gateway@^6.0.3: - version "6.0.3" - resolved "https://registry.yarnpkg.com/default-gateway/-/default-gateway-6.0.3.tgz#819494c888053bdb743edbf343d6cdf7f2943a71" - integrity sha512-fwSOJsbbNzZ/CUFpqFBqYfYNLj1NbMPm8MMCIzHjC83iSJRBEGmDUxU+WP661BaBQImeC2yHwXtz+P/O9o+XEg== - dependencies: - execa "^5.0.0" - define-data-property@^1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/define-data-property/-/define-data-property-1.1.4.tgz#894dc141bb7d3060ae4366f6a0107e68fbe48c5e" @@ -5868,11 +4629,6 @@ define-data-property@^1.1.4: es-errors "^1.3.0" gopd "^1.0.1" -define-lazy-prop@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz#3f7ae421129bcaaac9bc74905c98a0009ec9ee7f" - integrity sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og== - define-properties@^1.1.3, define-properties@^1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.4.tgz#0b14d7bd7fbeb2f3572c3a7eda80ea5d57fb05b1" @@ -5881,134 +4637,30 @@ define-properties@^1.1.3, define-properties@^1.1.4: has-property-descriptors "^1.0.0" object-keys "^1.1.1" -defined@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/defined/-/defined-1.0.1.tgz#c0b9db27bfaffd95d6f61399419b893df0f91ebf" - integrity sha512-hsBd2qSVCRE+5PmNdHt1uzyrFu5d3RwmFDKzyNZMFq/EwDNJF7Ee5+D5oEKF0hU6LhtoUF1macFvOe4AskQC1Q== - -delaunator@5: - version "5.0.1" - resolved "https://registry.yarnpkg.com/delaunator/-/delaunator-5.0.1.tgz#39032b08053923e924d6094fe2cde1a99cc51278" - integrity sha512-8nvh+XBe96aCESrGOqMp/84b13H9cdKbG5P2ejQCh4d4sK9RL4371qou9drQjMhvnPmhWl5hnmqbEE0fXr9Xnw== - dependencies: - robust-predicates "^3.0.2" - delayed-stream@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== -depd@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df" - integrity sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw== - -depd@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" - integrity sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ== - -destroy@1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.2.0.tgz#4803735509ad8be552934c67df614f94e66fa015" - integrity sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg== - -detect-libc@^2.0.0: - version "2.0.2" - resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.0.2.tgz#8ccf2ba9315350e1241b88d0ac3b0e1fbd99605d" - integrity sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw== +dequal@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/dequal/-/dequal-2.0.3.tgz#2644214f1997d39ed0ee0ece72335490a7ac67be" + integrity sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA== -detect-newline@^3.0.0: +detect-newline@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651" integrity sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA== -detect-node@^2.0.4: - version "2.1.0" - resolved "https://registry.yarnpkg.com/detect-node/-/detect-node-2.1.0.tgz#c9c70775a49c3d03bc2c06d9a73be550f978f8b1" - integrity sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g== - -detect-port-alt@^1.1.6: - version "1.1.6" - resolved "https://registry.yarnpkg.com/detect-port-alt/-/detect-port-alt-1.1.6.tgz#24707deabe932d4a3cf621302027c2b266568275" - integrity sha512-5tQykt+LqfJFBEYaDITx7S7cR7mJ/zQmLXZ2qt5w04ainYZw6tBf9dBunMjVeVOdYVRUzUOE4HkY5J7+uttb5Q== - dependencies: - address "^1.0.1" - debug "^2.6.0" - -detective@^5.2.1: - version "5.2.1" - resolved "https://registry.yarnpkg.com/detective/-/detective-5.2.1.tgz#6af01eeda11015acb0e73f933242b70f24f91034" - integrity sha512-v9XE1zRnz1wRtgurGu0Bs8uHKFSTdteYZNbIPFVhUZ39L/S79ppMpdmVOZAnoz1jfEFodc48n6MX483Xo3t1yw== - dependencies: - acorn-node "^1.8.2" - defined "^1.0.0" - minimist "^1.2.6" - -didyoumean@^1.2.2: - version "1.2.2" - resolved "https://registry.yarnpkg.com/didyoumean/-/didyoumean-1.2.2.tgz#989346ffe9e839b4555ecf5666edea0d3e8ad037" - integrity sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw== - -diff-sequences@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-27.5.1.tgz#eaecc0d327fd68c8d9672a1e64ab8dccb2ef5327" - integrity sha512-k1gCAXAsNgLwEL+Y8Wvl+M6oEFj5bgazfZULpS5CneoPPXRaCCW7dm+q21Ky2VEE5X+VeRDBVg1Pcvvsr4TtNQ== - -diff-sequences@^29.6.3: - version "29.6.3" - resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-29.6.3.tgz#4deaf894d11407c51efc8418012f9e70b84ea921" - integrity sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q== - -dir-glob@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f" - integrity sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA== - dependencies: - path-type "^4.0.0" - -dlv@^1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/dlv/-/dlv-1.1.3.tgz#5c198a8a11453596e751494d49874bc7732f2e79" - integrity sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA== - -dns-equal@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/dns-equal/-/dns-equal-1.0.0.tgz#b39e7f1da6eb0a75ba9c17324b34753c47e0654d" - integrity sha512-z+paD6YUQsk+AbGCEM4PrOXSss5gd66QfcVBFTKR/HpFL9jCqikS94HYwKww6fQyO7IxrIIyUu+g0Ka9tUS2Cg== - -dns-packet@^5.2.2: - version "5.4.0" - resolved "https://registry.yarnpkg.com/dns-packet/-/dns-packet-5.4.0.tgz#1f88477cf9f27e78a213fb6d118ae38e759a879b" - integrity sha512-EgqGeaBB8hLiHLZtp/IbaDQTL8pZ0+IvwzSHA6d7VyMDM+B9hgddEMa9xjK5oYnw0ci0JQ6g2XCD7/f6cafU6g== - dependencies: - "@leichtgewicht/ip-codec" "^2.0.1" - -doctrine@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-2.1.0.tgz#5cd01fc101621b42c4cd7f5d1a66243716d3f39d" - integrity sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw== - dependencies: - esutils "^2.0.2" - -doctrine@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-3.0.0.tgz#addebead72a6574db783639dc87a121773973961" - integrity sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w== - dependencies: - esutils "^2.0.2" - -dom-accessibility-api@^0.5.6, dom-accessibility-api@^0.5.9: - version "0.5.14" - resolved "https://registry.yarnpkg.com/dom-accessibility-api/-/dom-accessibility-api-0.5.14.tgz#56082f71b1dc7aac69d83c4285eef39c15d93f56" - integrity sha512-NMt+m9zFMPZe0JcY9gN224Qvk6qLIdqex29clBvc/y75ZBX9YA9wNK3frsYvu2DI1xcCIwxwnX+TlsJ2DSOADg== +dom-accessibility-api@^0.5.9: + version "0.5.16" + resolved "https://registry.yarnpkg.com/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz#5a7429e6066eb3664d911e33fb0e45de8eb08453" + integrity sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg== -dom-converter@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/dom-converter/-/dom-converter-0.2.0.tgz#6721a9daee2e293682955b6afe416771627bb768" - integrity sha512-gd3ypIPfOMr9h5jIKq8E3sHOTCjeirnl0WK5ZdS1AW0Odt0b1PaWaHdJ4Qk4klv+YB9aJBS7mESXjFoDQPu6DA== - dependencies: - utila "~0.4" +dom-accessibility-api@^0.6.3: + version "0.6.3" + resolved "https://registry.yarnpkg.com/dom-accessibility-api/-/dom-accessibility-api-0.6.3.tgz#993e925cc1d73f2c662e7d75dd5a5445259a8fd8" + integrity sha512-7ZgogeTnjuHbo+ct10G9Ffp0mif17idi0IyWNVA/wcwcm7NPOD/WEHVP3n7n3MhXqxoIYm8d6MuZohYWIZ4T3w== dom-helpers@^5.0.1: version "5.2.1" @@ -6018,147 +4670,47 @@ dom-helpers@^5.0.1: "@babel/runtime" "^7.8.7" csstype "^3.0.2" -dom-serializer@0: - version "0.2.2" - resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.2.2.tgz#1afb81f533717175d478655debc5e332d9f9bb51" - integrity sha512-2/xPb3ORsQ42nHYiSunXkDjPLBaEj/xTwUO4B7XCZQTRk7EBtTOPaygh10YAAh2OI1Qrp6NWfpAhzswj0ydt9g== - dependencies: - domelementtype "^2.0.1" - entities "^2.0.0" - -dom-serializer@^1.0.1: - version "1.4.1" - resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-1.4.1.tgz#de5d41b1aea290215dc45a6dae8adcf1d32e2d30" - integrity sha512-VHwB3KfrcOOkelEG2ZOfxqLZdfkil8PtJi4P8N2MMXucZq2yLp75ClViUlOVwyoHEDjYU433Aq+5zWP61+RGag== - dependencies: - domelementtype "^2.0.1" - domhandler "^4.2.0" - entities "^2.0.0" - -domelementtype@1: - version "1.3.1" - resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.3.1.tgz#d048c44b37b0d10a7f2a3d5fee3f4333d790481f" - integrity sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w== - -domelementtype@^2.0.1, domelementtype@^2.2.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.3.0.tgz#5c45e8e869952626331d7aab326d01daf65d589d" - integrity sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw== - -domexception@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/domexception/-/domexception-2.0.1.tgz#fb44aefba793e1574b0af6aed2801d057529f304" - integrity sha512-yxJ2mFy/sibVQlu5qHjOkf9J3K6zgmCxgJ94u2EdvDOV09H+32LtRswEcUsmUWN72pVLOEnTSRaIVVzVQgS0dg== - dependencies: - webidl-conversions "^5.0.0" - -domhandler@^4.0.0, domhandler@^4.2.0, domhandler@^4.3.1: - version "4.3.1" - resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-4.3.1.tgz#8d792033416f59d68bc03a5aa7b018c1ca89279c" - integrity sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ== - dependencies: - domelementtype "^2.2.0" - -dompurify@=3.0.6: - version "3.0.6" - resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-3.0.6.tgz#925ebd576d54a9531b5d76f0a5bef32548351dae" - integrity sha512-ilkD8YEnnGh1zJ240uJsW7AzE+2qpbOUYjacomn3AvJ6J4JhKGSZ2nh4wUIXPZrEPppaCLx5jFe8T89Rk8tQ7w== - -domutils@^1.7.0: - version "1.7.0" - resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.7.0.tgz#56ea341e834e06e6748af7a1cb25da67ea9f8c2a" - integrity sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg== - dependencies: - dom-serializer "0" - domelementtype "1" - -domutils@^2.5.2, domutils@^2.8.0: - version "2.8.0" - resolved "https://registry.yarnpkg.com/domutils/-/domutils-2.8.0.tgz#4437def5db6e2d1f5d6ee859bd95ca7d02048135" - integrity sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A== - dependencies: - dom-serializer "^1.0.1" - domelementtype "^2.2.0" - domhandler "^4.2.0" - -dot-case@^3.0.4: - version "3.0.4" - resolved "https://registry.yarnpkg.com/dot-case/-/dot-case-3.0.4.tgz#9b2b670d00a431667a8a75ba29cd1b98809ce751" - integrity sha512-Kv5nKlh6yRrdrGvxeJ2e5y2eRUpkUosIW4A2AS38zwSz27zu7ufDwQPi5Jhs3XAlGNetl3bmnGhQsMtkKJnj3w== - dependencies: - no-case "^3.0.4" - tslib "^2.0.3" - -dotenv-expand@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/dotenv-expand/-/dotenv-expand-5.1.0.tgz#3fbaf020bfd794884072ea26b1e9791d45a629f0" - integrity sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA== - -dotenv@^10.0.0: - version "10.0.0" - resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-10.0.0.tgz#3d4227b8fb95f81096cdd2b66653fb2c7085ba81" - integrity sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q== +dompurify@3.2.7, dompurify@3.3.1, dompurify@=3.2.6: + version "3.3.1" + resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-3.3.1.tgz#c7e1ddebfe3301eacd6c0c12a4af284936dbbb86" + integrity sha512-qkdCKzLNtrgPFP1Vo+98FRzJnBRGe4ffyCea9IwHB1fyxPOeNTHpLKYGd4Uk9xvNoH0ZoOjwZxNptyMwqrId1Q== + optionalDependencies: + "@types/trusted-types" "^2.0.7" drange@^1.0.2: version "1.1.1" resolved "https://registry.yarnpkg.com/drange/-/drange-1.1.1.tgz#b2aecec2aab82fcef11dbbd7b9e32b83f8f6c0b8" integrity sha512-pYxfDYpued//QpnLIm4Avk7rsNtAtQkUES2cwAYSvD/wd2pKD71gN2Ebj3e7klzXwjocvE8c5vx/1fxwpqmSxA== -duplexer@^0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/duplexer/-/duplexer-0.1.2.tgz#3abe43aef3835f8ae077d136ddce0f276b0400e6" - integrity sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg== - -ecc-jsbn@~0.1.1: - version "0.1.2" - resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" - integrity sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw== +dunder-proto@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/dunder-proto/-/dunder-proto-1.0.1.tgz#d7ae667e1dc83482f8b70fd0f6eefc50da30f58a" + integrity sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A== dependencies: - jsbn "~0.1.0" - safer-buffer "^2.1.0" - -ee-first@1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" - integrity sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow== + call-bind-apply-helpers "^1.0.1" + es-errors "^1.3.0" + gopd "^1.2.0" -ejs@^3.1.6: - version "3.1.10" - resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.10.tgz#69ab8358b14e896f80cc39e62087b88500c3ac3b" - integrity sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA== - dependencies: - jake "^10.8.5" +eastasianwidth@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/eastasianwidth/-/eastasianwidth-0.2.0.tgz#696ce2ec0aa0e6ea93a397ffcf24aa7840c827cb" + integrity sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA== electron-to-chromium@^1.4.251: version "1.4.284" resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.284.tgz#61046d1e4cab3a25238f6bf7413795270f125592" integrity sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA== -electron-to-chromium@^1.4.535: - version "1.4.563" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.563.tgz#dabb424202754c1fed2d2938ff564b23d3bbf0d3" - integrity sha512-dg5gj5qOgfZNkPNeyKBZQAQitIQ/xwfIDmEQJHCbXaD9ebTZxwJXUsDYcBlAvZGZLi+/354l35J1wkmP6CqYaw== - -electron-to-chromium@^1.5.4: - version "1.5.13" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.5.13.tgz#1abf0410c5344b2b829b7247e031f02810d442e6" - integrity sha512-lbBcvtIJ4J6sS4tb5TLp1b4LyfCdMkwStzXPyAgVgTRAsep4bvrAGaBOP7ZJtQMNJpSQ9SqG4brWOroNaQtm7Q== - -emittery@^0.10.2: - version "0.10.2" - resolved "https://registry.yarnpkg.com/emittery/-/emittery-0.10.2.tgz#902eec8aedb8c41938c46e9385e9db7e03182933" - integrity sha512-aITqOwnLanpHLNXZJENbOgjUBeHocD+xsSJmNrjovKBW5HbSpW3d1pEls7GFQPUWXiwG9+0P4GtHfEqC/4M0Iw== +electron-to-chromium@^1.5.249: + version "1.5.252" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.5.252.tgz#65d8fe59767e68ef90b4b6380cb5c227e6ba860f" + integrity sha512-53uTpjtRgS7gjIxZ4qCgFdNO2q+wJt/Z8+xAvxbCqXPJrY6h7ighUkadQmNMXH96crtpa6gPFNP7BF4UBGDuaA== emittery@^0.13.1: version "0.13.1" resolved "https://registry.yarnpkg.com/emittery/-/emittery-0.13.1.tgz#c04b8c3457490e0847ae51fced3af52d338e3dad" integrity sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ== -emittery@^0.8.1: - version "0.8.1" - resolved "https://registry.yarnpkg.com/emittery/-/emittery-0.8.1.tgz#bb23cc86d03b30aa75a7f734819dee2e1ba70860" - integrity sha512-uDfvUjVrfGJJhymx/kz6prltenw1u7WrCg1oa94zYY8xxVpLLUu045LAT0dhDZdXG58/EpPL/5kA180fQ/qudg== - emoji-regex@^8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" @@ -6169,47 +4721,18 @@ emoji-regex@^9.2.2: resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-9.2.2.tgz#840c8803b0d8047f4ff0cf963176b32d4ef3ed72" integrity sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg== -emojis-list@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/emojis-list/-/emojis-list-3.0.0.tgz#5570662046ad29e2e916e71aae260abdff4f6a78" - integrity sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q== - -encodeurl@~1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" - integrity sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w== - -encodeurl@~2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-2.0.0.tgz#7b8ea898077d7e409d3ac45474ea38eaf0857a58" - integrity sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg== - -end-of-stream@^1.1.0, end-of-stream@^1.4.1: - version "1.4.4" - resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" - integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== - dependencies: - once "^1.4.0" - -enhanced-resolve@^5.17.1: - version "5.17.1" - resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.17.1.tgz#67bfbbcc2f81d511be77d686a90267ef7f898a15" - integrity sha512-LMHl3dXhTcfv8gM4kEzIUeTQ+7fpdA0l2tUf34BddXPkz2A5xJ5L/Pchd5BL6rdccM9QGvu0sWZzK1Z1t4wwyg== +enhanced-resolve@^5.19.0: + version "5.19.0" + resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.19.0.tgz#6687446a15e969eaa63c2fa2694510e17ae6d97c" + integrity sha512-phv3E1Xl4tQOShqSte26C7Fl84EwUdZsyOuSSk9qtAGyyQs2s3jJzComh+Abf4g187lUUAvH+H26omrqia2aGg== dependencies: graceful-fs "^4.2.4" - tapable "^2.2.0" - -enquirer@^2.3.6: - version "2.3.6" - resolved "https://registry.yarnpkg.com/enquirer/-/enquirer-2.3.6.tgz#2a7fe5dd634a1e4125a975ec994ff5456dc3734d" - integrity sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg== - dependencies: - ansi-colors "^4.1.1" + tapable "^2.3.0" -entities@^2.0.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55" - integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A== +entities@^6.0.0: + version "6.0.1" + resolved "https://registry.yarnpkg.com/entities/-/entities-6.0.1.tgz#c28c34a43379ca7f61d074130b2f5f7020a30694" + integrity sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g== error-ex@^1.3.1: version "1.3.2" @@ -6218,55 +4741,10 @@ error-ex@^1.3.1: dependencies: is-arrayish "^0.2.1" -error-stack-parser@^2.0.6: - version "2.1.4" - resolved "https://registry.yarnpkg.com/error-stack-parser/-/error-stack-parser-2.1.4.tgz#229cb01cdbfa84440bfa91876285b94680188286" - integrity sha512-Sk5V6wVazPhq5MhpO+AUxJn5x7XSXGl1R93Vn7i+zS15KDVxQijejNCrz8340/2bgLBjR9GtEG8ZVKONDjcqGQ== - dependencies: - stackframe "^1.3.4" - -es-abstract@^1.17.2, es-abstract@^1.19.0, es-abstract@^1.20.4: - version "1.20.5" - resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.20.5.tgz#e6dc99177be37cacda5988e692c3fa8b218e95d2" - integrity sha512-7h8MM2EQhsCA7pU/Nv78qOXFpD8Rhqd12gYiSJVkrH9+e8VuA8JlPJK/hQjjlLv6pJvx/z1iRFKzYb0XT/RuAQ== - dependencies: - call-bind "^1.0.2" - es-to-primitive "^1.2.1" - function-bind "^1.1.1" - function.prototype.name "^1.1.5" - get-intrinsic "^1.1.3" - get-symbol-description "^1.0.0" - gopd "^1.0.1" - has "^1.0.3" - has-property-descriptors "^1.0.0" - has-symbols "^1.0.3" - internal-slot "^1.0.3" - is-callable "^1.2.7" - is-negative-zero "^2.0.2" - is-regex "^1.1.4" - is-shared-array-buffer "^1.0.2" - is-string "^1.0.7" - is-weakref "^1.0.2" - object-inspect "^1.12.2" - object-keys "^1.1.1" - object.assign "^4.1.4" - regexp.prototype.flags "^1.4.3" - safe-regex-test "^1.0.0" - string.prototype.trimend "^1.0.6" - string.prototype.trimstart "^1.0.6" - unbox-primitive "^1.0.2" - -es-array-method-boxes-properly@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/es-array-method-boxes-properly/-/es-array-method-boxes-properly-1.0.0.tgz#873f3e84418de4ee19c5be752990b2e44718d09e" - integrity sha512-wd6JXUmyHmt8T5a2xreUwKcGPq6f1f+WwIJkijUqiGcJz1qqnZgP6XIK+QyIWU5lT7imeNxUll48bziG+TSYcA== - -es-define-property@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/es-define-property/-/es-define-property-1.0.0.tgz#c7faefbdff8b2696cf5f46921edfb77cc4ba3845" - integrity sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ== - dependencies: - get-intrinsic "^1.2.4" +es-define-property@^1.0.0, es-define-property@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/es-define-property/-/es-define-property-1.0.1.tgz#983eb2f9a6724e9303f61addf011c72e09e0b0fa" + integrity sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g== es-errors@^1.3.0: version "1.3.0" @@ -6287,41 +4765,64 @@ es-get-iterator@^1.1.2: is-string "^1.0.5" isarray "^2.0.5" -es-module-lexer@^1.2.1: - version "1.5.4" - resolved "https://registry.yarnpkg.com/es-module-lexer/-/es-module-lexer-1.5.4.tgz#a8efec3a3da991e60efa6b633a7cad6ab8d26b78" - integrity sha512-MVNK56NiMrOwitFB7cqDwq0CQutbw+0BvLshJSse0MUNU+y1FC3bUS/AQg7oUng+/wKrrki7JfmwtVHkVfPLlw== - -es-shim-unscopables@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/es-shim-unscopables/-/es-shim-unscopables-1.0.0.tgz#702e632193201e3edf8713635d083d378e510241" - integrity sha512-Jm6GPcCdC30eMLbZ2x8z2WuRwAws3zTBBKuusffYVUrNj/GVSUAZ+xKMaUpfNDR5IbyNA5LJbaecoUVbmUcB1w== +es-object-atoms@^1.0.0, es-object-atoms@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/es-object-atoms/-/es-object-atoms-1.1.1.tgz#1c4f2c4837327597ce69d2ca190a7fdd172338c1" + integrity sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA== dependencies: - has "^1.0.3" + es-errors "^1.3.0" -es-to-primitive@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.2.1.tgz#e55cd4c9cdc188bcefb03b366c736323fc5c898a" - integrity sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA== +es-set-tostringtag@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz#f31dbbe0c183b00a6d26eb6325c810c0fd18bd4d" + integrity sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA== dependencies: - is-callable "^1.1.4" - is-date-object "^1.0.1" - is-symbol "^1.0.2" + es-errors "^1.3.0" + get-intrinsic "^1.2.6" + has-tostringtag "^1.0.2" + hasown "^2.0.2" + +esbuild@^0.27.0: + version "0.27.3" + resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.27.3.tgz#5859ca8e70a3af956b26895ce4954d7e73bd27a8" + integrity sha512-8VwMnyGCONIs6cWue2IdpHxHnAjzxnw2Zr7MkVxB2vjmQ2ivqGFb4LEG3SMnv0Gb2F/G/2yA8zUaiL1gywDCCg== + optionalDependencies: + "@esbuild/aix-ppc64" "0.27.3" + "@esbuild/android-arm" "0.27.3" + "@esbuild/android-arm64" "0.27.3" + "@esbuild/android-x64" "0.27.3" + "@esbuild/darwin-arm64" "0.27.3" + "@esbuild/darwin-x64" "0.27.3" + "@esbuild/freebsd-arm64" "0.27.3" + "@esbuild/freebsd-x64" "0.27.3" + "@esbuild/linux-arm" "0.27.3" + "@esbuild/linux-arm64" "0.27.3" + "@esbuild/linux-ia32" "0.27.3" + "@esbuild/linux-loong64" "0.27.3" + "@esbuild/linux-mips64el" "0.27.3" + "@esbuild/linux-ppc64" "0.27.3" + "@esbuild/linux-riscv64" "0.27.3" + "@esbuild/linux-s390x" "0.27.3" + "@esbuild/linux-x64" "0.27.3" + "@esbuild/netbsd-arm64" "0.27.3" + "@esbuild/netbsd-x64" "0.27.3" + "@esbuild/openbsd-arm64" "0.27.3" + "@esbuild/openbsd-x64" "0.27.3" + "@esbuild/openharmony-arm64" "0.27.3" + "@esbuild/sunos-x64" "0.27.3" + "@esbuild/win32-arm64" "0.27.3" + "@esbuild/win32-ia32" "0.27.3" + "@esbuild/win32-x64" "0.27.3" escalade@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" integrity sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw== -escalade@^3.1.2: - version "3.1.2" - resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.2.tgz#54076e9ab29ea5bf3d8f1ed62acffbb88272df27" - integrity sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA== - -escape-html@~1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988" - integrity sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow== +escalade@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.2.0.tgz#011a3f69856ba189dffa7dc8fcce99d2a87903e5" + integrity sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA== escape-string-regexp@^1.0.5: version "1.0.5" @@ -6338,317 +4839,17 @@ escape-string-regexp@^4.0.0: resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz#14ba83a5d373e3d311e5afca29cf5bfad965bf34" integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA== -escodegen@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-2.0.0.tgz#5e32b12833e8aa8fa35e1bf0befa89380484c7dd" - integrity sha512-mmHKys/C8BFUGI+MAWNcSYoORYLMdPzjrknd2Vc+bUsjN5bXcr8EhrNB+UTqfL1y3I9c4fw2ihgtMPQLBRiQxw== - dependencies: - esprima "^4.0.1" - estraverse "^5.2.0" - esutils "^2.0.2" - optionator "^0.8.1" - optionalDependencies: - source-map "~0.6.1" - -eslint-config-react-app@^7.0.1: - version "7.0.1" - resolved "https://registry.yarnpkg.com/eslint-config-react-app/-/eslint-config-react-app-7.0.1.tgz#73ba3929978001c5c86274c017ea57eb5fa644b4" - integrity sha512-K6rNzvkIeHaTd8m/QEh1Zko0KI7BACWkkneSs6s9cKZC/J27X3eZR6Upt1jkmZ/4FK+XUOPPxMEN7+lbUXfSlA== - dependencies: - "@babel/core" "^7.16.0" - "@babel/eslint-parser" "^7.16.3" - "@rushstack/eslint-patch" "^1.1.0" - "@typescript-eslint/eslint-plugin" "^5.5.0" - "@typescript-eslint/parser" "^5.5.0" - babel-preset-react-app "^10.0.1" - confusing-browser-globals "^1.0.11" - eslint-plugin-flowtype "^8.0.3" - eslint-plugin-import "^2.25.3" - eslint-plugin-jest "^25.3.0" - eslint-plugin-jsx-a11y "^6.5.1" - eslint-plugin-react "^7.27.1" - eslint-plugin-react-hooks "^4.3.0" - eslint-plugin-testing-library "^5.0.1" - -eslint-import-resolver-node@^0.3.6: - version "0.3.6" - resolved "https://registry.yarnpkg.com/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.6.tgz#4048b958395da89668252001dbd9eca6b83bacbd" - integrity sha512-0En0w03NRVMn9Uiyn8YRPDKvWjxCWkslUEhGNTdGx15RvPJYQ+lbOlqrlNI2vEAs4pDYK4f/HN2TbDmk5TP0iw== - dependencies: - debug "^3.2.7" - resolve "^1.20.0" - -eslint-module-utils@^2.7.3: - version "2.7.4" - resolved "https://registry.yarnpkg.com/eslint-module-utils/-/eslint-module-utils-2.7.4.tgz#4f3e41116aaf13a20792261e61d3a2e7e0583974" - integrity sha512-j4GT+rqzCoRKHwURX7pddtIPGySnX9Si/cgMI5ztrcqOPtk5dDEeZ34CQVPphnqkJytlc97Vuk05Um2mJ3gEQA== - dependencies: - debug "^3.2.7" - -eslint-plugin-flowtype@^8.0.3: - version "8.0.3" - resolved "https://registry.yarnpkg.com/eslint-plugin-flowtype/-/eslint-plugin-flowtype-8.0.3.tgz#e1557e37118f24734aa3122e7536a038d34a4912" - integrity sha512-dX8l6qUL6O+fYPtpNRideCFSpmWOUVx5QcaGLVqe/vlDiBSe4vYljDWDETwnyFzpl7By/WVIu6rcrniCgH9BqQ== - dependencies: - lodash "^4.17.21" - string-natural-compare "^3.0.1" - -eslint-plugin-import@^2.25.3: - version "2.26.0" - resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.26.0.tgz#f812dc47be4f2b72b478a021605a59fc6fe8b88b" - integrity sha512-hYfi3FXaM8WPLf4S1cikh/r4IxnO6zrhZbEGz2b660EJRbuxgpDS5gkCuYgGWg2xxh2rBuIr4Pvhve/7c31koA== - dependencies: - array-includes "^3.1.4" - array.prototype.flat "^1.2.5" - debug "^2.6.9" - doctrine "^2.1.0" - eslint-import-resolver-node "^0.3.6" - eslint-module-utils "^2.7.3" - has "^1.0.3" - is-core-module "^2.8.1" - is-glob "^4.0.3" - minimatch "^3.1.2" - object.values "^1.1.5" - resolve "^1.22.0" - tsconfig-paths "^3.14.1" - -eslint-plugin-jest@^25.3.0: - version "25.7.0" - resolved "https://registry.yarnpkg.com/eslint-plugin-jest/-/eslint-plugin-jest-25.7.0.tgz#ff4ac97520b53a96187bad9c9814e7d00de09a6a" - integrity sha512-PWLUEXeeF7C9QGKqvdSbzLOiLTx+bno7/HC9eefePfEb257QFHg7ye3dh80AZVkaa/RQsBB1Q/ORQvg2X7F0NQ== - dependencies: - "@typescript-eslint/experimental-utils" "^5.0.0" - -eslint-plugin-jsx-a11y@^6.5.1: - version "6.6.1" - resolved "https://registry.yarnpkg.com/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.6.1.tgz#93736fc91b83fdc38cc8d115deedfc3091aef1ff" - integrity sha512-sXgFVNHiWffBq23uiS/JaP6eVR622DqwB4yTzKvGZGcPq6/yZ3WmOZfuBks/vHWo9GaFOqC2ZK4i6+C35knx7Q== - dependencies: - "@babel/runtime" "^7.18.9" - aria-query "^4.2.2" - array-includes "^3.1.5" - ast-types-flow "^0.0.7" - axe-core "^4.4.3" - axobject-query "^2.2.0" - damerau-levenshtein "^1.0.8" - emoji-regex "^9.2.2" - has "^1.0.3" - jsx-ast-utils "^3.3.2" - language-tags "^1.0.5" - minimatch "^3.1.2" - semver "^6.3.0" - -eslint-plugin-react-hooks@^4.3.0: - version "4.6.0" - resolved "https://registry.yarnpkg.com/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-4.6.0.tgz#4c3e697ad95b77e93f8646aaa1630c1ba607edd3" - integrity sha512-oFc7Itz9Qxh2x4gNHStv3BqJq54ExXmfC+a1NjAta66IAN87Wu0R/QArgIS9qKzX3dXKPI9H5crl9QchNMY9+g== - -eslint-plugin-react@^7.27.1: - version "7.31.11" - resolved "https://registry.yarnpkg.com/eslint-plugin-react/-/eslint-plugin-react-7.31.11.tgz#011521d2b16dcf95795df688a4770b4eaab364c8" - integrity sha512-TTvq5JsT5v56wPa9OYHzsrOlHzKZKjV+aLgS+55NJP/cuzdiQPC7PfYoUjMoxlffKtvijpk7vA/jmuqRb9nohw== - dependencies: - array-includes "^3.1.6" - array.prototype.flatmap "^1.3.1" - array.prototype.tosorted "^1.1.1" - doctrine "^2.1.0" - estraverse "^5.3.0" - jsx-ast-utils "^2.4.1 || ^3.0.0" - minimatch "^3.1.2" - object.entries "^1.1.6" - object.fromentries "^2.0.6" - object.hasown "^1.1.2" - object.values "^1.1.6" - prop-types "^15.8.1" - resolve "^2.0.0-next.3" - semver "^6.3.0" - string.prototype.matchall "^4.0.8" - -eslint-plugin-testing-library@^5.0.1: - version "5.9.1" - resolved "https://registry.yarnpkg.com/eslint-plugin-testing-library/-/eslint-plugin-testing-library-5.9.1.tgz#12e4bd34c48683ee98af4df2e3318ec9f51dcf8a" - integrity sha512-6BQp3tmb79jLLasPHJmy8DnxREe+2Pgf7L+7o09TSWPfdqqtQfRZmZNetr5mOs3yqZk/MRNxpN3RUpJe0wB4LQ== - dependencies: - "@typescript-eslint/utils" "^5.13.0" - -eslint-scope@5.1.1, eslint-scope@^5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-5.1.1.tgz#e786e59a66cb92b3f6c1fb0d508aab174848f48c" - integrity sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw== - dependencies: - esrecurse "^4.3.0" - estraverse "^4.1.1" - -eslint-scope@^7.1.1: - version "7.1.1" - resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-7.1.1.tgz#fff34894c2f65e5226d3041ac480b4513a163642" - integrity sha512-QKQM/UXpIiHcLqJ5AOyIW7XZmzjkzQXYE54n1++wb0u9V/abW3l9uQnxX8Z5Xd18xyKIMTUAyQ0k1e8pz6LUrw== - dependencies: - esrecurse "^4.3.0" - estraverse "^5.2.0" - -eslint-utils@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/eslint-utils/-/eslint-utils-3.0.0.tgz#8aebaface7345bb33559db0a1f13a1d2d48c3672" - integrity sha512-uuQC43IGctw68pJA1RgbQS8/NP7rch6Cwd4j3ZBtgo4/8Flj4eGE7ZYSZRN3iq5pVUv6GPdW5Z1RFleo84uLDA== - dependencies: - eslint-visitor-keys "^2.0.0" - -eslint-visitor-keys@^2.0.0, eslint-visitor-keys@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz#f65328259305927392c938ed44eb0a5c9b2bd303" - integrity sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw== - -eslint-visitor-keys@^3.3.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz#f6480fa6b1f30efe2d1968aa8ac745b862469826" - integrity sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA== - -eslint-visitor-keys@^3.4.1: - version "3.4.3" - resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz#0cd72fe8550e3c2eae156a96a4dddcd1c8ac5800" - integrity sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag== - -eslint-webpack-plugin@^3.1.1: - version "3.2.0" - resolved "https://registry.yarnpkg.com/eslint-webpack-plugin/-/eslint-webpack-plugin-3.2.0.tgz#1978cdb9edc461e4b0195a20da950cf57988347c" - integrity sha512-avrKcGncpPbPSUHX6B3stNGzkKFto3eL+DKM4+VyMrVnhPc3vRczVlCq3uhuFOdRvDHTVXuzwk1ZKUrqDQHQ9w== - dependencies: - "@types/eslint" "^7.29.0 || ^8.4.1" - jest-worker "^28.0.2" - micromatch "^4.0.5" - normalize-path "^3.0.0" - schema-utils "^4.0.0" - -eslint@^8.3.0: - version "8.30.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.30.0.tgz#83a506125d089eef7c5b5910eeea824273a33f50" - integrity sha512-MGADB39QqYuzEGov+F/qb18r4i7DohCDOfatHaxI2iGlPuC65bwG2gxgO+7DkyL38dRFaRH7RaRAgU6JKL9rMQ== - dependencies: - "@eslint/eslintrc" "^1.4.0" - "@humanwhocodes/config-array" "^0.11.8" - "@humanwhocodes/module-importer" "^1.0.1" - "@nodelib/fs.walk" "^1.2.8" - ajv "^6.10.0" - chalk "^4.0.0" - cross-spawn "^7.0.2" - debug "^4.3.2" - doctrine "^3.0.0" - escape-string-regexp "^4.0.0" - eslint-scope "^7.1.1" - eslint-utils "^3.0.0" - eslint-visitor-keys "^3.3.0" - espree "^9.4.0" - esquery "^1.4.0" - esutils "^2.0.2" - fast-deep-equal "^3.1.3" - file-entry-cache "^6.0.1" - find-up "^5.0.0" - glob-parent "^6.0.2" - globals "^13.19.0" - grapheme-splitter "^1.0.4" - ignore "^5.2.0" - import-fresh "^3.0.0" - imurmurhash "^0.1.4" - is-glob "^4.0.0" - is-path-inside "^3.0.3" - js-sdsl "^4.1.4" - js-yaml "^4.1.0" - json-stable-stringify-without-jsonify "^1.0.1" - levn "^0.4.1" - lodash.merge "^4.6.2" - minimatch "^3.1.2" - natural-compare "^1.4.0" - optionator "^0.9.1" - regexpp "^3.2.0" - strip-ansi "^6.0.1" - strip-json-comments "^3.1.0" - text-table "^0.2.0" - -espree@^9.4.0: - version "9.4.1" - resolved "https://registry.yarnpkg.com/espree/-/espree-9.4.1.tgz#51d6092615567a2c2cff7833445e37c28c0065bd" - integrity sha512-XwctdmTO6SIvCzd9810yyNzIrOrqNYV9Koizx4C/mRhf9uq0o4yHoCEU/670pOxOL/MSraektvSAji79kX90Vg== - dependencies: - acorn "^8.8.0" - acorn-jsx "^5.3.2" - eslint-visitor-keys "^3.3.0" - -esprima@^4.0.0, esprima@^4.0.1: +esprima@^4.0.0: version "4.0.1" resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71" integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A== -esquery@^1.4.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/esquery/-/esquery-1.4.0.tgz#2148ffc38b82e8c7057dfed48425b3e61f0f24a5" - integrity sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w== - dependencies: - estraverse "^5.1.0" - -esrecurse@^4.3.0: - version "4.3.0" - resolved "https://registry.yarnpkg.com/esrecurse/-/esrecurse-4.3.0.tgz#7ad7964d679abb28bee72cec63758b1c5d2c9921" - integrity sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag== - dependencies: - estraverse "^5.2.0" - -estraverse@^4.1.1: - version "4.3.0" - resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-4.3.0.tgz#398ad3f3c5a24948be7725e83d11a7de28cdbd1d" - integrity sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw== - -estraverse@^5.1.0, estraverse@^5.2.0, estraverse@^5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-5.3.0.tgz#2eea5290702f26ab8fe5370370ff86c965d21123" - integrity sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA== - -estree-walker@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-1.0.1.tgz#31bc5d612c96b704106b477e6dd5d8aa138cb700" - integrity sha512-1fMXF3YP4pZZVozF8j/ZLfvnR8NSIljt56UhbZ5PeeDmmGHpgpdwQt7ITlGvYaQukCvuBRMLEiKiYC+oeIg4cg== - esutils@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== -etag@~1.8.1: - version "1.8.1" - resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" - integrity sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg== - -eventemitter2@6.4.7: - version "6.4.7" - resolved "https://registry.yarnpkg.com/eventemitter2/-/eventemitter2-6.4.7.tgz#a7f6c4d7abf28a14c1ef3442f21cb306a054271d" - integrity sha512-tYUSVOGeQPKt/eC1ABfhHy5Xd96N3oIijJvN3O9+TsC28T5V9yX9oEfEK5faP0EFSNVOG97qtAS68GBrQB2hDg== - -eventemitter3@^4.0.0: - version "4.0.7" - resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f" - integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw== - -events@^3.2.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" - integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== - -execa@4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/execa/-/execa-4.1.0.tgz#4e5491ad1572f2f17a77d388c6c857135b22847a" - integrity sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA== - dependencies: - cross-spawn "^7.0.0" - get-stream "^5.0.0" - human-signals "^1.1.1" - is-stream "^2.0.0" - merge-stream "^2.0.0" - npm-run-path "^4.0.0" - onetime "^5.1.0" - signal-exit "^3.0.2" - strip-final-newline "^2.0.0" - -execa@^5.0.0: +execa@^5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/execa/-/execa-5.1.1.tgz#f80ad9cbf4298f7bd1d4c9555c21e93741c411dd" integrity sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg== @@ -6663,145 +4864,33 @@ execa@^5.0.0: signal-exit "^3.0.3" strip-final-newline "^2.0.0" -executable@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/executable/-/executable-4.1.1.tgz#41532bff361d3e57af4d763b70582db18f5d133c" - integrity sha512-8iA79xD3uAch729dUG8xaaBBFGaEa0wdD2VkYLFHwlqosEj/jT66AzcreRDSgV7ehnNLBW2WR5jIXwGKjVdTLg== - dependencies: - pify "^2.2.0" - -exit@^0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c" - integrity sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ== - -expand-template@^2.0.3: - version "2.0.3" - resolved "https://registry.yarnpkg.com/expand-template/-/expand-template-2.0.3.tgz#6e14b3fcee0f3a6340ecb57d2e8918692052a47c" - integrity sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg== - -expect@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/expect/-/expect-27.5.1.tgz#83ce59f1e5bdf5f9d2b94b61d2050db48f3fef74" - integrity sha512-E1q5hSUG2AmYQwQJ041nvgpkODHQvB+RKlB4IYdru6uJsyFTRyZAP463M+1lINorwbqAmUggi6+WwkD8lCS/Dw== - dependencies: - "@jest/types" "^27.5.1" - jest-get-type "^27.5.1" - jest-matcher-utils "^27.5.1" - jest-message-util "^27.5.1" - -expect@^29.0.0, expect@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/expect/-/expect-29.7.0.tgz#578874590dcb3214514084c08115d8aee61e11bc" - integrity sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw== - dependencies: - "@jest/expect-utils" "^29.7.0" - jest-get-type "^29.6.3" - jest-matcher-utils "^29.7.0" - jest-message-util "^29.7.0" - jest-util "^29.7.0" - -express@^4.17.3: - version "4.20.0" - resolved "https://registry.yarnpkg.com/express/-/express-4.20.0.tgz#f1d08e591fcec770c07be4767af8eb9bcfd67c48" - integrity sha512-pLdae7I6QqShF5PnNTCVn4hI91Dx0Grkn2+IAsMTgMIKuQVte2dN9PeGSSAME2FR8anOhVA62QDIUaWVfEXVLw== - dependencies: - accepts "~1.3.8" - array-flatten "1.1.1" - body-parser "1.20.3" - content-disposition "0.5.4" - content-type "~1.0.4" - cookie "0.6.0" - cookie-signature "1.0.6" - debug "2.6.9" - depd "2.0.0" - encodeurl "~2.0.0" - escape-html "~1.0.3" - etag "~1.8.1" - finalhandler "1.2.0" - fresh "0.5.2" - http-errors "2.0.0" - merge-descriptors "1.0.3" - methods "~1.1.2" - on-finished "2.4.1" - parseurl "~1.3.3" - path-to-regexp "0.1.10" - proxy-addr "~2.0.7" - qs "6.11.0" - range-parser "~1.2.1" - safe-buffer "5.2.1" - send "0.19.0" - serve-static "1.16.0" - setprototypeof "1.2.0" - statuses "2.0.1" - type-is "~1.6.18" - utils-merge "1.0.1" - vary "~1.1.2" - -extend@~3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" - integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== - -extract-zip@2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a" - integrity sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg== - dependencies: - debug "^4.1.1" - get-stream "^5.1.0" - yauzl "^2.10.0" - optionalDependencies: - "@types/yauzl" "^2.9.1" - -extsprintf@1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" - integrity sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g== - -extsprintf@^1.2.0: - version "1.4.1" - resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.1.tgz#8d172c064867f235c0c84a596806d279bf4bcc07" - integrity sha512-Wrk35e8ydCKDj/ArClo1VrPVmN8zph5V4AtHwIuHhvMXsKf73UT3BOD+azBIW+3wOJ4FhEH7zyaJCFvChjYvMA== - -fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3: - version "3.1.3" - resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" - integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q== +exit-x@^0.2.2: + version "0.2.2" + resolved "https://registry.yarnpkg.com/exit-x/-/exit-x-0.2.2.tgz#1f9052de3b8d99a696b10dad5bced9bdd5c3aa64" + integrity sha512-+I6B/IkJc1o/2tiURyz/ivu/O0nKNEArIUB5O7zBrlDVJr22SCLH3xTeEry428LvFhRzIA1g8izguxJ/gbNcVQ== -fast-glob@^3.2.12, fast-glob@^3.2.9: - version "3.2.12" - resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.12.tgz#7f39ec99c2e6ab030337142da9e0c18f37afae80" - integrity sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w== +expect@30.2.0, expect@^30.0.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/expect/-/expect-30.2.0.tgz#d4013bed267013c14bc1199cec8aa57cee9b5869" + integrity sha512-u/feCi0GPsI+988gU2FLcsHyAHTU0MX1Wg68NhAnN7z/+C5wqG+CY8J53N9ioe8RXgaoz0nBR/TYMf3AycUuPw== dependencies: - "@nodelib/fs.stat" "^2.0.2" - "@nodelib/fs.walk" "^1.2.3" - glob-parent "^5.1.2" - merge2 "^1.3.0" - micromatch "^4.0.4" + "@jest/expect-utils" "30.2.0" + "@jest/get-type" "30.1.0" + jest-matcher-utils "30.2.0" + jest-message-util "30.2.0" + jest-mock "30.2.0" + jest-util "30.2.0" fast-json-patch@^3.0.0-1: version "3.1.1" resolved "https://registry.yarnpkg.com/fast-json-patch/-/fast-json-patch-3.1.1.tgz#85064ea1b1ebf97a3f7ad01e23f9337e72c66947" integrity sha512-vf6IHUX2SBcA+5/+4883dsIjpBTqmfBjmYiWK1savxQmFk4JfBMLa7ynTYOs1Rolp/T1betJxHiGD3g1Mn8lUQ== -fast-json-stable-stringify@2.x, fast-json-stable-stringify@^2.0.0, fast-json-stable-stringify@^2.1.0: +fast-json-stable-stringify@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== -fast-levenshtein@^2.0.6, fast-levenshtein@~2.0.6: - version "2.0.6" - resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" - integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw== - -fastq@^1.6.0: - version "1.14.0" - resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.14.0.tgz#107f69d7295b11e0fccc264e1fc6389f623731ce" - integrity sha512-eR2D+V9/ExcbF9ls441yIuN6TI2ED1Y2ZcA5BmMtJsOkWOFRJQ0Jt0g1UwqXJJVAb+V+umH5Dfr8oh4EVP7VVg== - dependencies: - reusify "^1.0.4" - fault@^1.0.0: version "1.0.4" resolved "https://registry.yarnpkg.com/fault/-/fault-1.0.4.tgz#eafcfc0a6d214fc94601e170df29954a4f842f13" @@ -6809,60 +4898,17 @@ fault@^1.0.0: dependencies: format "^0.2.0" -faye-websocket@^0.11.3: - version "0.11.4" - resolved "https://registry.yarnpkg.com/faye-websocket/-/faye-websocket-0.11.4.tgz#7f0d9275cfdd86a1c963dc8b65fcc451edcbb1da" - integrity sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g== - dependencies: - websocket-driver ">=0.5.1" - -fb-watchman@^2.0.0: +fb-watchman@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/fb-watchman/-/fb-watchman-2.0.2.tgz#e9524ee6b5c77e9e5001af0f85f3adbb8623255c" integrity sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA== dependencies: bser "2.1.1" -fd-slicer@~1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.1.0.tgz#25c7c89cb1f9077f8891bbe61d8f390eae256f1e" - integrity sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g== - dependencies: - pend "~1.2.0" - -figures@^3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/figures/-/figures-3.2.0.tgz#625c18bd293c604dc4a8ddb2febf0c88341746af" - integrity sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg== - dependencies: - escape-string-regexp "^1.0.5" - -file-entry-cache@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027" - integrity sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg== - dependencies: - flat-cache "^3.0.4" - -file-loader@^6.2.0: - version "6.2.0" - resolved "https://registry.yarnpkg.com/file-loader/-/file-loader-6.2.0.tgz#baef7cf8e1840df325e4390b4484879480eebe4d" - integrity sha512-qo3glqyTa61Ytg4u73GultjHGjdRyig3tG6lPtyX/jOEJvHif9uB0/OCI2Kif6ctF3caQTW2G5gym21oAsI4pw== - dependencies: - loader-utils "^2.0.0" - schema-utils "^3.0.0" - -filelist@^1.0.1: - version "1.0.4" - resolved "https://registry.yarnpkg.com/filelist/-/filelist-1.0.4.tgz#f78978a1e944775ff9e62e744424f215e58352b5" - integrity sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q== - dependencies: - minimatch "^5.0.1" - -filesize@^8.0.6: - version "8.0.7" - resolved "https://registry.yarnpkg.com/filesize/-/filesize-8.0.7.tgz#695e70d80f4e47012c132d57a059e80c6b580bd8" - integrity sha512-pjmC+bkIF8XI7fWaH8KxHcZL3DPybs1roSKP4rKDvy20tAWwIObE4+JIseG2byfGKhud5ZnM4YSGKBz7Sh0ndQ== +fdir@^6.5.0: + version "6.5.0" + resolved "https://registry.yarnpkg.com/fdir/-/fdir-6.5.0.tgz#ed2ab967a331ade62f18d077dae192684d50d350" + integrity sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg== fill-range@^7.1.1: version "7.1.1" @@ -6871,40 +4917,11 @@ fill-range@^7.1.1: dependencies: to-regex-range "^5.0.1" -finalhandler@1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.2.0.tgz#7d23fe5731b207b4640e4fcd00aec1f9207a7b32" - integrity sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg== - dependencies: - debug "2.6.9" - encodeurl "~1.0.2" - escape-html "~1.0.3" - on-finished "2.4.1" - parseurl "~1.3.3" - statuses "2.0.1" - unpipe "~1.0.0" - -find-cache-dir@^3.3.1: - version "3.3.2" - resolved "https://registry.yarnpkg.com/find-cache-dir/-/find-cache-dir-3.3.2.tgz#b30c5b6eff0730731aea9bbd9dbecbd80256d64b" - integrity sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig== - dependencies: - commondir "^1.0.1" - make-dir "^3.0.2" - pkg-dir "^4.1.0" - find-root@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/find-root/-/find-root-1.1.0.tgz#abcfc8ba76f708c42a97b3d685b7e9450bfb9ce4" integrity sha512-NKfW6bec6GfKc0SGx1e07QZY9PE99u0Bft/0rzSD5k3sO/vwkVUpDUKVm5Gpp5Ue3YfShPFTX2070tDs5kB9Ng== -find-up@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/find-up/-/find-up-3.0.0.tgz#49169f1d7993430646da61ecc5ae355c21c97b73" - integrity sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg== - dependencies: - locate-path "^3.0.0" - find-up@^4.0.0, find-up@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/find-up/-/find-up-4.1.0.tgz#97afe7d6cdc0bc5928584b7c8d7b16e8a9aa5d19" @@ -6913,43 +4930,15 @@ find-up@^4.0.0, find-up@^4.1.0: locate-path "^5.0.0" path-exists "^4.0.0" -find-up@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/find-up/-/find-up-5.0.0.tgz#4c92819ecb7083561e4f4a240a86be5198f536fc" - integrity sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng== - dependencies: - locate-path "^6.0.0" - path-exists "^4.0.0" - -find-yarn-workspace-root@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/find-yarn-workspace-root/-/find-yarn-workspace-root-2.0.0.tgz#f47fb8d239c900eb78179aa81b66673eac88f7bd" - integrity sha512-1IMnbjt4KzsQfnhnzNd8wUEgXZ44IzZaZmnLYx7D5FZlaHt2gW20Cri8Q+E/t5tIj4+epTBub+2Zxu/vNILzqQ== - dependencies: - micromatch "^4.0.2" - -flat-cache@^3.0.4: - version "3.0.4" - resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-3.0.4.tgz#61b0338302b2fe9f957dcc32fc2a87f1c3048b11" - integrity sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg== - dependencies: - flatted "^3.1.0" - rimraf "^3.0.2" - -flatted@^3.1.0: - version "3.2.7" - resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.7.tgz#609f39207cb614b89d0765b477cb2d437fbf9787" - integrity sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ== - -follow-redirects@^1.0.0: - version "1.15.6" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" - integrity sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA== +flatqueue@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/flatqueue/-/flatqueue-3.0.0.tgz#39679f344814b34880bd149c7510452d84485e9a" + integrity sha512-y1deYaVt+lIc/d2uIcWDNd0CrdQTO5xoCjeFdhX0kSXvm2Acm0o+3bAOiYklTEoRyzwio3sv3/IiBZdusbAe2Q== -follow-redirects@^1.15.6: - version "1.15.9" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.9.tgz#a604fa10e443bf98ca94228d9eebcc2e8a2c8ee1" - integrity sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ== +follow-redirects@^1.15.11: + version "1.15.11" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.11.tgz#777d73d72a92f8ec4d2e410eb47352a56b8e8340" + integrity sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ== for-each@^0.3.3: version "0.3.3" @@ -6958,55 +4947,30 @@ for-each@^0.3.3: dependencies: is-callable "^1.1.3" -forever-agent@~0.6.1: - version "0.6.1" - resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" - integrity sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw== - -fork-ts-checker-webpack-plugin@^6.5.0: - version "6.5.2" - resolved "https://registry.yarnpkg.com/fork-ts-checker-webpack-plugin/-/fork-ts-checker-webpack-plugin-6.5.2.tgz#4f67183f2f9eb8ba7df7177ce3cf3e75cdafb340" - integrity sha512-m5cUmF30xkZ7h4tWUgTAcEaKmUW7tfyUyTqNNOz7OxWJ0v1VWKTcOvH8FWHUwSjlW/356Ijc9vi3XfcPstpQKA== - dependencies: - "@babel/code-frame" "^7.8.3" - "@types/json-schema" "^7.0.5" - chalk "^4.1.0" - chokidar "^3.4.2" - cosmiconfig "^6.0.0" - deepmerge "^4.2.2" - fs-extra "^9.0.0" - glob "^7.1.6" - memfs "^3.1.2" - minimatch "^3.0.4" - schema-utils "2.7.0" - semver "^7.3.2" - tapable "^1.0.0" - -form-data@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-3.0.1.tgz#ebd53791b78356a99af9a300d4282c4d5eb9755f" - integrity sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg== +for-each@^0.3.5: + version "0.3.5" + resolved "https://registry.yarnpkg.com/for-each/-/for-each-0.3.5.tgz#d650688027826920feeb0af747ee7b9421a41d47" + integrity sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg== dependencies: - asynckit "^0.4.0" - combined-stream "^1.0.8" - mime-types "^2.1.12" + is-callable "^1.2.7" -form-data@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" - integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== +foreground-child@^3.3.1: + version "3.3.1" + resolved "https://registry.yarnpkg.com/foreground-child/-/foreground-child-3.3.1.tgz#32e8e9ed1b68a3497befb9ac2b6adf92a638576f" + integrity sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw== dependencies: - asynckit "^0.4.0" - combined-stream "^1.0.8" - mime-types "^2.1.12" + cross-spawn "^7.0.6" + signal-exit "^4.0.1" -form-data@~2.3.2: - version "2.3.3" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" - integrity sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ== +form-data@^4.0.5: + version "4.0.5" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.5.tgz#b49e48858045ff4cbf6b03e1805cebcad3679053" + integrity sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w== dependencies: asynckit "^0.4.0" - combined-stream "^1.0.6" + combined-stream "^1.0.8" + es-set-tostringtag "^2.1.0" + hasown "^2.0.2" mime-types "^2.1.12" format@^0.2.0: @@ -7014,60 +4978,16 @@ format@^0.2.0: resolved "https://registry.yarnpkg.com/format/-/format-0.2.2.tgz#d6170107e9efdc4ed30c9dc39016df942b5cb58b" integrity sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww== -forwarded@0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811" - integrity sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow== - -fraction.js@^4.2.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/fraction.js/-/fraction.js-4.2.0.tgz#448e5109a313a3527f5a3ab2119ec4cf0e0e2950" - integrity sha512-MhLuK+2gUcnZe8ZHlaaINnQLl0xRIGRfcGk2yl8xoQAfHrSsL3rYu6FCmBdkdbhc9EPlwyGHewaRsvwRMJtAlA== - -fresh@0.5.2: - version "0.5.2" - resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" - integrity sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q== - -fs-constants@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" - integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow== - -fs-extra@^10.0.0: - version "10.1.0" - resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.1.0.tgz#02873cfbc4084dde127eaa5f9905eef2325d1abf" - integrity sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ== - dependencies: - graceful-fs "^4.2.0" - jsonfile "^6.0.1" - universalify "^2.0.0" - -fs-extra@^9.0.0, fs-extra@^9.0.1, fs-extra@^9.1.0: - version "9.1.0" - resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.1.0.tgz#5954460c764a8da2094ba3554bf839e6b9a7c86d" - integrity sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ== - dependencies: - at-least-node "^1.0.0" - graceful-fs "^4.2.0" - jsonfile "^6.0.1" - universalify "^2.0.0" - -fs-monkey@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/fs-monkey/-/fs-monkey-1.0.3.tgz#ae3ac92d53bb328efe0e9a1d9541f6ad8d48e2d3" - integrity sha512-cybjIfiiE+pTWicSCLFHSrXZ6EilF30oh91FDP9S2B051prEa7QWfrVTQm10/dDpswBDXZugPa1Ogu8Yh+HV0Q== - -fs.realpath@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" - integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== - -fsevents@^2.3.2, fsevents@~2.3.2: +fsevents@2.3.2, fsevents@~2.3.2: version "2.3.2" resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== +fsevents@^2.3.3, fsevents@~2.3.3: + version "2.3.3" + resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6" + integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw== + function-bind@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" @@ -7078,16 +4998,6 @@ function-bind@^1.1.2: resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c" integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA== -function.prototype.name@^1.1.5: - version "1.1.5" - resolved "https://registry.yarnpkg.com/function.prototype.name/-/function.prototype.name-1.1.5.tgz#cce0505fe1ffb80503e6f9e46cc64e46a12a9621" - integrity sha512-uN7m/BzVKQnCUF/iW8jYea67v++2u7m5UgENbHRtdDVclOUP+FMPlCNdmk0h/ysGyo2tavMJEDqJAkJdRa1vMA== - dependencies: - call-bind "^1.0.2" - define-properties "^1.1.3" - es-abstract "^1.19.0" - functions-have-names "^1.2.2" - functions-have-names@^1.2.2: version "1.2.3" resolved "https://registry.yarnpkg.com/functions-have-names/-/functions-have-names-1.2.3.tgz#0404fe4ee2ba2f607f0e0ec3c80bae994133b834" @@ -7112,144 +5022,57 @@ get-intrinsic@^1.0.2, get-intrinsic@^1.1.0, get-intrinsic@^1.1.1, get-intrinsic@ has "^1.0.3" has-symbols "^1.0.3" -get-intrinsic@^1.2.4: - version "1.2.4" - resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.2.4.tgz#e385f5a4b5227d449c3eabbad05494ef0abbeadd" - integrity sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ== +get-intrinsic@^1.2.4, get-intrinsic@^1.2.6, get-intrinsic@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.3.0.tgz#743f0e3b6964a93a5491ed1bffaae054d7f98d01" + integrity sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ== dependencies: + call-bind-apply-helpers "^1.0.2" + es-define-property "^1.0.1" es-errors "^1.3.0" + es-object-atoms "^1.1.1" function-bind "^1.1.2" - has-proto "^1.0.1" - has-symbols "^1.0.3" - hasown "^2.0.0" - -get-own-enumerable-property-symbols@^3.0.0: - version "3.0.2" - resolved "https://registry.yarnpkg.com/get-own-enumerable-property-symbols/-/get-own-enumerable-property-symbols-3.0.2.tgz#b5fde77f22cbe35f390b4e089922c50bce6ef664" - integrity sha512-I0UBV/XOz1XkIJHEUDMZAbzCThU/H8DxmSfmdGcKPnVhu2VfFqr34jr9777IyaTYvxjedWhqVIilEDsCdP5G6g== + get-proto "^1.0.1" + gopd "^1.2.0" + has-symbols "^1.1.0" + hasown "^2.0.2" + math-intrinsics "^1.1.0" get-package-type@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/get-package-type/-/get-package-type-0.1.0.tgz#8de2d803cff44df3bc6c456e6668b36c3926e11a" integrity sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q== -get-stream@^5.0.0, get-stream@^5.1.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-5.2.0.tgz#4966a1795ee5ace65e706c4b7beb71257d6e22d3" - integrity sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA== +get-proto@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/get-proto/-/get-proto-1.0.1.tgz#150b3f2743869ef3e851ec0c49d15b1d14d00ee1" + integrity sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g== dependencies: - pump "^3.0.0" + dunder-proto "^1.0.1" + es-object-atoms "^1.0.0" get-stream@^6.0.0: version "6.0.1" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-6.0.1.tgz#a262d8eef67aced57c2852ad6167526a43cbf7b7" integrity sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg== -get-symbol-description@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/get-symbol-description/-/get-symbol-description-1.0.0.tgz#7fdb81c900101fbd564dd5f1a30af5aadc1e58d6" - integrity sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw== - dependencies: - call-bind "^1.0.2" - get-intrinsic "^1.1.1" - -getos@^3.2.1: - version "3.2.1" - resolved "https://registry.yarnpkg.com/getos/-/getos-3.2.1.tgz#0134d1f4e00eb46144c5a9c0ac4dc087cbb27dc5" - integrity sha512-U56CfOK17OKgTVqozZjUKNdkfEv6jk5WISBJ8SHoagjE6L69zOwl3Z+O8myjY9MEW3i2HPWQBt/LTbCgcC973Q== - dependencies: - async "^3.2.0" - -getpass@^0.1.1: - version "0.1.7" - resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" - integrity sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng== - dependencies: - assert-plus "^1.0.0" - -github-from-package@0.0.0: - version "0.0.0" - resolved "https://registry.yarnpkg.com/github-from-package/-/github-from-package-0.0.0.tgz#97fb5d96bfde8973313f20e8288ef9a167fa64ce" - integrity sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw== - -glob-parent@^5.1.2, glob-parent@~5.1.2: - version "5.1.2" - resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" - integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow== - dependencies: - is-glob "^4.0.1" - -glob-parent@^6.0.2: - version "6.0.2" - resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-6.0.2.tgz#6d237d99083950c79290f24c7642a3de9a28f9e3" - integrity sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A== - dependencies: - is-glob "^4.0.3" - -glob-to-regexp@^0.4.1: - version "0.4.1" - resolved "https://registry.yarnpkg.com/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz#c75297087c851b9a578bd217dd59a92f59fe546e" - integrity sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw== - -glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6: - version "7.2.3" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b" - integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q== - dependencies: - fs.realpath "^1.0.0" - inflight "^1.0.4" - inherits "2" - minimatch "^3.1.1" - once "^1.3.0" - path-is-absolute "^1.0.0" - -global-dirs@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/global-dirs/-/global-dirs-3.0.1.tgz#0c488971f066baceda21447aecb1a8b911d22485" - integrity sha512-NBcGGFbBA9s1VzD41QXDG+3++t9Mn5t1FpLdhESY6oKY4gYTFpX4wO3sqGUa0Srjtbfj3szX0RnemmrVRUdULA== - dependencies: - ini "2.0.0" - -global-modules@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-2.0.0.tgz#997605ad2345f27f51539bea26574421215c7780" - integrity sha512-NGbfmJBp9x8IxyJSd1P+otYK8vonoJactOogrVfFRIAEY1ukil8RSKDz2Yo7wh1oihl51l/r6W4epkeKJHqL8A== - dependencies: - global-prefix "^3.0.0" - -global-prefix@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/global-prefix/-/global-prefix-3.0.0.tgz#fc85f73064df69f50421f47f883fe5b913ba9b97" - integrity sha512-awConJSVCHVGND6x3tmMaKcQvwXLhjdkmomy2W+Goaui8YPgYgXJZewhg3fWC+DlfqqQuWg8AwqjGTD2nAPVWg== +glob@11.1.0, glob@^10.3.10, glob@^7.1.4: + version "11.1.0" + resolved "https://registry.yarnpkg.com/glob/-/glob-11.1.0.tgz#4f826576e4eb99c7dad383793d2f9f08f67e50a6" + integrity sha512-vuNwKSaKiqm7g0THUBu2x7ckSs3XJLXE+2ssL7/MfTGPLLcrJQ/4Uq1CjPTtO5cCIiRxqvN6Twy1qOwhL0Xjcw== dependencies: - ini "^1.3.5" - kind-of "^6.0.2" - which "^1.3.1" + foreground-child "^3.3.1" + jackspeak "^4.1.1" + minimatch "^10.1.1" + minipass "^7.1.2" + package-json-from-dist "^1.0.0" + path-scurry "^2.0.0" globals@^11.1.0: version "11.12.0" resolved "https://registry.yarnpkg.com/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e" integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA== -globals@^13.19.0: - version "13.19.0" - resolved "https://registry.yarnpkg.com/globals/-/globals-13.19.0.tgz#7a42de8e6ad4f7242fbcca27ea5b23aca367b5c8" - integrity sha512-dkQ957uSRWHw7CFXLUtUHQI3g3aWApYhfNR2O6jn/907riyTYKVBmxYVROkBcY614FSSeSJh7Xm7SrUWCxvJMQ== - dependencies: - type-fest "^0.20.2" - -globby@^11.0.4, globby@^11.1.0: - version "11.1.0" - resolved "https://registry.yarnpkg.com/globby/-/globby-11.1.0.tgz#bd4be98bb042f83d796f7e3811991fbe82a0d34b" - integrity sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g== - dependencies: - array-union "^2.1.0" - dir-glob "^3.0.1" - fast-glob "^3.2.9" - ignore "^5.2.0" - merge2 "^1.4.1" - slash "^3.0.0" - gopd@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/gopd/-/gopd-1.0.1.tgz#29ff76de69dac7489b7c0918a5788e56477c332c" @@ -7257,44 +5080,17 @@ gopd@^1.0.1: dependencies: get-intrinsic "^1.1.3" -graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.4, graceful-fs@^4.2.6, graceful-fs@^4.2.9: - version "4.2.10" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c" - integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== +gopd@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/gopd/-/gopd-1.2.0.tgz#89f56b8217bdbc8802bd299df6d7f1081d7e51a1" + integrity sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg== -graceful-fs@^4.2.11: +graceful-fs@^4.2.11, graceful-fs@^4.2.4: version "4.2.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== -grapheme-splitter@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz#9cf3a665c6247479896834af35cf1dbb4400767e" - integrity sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ== - -graphemer@^1.4.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/graphemer/-/graphemer-1.4.0.tgz#fb2f1d55e0e3a1849aeffc90c4fa0dd53a0e66c6" - integrity sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag== - -gzip-size@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/gzip-size/-/gzip-size-6.0.0.tgz#065367fd50c239c0671cbcbad5be3e2eeb10e462" - integrity sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q== - dependencies: - duplexer "^0.1.2" - -handle-thing@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/handle-thing/-/handle-thing-2.0.1.tgz#857f79ce359580c340d43081cc648970d0bb234e" - integrity sha512-9Qn4yBxelxoh2Ow62nP+Ka/kMnOXRi8BXnRaUwezLNhqelnN49xKz4F/dPP8OYLxLxq6JDtZb2i9XznUQbNPTg== - -harmony-reflect@^1.4.6: - version "1.6.2" - resolved "https://registry.yarnpkg.com/harmony-reflect/-/harmony-reflect-1.6.2.tgz#31ecbd32e648a34d030d86adb67d4d47547fe710" - integrity sha512-HIp/n38R9kQjDEziXyDTuW3vvoxxyxjxFzXLrBr18uB47GnSt+G9D29fqrpM5ZkspMcPICud3XsBJQ4Y2URg8g== - -has-bigints@^1.0.1, has-bigints@^1.0.2: +has-bigints@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/has-bigints/-/has-bigints-1.0.2.tgz#0871bd3e3d51626f6ca0966668ba35d5602d6eaa" integrity sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ== @@ -7323,16 +5119,16 @@ has-property-descriptors@^1.0.2: dependencies: es-define-property "^1.0.0" -has-proto@^1.0.1: - version "1.0.3" - resolved "https://registry.yarnpkg.com/has-proto/-/has-proto-1.0.3.tgz#b31ddfe9b0e6e9914536a6ab286426d0214f77fd" - integrity sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q== - has-symbols@^1.0.1, has-symbols@^1.0.2, has-symbols@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.3.tgz#bb7b2c4349251dce87b125f7bdf874aa7c8b39f8" integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A== +has-symbols@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.1.0.tgz#fc9c6a783a084951d0b971fe1018de813707a338" + integrity sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ== + has-tostringtag@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/has-tostringtag/-/has-tostringtag-1.0.0.tgz#7e133818a7d394734f941e73c3d3f9291e658b25" @@ -7340,6 +5136,13 @@ has-tostringtag@^1.0.0: dependencies: has-symbols "^1.0.2" +has-tostringtag@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/has-tostringtag/-/has-tostringtag-1.0.2.tgz#2cdc42d40bef2e5b4eeab7c01a73c54ce7ab5abc" + integrity sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw== + dependencies: + has-symbols "^1.0.3" + has@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796" @@ -7347,251 +5150,99 @@ has@^1.0.3: dependencies: function-bind "^1.1.1" -hasown@^2.0.0: +hasown@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003" integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ== dependencies: function-bind "^1.1.2" -hast-util-parse-selector@^2.0.0: - version "2.2.5" - resolved "https://registry.yarnpkg.com/hast-util-parse-selector/-/hast-util-parse-selector-2.2.5.tgz#d57c23f4da16ae3c63b3b6ca4616683313499c3a" - integrity sha512-7j6mrk/qqkSehsM92wQjdIgWM2/BW61u/53G6xmC8i1OmEdKLHbk419QKQUjz6LglWsfqoiHmyMRkP1BGjecNQ== - -hastscript@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/hastscript/-/hastscript-6.0.0.tgz#e8768d7eac56c3fdeac8a92830d58e811e5bf640" - integrity sha512-nDM6bvd7lIqDUiYEiu5Sl/+6ReP0BMk/2f4U/Rooccxkj0P5nm+acM5PrGJ/t5I8qPGiqZSE6hVAwZEdZIvP4w== +hast-util-parse-selector@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/hast-util-parse-selector/-/hast-util-parse-selector-4.0.0.tgz#352879fa86e25616036037dd8931fb5f34cb4a27" + integrity sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A== dependencies: - "@types/hast" "^2.0.0" - comma-separated-tokens "^1.0.0" - hast-util-parse-selector "^2.0.0" - property-information "^5.0.0" - space-separated-tokens "^1.0.0" + "@types/hast" "^3.0.0" -he@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f" - integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw== +hastscript@^9.0.0: + version "9.0.1" + resolved "https://registry.yarnpkg.com/hastscript/-/hastscript-9.0.1.tgz#dbc84bef6051d40084342c229c451cd9dc567dff" + integrity sha512-g7df9rMFX/SPi34tyGCyUBREQoKkapwdY/T04Qn9TDWfHhAYt4/I0gMVirzK5wEzeUqIjEB+LXC/ypb7Aqno5w== + dependencies: + "@types/hast" "^3.0.0" + comma-separated-tokens "^2.0.0" + hast-util-parse-selector "^4.0.0" + property-information "^7.0.0" + space-separated-tokens "^2.0.0" highlight.js@^10.4.1, highlight.js@~10.7.0: version "10.7.3" resolved "https://registry.yarnpkg.com/highlight.js/-/highlight.js-10.7.3.tgz#697272e3991356e40c3cac566a74eef681756531" integrity sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A== -hoist-non-react-statics@^3.0.0, hoist-non-react-statics@^3.3.0, hoist-non-react-statics@^3.3.1, hoist-non-react-statics@^3.3.2: +highlightjs-vue@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/highlightjs-vue/-/highlightjs-vue-1.0.0.tgz#fdfe97fbea6354e70ee44e3a955875e114db086d" + integrity sha512-PDEfEF102G23vHmPhLyPboFCD+BkMGu+GuJe2d9/eH4FsCwvgBpnc9n0pGE+ffKdph38s6foEZiEjdgHdzp+IA== + +hoist-non-react-statics@^3.3.1: version "3.3.2" resolved "https://registry.yarnpkg.com/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz#ece0acaf71d62c2969c2ec59feff42a4b1a85b45" integrity sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw== dependencies: react-is "^16.7.0" -hoopy@^0.1.4: - version "0.1.4" - resolved "https://registry.yarnpkg.com/hoopy/-/hoopy-0.1.4.tgz#609207d661100033a9a9402ad3dea677381c1b1d" - integrity sha512-HRcs+2mr52W0K+x8RzcLzuPPmVIKMSv97RGHy0Ea9y/mpcaK+xTrjICA04KAHi4GRzxliNqNJEFYWHghy3rSfQ== - -hpack.js@^2.1.6: - version "2.1.6" - resolved "https://registry.yarnpkg.com/hpack.js/-/hpack.js-2.1.6.tgz#87774c0949e513f42e84575b3c45681fade2a0b2" - integrity sha512-zJxVehUdMGIKsRaNt7apO2Gqp0BdqW5yaiGHXXmbpvxgBYVZnAql+BJb4RO5ad2MgpbZKn5G6nMnegrH1FcNYQ== - dependencies: - inherits "^2.0.1" - obuf "^1.0.0" - readable-stream "^2.0.1" - wbuf "^1.1.0" - -html-encoding-sniffer@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz#42a6dc4fd33f00281176e8b23759ca4e4fa185f3" - integrity sha512-D5JbOMBIR/TVZkubHT+OyT2705QvogUW4IBn6nHd756OwieSF9aDYFj4dv6HHEVGYbHaLETa3WggZYWWMyy3ZQ== +html-encoding-sniffer@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/html-encoding-sniffer/-/html-encoding-sniffer-4.0.0.tgz#696df529a7cfd82446369dc5193e590a3735b448" + integrity sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ== dependencies: - whatwg-encoding "^1.0.5" - -html-entities@^2.1.0, html-entities@^2.3.2: - version "2.3.3" - resolved "https://registry.yarnpkg.com/html-entities/-/html-entities-2.3.3.tgz#117d7626bece327fc8baace8868fa6f5ef856e46" - integrity sha512-DV5Ln36z34NNTDgnz0EWGBLZENelNAtkiFA4kyNOG2tDI6Mz1uSWiq1wAKdyjnJwyDiDO7Fa2SO1CTxPXL8VxA== + whatwg-encoding "^3.1.1" html-escaper@^2.0.0: version "2.0.2" resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-2.0.2.tgz#dfd60027da36a36dfcbe236262c00a5822681453" integrity sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg== -html-minifier-terser@^6.0.2: - version "6.1.0" - resolved "https://registry.yarnpkg.com/html-minifier-terser/-/html-minifier-terser-6.1.0.tgz#bfc818934cc07918f6b3669f5774ecdfd48f32ab" - integrity sha512-YXxSlJBZTP7RS3tWnQw74ooKa6L9b9i9QYXY21eUEvhZ3u9XLfv6OnFsQq6RxkhHygsaUMvYsZRV5rU/OVNZxw== - dependencies: - camel-case "^4.1.2" - clean-css "^5.2.2" - commander "^8.3.0" - he "^1.2.0" - param-case "^3.0.4" - relateurl "^0.2.7" - terser "^5.10.0" - -html-webpack-plugin@^5.5.0: - version "5.5.0" - resolved "https://registry.yarnpkg.com/html-webpack-plugin/-/html-webpack-plugin-5.5.0.tgz#c3911936f57681c1f9f4d8b68c158cd9dfe52f50" - integrity sha512-sy88PC2cRTVxvETRgUHFrL4No3UxvcH8G1NepGhqaTT+GXN2kTamqasot0inS5hXeg1cMbFDt27zzo9p35lZVw== - dependencies: - "@types/html-minifier-terser" "^6.0.0" - html-minifier-terser "^6.0.2" - lodash "^4.17.21" - pretty-error "^4.0.0" - tapable "^2.0.0" - -htmlparser2@^6.1.0: - version "6.1.0" - resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-6.1.0.tgz#c4d762b6c3371a05dbe65e94ae43a9f845fb8fb7" - integrity sha512-gyyPk6rgonLFEDGoeRgQNaEUvdJ4ktTmmUh/h2t7s+M8oPpIPxgNACWa+6ESR57kXstwqPiCut0V8NRpcwgU7A== - dependencies: - domelementtype "^2.0.1" - domhandler "^4.0.0" - domutils "^2.5.2" - entities "^2.0.0" - -http-deceiver@^1.2.7: - version "1.2.7" - resolved "https://registry.yarnpkg.com/http-deceiver/-/http-deceiver-1.2.7.tgz#fa7168944ab9a519d337cb0bec7284dc3e723d87" - integrity sha512-LmpOGxTfbpgtGVxJrj5k7asXHCgNZp5nLfp+hWc8QQRqtb7fUy6kRY3BO1h9ddF6yIPYUARgxGOwB42DnxIaNw== - -http-errors@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-2.0.0.tgz#b7774a1486ef73cf7667ac9ae0858c012c57b9d3" - integrity sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ== - dependencies: - depd "2.0.0" - inherits "2.0.4" - setprototypeof "1.2.0" - statuses "2.0.1" - toidentifier "1.0.1" - -http-errors@~1.6.2: - version "1.6.3" - resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.6.3.tgz#8b55680bb4be283a0b5bf4ea2e38580be1d9320d" - integrity sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A== - dependencies: - depd "~1.1.2" - inherits "2.0.3" - setprototypeof "1.1.0" - statuses ">= 1.4.0 < 2" - -http-parser-js@>=0.5.1: - version "0.5.8" - resolved "https://registry.yarnpkg.com/http-parser-js/-/http-parser-js-0.5.8.tgz#af23090d9ac4e24573de6f6aecc9d84a48bf20e3" - integrity sha512-SGeBX54F94Wgu5RH3X5jsDtf4eHyRogWX1XGT3b4HuW3tQPM4AaBzoUji/4AAJNXCEOWZ5O0DgZmJw1947gD5Q== - -http-proxy-agent@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz#8a8c8ef7f5932ccf953c296ca8291b95aa74aa3a" - integrity sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg== - dependencies: - "@tootallnate/once" "1" - agent-base "6" - debug "4" - -http-proxy-middleware@^2.0.3: - version "2.0.7" - resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz#915f236d92ae98ef48278a95dedf17e991936ec6" - integrity sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA== - dependencies: - "@types/http-proxy" "^1.17.8" - http-proxy "^1.18.1" - is-glob "^4.0.1" - is-plain-obj "^3.0.0" - micromatch "^4.0.2" - -http-proxy@^1.18.1: - version "1.18.1" - resolved "https://registry.yarnpkg.com/http-proxy/-/http-proxy-1.18.1.tgz#401541f0534884bbf95260334e72f88ee3976549" - integrity sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ== - dependencies: - eventemitter3 "^4.0.0" - follow-redirects "^1.0.0" - requires-port "^1.0.0" - -http-signature@~1.3.6: - version "1.3.6" - resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.3.6.tgz#cb6fbfdf86d1c974f343be94e87f7fc128662cf9" - integrity sha512-3adrsD6zqo4GsTqtO7FyrejHNv+NgiIfAfv68+jVlFmSr9OGy7zrxONceFRLKvnnZA5jbxQBX1u9PpB6Wi32Gw== +http-proxy-agent@^7.0.2: + version "7.0.2" + resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz#9a8b1f246866c028509486585f62b8f2c18c270e" + integrity sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig== dependencies: - assert-plus "^1.0.0" - jsprim "^2.0.2" - sshpk "^1.14.1" + agent-base "^7.1.0" + debug "^4.3.4" -https-proxy-agent@^5.0.0: - version "5.0.1" - resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz#c59ef224a04fe8b754f3db0063a25ea30d0005d6" - integrity sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA== +https-proxy-agent@^7.0.6: + version "7.0.6" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz#da8dfeac7da130b05c2ba4b59c9b6cd66611a6b9" + integrity sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw== dependencies: - agent-base "6" + agent-base "^7.1.2" debug "4" -human-signals@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-1.1.1.tgz#c5b1cd14f50aeae09ab6c59fe63ba3395fe4dfa3" - integrity sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw== - human-signals@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0" integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw== -iconv-lite@0.4.24: - version "0.4.24" - resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" - integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA== - dependencies: - safer-buffer ">= 2.1.2 < 3" - -iconv-lite@^0.6.3: +iconv-lite@0.6.3: version "0.6.3" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.3.tgz#a52f80bf38da1952eb5c681790719871a1a72501" integrity sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw== dependencies: safer-buffer ">= 2.1.2 < 3.0.0" -icss-utils@^5.0.0, icss-utils@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/icss-utils/-/icss-utils-5.1.0.tgz#c6be6858abd013d768e98366ae47e25d5887b1ae" - integrity sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA== - -idb@^7.0.1: - version "7.1.1" - resolved "https://registry.yarnpkg.com/idb/-/idb-7.1.1.tgz#d910ded866d32c7ced9befc5bfdf36f572ced72b" - integrity sha512-gchesWBzyvGHRO9W8tzUWFDycow5gwjvFKfyV9FF32Y7F50yZMp7mP+T2mJIWFx49zicqyC4uefHM17o6xKIVQ== - -identity-obj-proxy@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/identity-obj-proxy/-/identity-obj-proxy-3.0.0.tgz#94d2bda96084453ef36fbc5aaec37e0f79f1fc14" - integrity sha512-00n6YnVHKrinT9t0d9+5yZC6UBNJANpYEQvL2LlX6Ab9lnmxzIRcEmTPuyGScvl1+jKuCICX1Z0Ab1pPKKdikA== - dependencies: - harmony-reflect "^1.4.6" - -ieee754@^1.1.13, ieee754@^1.2.1: +ieee754@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== -ignore@^5.2.0, ignore@^5.2.4: - version "5.2.4" - resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.4.tgz#a291c0c6178ff1b960befe47fcdec301674a6324" - integrity sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ== - -immer@^9.0.7: - version "9.0.16" - resolved "https://registry.yarnpkg.com/immer/-/immer-9.0.16.tgz#8e7caab80118c2b54b37ad43e05758cdefad0198" - integrity sha512-qenGE7CstVm1NrHQbMh8YaSzTZTFNP3zPqr3YU0S0UY441j4bJTg4A2Hh5KAhwgaiU6ZZ1Ar6y/2f4TblnMReQ== - immutable@^3.x.x: - version "3.8.2" - resolved "https://registry.yarnpkg.com/immutable/-/immutable-3.8.2.tgz#c2439951455bb39913daf281376f1530e104adf3" - integrity sha512-15gZoQ38eYjEjxkorfbcgBKBL6R7T459OuK+CpcWt7O3KF4uPCx2tD0uFETlUDIyo+1789crbMhTvQBSR5yBMg== + version "3.8.3" + resolved "https://registry.yarnpkg.com/immutable/-/immutable-3.8.3.tgz#0a8d2494a94d4b2d4f0e99986e74dd25d1e9a859" + integrity sha512-AUY/VyX0E5XlibOmWt10uabJzam1zlYjwiEgQSDc5+UIkFNaF9WM0JxXKaNMGf+F/ffUF+7kRKXM9A7C0xXqMg== -import-fresh@^3.0.0, import-fresh@^3.1.0, import-fresh@^3.2.1: +import-fresh@^3.2.1: version "3.3.0" resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.0.tgz#37162c25fcb9ebaa2e6e53d5b4d88ce17d9e0c2b" integrity sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw== @@ -7599,10 +5250,10 @@ import-fresh@^3.0.0, import-fresh@^3.1.0, import-fresh@^3.2.1: parent-module "^1.0.0" resolve-from "^4.0.0" -import-local@^3.0.2: - version "3.1.0" - resolved "https://registry.yarnpkg.com/import-local/-/import-local-3.1.0.tgz#b4479df8a5fd44f6cdce24070675676063c95cb4" - integrity sha512-ASB07uLtnDs1o6EHjKpX34BKYDSqnFerfTOJL2HvMqF70LnxpjkzDB8J44oT9pu4AMPkQwf8jl6szgvNd2tRIg== +import-local@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/import-local/-/import-local-3.2.0.tgz#c3d5c745798c02a6f8b897726aba5100186ee260" + integrity sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA== dependencies: pkg-dir "^4.2.0" resolve-cwd "^3.0.0" @@ -7617,44 +5268,12 @@ indent-string@^4.0.0: resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-4.0.0.tgz#624f8f4497d619b2d9768531d58f4122854d7251" integrity sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg== -inflight@^1.0.4: - version "1.0.6" - resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" - integrity sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA== - dependencies: - once "^1.3.0" - wrappy "1" - -inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.3: +inherits@^2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== -inherits@2.0.3: - version "2.0.3" - resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" - integrity sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw== - -ini@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/ini/-/ini-2.0.0.tgz#e5fd556ecdd5726be978fa1001862eacb0a94bc5" - integrity sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA== - -ini@^1.3.5, ini@~1.3.0: - version "1.3.8" - resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c" - integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew== - -internal-slot@^1.0.3: - version "1.0.4" - resolved "https://registry.yarnpkg.com/internal-slot/-/internal-slot-1.0.4.tgz#8551e7baf74a7a6ba5f749cfb16aa60722f0d6f3" - integrity sha512-tA8URYccNzMo94s5MQZgH8NB/XTa6HsOo0MLfXTKKEnHVVdegzaQoFZ7Jp44bdvLvY2waT5dc+j5ICEswhi7UQ== - dependencies: - get-intrinsic "^1.1.3" - has "^1.0.3" - side-channel "^1.0.4" - -"internmap@1 - 2": +"internmap@1 - 2", internmap@^2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/internmap/-/internmap-2.0.3.tgz#6685f23755e43c524e251d29cbc97248e3061009" integrity sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg== @@ -7666,28 +5285,18 @@ invariant@^2.2.2: dependencies: loose-envify "^1.0.0" -ipaddr.js@1.9.1: - version "1.9.1" - resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz#bff38543eeb8984825079ff3a2a8e6cbd46781b3" - integrity sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g== - -ipaddr.js@^2.0.1: +is-alphabetical@^2.0.0: version "2.0.1" - resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-2.0.1.tgz#eca256a7a877e917aeb368b0a7497ddf42ef81c0" - integrity sha512-1qTgH9NG+IIJ4yfKs2e6Pp1bZg8wbDbKHT21HrLIeYBTRLgMYKnMTPAuI3Lcs61nfx5h1xlXnbJtH1kX5/d/ng== - -is-alphabetical@^1.0.0: - version "1.0.4" - resolved "https://registry.yarnpkg.com/is-alphabetical/-/is-alphabetical-1.0.4.tgz#9e7d6b94916be22153745d184c298cbf986a686d" - integrity sha512-DwzsA04LQ10FHTZuL0/grVDk4rFoVH1pjAToYwBrHSxcrBIGQuXrQMtD5U1b0U2XVgKZCTLLP8u2Qxqhy3l2Vg== + resolved "https://registry.yarnpkg.com/is-alphabetical/-/is-alphabetical-2.0.1.tgz#01072053ea7c1036df3c7d19a6daaec7f19e789b" + integrity sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ== -is-alphanumerical@^1.0.0: - version "1.0.4" - resolved "https://registry.yarnpkg.com/is-alphanumerical/-/is-alphanumerical-1.0.4.tgz#7eb9a2431f855f6b1ef1a78e326df515696c4dbf" - integrity sha512-UzoZUr+XfVz3t3v4KyGEniVL9BDRoQtY7tOyrRybkVNjDFWyo1yhXNGrrBTQxp3ib9BLAWs7k2YKBQsFRkZG9A== +is-alphanumerical@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz#7c03fbe96e3e931113e57f964b0a368cc2dfd875" + integrity sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw== dependencies: - is-alphabetical "^1.0.0" - is-decimal "^1.0.0" + is-alphabetical "^2.0.0" + is-decimal "^2.0.0" is-arguments@^1.1.0, is-arguments@^1.1.1: version "1.1.1" @@ -7709,13 +5318,6 @@ is-bigint@^1.0.1: dependencies: has-bigints "^1.0.1" -is-binary-path@~2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/is-binary-path/-/is-binary-path-2.1.0.tgz#ea1f7f3b80f064236e83470f86c09c254fb45b09" - integrity sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw== - dependencies: - binary-extensions "^2.0.0" - is-boolean-object@^1.1.0: version "1.1.2" resolved "https://registry.yarnpkg.com/is-boolean-object/-/is-boolean-object-1.1.2.tgz#5c6dc200246dd9321ae4b885a114bb1f75f63719" @@ -7724,92 +5326,50 @@ is-boolean-object@^1.1.0: call-bind "^1.0.2" has-tostringtag "^1.0.0" -is-callable@^1.1.3, is-callable@^1.1.4, is-callable@^1.2.7: +is-callable@^1.1.3, is-callable@^1.2.7: version "1.2.7" resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.7.tgz#3bc2a85ea742d9e36205dcacdd72ca1fdc51b055" integrity sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA== -is-ci@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/is-ci/-/is-ci-3.0.1.tgz#db6ecbed1bd659c43dac0f45661e7674103d1867" - integrity sha512-ZYvCgrefwqoQ6yTyYUbQu64HsITZ3NfKX1lzaEYdkTDcfKzzCI/wthRRYKkdjHKFVgNiXKAKm65Zo1pk2as/QQ== - dependencies: - ci-info "^3.2.0" - -is-core-module@^2.8.1, is-core-module@^2.9.0: +is-core-module@^2.9.0: version "2.11.0" resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.11.0.tgz#ad4cb3e3863e814523c96f3f58d26cc570ff0144" integrity sha512-RRjxlvLDkD1YJwDbroBHMb+cukurkDWNyHx7D3oNB5x9rb5ogcksMC5wHCadcXoo67gVr/+3GFySh3134zi6rw== dependencies: has "^1.0.3" -is-date-object@^1.0.1, is-date-object@^1.0.5: +is-date-object@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.5.tgz#0841d5536e724c25597bf6ea62e1bd38298df31f" integrity sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ== dependencies: has-tostringtag "^1.0.0" -is-decimal@^1.0.0: - version "1.0.4" - resolved "https://registry.yarnpkg.com/is-decimal/-/is-decimal-1.0.4.tgz#65a3a5958a1c5b63a706e1b333d7cd9f630d3fa5" - integrity sha512-RGdriMmQQvZ2aqaQq3awNA6dCGtKpiDFcOzrTWrDAT2MiWrKQVPmxLGHl7Y2nNu6led0kEyoX0enY0qXYsv9zw== - -is-docker@^2.0.0, is-docker@^2.1.1: - version "2.2.1" - resolved "https://registry.yarnpkg.com/is-docker/-/is-docker-2.2.1.tgz#33eeabe23cfe86f14bde4408a02c0cfb853acdaa" - integrity sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ== - -is-extglob@^2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2" - integrity sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ== +is-decimal@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-decimal/-/is-decimal-2.0.1.tgz#9469d2dc190d0214fd87d78b78caecc0cc14eef7" + integrity sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A== is-fullwidth-code-point@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d" integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg== -is-generator-fn@^2.0.0: +is-generator-fn@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/is-generator-fn/-/is-generator-fn-2.1.0.tgz#7d140adc389aaf3011a8f2a2a4cfa6faadffb118" integrity sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ== -is-glob@^4.0.0, is-glob@^4.0.1, is-glob@^4.0.3, is-glob@~4.0.1: - version "4.0.3" - resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.3.tgz#64f61e42cbbb2eec2071a9dac0b28ba1e65d5084" - integrity sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg== - dependencies: - is-extglob "^2.1.1" - -is-hexadecimal@^1.0.0: - version "1.0.4" - resolved "https://registry.yarnpkg.com/is-hexadecimal/-/is-hexadecimal-1.0.4.tgz#cc35c97588da4bd49a8eedd6bc4082d44dcb23a7" - integrity sha512-gyPJuv83bHMpocVYoqof5VDiZveEoGoFL8m3BXNb2VW8Xs+rz9kqO8LOQ5DH6EsuvilT1ApazU0pyl+ytbPtlw== - -is-installed-globally@~0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/is-installed-globally/-/is-installed-globally-0.4.0.tgz#9a0fd407949c30f86eb6959ef1b7994ed0b7b520" - integrity sha512-iwGqO3J21aaSkC7jWnHP/difazwS7SFeIqxv6wEtLU8Y5KlzFTjyqcSIT0d8s4+dDhKytsk9PJZ2BkS5eZwQRQ== - dependencies: - global-dirs "^3.0.0" - is-path-inside "^3.0.2" +is-hexadecimal@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-hexadecimal/-/is-hexadecimal-2.0.1.tgz#86b5bf668fca307498d319dfc03289d781a90027" + integrity sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg== is-map@^2.0.1, is-map@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/is-map/-/is-map-2.0.2.tgz#00922db8c9bf73e81b7a335827bc2a43f2b91127" integrity sha512-cOZFQQozTha1f4MxLFzlgKYPTyj26picdZTx82hbc/Xf4K/tZOOXSCkMvU4pKioRXGDLJRn0GM7Upe7kR721yg== -is-module@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-module/-/is-module-1.0.0.tgz#3258fb69f78c14d5b815d664336b4cffb6441591" - integrity sha512-51ypPSPCoTEIN9dy5Oy+h4pShgJmPCygKfyRCISBI+JoWT/2oJvK8QPxmwv7b/p239jXrm9M1mlQbyKJ5A152g== - -is-negative-zero@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.2.tgz#7bf6f03a28003b8b3965de3ac26f664d765f3150" - integrity sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA== - is-number-object@^1.0.4: version "1.0.7" resolved "https://registry.yarnpkg.com/is-number-object/-/is-number-object-1.0.7.tgz#59d50ada4c45251784e9904f5246c742f07a42fc" @@ -7822,26 +5382,6 @@ is-number@^7.0.0: resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b" integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng== -is-obj@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/is-obj/-/is-obj-1.0.1.tgz#3e4729ac1f5fde025cd7d83a896dab9f4f67db0f" - integrity sha512-l4RyHgRqGN4Y3+9JHVrNqO+tN0rV5My76uW5/nuO4K1b6vw5G8d/cmFjP9tRfEsdhZNt0IFdZuK/c2Vr4Nb+Qg== - -is-path-inside@^3.0.2, is-path-inside@^3.0.3: - version "3.0.3" - resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.3.tgz#d231362e53a07ff2b0e0ea7fed049161ffd16283" - integrity sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ== - -is-plain-obj@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-3.0.0.tgz#af6f2ea14ac5a646183a5bbdb5baabbc156ad9d7" - integrity sha512-gwsOE28k+23GP1B6vFl1oVh/WOzmawBrKwo5Ev6wMKzPkaXaCDIQKzLnvsA42DRlbVTWorkgTKIviAKCWkfUwA== - -is-plain-object@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-5.0.0.tgz#4427f50ab3429e9025ea7d52e9043a9ef4159344" - integrity sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q== - is-potential-custom-element-name@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz#171ed6f19e3ac554394edf78caa05784a45bebb5" @@ -7855,41 +5395,24 @@ is-regex@^1.1.4: call-bind "^1.0.2" has-tostringtag "^1.0.0" -is-regexp@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-regexp/-/is-regexp-1.0.0.tgz#fd2d883545c46bac5a633e7b9a09e87fa2cb5069" - integrity sha512-7zjFAPO4/gwyQAAgRRmqeEeyIICSdmCqa3tsVHMdBzaXXRiqopZL4Cyghg/XulGWrtABTpbnYYzzIRffLkP4oA== - -is-root@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/is-root/-/is-root-2.1.0.tgz#809e18129cf1129644302a4f8544035d51984a9c" - integrity sha512-AGOriNp96vNBd3HtU+RzFEc75FfR5ymiYv8E553I71SCeXBiMsVDUtdio1OEFvrPyLIQ9tVR5RxXIFe5PUFjMg== - is-set@^2.0.1, is-set@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/is-set/-/is-set-2.0.2.tgz#90755fa4c2562dc1c5d4024760d6119b94ca18ec" integrity sha512-+2cnTEZeY5z/iXGbLhPrOAaK/Mau5k5eXq9j14CpRTftq0pAJu2MwVRSZhyZWBzx3o6X795Lz6Bpb6R0GKf37g== -is-shared-array-buffer@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz#8f259c573b60b6a32d4058a1a07430c0a7344c79" - integrity sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA== - dependencies: - call-bind "^1.0.2" - is-stream@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.1.tgz#fac1e3d53b97ad5a9d0ae9cef2389f5810a5c077" integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg== -is-string@^1.0.5, is-string@^1.0.7: +is-string@^1.0.5: version "1.0.7" resolved "https://registry.yarnpkg.com/is-string/-/is-string-1.0.7.tgz#0dd12bf2006f255bb58f695110eff7491eebc0fd" integrity sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg== dependencies: has-tostringtag "^1.0.0" -is-symbol@^1.0.2, is-symbol@^1.0.3: +is-symbol@^1.0.3: version "1.0.4" resolved "https://registry.yarnpkg.com/is-symbol/-/is-symbol-1.0.4.tgz#a6dac93b635b063ca6872236de88910a57af139c" integrity sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg== @@ -7907,28 +5430,18 @@ is-typed-array@^1.1.10: gopd "^1.0.1" has-tostringtag "^1.0.0" -is-typedarray@^1.0.0, is-typedarray@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" - integrity sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA== - -is-unicode-supported@^0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz#3f26c76a809593b52bfa2ecb5710ed2779b522a7" - integrity sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw== +is-typed-array@^1.1.14: + version "1.1.15" + resolved "https://registry.yarnpkg.com/is-typed-array/-/is-typed-array-1.1.15.tgz#4bfb4a45b61cee83a5a46fba778e4e8d59c0ce0b" + integrity sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ== + dependencies: + which-typed-array "^1.1.16" is-weakmap@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/is-weakmap/-/is-weakmap-2.0.1.tgz#5008b59bdc43b698201d18f62b37b2ca243e8cf2" integrity sha512-NSBR4kH5oVj1Uwvv970ruUkCV7O1mzgVFO4/rev2cLRda9Tm9HrL70ZPut4rOHgY0FNrUu9BCbXA2sdQ+x0chA== -is-weakref@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/is-weakref/-/is-weakref-1.0.2.tgz#9529f383a9338205e89765e0392efc2f100f06f2" - integrity sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ== - dependencies: - call-bind "^1.0.2" - is-weakset@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/is-weakset/-/is-weakset-2.0.2.tgz#4569d67a747a1ce5a994dfd4ef6dcea76e7c0a1d" @@ -7937,49 +5450,21 @@ is-weakset@^2.0.1: call-bind "^1.0.2" get-intrinsic "^1.1.1" -is-wsl@^2.1.1, is-wsl@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/is-wsl/-/is-wsl-2.2.0.tgz#74a4c76e77ca9fd3f932f290c17ea326cd157271" - integrity sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww== - dependencies: - is-docker "^2.0.0" - isarray@^2.0.5: version "2.0.5" resolved "https://registry.yarnpkg.com/isarray/-/isarray-2.0.5.tgz#8af1e4c1221244cc62459faf38940d4e644a5723" integrity sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw== -isarray@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" - integrity sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ== - isexe@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" integrity sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw== -isstream@~0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" - integrity sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g== - istanbul-lib-coverage@^3.0.0, istanbul-lib-coverage@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.0.tgz#189e7909d0a39fa5a3dfad5b03f71947770191d3" integrity sha512-eOeJ5BHCmHYvQK7xt9GkdHuzuCGS1Y6g9Gvnx3Ym33fz/HpLRYxiS0wHNr+m/MBC8B647Xt608vCDEvhl9c6Mw== -istanbul-lib-instrument@^5.0.4, istanbul-lib-instrument@^5.1.0: - version "5.2.1" - resolved "https://registry.yarnpkg.com/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz#d10c8885c2125574e1c231cacadf955675e1ce3d" - integrity sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg== - dependencies: - "@babel/core" "^7.12.3" - "@babel/parser" "^7.14.7" - "@istanbuljs/schema" "^0.1.2" - istanbul-lib-coverage "^3.2.0" - semver "^6.3.0" - istanbul-lib-instrument@^6.0.0: version "6.0.1" resolved "https://registry.yarnpkg.com/istanbul-lib-instrument/-/istanbul-lib-instrument-6.0.1.tgz#71e87707e8041428732518c6fb5211761753fbdf" @@ -7991,6 +5476,17 @@ istanbul-lib-instrument@^6.0.0: istanbul-lib-coverage "^3.2.0" semver "^7.5.4" +istanbul-lib-instrument@^6.0.2: + version "6.0.3" + resolved "https://registry.yarnpkg.com/istanbul-lib-instrument/-/istanbul-lib-instrument-6.0.3.tgz#fa15401df6c15874bcb2105f773325d78c666765" + integrity sha512-Vtgk7L/R2JHyyGW07spoFlB8/lpjiOLTjMdms6AFMraYt3BaJauod/NGrfnVG/y4Ix1JEuMRPDPEj2ua+zz1/Q== + dependencies: + "@babel/core" "^7.23.9" + "@babel/parser" "^7.23.9" + "@istanbuljs/schema" "^0.1.3" + istanbul-lib-coverage "^3.2.0" + semver "^7.5.4" + istanbul-lib-report@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/istanbul-lib-report/-/istanbul-lib-report-3.0.0.tgz#7518fe52ea44de372f460a76b5ecda9ffb73d8a6" @@ -8000,14 +5496,14 @@ istanbul-lib-report@^3.0.0: make-dir "^3.0.0" supports-color "^7.1.0" -istanbul-lib-source-maps@^4.0.0: - version "4.0.1" - resolved "https://registry.yarnpkg.com/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz#895f3a709fcfba34c6de5a42939022f3e4358551" - integrity sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw== +istanbul-lib-source-maps@^5.0.0: + version "5.0.6" + resolved "https://registry.yarnpkg.com/istanbul-lib-source-maps/-/istanbul-lib-source-maps-5.0.6.tgz#acaef948df7747c8eb5fbf1265cb980f6353a441" + integrity sha512-yg2d+Em4KizZC5niWhQaIomgf5WlL4vOOjZ5xGCmF8SnPE/mDWWXgvRExdcpCgh9lLRRa1/fSYp2ymmbJ1pI+A== dependencies: + "@jridgewell/trace-mapping" "^0.3.23" debug "^4.1.1" istanbul-lib-coverage "^3.0.0" - source-map "^0.6.1" istanbul-reports@^3.1.3: version "3.1.5" @@ -8017,870 +5513,394 @@ istanbul-reports@^3.1.3: html-escaper "^2.0.0" istanbul-lib-report "^3.0.0" -jake@^10.8.5: - version "10.8.5" - resolved "https://registry.yarnpkg.com/jake/-/jake-10.8.5.tgz#f2183d2c59382cb274226034543b9c03b8164c46" - integrity sha512-sVpxYeuAhWt0OTWITwT98oyV0GsXyMlXCF+3L1SuafBVUIr/uILGRB+NqwkzhgXKvoJpDIpQvqkUALgdmQsQxw== - dependencies: - async "^3.2.3" - chalk "^4.0.2" - filelist "^1.0.1" - minimatch "^3.0.4" - -jest-changed-files@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-changed-files/-/jest-changed-files-27.5.1.tgz#a348aed00ec9bf671cc58a66fcbe7c3dfd6a68f5" - integrity sha512-buBLMiByfWGCoMsLLzGUUSpAmIAGnbR2KJoMN10ziLhOLvP4e0SlypHnAel8iqQXTrcbmfEY9sSqae5sgUsTvw== +jackspeak@^4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/jackspeak/-/jackspeak-4.1.1.tgz#96876030f450502047fc7e8c7fcf8ce8124e43ae" + integrity sha512-zptv57P3GpL+O0I7VdMJNBZCu+BPHVQUk55Ft8/QCJjTVxrnJHuVuX/0Bl2A6/+2oyR/ZMEuFKwmzqqZ/U5nPQ== dependencies: - "@jest/types" "^27.5.1" - execa "^5.0.0" - throat "^6.0.1" + "@isaacs/cliui" "^8.0.2" -jest-changed-files@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/jest-changed-files/-/jest-changed-files-29.7.0.tgz#1c06d07e77c78e1585d020424dedc10d6e17ac3a" - integrity sha512-fEArFiwf1BpQ+4bXSprcDc3/x4HSzL4al2tozwVpDFpsxALjLYdyiIK4e5Vz66GQJIbXJ82+35PtysofptNX2w== +jest-changed-files@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/jest-changed-files/-/jest-changed-files-30.2.0.tgz#602266e478ed554e1e1469944faa7efd37cee61c" + integrity sha512-L8lR1ChrRnSdfeOvTrwZMlnWV8G/LLjQ0nG9MBclwWZidA2N5FviRki0Bvh20WRMOX31/JYvzdqTJrk5oBdydQ== dependencies: - execa "^5.0.0" - jest-util "^29.7.0" + execa "^5.1.1" + jest-util "30.2.0" p-limit "^3.1.0" -jest-circus@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-circus/-/jest-circus-27.5.1.tgz#37a5a4459b7bf4406e53d637b49d22c65d125ecc" - integrity sha512-D95R7x5UtlMA5iBYsOHFFbMD/GVA4R/Kdq15f7xYWUfWHBto9NYRsOvnSauTgdF+ogCpJ4tyKOXhUifxS65gdw== +jest-circus@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/jest-circus/-/jest-circus-30.2.0.tgz#98b8198b958748a2f322354311023d1d02e7603f" + integrity sha512-Fh0096NC3ZkFx05EP2OXCxJAREVxj1BcW/i6EWqqymcgYKWjyyDpral3fMxVcHXg6oZM7iULer9wGRFvfpl+Tg== dependencies: - "@jest/environment" "^27.5.1" - "@jest/test-result" "^27.5.1" - "@jest/types" "^27.5.1" + "@jest/environment" "30.2.0" + "@jest/expect" "30.2.0" + "@jest/test-result" "30.2.0" + "@jest/types" "30.2.0" "@types/node" "*" - chalk "^4.0.0" - co "^4.6.0" - dedent "^0.7.0" - expect "^27.5.1" - is-generator-fn "^2.0.0" - jest-each "^27.5.1" - jest-matcher-utils "^27.5.1" - jest-message-util "^27.5.1" - jest-runtime "^27.5.1" - jest-snapshot "^27.5.1" - jest-util "^27.5.1" - pretty-format "^27.5.1" - slash "^3.0.0" - stack-utils "^2.0.3" - throat "^6.0.1" - -jest-circus@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/jest-circus/-/jest-circus-29.7.0.tgz#b6817a45fcc835d8b16d5962d0c026473ee3668a" - integrity sha512-3E1nCMgipcTkCocFwM90XXQab9bS+GMsjdpmPrlelaxwD93Ad8iVEjX/vvHPdLPnFf+L40u+5+iutRdA1N9myw== - dependencies: - "@jest/environment" "^29.7.0" - "@jest/expect" "^29.7.0" - "@jest/test-result" "^29.7.0" - "@jest/types" "^29.6.3" - "@types/node" "*" - chalk "^4.0.0" + chalk "^4.1.2" co "^4.6.0" - dedent "^1.0.0" - is-generator-fn "^2.0.0" - jest-each "^29.7.0" - jest-matcher-utils "^29.7.0" - jest-message-util "^29.7.0" - jest-runtime "^29.7.0" - jest-snapshot "^29.7.0" - jest-util "^29.7.0" + dedent "^1.6.0" + is-generator-fn "^2.1.0" + jest-each "30.2.0" + jest-matcher-utils "30.2.0" + jest-message-util "30.2.0" + jest-runtime "30.2.0" + jest-snapshot "30.2.0" + jest-util "30.2.0" p-limit "^3.1.0" - pretty-format "^29.7.0" - pure-rand "^6.0.0" + pretty-format "30.2.0" + pure-rand "^7.0.0" slash "^3.0.0" - stack-utils "^2.0.3" + stack-utils "^2.0.6" -jest-cli@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-cli/-/jest-cli-27.5.1.tgz#278794a6e6458ea8029547e6c6cbf673bd30b145" - integrity sha512-Hc6HOOwYq4/74/c62dEE3r5elx8wjYqxY0r0G/nFrLDPMFRu6RA/u8qINOIkvhxG7mMQ5EJsOGfRpI8L6eFUVw== - dependencies: - "@jest/core" "^27.5.1" - "@jest/test-result" "^27.5.1" - "@jest/types" "^27.5.1" - chalk "^4.0.0" - exit "^0.1.2" - graceful-fs "^4.2.9" - import-local "^3.0.2" - jest-config "^27.5.1" - jest-util "^27.5.1" - jest-validate "^27.5.1" - prompts "^2.0.1" - yargs "^16.2.0" - -jest-cli@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/jest-cli/-/jest-cli-29.7.0.tgz#5592c940798e0cae677eec169264f2d839a37995" - integrity sha512-OVVobw2IubN/GSYsxETi+gOe7Ka59EFMR/twOU3Jb2GnKKeMGJB5SGUUrEz3SFVmJASUdZUzy83sLNNQ2gZslg== - dependencies: - "@jest/core" "^29.7.0" - "@jest/test-result" "^29.7.0" - "@jest/types" "^29.6.3" - chalk "^4.0.0" - create-jest "^29.7.0" - exit "^0.1.2" - import-local "^3.0.2" - jest-config "^29.7.0" - jest-util "^29.7.0" - jest-validate "^29.7.0" - yargs "^17.3.1" - -jest-config@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-config/-/jest-config-27.5.1.tgz#5c387de33dca3f99ad6357ddeccd91bf3a0e4a41" - integrity sha512-5sAsjm6tGdsVbW9ahcChPAFCk4IlkQUknH5AvKjuLTSlcO/wCZKyFdn7Rg0EkC+OGgWODEy2hDpWB1PgzH0JNA== - dependencies: - "@babel/core" "^7.8.0" - "@jest/test-sequencer" "^27.5.1" - "@jest/types" "^27.5.1" - babel-jest "^27.5.1" - chalk "^4.0.0" - ci-info "^3.2.0" - deepmerge "^4.2.2" - glob "^7.1.1" - graceful-fs "^4.2.9" - jest-circus "^27.5.1" - jest-environment-jsdom "^27.5.1" - jest-environment-node "^27.5.1" - jest-get-type "^27.5.1" - jest-jasmine2 "^27.5.1" - jest-regex-util "^27.5.1" - jest-resolve "^27.5.1" - jest-runner "^27.5.1" - jest-util "^27.5.1" - jest-validate "^27.5.1" - micromatch "^4.0.4" +jest-cli@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/jest-cli/-/jest-cli-30.2.0.tgz#1780f8e9d66bf84a10b369aea60aeda7697dcc67" + integrity sha512-Os9ukIvADX/A9sLt6Zse3+nmHtHaE6hqOsjQtNiugFTbKRHYIYtZXNGNK9NChseXy7djFPjndX1tL0sCTlfpAA== + dependencies: + "@jest/core" "30.2.0" + "@jest/test-result" "30.2.0" + "@jest/types" "30.2.0" + chalk "^4.1.2" + exit-x "^0.2.2" + import-local "^3.2.0" + jest-config "30.2.0" + jest-util "30.2.0" + jest-validate "30.2.0" + yargs "^17.7.2" + +jest-config@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/jest-config/-/jest-config-30.2.0.tgz#29df8c50e2ad801cc59c406b50176c18c362a90b" + integrity sha512-g4WkyzFQVWHtu6uqGmQR4CQxz/CH3yDSlhzXMWzNjDx843gYjReZnMRanjRCq5XZFuQrGDxgUaiYWE8BRfVckA== + dependencies: + "@babel/core" "^7.27.4" + "@jest/get-type" "30.1.0" + "@jest/pattern" "30.0.1" + "@jest/test-sequencer" "30.2.0" + "@jest/types" "30.2.0" + babel-jest "30.2.0" + chalk "^4.1.2" + ci-info "^4.2.0" + deepmerge "^4.3.1" + glob "^10.3.10" + graceful-fs "^4.2.11" + jest-circus "30.2.0" + jest-docblock "30.2.0" + jest-environment-node "30.2.0" + jest-regex-util "30.0.1" + jest-resolve "30.2.0" + jest-runner "30.2.0" + jest-util "30.2.0" + jest-validate "30.2.0" + micromatch "^4.0.8" parse-json "^5.2.0" - pretty-format "^27.5.1" + pretty-format "30.2.0" slash "^3.0.0" strip-json-comments "^3.1.1" -jest-config@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/jest-config/-/jest-config-29.7.0.tgz#bcbda8806dbcc01b1e316a46bb74085a84b0245f" - integrity sha512-uXbpfeQ7R6TZBqI3/TxCU4q4ttk3u0PJeC+E0zbfSoSjq6bJ7buBPxzQPL0ifrkY4DNu4JUdk0ImlBUYi840eQ== - dependencies: - "@babel/core" "^7.11.6" - "@jest/test-sequencer" "^29.7.0" - "@jest/types" "^29.6.3" - babel-jest "^29.7.0" - chalk "^4.0.0" - ci-info "^3.2.0" - deepmerge "^4.2.2" - glob "^7.1.3" - graceful-fs "^4.2.9" - jest-circus "^29.7.0" - jest-environment-node "^29.7.0" - jest-get-type "^29.6.3" - jest-regex-util "^29.6.3" - jest-resolve "^29.7.0" - jest-runner "^29.7.0" - jest-util "^29.7.0" - jest-validate "^29.7.0" - micromatch "^4.0.4" - parse-json "^5.2.0" - pretty-format "^29.7.0" - slash "^3.0.0" - strip-json-comments "^3.1.1" - -jest-diff@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-27.5.1.tgz#a07f5011ac9e6643cf8a95a462b7b1ecf6680def" - integrity sha512-m0NvkX55LDt9T4mctTEgnZk3fmEg3NRYutvMPWM/0iPnkFj2wIeF45O1718cMSOFO1vINkqmxqD8vE37uTEbqw== - dependencies: - chalk "^4.0.0" - diff-sequences "^27.5.1" - jest-get-type "^27.5.1" - pretty-format "^27.5.1" - -jest-diff@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-29.7.0.tgz#017934a66ebb7ecf6f205e84699be10afd70458a" - integrity sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw== - dependencies: - chalk "^4.0.0" - diff-sequences "^29.6.3" - jest-get-type "^29.6.3" - pretty-format "^29.7.0" - -jest-docblock@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-docblock/-/jest-docblock-27.5.1.tgz#14092f364a42c6108d42c33c8cf30e058e25f6c0" - integrity sha512-rl7hlABeTsRYxKiUfpHrQrG4e2obOiTQWfMEH3PxPjOtdsfLQO4ReWSZaQ7DETm4xu07rl4q/h4zcKXyU0/OzQ== +jest-diff@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-30.2.0.tgz#e3ec3a6ea5c5747f605c9e874f83d756cba36825" + integrity sha512-dQHFo3Pt4/NLlG5z4PxZ/3yZTZ1C7s9hveiOj+GCN+uT109NC2QgsoVZsVOAvbJ3RgKkvyLGXZV9+piDpWbm6A== dependencies: - detect-newline "^3.0.0" - -jest-docblock@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/jest-docblock/-/jest-docblock-29.7.0.tgz#8fddb6adc3cdc955c93e2a87f61cfd350d5d119a" - integrity sha512-q617Auw3A612guyaFgsbFeYpNP5t2aoUNLwBUbc/0kD1R4t9ixDbyFTHd1nok4epoVFpr7PmeWHrhvuV3XaJ4g== - dependencies: - detect-newline "^3.0.0" + "@jest/diff-sequences" "30.0.1" + "@jest/get-type" "30.1.0" + chalk "^4.1.2" + pretty-format "30.2.0" -jest-each@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-each/-/jest-each-27.5.1.tgz#5bc87016f45ed9507fed6e4702a5b468a5b2c44e" - integrity sha512-1Ff6p+FbhT/bXQnEouYy00bkNSY7OUpfIcmdl8vZ31A1UUaurOLPA8a8BbJOF2RDUElwJhmeaV7LnagI+5UwNQ== - dependencies: - "@jest/types" "^27.5.1" - chalk "^4.0.0" - jest-get-type "^27.5.1" - jest-util "^27.5.1" - pretty-format "^27.5.1" - -jest-each@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/jest-each/-/jest-each-29.7.0.tgz#162a9b3f2328bdd991beaabffbb74745e56577d1" - integrity sha512-gns+Er14+ZrEoC5fhOfYCY1LOHHr0TI+rQUHZS8Ttw2l7gl+80eHc/gFf2Ktkw0+SIACDTeWvpFcv3B04VembQ== - dependencies: - "@jest/types" "^29.6.3" - chalk "^4.0.0" - jest-get-type "^29.6.3" - jest-util "^29.7.0" - pretty-format "^29.7.0" - -jest-environment-jsdom@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-environment-jsdom/-/jest-environment-jsdom-27.5.1.tgz#ea9ccd1fc610209655a77898f86b2b559516a546" - integrity sha512-TFBvkTC1Hnnnrka/fUb56atfDtJ9VMZ94JkjTbggl1PEpwrYtUBKMezB3inLmWqQsXYLcMwNoDQwoBTAvFfsfw== +jest-docblock@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/jest-docblock/-/jest-docblock-30.2.0.tgz#42cd98d69f887e531c7352309542b1ce4ee10256" + integrity sha512-tR/FFgZKS1CXluOQzZvNH3+0z9jXr3ldGSD8bhyuxvlVUwbeLOGynkunvlTMxchC5urrKndYiwCFC0DLVjpOCA== dependencies: - "@jest/environment" "^27.5.1" - "@jest/fake-timers" "^27.5.1" - "@jest/types" "^27.5.1" - "@types/node" "*" - jest-mock "^27.5.1" - jest-util "^27.5.1" - jsdom "^16.6.0" + detect-newline "^3.1.0" -jest-environment-node@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-environment-node/-/jest-environment-node-27.5.1.tgz#dedc2cfe52fab6b8f5714b4808aefa85357a365e" - integrity sha512-Jt4ZUnxdOsTGwSRAfKEnE6BcwsSPNOijjwifq5sDFSA2kesnXTvNqKHYgM0hDq3549Uf/KzdXNYn4wMZJPlFLw== +jest-each@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/jest-each/-/jest-each-30.2.0.tgz#39e623ae71641c2ac3ee69b3ba3d258fce8e768d" + integrity sha512-lpWlJlM7bCUf1mfmuqTA8+j2lNURW9eNafOy99knBM01i5CQeY5UH1vZjgT9071nDJac1M4XsbyI44oNOdhlDQ== dependencies: - "@jest/environment" "^27.5.1" - "@jest/fake-timers" "^27.5.1" - "@jest/types" "^27.5.1" - "@types/node" "*" - jest-mock "^27.5.1" - jest-util "^27.5.1" + "@jest/get-type" "30.1.0" + "@jest/types" "30.2.0" + chalk "^4.1.2" + jest-util "30.2.0" + pretty-format "30.2.0" -jest-environment-node@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/jest-environment-node/-/jest-environment-node-29.7.0.tgz#0b93e111dda8ec120bc8300e6d1fb9576e164376" - integrity sha512-DOSwCRqXirTOyheM+4d5YZOrWcdu0LNZ87ewUoywbcb2XR4wKgqiG8vNeYwhjFMbEkfju7wx2GYH0P2gevGvFw== +jest-environment-jsdom@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/jest-environment-jsdom/-/jest-environment-jsdom-30.2.0.tgz#e95e0921ed22be974f1d8a324766d12b1844cb2c" + integrity sha512-zbBTiqr2Vl78pKp/laGBREYzbZx9ZtqPjOK4++lL4BNDhxRnahg51HtoDrk9/VjIy9IthNEWdKVd7H5bqBhiWQ== dependencies: - "@jest/environment" "^29.7.0" - "@jest/fake-timers" "^29.7.0" - "@jest/types" "^29.6.3" + "@jest/environment" "30.2.0" + "@jest/environment-jsdom-abstract" "30.2.0" + "@types/jsdom" "^21.1.7" "@types/node" "*" - jest-mock "^29.7.0" - jest-util "^29.7.0" - -jest-get-type@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-27.5.1.tgz#3cd613c507b0f7ace013df407a1c1cd578bcb4f1" - integrity sha512-2KY95ksYSaK7DMBWQn6dQz3kqAf3BB64y2udeG+hv4KfSOb9qwcYQstTJc1KCbsix+wLZWZYN8t7nwX3GOBLRw== - -jest-get-type@^29.6.3: - version "29.6.3" - resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-29.6.3.tgz#36f499fdcea197c1045a127319c0481723908fd1" - integrity sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw== + jsdom "^26.1.0" -jest-haste-map@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-haste-map/-/jest-haste-map-27.5.1.tgz#9fd8bd7e7b4fa502d9c6164c5640512b4e811e7f" - integrity sha512-7GgkZ4Fw4NFbMSDSpZwXeBiIbx+t/46nJ2QitkOjvwPYyZmqttu2TDSimMHP1EkPOi4xUZAN1doE5Vd25H4Jng== +jest-environment-node@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/jest-environment-node/-/jest-environment-node-30.2.0.tgz#3def7980ebd2fd86e74efd4d2e681f55ab38da0f" + integrity sha512-ElU8v92QJ9UrYsKrxDIKCxu6PfNj4Hdcktcn0JX12zqNdqWHB0N+hwOnnBBXvjLd2vApZtuLUGs1QSY+MsXoNA== dependencies: - "@jest/types" "^27.5.1" - "@types/graceful-fs" "^4.1.2" + "@jest/environment" "30.2.0" + "@jest/fake-timers" "30.2.0" + "@jest/types" "30.2.0" "@types/node" "*" - anymatch "^3.0.3" - fb-watchman "^2.0.0" - graceful-fs "^4.2.9" - jest-regex-util "^27.5.1" - jest-serializer "^27.5.1" - jest-util "^27.5.1" - jest-worker "^27.5.1" - micromatch "^4.0.4" - walker "^1.0.7" - optionalDependencies: - fsevents "^2.3.2" + jest-mock "30.2.0" + jest-util "30.2.0" + jest-validate "30.2.0" -jest-haste-map@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/jest-haste-map/-/jest-haste-map-29.7.0.tgz#3c2396524482f5a0506376e6c858c3bbcc17b104" - integrity sha512-fP8u2pyfqx0K1rGn1R9pyE0/KTn+G7PxktWidOBTqFPLYX0b9ksaMFkhK5vrS3DVun09pckLdlx90QthlW7AmA== +jest-haste-map@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/jest-haste-map/-/jest-haste-map-30.2.0.tgz#808e3889f288603ac70ff0ac047598345a66022e" + integrity sha512-sQA/jCb9kNt+neM0anSj6eZhLZUIhQgwDt7cPGjumgLM4rXsfb9kpnlacmvZz3Q5tb80nS+oG/if+NBKrHC+Xw== dependencies: - "@jest/types" "^29.6.3" - "@types/graceful-fs" "^4.1.3" + "@jest/types" "30.2.0" "@types/node" "*" - anymatch "^3.0.3" - fb-watchman "^2.0.0" - graceful-fs "^4.2.9" - jest-regex-util "^29.6.3" - jest-util "^29.7.0" - jest-worker "^29.7.0" - micromatch "^4.0.4" + anymatch "^3.1.3" + fb-watchman "^2.0.2" + graceful-fs "^4.2.11" + jest-regex-util "30.0.1" + jest-util "30.2.0" + jest-worker "30.2.0" + micromatch "^4.0.8" walker "^1.0.8" optionalDependencies: - fsevents "^2.3.2" - -jest-jasmine2@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-jasmine2/-/jest-jasmine2-27.5.1.tgz#a037b0034ef49a9f3d71c4375a796f3b230d1ac4" - integrity sha512-jtq7VVyG8SqAorDpApwiJJImd0V2wv1xzdheGHRGyuT7gZm6gG47QEskOlzsN1PG/6WNaCo5pmwMHDf3AkG2pQ== - dependencies: - "@jest/environment" "^27.5.1" - "@jest/source-map" "^27.5.1" - "@jest/test-result" "^27.5.1" - "@jest/types" "^27.5.1" - "@types/node" "*" - chalk "^4.0.0" - co "^4.6.0" - expect "^27.5.1" - is-generator-fn "^2.0.0" - jest-each "^27.5.1" - jest-matcher-utils "^27.5.1" - jest-message-util "^27.5.1" - jest-runtime "^27.5.1" - jest-snapshot "^27.5.1" - jest-util "^27.5.1" - pretty-format "^27.5.1" - throat "^6.0.1" - -jest-leak-detector@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-leak-detector/-/jest-leak-detector-27.5.1.tgz#6ec9d54c3579dd6e3e66d70e3498adf80fde3fb8" - integrity sha512-POXfWAMvfU6WMUXftV4HolnJfnPOGEu10fscNCA76KBpRRhcMN2c8d3iT2pxQS3HLbA+5X4sOUPzYO2NUyIlHQ== - dependencies: - jest-get-type "^27.5.1" - pretty-format "^27.5.1" + fsevents "^2.3.3" -jest-leak-detector@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/jest-leak-detector/-/jest-leak-detector-29.7.0.tgz#5b7ec0dadfdfec0ca383dc9aa016d36b5ea4c728" - integrity sha512-kYA8IJcSYtST2BY9I+SMC32nDpBT3J2NvWJx8+JCuCdl/CR1I4EKUJROiP8XtCcxqgTTBGJNdbB1A8XRKbTetw== +jest-leak-detector@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/jest-leak-detector/-/jest-leak-detector-30.2.0.tgz#292fdca7b7c9cf594e1e570ace140b01d8beb736" + integrity sha512-M6jKAjyzjHG0SrQgwhgZGy9hFazcudwCNovY/9HPIicmNSBuockPSedAP9vlPK6ONFJ1zfyH/M2/YYJxOz5cdQ== dependencies: - jest-get-type "^29.6.3" - pretty-format "^29.7.0" + "@jest/get-type" "30.1.0" + pretty-format "30.2.0" -jest-matcher-utils@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-27.5.1.tgz#9c0cdbda8245bc22d2331729d1091308b40cf8ab" - integrity sha512-z2uTx/T6LBaCoNWNFWwChLBKYxTMcGBRjAt+2SbP929/Fflb9aa5LGma654Rz8z9HLxsrUaYzxE9T/EFIL/PAw== +jest-matcher-utils@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-30.2.0.tgz#69a0d4c271066559ec8b0d8174829adc3f23a783" + integrity sha512-dQ94Nq4dbzmUWkQ0ANAWS9tBRfqCrn0bV9AMYdOi/MHW726xn7eQmMeRTpX2ViC00bpNaWXq+7o4lIQ3AX13Hg== dependencies: - chalk "^4.0.0" - jest-diff "^27.5.1" - jest-get-type "^27.5.1" - pretty-format "^27.5.1" + "@jest/get-type" "30.1.0" + chalk "^4.1.2" + jest-diff "30.2.0" + pretty-format "30.2.0" -jest-matcher-utils@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-29.7.0.tgz#ae8fec79ff249fd592ce80e3ee474e83a6c44f12" - integrity sha512-sBkD+Xi9DtcChsI3L3u0+N0opgPYnCRPtGcQYrgXmR+hmt/fYfWAL0xRXYU8eWOdfuLgBe0YCW3AFtnRLagq/g== +jest-message-util@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/jest-message-util/-/jest-message-util-30.2.0.tgz#fc97bf90d11f118b31e6131e2b67fc4f39f92152" + integrity sha512-y4DKFLZ2y6DxTWD4cDe07RglV88ZiNEdlRfGtqahfbIjfsw1nMCPx49Uev4IA/hWn3sDKyAnSPwoYSsAEdcimw== dependencies: - chalk "^4.0.0" - jest-diff "^29.7.0" - jest-get-type "^29.6.3" - pretty-format "^29.7.0" - -jest-message-util@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-message-util/-/jest-message-util-27.5.1.tgz#bdda72806da10d9ed6425e12afff38cd1458b6cf" - integrity sha512-rMyFe1+jnyAAf+NHwTclDz0eAaLkVDdKVHHBFWsBWHnnh5YeJMNWWsv7AbFYXfK3oTqvL7VTWkhNLu1jX24D+g== - dependencies: - "@babel/code-frame" "^7.12.13" - "@jest/types" "^27.5.1" - "@types/stack-utils" "^2.0.0" - chalk "^4.0.0" - graceful-fs "^4.2.9" - micromatch "^4.0.4" - pretty-format "^27.5.1" - slash "^3.0.0" - stack-utils "^2.0.3" - -jest-message-util@^28.1.3: - version "28.1.3" - resolved "https://registry.yarnpkg.com/jest-message-util/-/jest-message-util-28.1.3.tgz#232def7f2e333f1eecc90649b5b94b0055e7c43d" - integrity sha512-PFdn9Iewbt575zKPf1286Ht9EPoJmYT7P0kY+RibeYZ2XtOr53pDLEFoTWXbd1h4JiGiWpTBC84fc8xMXQMb7g== - dependencies: - "@babel/code-frame" "^7.12.13" - "@jest/types" "^28.1.3" - "@types/stack-utils" "^2.0.0" - chalk "^4.0.0" - graceful-fs "^4.2.9" - micromatch "^4.0.4" - pretty-format "^28.1.3" - slash "^3.0.0" - stack-utils "^2.0.3" - -jest-message-util@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/jest-message-util/-/jest-message-util-29.7.0.tgz#8bc392e204e95dfe7564abbe72a404e28e51f7f3" - integrity sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w== - dependencies: - "@babel/code-frame" "^7.12.13" - "@jest/types" "^29.6.3" - "@types/stack-utils" "^2.0.0" - chalk "^4.0.0" - graceful-fs "^4.2.9" - micromatch "^4.0.4" - pretty-format "^29.7.0" + "@babel/code-frame" "^7.27.1" + "@jest/types" "30.2.0" + "@types/stack-utils" "^2.0.3" + chalk "^4.1.2" + graceful-fs "^4.2.11" + micromatch "^4.0.8" + pretty-format "30.2.0" slash "^3.0.0" - stack-utils "^2.0.3" - -jest-mock@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-mock/-/jest-mock-27.5.1.tgz#19948336d49ef4d9c52021d34ac7b5f36ff967d6" - integrity sha512-K4jKbY1d4ENhbrG2zuPWaQBvDly+iZ2yAW+T1fATN78hc0sInwn7wZB8XtlNnvHug5RMwV897Xm4LqmPM4e2Og== - dependencies: - "@jest/types" "^27.5.1" - "@types/node" "*" + stack-utils "^2.0.6" -jest-mock@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/jest-mock/-/jest-mock-29.7.0.tgz#4e836cf60e99c6fcfabe9f99d017f3fdd50a6347" - integrity sha512-ITOMZn+UkYS4ZFh83xYAOzWStloNzJFO2s8DWrE4lhtGD+AorgnbkiKERe4wQVBydIGPx059g6riW5Btp6Llnw== +jest-mock@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/jest-mock/-/jest-mock-30.2.0.tgz#69f991614eeb4060189459d3584f710845bff45e" + integrity sha512-JNNNl2rj4b5ICpmAcq+WbLH83XswjPbjH4T7yvGzfAGCPh1rw+xVNbtk+FnRslvt9lkCcdn9i1oAoKUuFsOxRw== dependencies: - "@jest/types" "^29.6.3" + "@jest/types" "30.2.0" "@types/node" "*" - jest-util "^29.7.0" + jest-util "30.2.0" -jest-pnp-resolver@^1.2.2: +jest-pnp-resolver@^1.2.3: version "1.2.3" resolved "https://registry.yarnpkg.com/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz#930b1546164d4ad5937d5540e711d4d38d4cad2e" integrity sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w== -jest-regex-util@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-regex-util/-/jest-regex-util-27.5.1.tgz#4da143f7e9fd1e542d4aa69617b38e4a78365b95" - integrity sha512-4bfKq2zie+x16okqDXjXn9ql2B0dScQu+vcwe4TvFVhkVyuWLqpZrZtXxLLWoXYgn0E87I6r6GRYHF7wFZBUvg== - -jest-regex-util@^28.0.0: - version "28.0.2" - resolved "https://registry.yarnpkg.com/jest-regex-util/-/jest-regex-util-28.0.2.tgz#afdc377a3b25fb6e80825adcf76c854e5bf47ead" - integrity sha512-4s0IgyNIy0y9FK+cjoVYoxamT7Zeo7MhzqRGx7YDYmaQn1wucY9rotiGkBzzcMXTtjrCAP/f7f+E0F7+fxPNdw== +jest-regex-util@30.0.1: + version "30.0.1" + resolved "https://registry.yarnpkg.com/jest-regex-util/-/jest-regex-util-30.0.1.tgz#f17c1de3958b67dfe485354f5a10093298f2a49b" + integrity sha512-jHEQgBXAgc+Gh4g0p3bCevgRCVRkB4VB70zhoAE48gxeSr1hfUOsM/C2WoJgVL7Eyg//hudYENbm3Ne+/dRVVA== -jest-regex-util@^29.6.3: - version "29.6.3" - resolved "https://registry.yarnpkg.com/jest-regex-util/-/jest-regex-util-29.6.3.tgz#4a556d9c776af68e1c5f48194f4d0327d24e8a52" - integrity sha512-KJJBsRCyyLNWCNBOvZyRDnAIfUiRJ8v+hOBQYGn8gDyF3UegwiP4gwRR3/SDa42g1YbVycTidUF3rKjyLFDWbg== - -jest-resolve-dependencies@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-resolve-dependencies/-/jest-resolve-dependencies-27.5.1.tgz#d811ecc8305e731cc86dd79741ee98fed06f1da8" - integrity sha512-QQOOdY4PE39iawDn5rzbIePNigfe5B9Z91GDD1ae/xNDlu9kaat8QQ5EKnNmVWPV54hUdxCVwwj6YMgR2O7IOg== +jest-resolve-dependencies@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/jest-resolve-dependencies/-/jest-resolve-dependencies-30.2.0.tgz#3370e2c0b49cc560f6a7e8ec3a59dd99525e1a55" + integrity sha512-xTOIGug/0RmIe3mmCqCT95yO0vj6JURrn1TKWlNbhiAefJRWINNPgwVkrVgt/YaerPzY3iItufd80v3lOrFJ2w== dependencies: - "@jest/types" "^27.5.1" - jest-regex-util "^27.5.1" - jest-snapshot "^27.5.1" + jest-regex-util "30.0.1" + jest-snapshot "30.2.0" -jest-resolve-dependencies@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/jest-resolve-dependencies/-/jest-resolve-dependencies-29.7.0.tgz#1b04f2c095f37fc776ff40803dc92921b1e88428" - integrity sha512-un0zD/6qxJ+S0et7WxeI3H5XSe9lTBBR7bOHCHXkKR6luG5mwDDlIzVQ0V5cZCuoTgEdcdwzTghYkTWfubi+nA== +jest-resolve@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/jest-resolve/-/jest-resolve-30.2.0.tgz#2e2009cbd61e8f1f003355d5ec87225412cebcd7" + integrity sha512-TCrHSxPlx3tBY3hWNtRQKbtgLhsXa1WmbJEqBlTBrGafd5fiQFByy2GNCEoGR+Tns8d15GaL9cxEzKOO3GEb2A== dependencies: - jest-regex-util "^29.6.3" - jest-snapshot "^29.7.0" - -jest-resolve@^27.4.2, jest-resolve@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-resolve/-/jest-resolve-27.5.1.tgz#a2f1c5a0796ec18fe9eb1536ac3814c23617b384" - integrity sha512-FFDy8/9E6CV83IMbDpcjOhumAQPDyETnU2KZ1O98DwTnz8AOBsW/Xv3GySr1mOZdItLR+zDZ7I/UdTFbgSOVCw== - dependencies: - "@jest/types" "^27.5.1" - chalk "^4.0.0" - graceful-fs "^4.2.9" - jest-haste-map "^27.5.1" - jest-pnp-resolver "^1.2.2" - jest-util "^27.5.1" - jest-validate "^27.5.1" - resolve "^1.20.0" - resolve.exports "^1.1.0" - slash "^3.0.0" - -jest-resolve@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/jest-resolve/-/jest-resolve-29.7.0.tgz#64d6a8992dd26f635ab0c01e5eef4399c6bcbc30" - integrity sha512-IOVhZSrg+UvVAshDSDtHyFCCBUl/Q3AAJv8iZ6ZjnZ74xzvwuzLXid9IIIPgTnY62SJjfuupMKZsZQRsCvxEgA== - dependencies: - chalk "^4.0.0" - graceful-fs "^4.2.9" - jest-haste-map "^29.7.0" - jest-pnp-resolver "^1.2.2" - jest-util "^29.7.0" - jest-validate "^29.7.0" - resolve "^1.20.0" - resolve.exports "^2.0.0" + chalk "^4.1.2" + graceful-fs "^4.2.11" + jest-haste-map "30.2.0" + jest-pnp-resolver "^1.2.3" + jest-util "30.2.0" + jest-validate "30.2.0" slash "^3.0.0" - -jest-runner@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-runner/-/jest-runner-27.5.1.tgz#071b27c1fa30d90540805c5645a0ec167c7b62e5" - integrity sha512-g4NPsM4mFCOwFKXO4p/H/kWGdJp9V8kURY2lX8Me2drgXqG7rrZAx5kv+5H7wtt/cdFIjhqYx1HrlqWHaOvDaQ== - dependencies: - "@jest/console" "^27.5.1" - "@jest/environment" "^27.5.1" - "@jest/test-result" "^27.5.1" - "@jest/transform" "^27.5.1" - "@jest/types" "^27.5.1" + unrs-resolver "^1.7.11" + +jest-runner@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/jest-runner/-/jest-runner-30.2.0.tgz#c62b4c3130afa661789705e13a07bdbcec26a114" + integrity sha512-PqvZ2B2XEyPEbclp+gV6KO/F1FIFSbIwewRgmROCMBo/aZ6J1w8Qypoj2pEOcg3G2HzLlaP6VUtvwCI8dM3oqQ== + dependencies: + "@jest/console" "30.2.0" + "@jest/environment" "30.2.0" + "@jest/test-result" "30.2.0" + "@jest/transform" "30.2.0" + "@jest/types" "30.2.0" "@types/node" "*" - chalk "^4.0.0" - emittery "^0.8.1" - graceful-fs "^4.2.9" - jest-docblock "^27.5.1" - jest-environment-jsdom "^27.5.1" - jest-environment-node "^27.5.1" - jest-haste-map "^27.5.1" - jest-leak-detector "^27.5.1" - jest-message-util "^27.5.1" - jest-resolve "^27.5.1" - jest-runtime "^27.5.1" - jest-util "^27.5.1" - jest-worker "^27.5.1" - source-map-support "^0.5.6" - throat "^6.0.1" - -jest-runner@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/jest-runner/-/jest-runner-29.7.0.tgz#809af072d408a53dcfd2e849a4c976d3132f718e" - integrity sha512-fsc4N6cPCAahybGBfTRcq5wFR6fpLznMg47sY5aDpsoejOcVYFb07AHuSnR0liMcPTgBsA3ZJL6kFOjPdoNipQ== - dependencies: - "@jest/console" "^29.7.0" - "@jest/environment" "^29.7.0" - "@jest/test-result" "^29.7.0" - "@jest/transform" "^29.7.0" - "@jest/types" "^29.6.3" - "@types/node" "*" - chalk "^4.0.0" + chalk "^4.1.2" emittery "^0.13.1" - graceful-fs "^4.2.9" - jest-docblock "^29.7.0" - jest-environment-node "^29.7.0" - jest-haste-map "^29.7.0" - jest-leak-detector "^29.7.0" - jest-message-util "^29.7.0" - jest-resolve "^29.7.0" - jest-runtime "^29.7.0" - jest-util "^29.7.0" - jest-watcher "^29.7.0" - jest-worker "^29.7.0" + exit-x "^0.2.2" + graceful-fs "^4.2.11" + jest-docblock "30.2.0" + jest-environment-node "30.2.0" + jest-haste-map "30.2.0" + jest-leak-detector "30.2.0" + jest-message-util "30.2.0" + jest-resolve "30.2.0" + jest-runtime "30.2.0" + jest-util "30.2.0" + jest-watcher "30.2.0" + jest-worker "30.2.0" p-limit "^3.1.0" source-map-support "0.5.13" -jest-runtime@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-runtime/-/jest-runtime-27.5.1.tgz#4896003d7a334f7e8e4a53ba93fb9bcd3db0a1af" - integrity sha512-o7gxw3Gf+H2IGt8fv0RiyE1+r83FJBRruoA+FXrlHw6xEyBsU8ugA6IPfTdVyA0w8HClpbK+DGJxH59UrNMx8A== - dependencies: - "@jest/environment" "^27.5.1" - "@jest/fake-timers" "^27.5.1" - "@jest/globals" "^27.5.1" - "@jest/source-map" "^27.5.1" - "@jest/test-result" "^27.5.1" - "@jest/transform" "^27.5.1" - "@jest/types" "^27.5.1" - chalk "^4.0.0" - cjs-module-lexer "^1.0.0" - collect-v8-coverage "^1.0.0" - execa "^5.0.0" - glob "^7.1.3" - graceful-fs "^4.2.9" - jest-haste-map "^27.5.1" - jest-message-util "^27.5.1" - jest-mock "^27.5.1" - jest-regex-util "^27.5.1" - jest-resolve "^27.5.1" - jest-snapshot "^27.5.1" - jest-util "^27.5.1" - slash "^3.0.0" - strip-bom "^4.0.0" - -jest-runtime@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/jest-runtime/-/jest-runtime-29.7.0.tgz#efecb3141cf7d3767a3a0cc8f7c9990587d3d817" - integrity sha512-gUnLjgwdGqW7B4LvOIkbKs9WGbn+QLqRQQ9juC6HndeDiezIwhDP+mhMwHWCEcfQ5RUXa6OPnFF8BJh5xegwwQ== - dependencies: - "@jest/environment" "^29.7.0" - "@jest/fake-timers" "^29.7.0" - "@jest/globals" "^29.7.0" - "@jest/source-map" "^29.6.3" - "@jest/test-result" "^29.7.0" - "@jest/transform" "^29.7.0" - "@jest/types" "^29.6.3" +jest-runtime@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/jest-runtime/-/jest-runtime-30.2.0.tgz#395ea792cde048db1b0cd1a92dc9cb9f1921bf8a" + integrity sha512-p1+GVX/PJqTucvsmERPMgCPvQJpFt4hFbM+VN3n8TMo47decMUcJbt+rgzwrEme0MQUA/R+1de2axftTHkKckg== + dependencies: + "@jest/environment" "30.2.0" + "@jest/fake-timers" "30.2.0" + "@jest/globals" "30.2.0" + "@jest/source-map" "30.0.1" + "@jest/test-result" "30.2.0" + "@jest/transform" "30.2.0" + "@jest/types" "30.2.0" "@types/node" "*" - chalk "^4.0.0" - cjs-module-lexer "^1.0.0" - collect-v8-coverage "^1.0.0" - glob "^7.1.3" - graceful-fs "^4.2.9" - jest-haste-map "^29.7.0" - jest-message-util "^29.7.0" - jest-mock "^29.7.0" - jest-regex-util "^29.6.3" - jest-resolve "^29.7.0" - jest-snapshot "^29.7.0" - jest-util "^29.7.0" + chalk "^4.1.2" + cjs-module-lexer "^2.1.0" + collect-v8-coverage "^1.0.2" + glob "^10.3.10" + graceful-fs "^4.2.11" + jest-haste-map "30.2.0" + jest-message-util "30.2.0" + jest-mock "30.2.0" + jest-regex-util "30.0.1" + jest-resolve "30.2.0" + jest-snapshot "30.2.0" + jest-util "30.2.0" slash "^3.0.0" strip-bom "^4.0.0" -jest-serializer@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-serializer/-/jest-serializer-27.5.1.tgz#81438410a30ea66fd57ff730835123dea1fb1f64" - integrity sha512-jZCyo6iIxO1aqUxpuBlwTDMkzOAJS4a3eYz3YzgxxVQFwLeSA7Jfq5cbqCY+JLvTDrWirgusI/0KwxKMgrdf7w== - dependencies: - "@types/node" "*" - graceful-fs "^4.2.9" - -jest-snapshot@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-snapshot/-/jest-snapshot-27.5.1.tgz#b668d50d23d38054a51b42c4039cab59ae6eb6a1" - integrity sha512-yYykXI5a0I31xX67mgeLw1DZ0bJB+gpq5IpSuCAoyDi0+BhgU/RIrL+RTzDmkNTchvDFWKP8lp+w/42Z3us5sA== - dependencies: - "@babel/core" "^7.7.2" - "@babel/generator" "^7.7.2" - "@babel/plugin-syntax-typescript" "^7.7.2" - "@babel/traverse" "^7.7.2" - "@babel/types" "^7.0.0" - "@jest/transform" "^27.5.1" - "@jest/types" "^27.5.1" - "@types/babel__traverse" "^7.0.4" - "@types/prettier" "^2.1.5" - babel-preset-current-node-syntax "^1.0.0" - chalk "^4.0.0" - expect "^27.5.1" - graceful-fs "^4.2.9" - jest-diff "^27.5.1" - jest-get-type "^27.5.1" - jest-haste-map "^27.5.1" - jest-matcher-utils "^27.5.1" - jest-message-util "^27.5.1" - jest-util "^27.5.1" - natural-compare "^1.4.0" - pretty-format "^27.5.1" - semver "^7.3.2" - -jest-snapshot@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/jest-snapshot/-/jest-snapshot-29.7.0.tgz#c2c574c3f51865da1bb329036778a69bf88a6be5" - integrity sha512-Rm0BMWtxBcioHr1/OX5YCP8Uov4riHvKPknOGs804Zg9JGZgmIBkbtlxJC/7Z4msKYVbIJtfU+tKb8xlYNfdkw== - dependencies: - "@babel/core" "^7.11.6" - "@babel/generator" "^7.7.2" - "@babel/plugin-syntax-jsx" "^7.7.2" - "@babel/plugin-syntax-typescript" "^7.7.2" - "@babel/types" "^7.3.3" - "@jest/expect-utils" "^29.7.0" - "@jest/transform" "^29.7.0" - "@jest/types" "^29.6.3" - babel-preset-current-node-syntax "^1.0.0" - chalk "^4.0.0" - expect "^29.7.0" - graceful-fs "^4.2.9" - jest-diff "^29.7.0" - jest-get-type "^29.6.3" - jest-matcher-utils "^29.7.0" - jest-message-util "^29.7.0" - jest-util "^29.7.0" - natural-compare "^1.4.0" - pretty-format "^29.7.0" - semver "^7.5.3" - -jest-util@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-27.5.1.tgz#3ba9771e8e31a0b85da48fe0b0891fb86c01c2f9" - integrity sha512-Kv2o/8jNvX1MQ0KGtw480E/w4fBCDOnH6+6DmeKi6LZUIlKA5kwY0YNdlzaWTiVgxqAqik11QyxDOKk543aKXw== - dependencies: - "@jest/types" "^27.5.1" - "@types/node" "*" - chalk "^4.0.0" - ci-info "^3.2.0" - graceful-fs "^4.2.9" - picomatch "^2.2.3" - -jest-util@^28.1.3: - version "28.1.3" - resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-28.1.3.tgz#f4f932aa0074f0679943220ff9cbba7e497028b0" - integrity sha512-XdqfpHwpcSRko/C35uLYFM2emRAltIIKZiJ9eAmhjsj0CqZMa0p1ib0R5fWIqGhn1a103DebTbpqIaP1qCQ6tQ== - dependencies: - "@jest/types" "^28.1.3" - "@types/node" "*" - chalk "^4.0.0" - ci-info "^3.2.0" - graceful-fs "^4.2.9" - picomatch "^2.2.3" - -jest-util@^29.0.0, jest-util@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-29.7.0.tgz#23c2b62bfb22be82b44de98055802ff3710fc0bc" - integrity sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA== - dependencies: - "@jest/types" "^29.6.3" +jest-snapshot@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/jest-snapshot/-/jest-snapshot-30.2.0.tgz#266fbbb4b95fc4665ce6f32f1f38eeb39f4e26d0" + integrity sha512-5WEtTy2jXPFypadKNpbNkZ72puZCa6UjSr/7djeecHWOu7iYhSXSnHScT8wBz3Rn8Ena5d5RYRcsyKIeqG1IyA== + dependencies: + "@babel/core" "^7.27.4" + "@babel/generator" "^7.27.5" + "@babel/plugin-syntax-jsx" "^7.27.1" + "@babel/plugin-syntax-typescript" "^7.27.1" + "@babel/types" "^7.27.3" + "@jest/expect-utils" "30.2.0" + "@jest/get-type" "30.1.0" + "@jest/snapshot-utils" "30.2.0" + "@jest/transform" "30.2.0" + "@jest/types" "30.2.0" + babel-preset-current-node-syntax "^1.2.0" + chalk "^4.1.2" + expect "30.2.0" + graceful-fs "^4.2.11" + jest-diff "30.2.0" + jest-matcher-utils "30.2.0" + jest-message-util "30.2.0" + jest-util "30.2.0" + pretty-format "30.2.0" + semver "^7.7.2" + synckit "^0.11.8" + +jest-util@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-30.2.0.tgz#5142adbcad6f4e53c2776c067a4db3c14f913705" + integrity sha512-QKNsM0o3Xe6ISQU869e+DhG+4CK/48aHYdJZGlFQVTjnbvgpcKyxpzk29fGiO7i/J8VENZ+d2iGnSsvmuHywlA== + dependencies: + "@jest/types" "30.2.0" "@types/node" "*" - chalk "^4.0.0" - ci-info "^3.2.0" - graceful-fs "^4.2.9" - picomatch "^2.2.3" - -jest-validate@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-validate/-/jest-validate-27.5.1.tgz#9197d54dc0bdb52260b8db40b46ae668e04df067" - integrity sha512-thkNli0LYTmOI1tDB3FI1S1RTp/Bqyd9pTarJwL87OIBFuqEb5Apv5EaApEudYg4g86e3CT6kM0RowkhtEnCBQ== - dependencies: - "@jest/types" "^27.5.1" - camelcase "^6.2.0" - chalk "^4.0.0" - jest-get-type "^27.5.1" - leven "^3.1.0" - pretty-format "^27.5.1" + chalk "^4.1.2" + ci-info "^4.2.0" + graceful-fs "^4.2.11" + picomatch "^4.0.2" -jest-validate@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/jest-validate/-/jest-validate-29.7.0.tgz#7bf705511c64da591d46b15fce41400d52147d9c" - integrity sha512-ZB7wHqaRGVw/9hST/OuFUReG7M8vKeq0/J2egIGLdvjHCmYqGARhzXmtgi+gVeZ5uXFF219aOc3Ls2yLg27tkw== +jest-validate@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/jest-validate/-/jest-validate-30.2.0.tgz#273eaaed4c0963b934b5b31e96289edda6e0a2ef" + integrity sha512-FBGWi7dP2hpdi8nBoWxSsLvBFewKAg0+uSQwBaof4Y4DPgBabXgpSYC5/lR7VmnIlSpASmCi/ntRWPbv7089Pw== dependencies: - "@jest/types" "^29.6.3" - camelcase "^6.2.0" - chalk "^4.0.0" - jest-get-type "^29.6.3" + "@jest/get-type" "30.1.0" + "@jest/types" "30.2.0" + camelcase "^6.3.0" + chalk "^4.1.2" leven "^3.1.0" - pretty-format "^29.7.0" - -jest-watch-typeahead@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/jest-watch-typeahead/-/jest-watch-typeahead-1.1.0.tgz#b4a6826dfb9c9420da2f7bc900de59dad11266a9" - integrity sha512-Va5nLSJTN7YFtC2jd+7wsoe1pNe5K4ShLux/E5iHEwlB9AxaxmggY7to9KUqKojhaJw3aXqt5WAb4jGPOolpEw== - dependencies: - ansi-escapes "^4.3.1" - chalk "^4.0.0" - jest-regex-util "^28.0.0" - jest-watcher "^28.0.0" - slash "^4.0.0" - string-length "^5.0.1" - strip-ansi "^7.0.1" + pretty-format "30.2.0" -jest-watcher@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-watcher/-/jest-watcher-27.5.1.tgz#71bd85fb9bde3a2c2ec4dc353437971c43c642a2" - integrity sha512-z676SuD6Z8o8qbmEGhoEUFOM1+jfEiL3DXHK/xgEiG2EyNYfFG60jluWcupY6dATjfEsKQuibReS1djInQnoVw== +jest-watcher@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/jest-watcher/-/jest-watcher-30.2.0.tgz#f9c055de48e18c979e7756a3917e596e2d69b07b" + integrity sha512-PYxa28dxJ9g777pGm/7PrbnMeA0Jr7osHP9bS7eJy9DuAjMgdGtxgf0uKMyoIsTWAkIbUW5hSDdJ3urmgXBqxg== dependencies: - "@jest/test-result" "^27.5.1" - "@jest/types" "^27.5.1" - "@types/node" "*" - ansi-escapes "^4.2.1" - chalk "^4.0.0" - jest-util "^27.5.1" - string-length "^4.0.1" - -jest-watcher@^28.0.0: - version "28.1.3" - resolved "https://registry.yarnpkg.com/jest-watcher/-/jest-watcher-28.1.3.tgz#c6023a59ba2255e3b4c57179fc94164b3e73abd4" - integrity sha512-t4qcqj9hze+jviFPUN3YAtAEeFnr/azITXQEMARf5cMwKY2SMBRnCQTXLixTl20OR6mLh9KLMrgVJgJISym+1g== - dependencies: - "@jest/test-result" "^28.1.3" - "@jest/types" "^28.1.3" - "@types/node" "*" - ansi-escapes "^4.2.1" - chalk "^4.0.0" - emittery "^0.10.2" - jest-util "^28.1.3" - string-length "^4.0.1" - -jest-watcher@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/jest-watcher/-/jest-watcher-29.7.0.tgz#7810d30d619c3a62093223ce6bb359ca1b28a2f2" - integrity sha512-49Fg7WXkU3Vl2h6LbLtMQ/HyB6rXSIX7SqvBLQmssRBGN9I0PNvPmAmCWSOY6SOvrjhI/F7/bGAv9RtnsPA03g== - dependencies: - "@jest/test-result" "^29.7.0" - "@jest/types" "^29.6.3" + "@jest/test-result" "30.2.0" + "@jest/types" "30.2.0" "@types/node" "*" - ansi-escapes "^4.2.1" - chalk "^4.0.0" + ansi-escapes "^4.3.2" + chalk "^4.1.2" emittery "^0.13.1" - jest-util "^29.7.0" - string-length "^4.0.1" - -jest-worker@^26.2.1: - version "26.6.2" - resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-26.6.2.tgz#7f72cbc4d643c365e27b9fd775f9d0eaa9c7a8ed" - integrity sha512-KWYVV1c4i+jbMpaBC+U++4Va0cp8OisU185o73T1vo99hqi7w8tSJfUXYswwqqrjzwxa6KpRK54WhPvwf5w6PQ== - dependencies: - "@types/node" "*" - merge-stream "^2.0.0" - supports-color "^7.0.0" - -jest-worker@^27.0.2, jest-worker@^27.4.5, jest-worker@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-27.5.1.tgz#8d146f0900e8973b106b6f73cc1e9a8cb86f8db0" - integrity sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg== - dependencies: - "@types/node" "*" - merge-stream "^2.0.0" - supports-color "^8.0.0" - -jest-worker@^28.0.2: - version "28.1.3" - resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-28.1.3.tgz#7e3c4ce3fa23d1bb6accb169e7f396f98ed4bb98" - integrity sha512-CqRA220YV/6jCo8VWvAt1KKx6eek1VIHMPeLEbpcfSfkEeWyBNppynM/o6q+Wmw+sOhos2ml34wZbSX3G13//g== - dependencies: - "@types/node" "*" - merge-stream "^2.0.0" - supports-color "^8.0.0" + jest-util "30.2.0" + string-length "^4.0.2" -jest-worker@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-29.7.0.tgz#acad073acbbaeb7262bd5389e1bcf43e10058d4a" - integrity sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw== +jest-worker@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-30.2.0.tgz#fd5c2a36ff6058ec8f74366ec89538cc99539d26" + integrity sha512-0Q4Uk8WF7BUwqXHuAjc23vmopWJw5WH7w2tqBoUOZpOjW/ZnR44GXXd1r82RvnmI2GZge3ivrYXk/BE2+VtW2g== dependencies: "@types/node" "*" - jest-util "^29.7.0" + "@ungap/structured-clone" "^1.3.0" + jest-util "30.2.0" merge-stream "^2.0.0" - supports-color "^8.0.0" - -jest@^27.4.3: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest/-/jest-27.5.1.tgz#dadf33ba70a779be7a6fc33015843b51494f63fc" - integrity sha512-Yn0mADZB89zTtjkPJEXwrac3LHudkQMR+Paqa8uxJHCBr9agxztUifWCyiYrjhMPBoUVBjyny0I7XH6ozDr7QQ== - dependencies: - "@jest/core" "^27.5.1" - import-local "^3.0.2" - jest-cli "^27.5.1" + supports-color "^8.1.1" -jest@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/jest/-/jest-29.7.0.tgz#994676fc24177f088f1c5e3737f5697204ff2613" - integrity sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw== +jest@30.2.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/jest/-/jest-30.2.0.tgz#9f0a71e734af968f26952b5ae4b724af82681630" + integrity sha512-F26gjC0yWN8uAA5m5Ss8ZQf5nDHWGlN/xWZIh8S5SRbsEKBovwZhxGd6LJlbZYxBgCYOtreSUyb8hpXyGC5O4A== dependencies: - "@jest/core" "^29.7.0" - "@jest/types" "^29.6.3" - import-local "^3.0.2" - jest-cli "^29.7.0" + "@jest/core" "30.2.0" + "@jest/types" "30.2.0" + import-local "^3.2.0" + jest-cli "30.2.0" js-file-download@^0.4.12: version "0.4.12" resolved "https://registry.yarnpkg.com/js-file-download/-/js-file-download-0.4.12.tgz#10c70ef362559a5b23cdbdc3bd6f399c3d91d821" integrity sha512-rML+NkoD08p5Dllpjo0ffy4jRHeY6Zsapvr/W86N7E0yuzAO6qa5X9+xog6zQNlH102J7IXljNY2FtS6Lj3ucg== -js-sdsl@^4.1.4: - version "4.2.0" - resolved "https://registry.yarnpkg.com/js-sdsl/-/js-sdsl-4.2.0.tgz#278e98b7bea589b8baaf048c20aeb19eb7ad09d0" - integrity sha512-dyBIzQBDkCqCu+0upx25Y2jGdbTGxE9fshMsCdK0ViOongpV+n5tXRcZY9v7CaVQ79AGS9KA1KHtojxiM7aXSQ== - "js-tokens@^3.0.0 || ^4.0.0", js-tokens@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== -js-yaml@=4.1.0, js-yaml@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.0.tgz#c1fb65f8f5017901cdd2c951864ba18458a10602" - integrity sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA== +js-yaml@=4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.1.tgz#854c292467705b699476e1a2decc0c8a3458806b" + integrity sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA== dependencies: argparse "^2.0.1" @@ -8892,99 +5912,60 @@ js-yaml@^3.13.1: argparse "^1.0.7" esprima "^4.0.0" -jsbn@~0.1.0: - version "0.1.1" - resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" - integrity sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg== - -jsdom@^16.6.0: - version "16.7.0" - resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-16.7.0.tgz#918ae71965424b197c819f8183a754e18977b710" - integrity sha512-u9Smc2G1USStM+s/x1ru5Sxrl6mPYCbByG1U/hUmqaVsm4tbNyS7CicOSRyuGQYZhTu0h84qkZZQ/I+dzizSVw== - dependencies: - abab "^2.0.5" - acorn "^8.2.4" - acorn-globals "^6.0.0" - cssom "^0.4.4" - cssstyle "^2.3.0" - data-urls "^2.0.0" - decimal.js "^10.2.1" - domexception "^2.0.1" - escodegen "^2.0.0" - form-data "^3.0.0" - html-encoding-sniffer "^2.0.1" - http-proxy-agent "^4.0.1" - https-proxy-agent "^5.0.0" +js-yaml@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.0.tgz#c1fb65f8f5017901cdd2c951864ba18458a10602" + integrity sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA== + dependencies: + argparse "^2.0.1" + +jsdom@^26.1.0: + version "26.1.0" + resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-26.1.0.tgz#ab5f1c1cafc04bd878725490974ea5e8bf0c72b3" + integrity sha512-Cvc9WUhxSMEo4McES3P7oK3QaXldCfNWp7pl2NNeiIFlCoLr3kfq9kb1fxftiwk1FLV7CvpvDfonxtzUDeSOPg== + dependencies: + cssstyle "^4.2.1" + data-urls "^5.0.0" + decimal.js "^10.5.0" + html-encoding-sniffer "^4.0.0" + http-proxy-agent "^7.0.2" + https-proxy-agent "^7.0.6" is-potential-custom-element-name "^1.0.1" - nwsapi "^2.2.0" - parse5 "6.0.1" - saxes "^5.0.1" + nwsapi "^2.2.16" + parse5 "^7.2.1" + rrweb-cssom "^0.8.0" + saxes "^6.0.0" symbol-tree "^3.2.4" - tough-cookie "^4.0.0" - w3c-hr-time "^1.0.2" - w3c-xmlserializer "^2.0.0" - webidl-conversions "^6.1.0" - whatwg-encoding "^1.0.5" - whatwg-mimetype "^2.3.0" - whatwg-url "^8.5.0" - ws "^7.4.6" - xml-name-validator "^3.0.0" + tough-cookie "^5.1.1" + w3c-xmlserializer "^5.0.0" + webidl-conversions "^7.0.0" + whatwg-encoding "^3.1.1" + whatwg-mimetype "^4.0.0" + whatwg-url "^14.1.1" + ws "^8.18.0" + xml-name-validator "^5.0.0" jsesc@^2.5.1: version "2.5.2" resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA== +jsesc@^3.0.2: + version "3.1.0" + resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-3.1.0.tgz#74d335a234f67ed19907fdadfac7ccf9d409825d" + integrity sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA== + jsesc@~0.5.0: version "0.5.0" resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-0.5.0.tgz#e7dee66e35d6fc16f710fe91d5cf69f70f08911d" integrity sha512-uZz5UnB7u4T9LvwmFqXii7pZSouaRPorGs5who1Ip7VO0wxanFvBL7GkM6dTHlgX+jhBApRetaWpnDabOeTcnA== -json-parse-even-better-errors@^2.3.0, json-parse-even-better-errors@^2.3.1: +json-parse-even-better-errors@^2.3.0: version "2.3.1" resolved "https://registry.yarnpkg.com/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz#7c47805a94319928e05777405dc12e1f7a4ee02d" integrity sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w== -json-schema-traverse@^0.4.1: - version "0.4.1" - resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" - integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== - -json-schema-traverse@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz#ae7bcb3656ab77a73ba5c49bf654f38e6b6860e2" - integrity sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug== - -json-schema@0.4.0, json-schema@^0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.4.0.tgz#f7de4cf6efab838ebaeb3236474cbba5a1930ab5" - integrity sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA== - -json-stable-stringify-without-jsonify@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651" - integrity sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw== - -json-stable-stringify@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/json-stable-stringify/-/json-stable-stringify-1.0.2.tgz#e06f23128e0bbe342dc996ed5a19e28b57b580e0" - integrity sha512-eunSSaEnxV12z+Z73y/j5N37/In40GK4GmsSy+tEHJMxknvqnA7/djeYtAgW0GsWHUfg+847WJjKaEylk2y09g== - dependencies: - jsonify "^0.0.1" - -json-stringify-safe@~5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" - integrity sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA== - -json5@^1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.2.tgz#63d98d60f21b313b77c4d6da18bfa69d80e1d593" - integrity sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA== - dependencies: - minimist "^1.2.0" - -json5@^2.1.2, json5@^2.2.0, json5@^2.2.1: +json5@^2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.2.tgz#64471c5bdcc564c18f7c1d4df2e2297f2457c5ab" integrity sha512-46Tk9JiOL2z7ytNQWFLpj99RZkVgeHf87yGQKsIkaPz1qSH9UczKH1rO7K3wgRselo0tYMUNfecYpm/p1vC7tQ== @@ -8994,153 +5975,20 @@ json5@^2.2.3: resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283" integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg== -jsonfile@^6.0.1: - version "6.1.0" - resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae" - integrity sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ== - dependencies: - universalify "^2.0.0" - optionalDependencies: - graceful-fs "^4.1.6" - -jsonify@^0.0.1: - version "0.0.1" - resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.1.tgz#2aa3111dae3d34a0f151c63f3a45d995d9420978" - integrity sha512-2/Ki0GcmuqSrgFyelQq9M05y7PS0mEwuIzrf3f1fPqkVDVRvZrPZtVSMHxdgo8Aq0sxAOb/cr2aqqA3LeWHVPg== - -jsonpointer@^5.0.0: - version "5.0.1" - resolved "https://registry.yarnpkg.com/jsonpointer/-/jsonpointer-5.0.1.tgz#2110e0af0900fd37467b5907ecd13a7884a1b559" - integrity sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ== - -jsprim@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-2.0.2.tgz#77ca23dbcd4135cd364800d22ff82c2185803d4d" - integrity sha512-gqXddjPqQ6G40VdnI6T6yObEC+pDNvyP95wdQhkWkg7crHH3km5qP1FsOXEkzEQwnz6gz5qGTn1c2Y52wP3OyQ== - dependencies: - assert-plus "1.0.0" - extsprintf "1.3.0" - json-schema "0.4.0" - verror "1.10.0" - -"jsx-ast-utils@^2.4.1 || ^3.0.0", jsx-ast-utils@^3.3.2: - version "3.3.3" - resolved "https://registry.yarnpkg.com/jsx-ast-utils/-/jsx-ast-utils-3.3.3.tgz#76b3e6e6cece5c69d49a5792c3d01bd1a0cdc7ea" - integrity sha512-fYQHZTZ8jSfmWZ0iyzfwiU4WDX4HpHbMCZ3gPlWYiCl3BoeOTsqKBqnTVfH2rYT7eP5c3sVbeSPHnnJOaTrWiw== - dependencies: - array-includes "^3.1.5" - object.assign "^4.1.3" - -kind-of@^6.0.2: - version "6.0.3" - resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd" - integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw== - -klaw-sync@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/klaw-sync/-/klaw-sync-6.0.0.tgz#1fd2cfd56ebb6250181114f0a581167099c2b28c" - integrity sha512-nIeuVSzdCCs6TDPTqI8w1Yre34sSq7AkZ4B3sfOBbI2CgVSB4Du4aLQijFU2+lhAFCwt9+42Hel6lQNIv6AntQ== - dependencies: - graceful-fs "^4.1.11" - -kleur@^3.0.3: - version "3.0.3" - resolved "https://registry.yarnpkg.com/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e" - integrity sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w== - -klona@^2.0.4, klona@^2.0.5: - version "2.0.5" - resolved "https://registry.yarnpkg.com/klona/-/klona-2.0.5.tgz#d166574d90076395d9963aa7a928fabb8d76afbc" - integrity sha512-pJiBpiXMbt7dkzXe8Ghj/u4FfXOOa98fPW+bihOJ4SjnoijweJrNThJfd3ifXpXhREjpoF2mZVH1GfS9LV3kHQ== - -language-subtag-registry@^0.3.20: - version "0.3.22" - resolved "https://registry.yarnpkg.com/language-subtag-registry/-/language-subtag-registry-0.3.22.tgz#2e1500861b2e457eba7e7ae86877cbd08fa1fd1d" - integrity sha512-tN0MCzyWnoz/4nHS6uxdlFWoUZT7ABptwKPQ52Ea7URk6vll88bWBVhodtnlfEuCcKWNGoc+uGbw1cwa9IKh/w== - -language-tags@^1.0.5: - version "1.0.7" - resolved "https://registry.yarnpkg.com/language-tags/-/language-tags-1.0.7.tgz#41cc248730f3f12a452c2e2efe32bc0bbce67967" - integrity sha512-bSytju1/657hFjgUzPAPqszxH62ouE8nQFoFaVlIQfne4wO/wXC9A4+m8jYve7YBBvi59eq0SUpcshvG8h5Usw== - dependencies: - language-subtag-registry "^0.3.20" - -lazy-ass@^1.6.0: - version "1.6.0" - resolved "https://registry.yarnpkg.com/lazy-ass/-/lazy-ass-1.6.0.tgz#7999655e8646c17f089fdd187d150d3324d54513" - integrity sha512-cc8oEVoctTvsFZ/Oje/kGnHbpWHYBe8IAJe4C0QNc3t8uM/0Y8+erSz/7Y1ALuXTEZTMvxXwO6YbX1ey3ujiZw== - leven@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/leven/-/leven-3.1.0.tgz#77891de834064cccba82ae7842bb6b14a13ed7f2" integrity sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A== -levn@^0.4.1: - version "0.4.1" - resolved "https://registry.yarnpkg.com/levn/-/levn-0.4.1.tgz#ae4562c007473b932a6200d403268dd2fffc6ade" - integrity sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ== - dependencies: - prelude-ls "^1.2.1" - type-check "~0.4.0" - -levn@~0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/levn/-/levn-0.3.0.tgz#3b09924edf9f083c0490fdd4c0bc4421e04764ee" - integrity sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA== - dependencies: - prelude-ls "~1.1.2" - type-check "~0.3.2" - -lilconfig@^2.0.3, lilconfig@^2.0.5, lilconfig@^2.0.6: - version "2.0.6" - resolved "https://registry.yarnpkg.com/lilconfig/-/lilconfig-2.0.6.tgz#32a384558bd58af3d4c6e077dd1ad1d397bc69d4" - integrity sha512-9JROoBW7pobfsx+Sq2JsASvCo6Pfo6WWoUW79HuB1BCoBXD4PLWJPqDF6fNj67pqBYTbAHkE57M1kS/+L1neOg== - lines-and-columns@^1.1.6: version "1.2.4" resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632" integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg== -listr2@^3.8.3: - version "3.14.0" - resolved "https://registry.yarnpkg.com/listr2/-/listr2-3.14.0.tgz#23101cc62e1375fd5836b248276d1d2b51fdbe9e" - integrity sha512-TyWI8G99GX9GjE54cJ+RrNMcIFBfwMPxc3XTFiAYGN4s10hWROGtOg7+O6u6LE3mNkyld7RSLE6nrKBvTfcs3g== - dependencies: - cli-truncate "^2.1.0" - colorette "^2.0.16" - log-update "^4.0.0" - p-map "^4.0.0" - rfdc "^1.3.0" - rxjs "^7.5.1" - through "^2.3.8" - wrap-ansi "^7.0.0" - -loader-runner@^4.2.0: - version "4.3.0" - resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-4.3.0.tgz#c1b4a163b99f614830353b16755e7149ac2314e1" - integrity sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg== - -loader-utils@^2.0.0, loader-utils@^2.0.2, loader-utils@^2.0.4: - version "2.0.4" - resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-2.0.4.tgz#8b5cb38b5c34a9a018ee1fc0e6a066d1dfcc528c" - integrity sha512-xXqpXoINfFhgua9xiqD8fPFHgkoq1mmmpE92WlDbm9rNRd/EbRb+Gqf908T2DMfuHjjJlksiK2RbHVOdD/MqSw== - dependencies: - big.js "^5.2.2" - emojis-list "^3.0.0" - json5 "^2.1.2" - -loader-utils@^3.2.0: - version "3.2.1" - resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-3.2.1.tgz#4fb104b599daafd82ef3e1a41fb9265f87e1f576" - integrity sha512-ZvFw1KWS3GVyYBYb7qkmRM/WwL2TQQBxgCK62rlvm4WpVQ23Nb4tYjApUlfjrEGvOs7KHEsmyUn75OHZrJMWPw== - -locate-path@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-3.0.0.tgz#dbec3b3ab759758071b58fe59fc41871af21400e" - integrity sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A== - dependencies: - p-locate "^3.0.0" - path-exists "^3.0.0" +loader-utils@^3.3.1: + version "3.3.1" + resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-3.3.1.tgz#735b9a19fd63648ca7adbd31c2327dfe281304e5" + integrity sha512-FMJTLMXfCLMLfJxcX9PFqX5qD88Z5MRGaZCVzfuqeZSPsyiBzs+pahDQjbIWz2QIzPZz0NX9Zy4FX3lmK6YHIg== locate-path@^5.0.0: version "5.0.0" @@ -9149,80 +5997,23 @@ locate-path@^5.0.0: dependencies: p-locate "^4.1.0" -locate-path@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-6.0.0.tgz#55321eb309febbc59c4801d931a72452a681d286" - integrity sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw== - dependencies: - p-locate "^5.0.0" - lodash.debounce@^4, lodash.debounce@^4.0.8: version "4.0.8" resolved "https://registry.yarnpkg.com/lodash.debounce/-/lodash.debounce-4.0.8.tgz#82d79bff30a67c4005ffd5e2515300ad9ca4d7af" integrity sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow== -lodash.memoize@4.x, lodash.memoize@^4.1.2: - version "4.1.2" - resolved "https://registry.yarnpkg.com/lodash.memoize/-/lodash.memoize-4.1.2.tgz#bcc6c49a42a2840ed997f323eada5ecd182e0bfe" - integrity sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag== - -lodash.merge@^4.6.2: - version "4.6.2" - resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" - integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== - -lodash.once@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac" - integrity sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg== - -lodash.sortby@^4.7.0: - version "4.7.0" - resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438" - integrity sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA== - -lodash.uniq@^4.5.0: - version "4.5.0" - resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773" - integrity sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ== - -lodash@^4.15.0, lodash@^4.17.11, lodash@^4.17.15, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.7.0: - version "4.17.21" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" - integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== - -log-symbols@^4.0.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-4.1.0.tgz#3fbdbb95b4683ac9fc785111e792e558d4abd503" - integrity sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg== - dependencies: - chalk "^4.1.0" - is-unicode-supported "^0.1.0" - -log-update@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/log-update/-/log-update-4.0.0.tgz#589ecd352471f2a1c0c570287543a64dfd20e0a1" - integrity sha512-9fkkDevMefjg0mmzWFBW8YkFP91OrizzkW3diF7CpG+S2EYdy4+TVfGwz1zeF8x7hCx1ovSPTOE9Ngib74qqUg== - dependencies: - ansi-escapes "^4.3.0" - cli-cursor "^3.1.0" - slice-ansi "^4.0.0" - wrap-ansi "^6.2.0" +lodash@^4.15.0, lodash@^4.17.21: + version "4.17.23" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.23.tgz#f113b0378386103be4f6893388c73d0bde7f2c5a" + integrity sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w== -loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.4.0: +loose-envify@^1.0.0, loose-envify@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf" integrity sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q== dependencies: js-tokens "^3.0.0 || ^4.0.0" -lower-case@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/lower-case/-/lower-case-2.0.2.tgz#6fa237c63dbdc4a82ca0fd882e4722dc5e634e28" - integrity sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg== - dependencies: - tslib "^2.0.3" - lowlight@^1.17.0: version "1.20.0" resolved "https://registry.yarnpkg.com/lowlight/-/lowlight-1.20.0.tgz#ddb197d33462ad0d93bf19d17b6c301aa3941888" @@ -9231,6 +6022,16 @@ lowlight@^1.17.0: fault "^1.0.0" highlight.js "~10.7.0" +lru-cache@^10.4.3: + version "10.4.3" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-10.4.3.tgz#410fc8a17b70e598013df257c2446b7f3383f119" + integrity sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ== + +lru-cache@^11.0.0: + version "11.2.2" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-11.2.2.tgz#40fd37edffcfae4b2940379c0722dc6eeaa75f24" + integrity sha512-F9ODfyqML2coTIsQpSkRHnLSZMtkU8Q+mSfcaIyKwy58u+8k5nvAYeiNhsyMARvzNcXJ9QfWVrcPsC9e9rAxtg== + lru-cache@^5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-5.1.1.tgz#1da27e6710271947695daf6848e847f01d84b920" @@ -9245,30 +6046,18 @@ lru-cache@^6.0.0: dependencies: yallist "^4.0.0" -lz-string@^1.4.4: - version "1.4.4" - resolved "https://registry.yarnpkg.com/lz-string/-/lz-string-1.4.4.tgz#c0d8eaf36059f705796e1e344811cf4c498d3a26" - integrity sha512-0ckx7ZHRPqb0oUm8zNr+90mtf9DQB60H1wMCjBtfi62Kl3a7JbHob6gA2bC+xRvZoOL+1hzUK8jeuEIQE8svEQ== - -magic-string@^0.25.0, magic-string@^0.25.7: - version "0.25.9" - resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.25.9.tgz#de7f9faf91ef8a1c91d02c2e5314c8277dbcdd1c" - integrity sha512-RmF0AsMzgt25qzqqLc1+MbHmhdx0ojF2Fvs4XnOqz2ZOBXzzkEwc/dJQZCYHAn7v1jbVOjAZfK8msRn4BxO4VQ== - dependencies: - sourcemap-codec "^1.4.8" +lz-string@^1.5.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/lz-string/-/lz-string-1.5.0.tgz#c1ab50f77887b712621201ba9fd4e3a6ed099941" + integrity sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ== -make-dir@^3.0.0, make-dir@^3.0.2, make-dir@^3.1.0: +make-dir@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f" integrity sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw== dependencies: semver "^6.0.0" -make-error@1.x: - version "1.3.6" - resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2" - integrity sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw== - makeerror@1.0.12: version "1.0.12" resolved "https://registry.yarnpkg.com/makeerror/-/makeerror-1.0.12.tgz#3e5dd2079a82e812e983cc6610c4a2cb0eaa801a" @@ -9276,49 +6065,22 @@ makeerror@1.0.12: dependencies: tmpl "1.0.5" -mdn-data@2.0.14: - version "2.0.14" - resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-2.0.14.tgz#7113fc4281917d63ce29b43446f701e68c25ba50" - integrity sha512-dn6wd0uw5GsdswPFfsgMp5NSB0/aDe6fK94YJV/AJDYXL6HVLWBsxeq7js7Ad+mU2K9LAlwpk6kN2D5mwCPVow== - -mdn-data@2.0.4: - version "2.0.4" - resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-2.0.4.tgz#699b3c38ac6f1d728091a64650b65d388502fd5b" - integrity sha512-iV3XNKw06j5Q7mi6h+9vbx23Tv7JkjEVgKHW4pimwyDGWm0OIQntJJ+u1C6mg6mK1EaTv42XQ7w76yuzH7M2cA== - -media-typer@0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" - integrity sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ== - -memfs@^3.1.2, memfs@^3.4.3: - version "3.4.12" - resolved "https://registry.yarnpkg.com/memfs/-/memfs-3.4.12.tgz#d00f8ad8dab132dc277c659dc85bfd14b07d03bd" - integrity sha512-BcjuQn6vfqP+k100e0E9m61Hyqa//Brp+I3f0OBmN0ATHlFA8vx3Lt8z57R3u2bPqe3WGDBC+nF72fTH7isyEw== - dependencies: - fs-monkey "^1.0.3" +marked@14.0.0: + version "14.0.0" + resolved "https://registry.yarnpkg.com/marked/-/marked-14.0.0.tgz#79a1477358a59e0660276f8fec76de2c33f35d83" + integrity sha512-uIj4+faQ+MgHgwUW1l2PsPglZLOLOT1uErt06dAPtx2kjteLAkbsd/0FiYg/MGS+i7ZKLb7w2WClxHkzOOuryQ== -merge-descriptors@1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.3.tgz#d80319a65f3c7935351e5cfdac8f9318504dbed5" - integrity sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ== +math-intrinsics@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/math-intrinsics/-/math-intrinsics-1.1.0.tgz#a0dd74be81e2aa5c2f27e65ce283605ee4e2b7f9" + integrity sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g== merge-stream@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-2.0.0.tgz#52823629a14dd00c9770fb6ad47dc6310f2c1f60" integrity sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w== -merge2@^1.3.0, merge2@^1.4.1: - version "1.4.1" - resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae" - integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg== - -methods@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" - integrity sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w== - -micromatch@^4.0.2, micromatch@^4.0.4, micromatch@^4.0.5: +micromatch@^4.0.8: version "4.0.8" resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.8.tgz#d66fa18f3a47076789320b9b1af32bd86d9fa202" integrity sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA== @@ -9326,45 +6088,28 @@ micromatch@^4.0.2, micromatch@^4.0.4, micromatch@^4.0.5: braces "^3.0.3" picomatch "^2.3.1" -mime-db@1.52.0, "mime-db@>= 1.43.0 < 2": +mime-db@1.52.0: version "1.52.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== -mime-types@^2.1.12, mime-types@^2.1.27, mime-types@^2.1.31, mime-types@~2.1.17, mime-types@~2.1.19, mime-types@~2.1.24, mime-types@~2.1.34: +mime-types@^2.1.12: version "2.1.35" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== dependencies: mime-db "1.52.0" -mime@1.6.0: - version "1.6.0" - resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" - integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== - mimic-fn@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== -mimic-response@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-3.1.0.tgz#2d1d59af9c1b129815accc2c46a022a5ce1fa3c9" - integrity sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ== - min-indent@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/min-indent/-/min-indent-1.0.1.tgz#a63f681673b30571fbe8bc25686ae746eefa9869" integrity sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg== -mini-css-extract-plugin@^2.4.5: - version "2.7.2" - resolved "https://registry.yarnpkg.com/mini-css-extract-plugin/-/mini-css-extract-plugin-2.7.2.tgz#e049d3ea7d3e4e773aad585c6cb329ce0c7b72d7" - integrity sha512-EdlUizq13o0Pd+uCp+WO/JpkLvHRVGt97RqfeGhXqAcorYo1ypJSpkV+WDT0vY/kmh/p7wRdJNJtuyK540PXDw== - dependencies: - schema-utils "^4.0.0" - minim@~0.23.8: version "0.23.8" resolved "https://registry.yarnpkg.com/minim/-/minim-0.23.8.tgz#a529837afe1654f119dfb68ce7487dd8d4866b9c" @@ -9372,155 +6117,86 @@ minim@~0.23.8: dependencies: lodash "^4.15.0" -minimalistic-assert@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz#2e194de044626d4a10e7f7fbc00ce73e83e4d5c7" - integrity sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A== +minimatch@^10.1.1: + version "10.1.1" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-10.1.1.tgz#e6e61b9b0c1dcab116b5a7d1458e8b6ae9e73a55" + integrity sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ== + dependencies: + "@isaacs/brace-expansion" "^5.0.0" + +minimatch@^10.2.1: + version "10.2.4" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-10.2.4.tgz#465b3accbd0218b8281f5301e27cedc697f96fde" + integrity sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg== + dependencies: + brace-expansion "^5.0.2" -minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1, minimatch@^3.1.2: +minimatch@^3.0.4: version "3.1.2" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== dependencies: brace-expansion "^1.1.7" -minimatch@^5.0.1: - version "5.1.2" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-5.1.2.tgz#0939d7d6f0898acbd1508abe534d1929368a8fff" - integrity sha512-bNH9mmM9qsJ2X4r2Nat1B//1dJVcn3+iBLa3IgqJ7EbGaDNepL9QSHOxN4ng33s52VMMhhIfgCYDk3C4ZmlDAg== - dependencies: - brace-expansion "^2.0.1" - -minimatch@^7.4.3: - version "7.4.6" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-7.4.6.tgz#845d6f254d8f4a5e4fd6baf44d5f10c8448365fb" - integrity sha512-sBz8G/YjVniEz6lKPNpKxXwazJe4c19fEfV2GDMX6AjFz+MX9uDWIZW8XreVhkFW3fkIdTv/gxWr/Kks5FFAVw== - dependencies: - brace-expansion "^2.0.1" - -minimist@^1.2.0, minimist@^1.2.6: - version "1.2.7" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.7.tgz#daa1c4d91f507390437c6a8bc01078e7000c4d18" - integrity sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g== - -minimist@^1.2.3, minimist@^1.2.8: - version "1.2.8" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" - integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== +minipass@^7.1.2: + version "7.1.2" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.1.2.tgz#93a9626ce5e5e66bd4db86849e7515e92340a707" + integrity sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw== -mkdirp-classic@^0.5.2, mkdirp-classic@^0.5.3: - version "0.5.3" - resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" - integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A== - -mkdirp@~0.5.1: - version "0.5.6" - resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.6.tgz#7def03d2432dcae4ba1d611445c48396062255f6" - integrity sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw== - dependencies: - minimist "^1.2.6" - -monaco-editor-webpack-plugin@^7.0.1: - version "7.1.0" - resolved "https://registry.yarnpkg.com/monaco-editor-webpack-plugin/-/monaco-editor-webpack-plugin-7.1.0.tgz#16f265c2b5dbb5fe08681b6b3b7d00d3c5b2ee97" - integrity sha512-ZjnGINHN963JQkFqjjcBtn1XBtUATDZBMgNQhDQwd78w2ukRhFXAPNgWuacaQiDZsUr4h1rWv5Mv6eriKuOSzA== +monaco-editor@0.55.1: + version "0.55.1" + resolved "https://registry.yarnpkg.com/monaco-editor/-/monaco-editor-0.55.1.tgz#e74c6fe5a6bf985b817d2de3eb88d56afc494a1b" + integrity sha512-jz4x+TJNFHwHtwuV9vA9rMujcZRb0CEilTEwG2rRSpe/A7Jdkuj8xPKttCgOh+v/lkHy7HsZ64oj+q3xoAFl9A== dependencies: - loader-utils "^2.0.2" - -monaco-editor@^0.34.1: - version "0.34.1" - resolved "https://registry.yarnpkg.com/monaco-editor/-/monaco-editor-0.34.1.tgz#1b75c4ad6bc4c1f9da656d740d98e0b850a22f87" - integrity sha512-FKc80TyiMaruhJKKPz5SpJPIjL+dflGvz4CpuThaPMc94AyN7SeC9HQ8hrvaxX7EyHdJcUY5i4D0gNyJj1vSZQ== - -ms@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" - integrity sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A== + dompurify "3.2.7" + marked "14.0.0" ms@2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@2.1.3, ms@^2.1.1: +ms@^2.1.3: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== -multicast-dns@^7.2.5: - version "7.2.5" - resolved "https://registry.yarnpkg.com/multicast-dns/-/multicast-dns-7.2.5.tgz#77eb46057f4d7adbd16d9290fa7299f6fa64cced" - integrity sha512-2eznPJP8z2BFLX50tf0LuODrpINqP1RVIm/CObbTcBRITQgmC/TjcREF1NeTBzIcR5XO/ukWo+YHOjBbFwIupg== - dependencies: - dns-packet "^5.2.2" - thunky "^1.0.2" - -nan@^2.14.0: - version "2.17.0" - resolved "https://registry.yarnpkg.com/nan/-/nan-2.17.0.tgz#c0150a2368a182f033e9aa5195ec76ea41a199cb" - integrity sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ== - -nan@^2.17.0, nan@^2.18.0: - version "2.18.0" - resolved "https://registry.yarnpkg.com/nan/-/nan-2.18.0.tgz#26a6faae7ffbeb293a39660e88a76b82e30b7554" - integrity sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w== +nanoid@^3.3.11, nanoid@^3.3.7: + version "3.3.11" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.11.tgz#4f4f112cefbe303202f2199838128936266d185b" + integrity sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w== -nanoid@^3.3.4: - version "3.3.8" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.8.tgz#b1be3030bee36aaff18bacb375e5cce521684baf" - integrity sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w== - -napi-build-utils@^1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/napi-build-utils/-/napi-build-utils-1.0.2.tgz#b1fddc0b2c46e380a0b7a76f984dd47c41a13806" - integrity sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg== - -natural-compare-lite@^1.4.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/natural-compare-lite/-/natural-compare-lite-1.4.0.tgz#17b09581988979fddafe0201e931ba933c96cbb4" - integrity sha512-Tj+HTDSJJKaZnfiuw+iaF9skdPpTo2GtEly5JHnWV/hfv2Qj/9RKsGISQtLh2ox3l5EAGw487hnBee0sIJ6v2g== +napi-postinstall@^0.3.0: + version "0.3.4" + resolved "https://registry.yarnpkg.com/napi-postinstall/-/napi-postinstall-0.3.4.tgz#7af256d6588b5f8e952b9190965d6b019653bbb9" + integrity sha512-PHI5f1O0EP5xJ9gQmFGMS6IZcrVvTjpXjz7Na41gTE7eE2hK11lg04CECCYEEjdc17EV4DO+fkGEtt7TpTaTiQ== natural-compare@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" integrity sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw== -negotiator@0.6.3: - version "0.6.3" - resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd" - integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg== - -neo-async@^2.6.2: - version "2.6.2" - resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f" - integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw== - -no-case@^3.0.4: - version "3.0.4" - resolved "https://registry.yarnpkg.com/no-case/-/no-case-3.0.4.tgz#d361fd5c9800f558551a8369fc0dcd4662b6124d" - integrity sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg== - dependencies: - lower-case "^2.0.2" - tslib "^2.0.3" - -node-abi@^3.3.0: - version "3.51.0" - resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-3.51.0.tgz#970bf595ef5a26a271307f8a4befa02823d4e87d" - integrity sha512-SQkEP4hmNWjlniS5zdnfIXTk1x7Ome85RDzHlTbBtzE97Gfwz/Ipw4v/Ryk20DWIy3yCNVLVlGKApCnmvYoJbA== - dependencies: - semver "^7.3.5" +neotraverse@=0.6.18: + version "0.6.18" + resolved "https://registry.yarnpkg.com/neotraverse/-/neotraverse-0.6.18.tgz#abcb33dda2e8e713cf6321b29405e822230cdb30" + integrity sha512-Z4SmBUweYa09+o6pG+eASabEpP6QkQ70yHj351pQoEXIs8uHbaU2DWVmzBANKgflPa47A50PtB2+NgRpQvr7vA== node-abort-controller@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.1.1.tgz#a94377e964a9a37ac3976d848cb5c765833b8548" integrity sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ== +node-addon-api@^8.0.0, node-addon-api@^8.2.2, node-addon-api@^8.3.0, node-addon-api@^8.3.1: + version "8.5.0" + resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-8.5.0.tgz#c91b2d7682fa457d2e1c388150f0dff9aafb8f3f" + integrity sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A== + node-domexception@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== -node-fetch-commonjs@^3.3.1: +node-fetch-commonjs@^3.3.2: version "3.3.2" resolved "https://registry.yarnpkg.com/node-fetch-commonjs/-/node-fetch-commonjs-3.3.2.tgz#0dd0fd4c4a314c5234f496ff7b5d9ce5a6c8feaa" integrity sha512-VBlAiynj3VMLrotgwOS3OyECFxas5y7ltLcK4t41lMUZeaK15Ym4QRkqN0EQKAFL42q9i21EPKjzLUPfltR72A== @@ -9528,92 +6204,53 @@ node-fetch-commonjs@^3.3.1: node-domexception "^1.0.0" web-streams-polyfill "^3.0.3" -node-forge@^1: - version "1.3.1" - resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" - integrity sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA== +node-gyp-build@^4.8.0, node-gyp-build@^4.8.2, node-gyp-build@^4.8.4: + version "4.8.4" + resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.8.4.tgz#8a70ee85464ae52327772a90d66c6077a900cfc8" + integrity sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ== node-int64@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b" integrity sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw== -node-releases@^2.0.13: - version "2.0.13" - resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.13.tgz#d5ed1627c23e3461e819b02e57b75e4899b1c81d" - integrity sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ== - -node-releases@^2.0.18: - version "2.0.18" - resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.18.tgz#f010e8d35e2fe8d6b2944f03f70213ecedc4ca3f" - integrity sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g== +node-releases@^2.0.27: + version "2.0.27" + resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.27.tgz#eedca519205cf20f650f61d56b070db111231e4e" + integrity sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA== node-releases@^2.0.6: version "2.0.8" resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.8.tgz#0f349cdc8fcfa39a92ac0be9bc48b7706292b9ae" integrity sha512-dFSmB8fFHEH/s81Xi+Y/15DQY6VHW81nXRj86EMSL3lmuTmK1e+aT4wrFCkTbm+gSwkw4KpX+rT/pMM2c1mF+A== -normalize-path@^3.0.0, normalize-path@~3.0.0: +normalize-path@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65" integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA== -normalize-range@^0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/normalize-range/-/normalize-range-0.1.2.tgz#2d10c06bdfd312ea9777695a4d28439456b75942" - integrity sha512-bdok/XvKII3nUpklnV6P2hxtMNrCboOjAcyBuQnWEhO665FwrSNRxU+AqpsyvO6LgGYPspN+lu5CLtw4jPRKNA== - -normalize-url@^6.0.1: - version "6.1.0" - resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-6.1.0.tgz#40d0885b535deffe3f3147bec877d05fe4c5668a" - integrity sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A== - -npm-run-path@^4.0.0, npm-run-path@^4.0.1: +npm-run-path@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-4.0.1.tgz#b7ecd1e5ed53da8e37a55e1c2269e0b97ed748ea" integrity sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw== dependencies: path-key "^3.0.0" -nth-check@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-1.0.2.tgz#b2bd295c37e3dd58a3bf0700376663ba4d9cf05c" - integrity sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg== - dependencies: - boolbase "~1.0.0" - -nth-check@^2.0.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-2.1.1.tgz#c9eab428effce36cd6b92c924bdb000ef1f1ed1d" - integrity sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w== - dependencies: - boolbase "^1.0.0" - -nwsapi@^2.2.0: - version "2.2.2" - resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.2.2.tgz#e5418863e7905df67d51ec95938d67bf801f0bb0" - integrity sha512-90yv+6538zuvUMnN+zCr8LuV6bPFdq50304114vJYJ8RDyK8D5O9Phpbd6SZWgI7PwzmmfN1upeOJlvybDSgCw== +nwsapi@^2.2.16: + version "2.2.22" + resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.2.22.tgz#109f9530cda6c156d6a713cdf5939e9f0de98b9d" + integrity sha512-ujSMe1OWVn55euT1ihwCI1ZcAaAU3nxUiDwfDQldc51ZXaB9m2AyOn6/jh1BLe2t/G8xd6uKG1UBF2aZJeg2SQ== object-assign@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" integrity sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg== -object-hash@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-3.0.0.tgz#73f97f753e7baffc0e2cc9d6e079079744ac82e9" - integrity sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw== - -object-inspect@^1.12.2, object-inspect@^1.9.0: +object-inspect@^1.9.0: version "1.12.2" resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.2.tgz#c0641f26394532f28ab8d796ab954e43c009a8ea" integrity sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ== -object-inspect@^1.13.1: - version "1.13.2" - resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.13.2.tgz#dea0088467fb991e67af4058147a24824a3043ff" - integrity sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g== - object-is@^1.1.5: version "1.1.5" resolved "https://registry.yarnpkg.com/object-is/-/object-is-1.1.5.tgz#b9deeaa5fc7f1846a0faecdceec138e5778f53ac" @@ -9627,7 +6264,7 @@ object-keys@^1.1.1: resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e" integrity sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA== -object.assign@^4.1.3, object.assign@^4.1.4: +object.assign@^4.1.4: version "4.1.4" resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.1.4.tgz#9673c7c7c351ab8c4d0b516f4343ebf4dfb7799f" integrity sha512-1mxKf0e58bvyjSCtKYY4sRe9itRk3PJpquJOjeIkz885CczcI4IvJJDLPS72oowuSh+pBxUFROpX+TU++hxhZQ== @@ -9637,154 +6274,41 @@ object.assign@^4.1.3, object.assign@^4.1.4: has-symbols "^1.0.3" object-keys "^1.1.1" -object.entries@^1.1.6: - version "1.1.6" - resolved "https://registry.yarnpkg.com/object.entries/-/object.entries-1.1.6.tgz#9737d0e5b8291edd340a3e3264bb8a3b00d5fa23" - integrity sha512-leTPzo4Zvg3pmbQ3rDK69Rl8GQvIqMWubrkxONG9/ojtFE2rD9fjMKfSI5BxW3osRH1m6VdzmqK8oAY9aT4x5w== - dependencies: - call-bind "^1.0.2" - define-properties "^1.1.4" - es-abstract "^1.20.4" - -object.fromentries@^2.0.6: - version "2.0.6" - resolved "https://registry.yarnpkg.com/object.fromentries/-/object.fromentries-2.0.6.tgz#cdb04da08c539cffa912dcd368b886e0904bfa73" - integrity sha512-VciD13dswC4j1Xt5394WR4MzmAQmlgN72phd/riNp9vtD7tp4QQWJ0R4wvclXcafgcYK8veHRed2W6XeGBvcfg== - dependencies: - call-bind "^1.0.2" - define-properties "^1.1.4" - es-abstract "^1.20.4" - -object.getownpropertydescriptors@^2.1.0: - version "2.1.5" - resolved "https://registry.yarnpkg.com/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.5.tgz#db5a9002489b64eef903df81d6623c07e5b4b4d3" - integrity sha512-yDNzckpM6ntyQiGTik1fKV1DcVDRS+w8bvpWNCBanvH5LfRX9O8WTHqQzG4RZwRAM4I0oU7TV11Lj5v0g20ibw== - dependencies: - array.prototype.reduce "^1.0.5" - call-bind "^1.0.2" - define-properties "^1.1.4" - es-abstract "^1.20.4" - -object.hasown@^1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/object.hasown/-/object.hasown-1.1.2.tgz#f919e21fad4eb38a57bc6345b3afd496515c3f92" - integrity sha512-B5UIT3J1W+WuWIU55h0mjlwaqxiE5vYENJXIXZ4VFe05pNYrkKuK0U/6aFcb0pKywYJh7IhfoqUfKVmrJJHZHw== - dependencies: - define-properties "^1.1.4" - es-abstract "^1.20.4" - -object.values@^1.1.0, object.values@^1.1.5, object.values@^1.1.6: - version "1.1.6" - resolved "https://registry.yarnpkg.com/object.values/-/object.values-1.1.6.tgz#4abbaa71eba47d63589d402856f908243eea9b1d" - integrity sha512-FVVTkD1vENCsAcwNs9k6jea2uHC/X0+JcjG8YA60FN5CMaJmG95wT9jek/xX9nornqGRrBkKtzuAu2wuHpKqvw== - dependencies: - call-bind "^1.0.2" - define-properties "^1.1.4" - es-abstract "^1.20.4" - -obuf@^1.0.0, obuf@^1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/obuf/-/obuf-1.1.2.tgz#09bea3343d41859ebd446292d11c9d4db619084e" - integrity sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg== - -on-finished@2.4.1: - version "2.4.1" - resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.4.1.tgz#58c8c44116e54845ad57f14ab10b03533184ac3f" - integrity sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg== - dependencies: - ee-first "1.1.1" - -on-headers@~1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/on-headers/-/on-headers-1.0.2.tgz#772b0ae6aaa525c399e489adfad90c403eb3c28f" - integrity sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA== - -once@^1.3.0, once@^1.3.1, once@^1.4.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" - integrity sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w== - dependencies: - wrappy "1" - -onetime@^5.1.0, onetime@^5.1.2: +onetime@^5.1.2: version "5.1.2" resolved "https://registry.yarnpkg.com/onetime/-/onetime-5.1.2.tgz#d0e96ebb56b07476df1dd9c4806e5237985ca45e" integrity sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg== dependencies: mimic-fn "^2.1.0" -open@^7.4.2: - version "7.4.2" - resolved "https://registry.yarnpkg.com/open/-/open-7.4.2.tgz#b8147e26dcf3e426316c730089fd71edd29c2321" - integrity sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q== - dependencies: - is-docker "^2.0.0" - is-wsl "^2.1.1" - -open@^8.0.9, open@^8.4.0: - version "8.4.0" - resolved "https://registry.yarnpkg.com/open/-/open-8.4.0.tgz#345321ae18f8138f82565a910fdc6b39e8c244f8" - integrity sha512-XgFPPM+B28FtCCgSb9I+s9szOC1vZRSwgWsRUA5ylIxRTgKozqjOCrVOqGsYABPYK5qnfqClxZTFBa8PKt2v6Q== - dependencies: - define-lazy-prop "^2.0.0" - is-docker "^2.1.1" - is-wsl "^2.2.0" - -optionator@^0.8.1: - version "0.8.3" - resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.8.3.tgz#84fa1d036fe9d3c7e21d99884b601167ec8fb495" - integrity sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA== +openapi-path-templating@^2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/openapi-path-templating/-/openapi-path-templating-2.2.1.tgz#57026767530667096d33d7362382a93d75d497f6" + integrity sha512-eN14VrDvl/YyGxxrkGOHkVkWEoPyhyeydOUrbvjoz8K5eIGgELASwN1eqFOJ2CTQMGCy2EntOK1KdtJ8ZMekcg== dependencies: - deep-is "~0.1.3" - fast-levenshtein "~2.0.6" - levn "~0.3.0" - prelude-ls "~1.1.2" - type-check "~0.3.2" - word-wrap "~1.2.3" + apg-lite "^1.0.4" -optionator@^0.9.1: - version "0.9.1" - resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.1.tgz#4f236a6373dae0566a6d43e1326674f50c291499" - integrity sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw== +openapi-server-url-templating@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/openapi-server-url-templating/-/openapi-server-url-templating-1.3.0.tgz#80bc6ea5209a3c4fe9d359673ba51635676e2503" + integrity sha512-DPlCms3KKEbjVQb0spV6Awfn6UWNheuG/+folQPzh/wUaKwuqvj8zt5gagD7qoyxtE03cIiKPgLFS3Q8Bz00uQ== dependencies: - deep-is "^0.1.3" - fast-levenshtein "^2.0.6" - levn "^0.4.1" - prelude-ls "^1.2.1" - type-check "^0.4.0" - word-wrap "^1.2.3" - -os-tmpdir@~1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" - integrity sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g== - -ospath@^1.2.2: - version "1.2.2" - resolved "https://registry.yarnpkg.com/ospath/-/ospath-1.2.2.tgz#1276639774a3f8ef2572f7fe4280e0ea4550c07b" - integrity sha512-o6E5qJV5zkAbIDNhGSIlyOhScKXgQrSRMilfph0clDfM0nEnBOlKlH4sWDmG95BW/CvwNz0vmm7dJVtU2KlMiA== + apg-lite "^1.0.4" -p-limit@^2.0.0, p-limit@^2.2.0: +p-limit@^2.2.0: version "2.3.0" resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1" integrity sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w== dependencies: p-try "^2.0.0" -p-limit@^3.0.2, p-limit@^3.1.0: +p-limit@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b" integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ== dependencies: yocto-queue "^0.1.0" -p-locate@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-3.0.0.tgz#322d69a05c0264b25997d9f40cd8a891ab0064a4" - integrity sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ== - dependencies: - p-limit "^2.0.0" - p-locate@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-4.1.0.tgz#a3428bb7088b3a60292f66919278b7c297ad4f07" @@ -9792,40 +6316,15 @@ p-locate@^4.1.0: dependencies: p-limit "^2.2.0" -p-locate@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-5.0.0.tgz#83c8315c6785005e3bd021839411c9e110e6d834" - integrity sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw== - dependencies: - p-limit "^3.0.2" - -p-map@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/p-map/-/p-map-4.0.0.tgz#bb2f95a5eda2ec168ec9274e06a747c3e2904d2b" - integrity sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ== - dependencies: - aggregate-error "^3.0.0" - -p-retry@^4.5.0: - version "4.6.2" - resolved "https://registry.yarnpkg.com/p-retry/-/p-retry-4.6.2.tgz#9baae7184057edd4e17231cee04264106e092a16" - integrity sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ== - dependencies: - "@types/retry" "0.12.0" - retry "^0.13.1" - p-try@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6" integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ== -param-case@^3.0.4: - version "3.0.4" - resolved "https://registry.yarnpkg.com/param-case/-/param-case-3.0.4.tgz#7d17fe4aa12bde34d4a77d91acfb6219caad01c5" - integrity sha512-RXlj7zCYokReqWpOPH9oYivUzLYZ5vAPIfEmCTNViosC78F8F0H9y7T7gG2M39ymgutxF5gcFEsyZQSph9Bp3A== - dependencies: - dot-case "^3.0.4" - tslib "^2.0.3" +package-json-from-dist@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz#4f1471a010827a86f94cfd9b0727e36d267de505" + integrity sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw== parent-module@^1.0.0: version "1.0.1" @@ -9834,17 +6333,18 @@ parent-module@^1.0.0: dependencies: callsites "^3.0.0" -parse-entities@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/parse-entities/-/parse-entities-2.0.0.tgz#53c6eb5b9314a1f4ec99fa0fdf7ce01ecda0cbe8" - integrity sha512-kkywGpCcRYhqQIchaWqZ875wzpS/bMKhz5HnN3p7wveJTkTtyAB/AlnS0f8DFSqYW1T82t6yEAkEcB+A1I3MbQ== +parse-entities@^4.0.0: + version "4.0.2" + resolved "https://registry.yarnpkg.com/parse-entities/-/parse-entities-4.0.2.tgz#61d46f5ed28e4ee62e9ddc43d6b010188443f159" + integrity sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw== dependencies: - character-entities "^1.0.0" - character-entities-legacy "^1.0.0" - character-reference-invalid "^1.0.0" - is-alphanumerical "^1.0.0" - is-decimal "^1.0.0" - is-hexadecimal "^1.0.0" + "@types/unist" "^2.0.0" + character-entities-legacy "^3.0.0" + character-reference-invalid "^2.0.0" + decode-named-character-reference "^1.0.0" + is-alphanumerical "^2.0.0" + is-decimal "^2.0.0" + is-hexadecimal "^2.0.0" parse-json@^5.0.0, parse-json@^5.2.0: version "5.2.0" @@ -9856,60 +6356,18 @@ parse-json@^5.0.0, parse-json@^5.2.0: json-parse-even-better-errors "^2.3.0" lines-and-columns "^1.1.6" -parse5@6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/parse5/-/parse5-6.0.1.tgz#e1a1c085c569b3dc08321184f19a39cc27f7c30b" - integrity sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw== - -parseurl@~1.3.2, parseurl@~1.3.3: - version "1.3.3" - resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4" - integrity sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ== - -pascal-case@^3.1.2: - version "3.1.2" - resolved "https://registry.yarnpkg.com/pascal-case/-/pascal-case-3.1.2.tgz#b48e0ef2b98e205e7c1dae747d0b1508237660eb" - integrity sha512-uWlGT3YSnK9x3BQJaOdcZwrnV6hPpd8jFH1/ucpiLRPh/2zCVJKS19E4GvYHvaCcACn3foXZ0cLB9Wrx1KGe5g== +parse5@^7.0.0, parse5@^7.2.1: + version "7.3.0" + resolved "https://registry.yarnpkg.com/parse5/-/parse5-7.3.0.tgz#d7e224fa72399c7a175099f45fc2ad024b05ec05" + integrity sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw== dependencies: - no-case "^3.0.4" - tslib "^2.0.3" - -patch-package@^8.0.0: - version "8.0.0" - resolved "https://registry.yarnpkg.com/patch-package/-/patch-package-8.0.0.tgz#d191e2f1b6e06a4624a0116bcb88edd6714ede61" - integrity sha512-da8BVIhzjtgScwDJ2TtKsfT5JFWz1hYoBl9rUQ1f38MC2HwnEIkK8VN3dKMKcP7P7bvvgzNDbfNHtx3MsQb5vA== - dependencies: - "@yarnpkg/lockfile" "^1.1.0" - chalk "^4.1.2" - ci-info "^3.7.0" - cross-spawn "^7.0.3" - find-yarn-workspace-root "^2.0.0" - fs-extra "^9.0.0" - json-stable-stringify "^1.0.2" - klaw-sync "^6.0.0" - minimist "^1.2.6" - open "^7.4.2" - rimraf "^2.6.3" - semver "^7.5.3" - slash "^2.0.0" - tmp "^0.0.33" - yaml "^2.2.2" - -path-exists@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-3.0.0.tgz#ce0ebeaa5f78cb18925ea7d810d7b59b010fd515" - integrity sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ== + entities "^6.0.0" path-exists@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-4.0.0.tgz#513bdbe2d3b95d7762e8c1137efa195c6c61b5b3" integrity sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w== -path-is-absolute@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" - integrity sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg== - path-key@^3.0.0, path-key@^3.1.0: version "3.1.1" resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375" @@ -9920,669 +6378,108 @@ path-parse@^1.0.7: resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== -path-to-regexp@0.1.10: - version "0.1.10" - resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.10.tgz#67e9108c5c0551b9e5326064387de4763c4d5f8b" - integrity sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w== +path-scurry@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/path-scurry/-/path-scurry-2.0.1.tgz#4b6572376cfd8b811fca9cd1f5c24b3cbac0fe10" + integrity sha512-oWyT4gICAu+kaA7QWk/jvCHWarMKNs6pXOGWKDTr7cw4IGcUbW+PeTfbaQiLGheFRpjo6O9J0PmyMfQPjH71oA== + dependencies: + lru-cache "^11.0.0" + minipass "^7.1.2" path-type@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== -pend@~1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" - integrity sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg== - performance-now@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" integrity sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow== -picocolors@^0.2.1: - version "0.2.1" - resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-0.2.1.tgz#570670f793646851d1ba135996962abad587859f" - integrity sha512-cMlDqaLEqfSaW8Z7N5Jw+lyIW869EzT73/F5lhtY9cLGoVxSXznfgfXMO0Z5K0o0Q2TkTXq+0KFsdnSe3jDViA== +picocolors@1.1.1, picocolors@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.1.1.tgz#3d321af3eab939b083c8f929a1d12cda81c26b6b" + integrity sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA== picocolors@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ== -picocolors@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.1.tgz#a8ad579b571952f0e5d25892de5445bcfe25aaa1" - integrity sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew== - -picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.2, picomatch@^2.2.3, picomatch@^2.3.0, picomatch@^2.3.1: +picomatch@^2.0.4, picomatch@^2.3.1: version "2.3.1" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== -pify@^2.2.0, pify@^2.3.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c" - integrity sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog== +picomatch@^4.0.2, picomatch@^4.0.3: + version "4.0.3" + resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-4.0.3.tgz#796c76136d1eead715db1e7bad785dedd695a042" + integrity sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q== -pirates@^4.0.4: - version "4.0.5" - resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.5.tgz#feec352ea5c3268fb23a37c702ab1699f35a5f3b" - integrity sha512-8V9+HQPupnaXMA23c5hvl69zXvTwTzyAYasnkb0Tts4XvO4CliqONMOnvlq26rkhLC3nWDFBJf73LU1e1VZLaQ== +pirates@^4.0.7: + version "4.0.7" + resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.7.tgz#643b4a18c4257c8a65104b73f3049ce9a0a15e22" + integrity sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA== -pkg-dir@^4.1.0, pkg-dir@^4.2.0: +pkg-dir@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3" integrity sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ== dependencies: find-up "^4.0.0" -pkg-up@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/pkg-up/-/pkg-up-3.1.0.tgz#100ec235cc150e4fd42519412596a28512a0def5" - integrity sha512-nDywThFk1i4BQK4twPQ6TA4RT8bDY96yeuCVBWL3ePARCiEKDRSrNGbFIgUJpLp+XeIR65v8ra7WuJOFUBtkMA== - dependencies: - find-up "^3.0.0" - -postcss-attribute-case-insensitive@^5.0.2: - version "5.0.2" - resolved "https://registry.yarnpkg.com/postcss-attribute-case-insensitive/-/postcss-attribute-case-insensitive-5.0.2.tgz#03d761b24afc04c09e757e92ff53716ae8ea2741" - integrity sha512-XIidXV8fDr0kKt28vqki84fRK8VW8eTuIa4PChv2MqKuT6C9UjmSKzen6KaWhWEoYvwxFCa7n/tC1SZ3tyq4SQ== - dependencies: - postcss-selector-parser "^6.0.10" - -postcss-browser-comments@^4: - version "4.0.0" - resolved "https://registry.yarnpkg.com/postcss-browser-comments/-/postcss-browser-comments-4.0.0.tgz#bcfc86134df5807f5d3c0eefa191d42136b5e72a" - integrity sha512-X9X9/WN3KIvY9+hNERUqX9gncsgBA25XaeR+jshHz2j8+sYyHktHw1JdKuMjeLpGktXidqDhA7b/qm1mrBDmgg== - -postcss-calc@^8.2.3: - version "8.2.4" - resolved "https://registry.yarnpkg.com/postcss-calc/-/postcss-calc-8.2.4.tgz#77b9c29bfcbe8a07ff6693dc87050828889739a5" - integrity sha512-SmWMSJmB8MRnnULldx0lQIyhSNvuDl9HfrZkaqqE/WHAhToYsAvDq+yAsA/kIyINDszOp3Rh0GFoNuH5Ypsm3Q== - dependencies: - postcss-selector-parser "^6.0.9" - postcss-value-parser "^4.2.0" - -postcss-clamp@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/postcss-clamp/-/postcss-clamp-4.1.0.tgz#7263e95abadd8c2ba1bd911b0b5a5c9c93e02363" - integrity sha512-ry4b1Llo/9zz+PKC+030KUnPITTJAHeOwjfAyyB60eT0AorGLdzp52s31OsPRHRf8NchkgFoG2y6fCfn1IV1Ow== - dependencies: - postcss-value-parser "^4.2.0" - -postcss-color-functional-notation@^4.2.4: - version "4.2.4" - resolved "https://registry.yarnpkg.com/postcss-color-functional-notation/-/postcss-color-functional-notation-4.2.4.tgz#21a909e8d7454d3612d1659e471ce4696f28caec" - integrity sha512-2yrTAUZUab9s6CpxkxC4rVgFEVaR6/2Pipvi6qcgvnYiVqZcbDHEoBDhrXzyb7Efh2CCfHQNtcqWcIruDTIUeg== - dependencies: - postcss-value-parser "^4.2.0" - -postcss-color-hex-alpha@^8.0.4: - version "8.0.4" - resolved "https://registry.yarnpkg.com/postcss-color-hex-alpha/-/postcss-color-hex-alpha-8.0.4.tgz#c66e2980f2fbc1a63f5b079663340ce8b55f25a5" - integrity sha512-nLo2DCRC9eE4w2JmuKgVA3fGL3d01kGq752pVALF68qpGLmx2Qrk91QTKkdUqqp45T1K1XV8IhQpcu1hoAQflQ== - dependencies: - postcss-value-parser "^4.2.0" - -postcss-color-rebeccapurple@^7.1.1: - version "7.1.1" - resolved "https://registry.yarnpkg.com/postcss-color-rebeccapurple/-/postcss-color-rebeccapurple-7.1.1.tgz#63fdab91d878ebc4dd4b7c02619a0c3d6a56ced0" - integrity sha512-pGxkuVEInwLHgkNxUc4sdg4g3py7zUeCQ9sMfwyHAT+Ezk8a4OaaVZ8lIY5+oNqA/BXXgLyXv0+5wHP68R79hg== - dependencies: - postcss-value-parser "^4.2.0" - -postcss-colormin@^5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/postcss-colormin/-/postcss-colormin-5.3.0.tgz#3cee9e5ca62b2c27e84fce63affc0cfb5901956a" - integrity sha512-WdDO4gOFG2Z8n4P8TWBpshnL3JpmNmJwdnfP2gbk2qBA8PWwOYcmjmI/t3CmMeL72a7Hkd+x/Mg9O2/0rD54Pg== - dependencies: - browserslist "^4.16.6" - caniuse-api "^3.0.0" - colord "^2.9.1" - postcss-value-parser "^4.2.0" - -postcss-convert-values@^5.1.3: - version "5.1.3" - resolved "https://registry.yarnpkg.com/postcss-convert-values/-/postcss-convert-values-5.1.3.tgz#04998bb9ba6b65aa31035d669a6af342c5f9d393" - integrity sha512-82pC1xkJZtcJEfiLw6UXnXVXScgtBrjlO5CBmuDQc+dlb88ZYheFsjTn40+zBVi3DkfF7iezO0nJUPLcJK3pvA== - dependencies: - browserslist "^4.21.4" - postcss-value-parser "^4.2.0" - -postcss-custom-media@^8.0.2: - version "8.0.2" - resolved "https://registry.yarnpkg.com/postcss-custom-media/-/postcss-custom-media-8.0.2.tgz#c8f9637edf45fef761b014c024cee013f80529ea" - integrity sha512-7yi25vDAoHAkbhAzX9dHx2yc6ntS4jQvejrNcC+csQJAXjj15e7VcWfMgLqBNAbOvqi5uIa9huOVwdHbf+sKqg== - dependencies: - postcss-value-parser "^4.2.0" - -postcss-custom-properties@^12.1.10: - version "12.1.11" - resolved "https://registry.yarnpkg.com/postcss-custom-properties/-/postcss-custom-properties-12.1.11.tgz#d14bb9b3989ac4d40aaa0e110b43be67ac7845cf" - integrity sha512-0IDJYhgU8xDv1KY6+VgUwuQkVtmYzRwu+dMjnmdMafXYv86SWqfxkc7qdDvWS38vsjaEtv8e0vGOUQrAiMBLpQ== - dependencies: - postcss-value-parser "^4.2.0" - -postcss-custom-selectors@^6.0.3: - version "6.0.3" - resolved "https://registry.yarnpkg.com/postcss-custom-selectors/-/postcss-custom-selectors-6.0.3.tgz#1ab4684d65f30fed175520f82d223db0337239d9" - integrity sha512-fgVkmyiWDwmD3JbpCmB45SvvlCD6z9CG6Ie6Iere22W5aHea6oWa7EM2bpnv2Fj3I94L3VbtvX9KqwSi5aFzSg== - dependencies: - postcss-selector-parser "^6.0.4" - -postcss-dir-pseudo-class@^6.0.5: - version "6.0.5" - resolved "https://registry.yarnpkg.com/postcss-dir-pseudo-class/-/postcss-dir-pseudo-class-6.0.5.tgz#2bf31de5de76added44e0a25ecf60ae9f7c7c26c" - integrity sha512-eqn4m70P031PF7ZQIvSgy9RSJ5uI2171O/OO/zcRNYpJbvaeKFUlar1aJ7rmgiQtbm0FSPsRewjpdS0Oew7MPA== - dependencies: - postcss-selector-parser "^6.0.10" - -postcss-discard-comments@^5.1.2: - version "5.1.2" - resolved "https://registry.yarnpkg.com/postcss-discard-comments/-/postcss-discard-comments-5.1.2.tgz#8df5e81d2925af2780075840c1526f0660e53696" - integrity sha512-+L8208OVbHVF2UQf1iDmRcbdjJkuBF6IS29yBDSiWUIzpYaAhtNl6JYnYm12FnkeCwQqF5LeklOu6rAqgfBZqQ== +playwright-core@1.58.2: + version "1.58.2" + resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.58.2.tgz#ac5f5b4b10d29bcf934415f0b8d133b34b0dcb13" + integrity sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg== -postcss-discard-duplicates@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/postcss-discard-duplicates/-/postcss-discard-duplicates-5.1.0.tgz#9eb4fe8456706a4eebd6d3b7b777d07bad03e848" - integrity sha512-zmX3IoSI2aoenxHV6C7plngHWWhUOV3sP1T8y2ifzxzbtnuhk1EdPwm0S1bIUNaJ2eNbWeGLEwzw8huPD67aQw== - -postcss-discard-empty@^5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/postcss-discard-empty/-/postcss-discard-empty-5.1.1.tgz#e57762343ff7f503fe53fca553d18d7f0c369c6c" - integrity sha512-zPz4WljiSuLWsI0ir4Mcnr4qQQ5e1Ukc3i7UfE2XcrwKK2LIPIqE5jxMRxO6GbI3cv//ztXDsXwEWT3BHOGh3A== - -postcss-discard-overridden@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/postcss-discard-overridden/-/postcss-discard-overridden-5.1.0.tgz#7e8c5b53325747e9d90131bb88635282fb4a276e" - integrity sha512-21nOL7RqWR1kasIVdKs8HNqQJhFxLsyRfAnUDm4Fe4t4mCWL9OJiHvlHPjcd8zc5Myu89b/7wZDnOSjFgeWRtw== - -postcss-double-position-gradients@^3.1.2: - version "3.1.2" - resolved "https://registry.yarnpkg.com/postcss-double-position-gradients/-/postcss-double-position-gradients-3.1.2.tgz#b96318fdb477be95997e86edd29c6e3557a49b91" - integrity sha512-GX+FuE/uBR6eskOK+4vkXgT6pDkexLokPaz/AbJna9s5Kzp/yl488pKPjhy0obB475ovfT1Wv8ho7U/cHNaRgQ== - dependencies: - "@csstools/postcss-progressive-custom-properties" "^1.1.0" - postcss-value-parser "^4.2.0" - -postcss-env-function@^4.0.6: - version "4.0.6" - resolved "https://registry.yarnpkg.com/postcss-env-function/-/postcss-env-function-4.0.6.tgz#7b2d24c812f540ed6eda4c81f6090416722a8e7a" - integrity sha512-kpA6FsLra+NqcFnL81TnsU+Z7orGtDTxcOhl6pwXeEq1yFPpRMkCDpHhrz8CFQDr/Wfm0jLiNQ1OsGGPjlqPwA== +playwright@1.58.2: + version "1.58.2" + resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.58.2.tgz#afe547164539b0bcfcb79957394a7a3fa8683cfd" + integrity sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A== dependencies: - postcss-value-parser "^4.2.0" - -postcss-flexbugs-fixes@^5.0.2: - version "5.0.2" - resolved "https://registry.yarnpkg.com/postcss-flexbugs-fixes/-/postcss-flexbugs-fixes-5.0.2.tgz#2028e145313074fc9abe276cb7ca14e5401eb49d" - integrity sha512-18f9voByak7bTktR2QgDveglpn9DTbBWPUzSOe9g0N4WR/2eSt6Vrcbf0hmspvMI6YWGywz6B9f7jzpFNJJgnQ== - -postcss-focus-visible@^6.0.4: - version "6.0.4" - resolved "https://registry.yarnpkg.com/postcss-focus-visible/-/postcss-focus-visible-6.0.4.tgz#50c9ea9afa0ee657fb75635fabad25e18d76bf9e" - integrity sha512-QcKuUU/dgNsstIK6HELFRT5Y3lbrMLEOwG+A4s5cA+fx3A3y/JTq3X9LaOj3OC3ALH0XqyrgQIgey/MIZ8Wczw== - dependencies: - postcss-selector-parser "^6.0.9" - -postcss-focus-within@^5.0.4: - version "5.0.4" - resolved "https://registry.yarnpkg.com/postcss-focus-within/-/postcss-focus-within-5.0.4.tgz#5b1d2ec603195f3344b716c0b75f61e44e8d2e20" - integrity sha512-vvjDN++C0mu8jz4af5d52CB184ogg/sSxAFS+oUJQq2SuCe7T5U2iIsVJtsCp2d6R4j0jr5+q3rPkBVZkXD9fQ== - dependencies: - postcss-selector-parser "^6.0.9" - -postcss-font-variant@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/postcss-font-variant/-/postcss-font-variant-5.0.0.tgz#efd59b4b7ea8bb06127f2d031bfbb7f24d32fa66" - integrity sha512-1fmkBaCALD72CK2a9i468mA/+tr9/1cBxRRMXOUaZqO43oWPR5imcyPjXwuv7PXbCid4ndlP5zWhidQVVa3hmA== - -postcss-gap-properties@^3.0.5: - version "3.0.5" - resolved "https://registry.yarnpkg.com/postcss-gap-properties/-/postcss-gap-properties-3.0.5.tgz#f7e3cddcf73ee19e94ccf7cb77773f9560aa2fff" - integrity sha512-IuE6gKSdoUNcvkGIqdtjtcMtZIFyXZhmFd5RUlg97iVEvp1BZKV5ngsAjCjrVy+14uhGBQl9tzmi1Qwq4kqVOg== - -postcss-image-set-function@^4.0.7: - version "4.0.7" - resolved "https://registry.yarnpkg.com/postcss-image-set-function/-/postcss-image-set-function-4.0.7.tgz#08353bd756f1cbfb3b6e93182c7829879114481f" - integrity sha512-9T2r9rsvYzm5ndsBE8WgtrMlIT7VbtTfE7b3BQnudUqnBcBo7L758oc+o+pdj/dUV0l5wjwSdjeOH2DZtfv8qw== - dependencies: - postcss-value-parser "^4.2.0" - -postcss-import@^14.1.0: - version "14.1.0" - resolved "https://registry.yarnpkg.com/postcss-import/-/postcss-import-14.1.0.tgz#a7333ffe32f0b8795303ee9e40215dac922781f0" - integrity sha512-flwI+Vgm4SElObFVPpTIT7SU7R3qk2L7PyduMcokiaVKuWv9d/U+Gm/QAd8NDLuykTWTkcrjOeD2Pp1rMeBTGw== - dependencies: - postcss-value-parser "^4.0.0" - read-cache "^1.0.0" - resolve "^1.1.7" - -postcss-initial@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/postcss-initial/-/postcss-initial-4.0.1.tgz#529f735f72c5724a0fb30527df6fb7ac54d7de42" - integrity sha512-0ueD7rPqX8Pn1xJIjay0AZeIuDoF+V+VvMt/uOnn+4ezUKhZM/NokDeP6DwMNyIoYByuN/94IQnt5FEkaN59xQ== - -postcss-js@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/postcss-js/-/postcss-js-4.0.0.tgz#31db79889531b80dc7bc9b0ad283e418dce0ac00" - integrity sha512-77QESFBwgX4irogGVPgQ5s07vLvFqWr228qZY+w6lW599cRlK/HmnlivnnVUxkjHnCu4J16PDMHcH+e+2HbvTQ== - dependencies: - camelcase-css "^2.0.1" - -postcss-lab-function@^4.2.1: - version "4.2.1" - resolved "https://registry.yarnpkg.com/postcss-lab-function/-/postcss-lab-function-4.2.1.tgz#6fe4c015102ff7cd27d1bd5385582f67ebdbdc98" - integrity sha512-xuXll4isR03CrQsmxyz92LJB2xX9n+pZJ5jE9JgcnmsCammLyKdlzrBin+25dy6wIjfhJpKBAN80gsTlCgRk2w== - dependencies: - "@csstools/postcss-progressive-custom-properties" "^1.1.0" - postcss-value-parser "^4.2.0" - -postcss-load-config@^3.1.4: - version "3.1.4" - resolved "https://registry.yarnpkg.com/postcss-load-config/-/postcss-load-config-3.1.4.tgz#1ab2571faf84bb078877e1d07905eabe9ebda855" - integrity sha512-6DiM4E7v4coTE4uzA8U//WhtPwyhiim3eyjEMFCnUpzbrkK9wJHgKDT2mR+HbtSrd/NubVaYTOpSpjUl8NQeRg== - dependencies: - lilconfig "^2.0.5" - yaml "^1.10.2" - -postcss-loader@^6.2.1: - version "6.2.1" - resolved "https://registry.yarnpkg.com/postcss-loader/-/postcss-loader-6.2.1.tgz#0895f7346b1702103d30fdc66e4d494a93c008ef" - integrity sha512-WbbYpmAaKcux/P66bZ40bpWsBucjx/TTgVVzRZ9yUO8yQfVBlameJ0ZGVaPfH64hNSBh63a+ICP5nqOpBA0w+Q== - dependencies: - cosmiconfig "^7.0.0" - klona "^2.0.5" - semver "^7.3.5" - -postcss-logical@^5.0.4: - version "5.0.4" - resolved "https://registry.yarnpkg.com/postcss-logical/-/postcss-logical-5.0.4.tgz#ec75b1ee54421acc04d5921576b7d8db6b0e6f73" - integrity sha512-RHXxplCeLh9VjinvMrZONq7im4wjWGlRJAqmAVLXyZaXwfDWP73/oq4NdIp+OZwhQUMj0zjqDfM5Fj7qby+B4g== - -postcss-media-minmax@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/postcss-media-minmax/-/postcss-media-minmax-5.0.0.tgz#7140bddec173e2d6d657edbd8554a55794e2a5b5" - integrity sha512-yDUvFf9QdFZTuCUg0g0uNSHVlJ5X1lSzDZjPSFaiCWvjgsvu8vEVxtahPrLMinIDEEGnx6cBe6iqdx5YWz08wQ== - -postcss-merge-longhand@^5.1.7: - version "5.1.7" - resolved "https://registry.yarnpkg.com/postcss-merge-longhand/-/postcss-merge-longhand-5.1.7.tgz#24a1bdf402d9ef0e70f568f39bdc0344d568fb16" - integrity sha512-YCI9gZB+PLNskrK0BB3/2OzPnGhPkBEwmwhfYk1ilBHYVAZB7/tkTHFBAnCrvBBOmeYyMYw3DMjT55SyxMBzjQ== - dependencies: - postcss-value-parser "^4.2.0" - stylehacks "^5.1.1" - -postcss-merge-rules@^5.1.3: - version "5.1.3" - resolved "https://registry.yarnpkg.com/postcss-merge-rules/-/postcss-merge-rules-5.1.3.tgz#8f97679e67cc8d08677a6519afca41edf2220894" - integrity sha512-LbLd7uFC00vpOuMvyZop8+vvhnfRGpp2S+IMQKeuOZZapPRY4SMq5ErjQeHbHsjCUgJkRNrlU+LmxsKIqPKQlA== - dependencies: - browserslist "^4.21.4" - caniuse-api "^3.0.0" - cssnano-utils "^3.1.0" - postcss-selector-parser "^6.0.5" - -postcss-minify-font-values@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/postcss-minify-font-values/-/postcss-minify-font-values-5.1.0.tgz#f1df0014a726083d260d3bd85d7385fb89d1f01b" - integrity sha512-el3mYTgx13ZAPPirSVsHqFzl+BBBDrXvbySvPGFnQcTI4iNslrPaFq4muTkLZmKlGk4gyFAYUBMH30+HurREyA== - dependencies: - postcss-value-parser "^4.2.0" - -postcss-minify-gradients@^5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/postcss-minify-gradients/-/postcss-minify-gradients-5.1.1.tgz#f1fe1b4f498134a5068240c2f25d46fcd236ba2c" - integrity sha512-VGvXMTpCEo4qHTNSa9A0a3D+dxGFZCYwR6Jokk+/3oB6flu2/PnPXAh2x7x52EkY5xlIHLm+Le8tJxe/7TNhzw== - dependencies: - colord "^2.9.1" - cssnano-utils "^3.1.0" - postcss-value-parser "^4.2.0" - -postcss-minify-params@^5.1.4: - version "5.1.4" - resolved "https://registry.yarnpkg.com/postcss-minify-params/-/postcss-minify-params-5.1.4.tgz#c06a6c787128b3208b38c9364cfc40c8aa5d7352" - integrity sha512-+mePA3MgdmVmv6g+30rn57USjOGSAyuxUmkfiWpzalZ8aiBkdPYjXWtHuwJGm1v5Ojy0Z0LaSYhHaLJQB0P8Jw== - dependencies: - browserslist "^4.21.4" - cssnano-utils "^3.1.0" - postcss-value-parser "^4.2.0" - -postcss-minify-selectors@^5.2.1: - version "5.2.1" - resolved "https://registry.yarnpkg.com/postcss-minify-selectors/-/postcss-minify-selectors-5.2.1.tgz#d4e7e6b46147b8117ea9325a915a801d5fe656c6" - integrity sha512-nPJu7OjZJTsVUmPdm2TcaiohIwxP+v8ha9NehQ2ye9szv4orirRU3SDdtUmKH+10nzn0bAyOXZ0UEr7OpvLehg== - dependencies: - postcss-selector-parser "^6.0.5" - -postcss-modules-extract-imports@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/postcss-modules-extract-imports/-/postcss-modules-extract-imports-3.0.0.tgz#cda1f047c0ae80c97dbe28c3e76a43b88025741d" - integrity sha512-bdHleFnP3kZ4NYDhuGlVK+CMrQ/pqUm8bx/oGL93K6gVwiclvX5x0n76fYMKuIGKzlABOy13zsvqjb0f92TEXw== - -postcss-modules-local-by-default@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/postcss-modules-local-by-default/-/postcss-modules-local-by-default-4.0.0.tgz#ebbb54fae1598eecfdf691a02b3ff3b390a5a51c" - integrity sha512-sT7ihtmGSF9yhm6ggikHdV0hlziDTX7oFoXtuVWeDd3hHObNkcHRo9V3yg7vCAY7cONyxJC/XXCmmiHHcvX7bQ== - dependencies: - icss-utils "^5.0.0" - postcss-selector-parser "^6.0.2" - postcss-value-parser "^4.1.0" - -postcss-modules-scope@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/postcss-modules-scope/-/postcss-modules-scope-3.0.0.tgz#9ef3151456d3bbfa120ca44898dfca6f2fa01f06" - integrity sha512-hncihwFA2yPath8oZ15PZqvWGkWf+XUfQgUGamS4LqoP1anQLOsOJw0vr7J7IwLpoY9fatA2qiGUGmuZL0Iqlg== - dependencies: - postcss-selector-parser "^6.0.4" - -postcss-modules-values@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/postcss-modules-values/-/postcss-modules-values-4.0.0.tgz#d7c5e7e68c3bb3c9b27cbf48ca0bb3ffb4602c9c" - integrity sha512-RDxHkAiEGI78gS2ofyvCsu7iycRv7oqw5xMWn9iMoR0N/7mf9D50ecQqUo5BZ9Zh2vH4bCUR/ktCqbB9m8vJjQ== - dependencies: - icss-utils "^5.0.0" - -postcss-nested@6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/postcss-nested/-/postcss-nested-6.0.0.tgz#1572f1984736578f360cffc7eb7dca69e30d1735" - integrity sha512-0DkamqrPcmkBDsLn+vQDIrtkSbNkv5AD/M322ySo9kqFkCIYklym2xEmWkwo+Y3/qZo34tzEPNUw4y7yMCdv5w== - dependencies: - postcss-selector-parser "^6.0.10" - -postcss-nesting@^10.2.0: - version "10.2.0" - resolved "https://registry.yarnpkg.com/postcss-nesting/-/postcss-nesting-10.2.0.tgz#0b12ce0db8edfd2d8ae0aaf86427370b898890be" - integrity sha512-EwMkYchxiDiKUhlJGzWsD9b2zvq/r2SSubcRrgP+jujMXFzqvANLt16lJANC+5uZ6hjI7lpRmI6O8JIl+8l1KA== - dependencies: - "@csstools/selector-specificity" "^2.0.0" - postcss-selector-parser "^6.0.10" - -postcss-normalize-charset@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/postcss-normalize-charset/-/postcss-normalize-charset-5.1.0.tgz#9302de0b29094b52c259e9b2cf8dc0879879f0ed" - integrity sha512-mSgUJ+pd/ldRGVx26p2wz9dNZ7ji6Pn8VWBajMXFf8jk7vUoSrZ2lt/wZR7DtlZYKesmZI680qjr2CeFF2fbUg== - -postcss-normalize-display-values@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/postcss-normalize-display-values/-/postcss-normalize-display-values-5.1.0.tgz#72abbae58081960e9edd7200fcf21ab8325c3da8" - integrity sha512-WP4KIM4o2dazQXWmFaqMmcvsKmhdINFblgSeRgn8BJ6vxaMyaJkwAzpPpuvSIoG/rmX3M+IrRZEz2H0glrQNEA== - dependencies: - postcss-value-parser "^4.2.0" - -postcss-normalize-positions@^5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/postcss-normalize-positions/-/postcss-normalize-positions-5.1.1.tgz#ef97279d894087b59325b45c47f1e863daefbb92" - integrity sha512-6UpCb0G4eofTCQLFVuI3EVNZzBNPiIKcA1AKVka+31fTVySphr3VUgAIULBhxZkKgwLImhzMR2Bw1ORK+37INg== - dependencies: - postcss-value-parser "^4.2.0" - -postcss-normalize-repeat-style@^5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/postcss-normalize-repeat-style/-/postcss-normalize-repeat-style-5.1.1.tgz#e9eb96805204f4766df66fd09ed2e13545420fb2" - integrity sha512-mFpLspGWkQtBcWIRFLmewo8aC3ImN2i/J3v8YCFUwDnPu3Xz4rLohDO26lGjwNsQxB3YF0KKRwspGzE2JEuS0g== - dependencies: - postcss-value-parser "^4.2.0" - -postcss-normalize-string@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/postcss-normalize-string/-/postcss-normalize-string-5.1.0.tgz#411961169e07308c82c1f8c55f3e8a337757e228" - integrity sha512-oYiIJOf4T9T1N4i+abeIc7Vgm/xPCGih4bZz5Nm0/ARVJ7K6xrDlLwvwqOydvyL3RHNf8qZk6vo3aatiw/go3w== - dependencies: - postcss-value-parser "^4.2.0" - -postcss-normalize-timing-functions@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/postcss-normalize-timing-functions/-/postcss-normalize-timing-functions-5.1.0.tgz#d5614410f8f0b2388e9f240aa6011ba6f52dafbb" - integrity sha512-DOEkzJ4SAXv5xkHl0Wa9cZLF3WCBhF3o1SKVxKQAa+0pYKlueTpCgvkFAHfk+Y64ezX9+nITGrDZeVGgITJXjg== - dependencies: - postcss-value-parser "^4.2.0" - -postcss-normalize-unicode@^5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/postcss-normalize-unicode/-/postcss-normalize-unicode-5.1.1.tgz#f67297fca3fea7f17e0d2caa40769afc487aa030" - integrity sha512-qnCL5jzkNUmKVhZoENp1mJiGNPcsJCs1aaRmURmeJGES23Z/ajaln+EPTD+rBeNkSryI+2WTdW+lwcVdOikrpA== - dependencies: - browserslist "^4.21.4" - postcss-value-parser "^4.2.0" - -postcss-normalize-url@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/postcss-normalize-url/-/postcss-normalize-url-5.1.0.tgz#ed9d88ca82e21abef99f743457d3729a042adcdc" - integrity sha512-5upGeDO+PVthOxSmds43ZeMeZfKH+/DKgGRD7TElkkyS46JXAUhMzIKiCa7BabPeIy3AQcTkXwVVN7DbqsiCew== - dependencies: - normalize-url "^6.0.1" - postcss-value-parser "^4.2.0" - -postcss-normalize-whitespace@^5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/postcss-normalize-whitespace/-/postcss-normalize-whitespace-5.1.1.tgz#08a1a0d1ffa17a7cc6efe1e6c9da969cc4493cfa" - integrity sha512-83ZJ4t3NUDETIHTa3uEg6asWjSBYL5EdkVB0sDncx9ERzOKBVJIUeDO9RyA9Zwtig8El1d79HBp0JEi8wvGQnA== - dependencies: - postcss-value-parser "^4.2.0" - -postcss-normalize@^10.0.1: - version "10.0.1" - resolved "https://registry.yarnpkg.com/postcss-normalize/-/postcss-normalize-10.0.1.tgz#464692676b52792a06b06880a176279216540dd7" - integrity sha512-+5w18/rDev5mqERcG3W5GZNMJa1eoYYNGo8gB7tEwaos0ajk3ZXAI4mHGcNT47NE+ZnZD1pEpUOFLvltIwmeJA== - dependencies: - "@csstools/normalize.css" "*" - postcss-browser-comments "^4" - sanitize.css "*" - -postcss-opacity-percentage@^1.1.2: - version "1.1.3" - resolved "https://registry.yarnpkg.com/postcss-opacity-percentage/-/postcss-opacity-percentage-1.1.3.tgz#5b89b35551a556e20c5d23eb5260fbfcf5245da6" - integrity sha512-An6Ba4pHBiDtyVpSLymUUERMo2cU7s+Obz6BTrS+gxkbnSBNKSuD0AVUc+CpBMrpVPKKfoVz0WQCX+Tnst0i4A== - -postcss-ordered-values@^5.1.3: - version "5.1.3" - resolved "https://registry.yarnpkg.com/postcss-ordered-values/-/postcss-ordered-values-5.1.3.tgz#b6fd2bd10f937b23d86bc829c69e7732ce76ea38" - integrity sha512-9UO79VUhPwEkzbb3RNpqqghc6lcYej1aveQteWY+4POIwlqkYE21HKWaLDF6lWNuqCobEAyTovVhtI32Rbv2RQ== - dependencies: - cssnano-utils "^3.1.0" - postcss-value-parser "^4.2.0" - -postcss-overflow-shorthand@^3.0.4: - version "3.0.4" - resolved "https://registry.yarnpkg.com/postcss-overflow-shorthand/-/postcss-overflow-shorthand-3.0.4.tgz#7ed6486fec44b76f0eab15aa4866cda5d55d893e" - integrity sha512-otYl/ylHK8Y9bcBnPLo3foYFLL6a6Ak+3EQBPOTR7luMYCOsiVTUk1iLvNf6tVPNGXcoL9Hoz37kpfriRIFb4A== - dependencies: - postcss-value-parser "^4.2.0" - -postcss-page-break@^3.0.4: - version "3.0.4" - resolved "https://registry.yarnpkg.com/postcss-page-break/-/postcss-page-break-3.0.4.tgz#7fbf741c233621622b68d435babfb70dd8c1ee5f" - integrity sha512-1JGu8oCjVXLa9q9rFTo4MbeeA5FMe00/9C7lN4va606Rdb+HkxXtXsmEDrIraQ11fGz/WvKWa8gMuCKkrXpTsQ== - -postcss-place@^7.0.5: - version "7.0.5" - resolved "https://registry.yarnpkg.com/postcss-place/-/postcss-place-7.0.5.tgz#95dbf85fd9656a3a6e60e832b5809914236986c4" - integrity sha512-wR8igaZROA6Z4pv0d+bvVrvGY4GVHihBCBQieXFY3kuSuMyOmEnnfFzHl/tQuqHZkfkIVBEbDvYcFfHmpSet9g== - dependencies: - postcss-value-parser "^4.2.0" - -postcss-preset-env@^7.0.1: - version "7.8.3" - resolved "https://registry.yarnpkg.com/postcss-preset-env/-/postcss-preset-env-7.8.3.tgz#2a50f5e612c3149cc7af75634e202a5b2ad4f1e2" - integrity sha512-T1LgRm5uEVFSEF83vHZJV2z19lHg4yJuZ6gXZZkqVsqv63nlr6zabMH3l4Pc01FQCyfWVrh2GaUeCVy9Po+Aag== - dependencies: - "@csstools/postcss-cascade-layers" "^1.1.1" - "@csstools/postcss-color-function" "^1.1.1" - "@csstools/postcss-font-format-keywords" "^1.0.1" - "@csstools/postcss-hwb-function" "^1.0.2" - "@csstools/postcss-ic-unit" "^1.0.1" - "@csstools/postcss-is-pseudo-class" "^2.0.7" - "@csstools/postcss-nested-calc" "^1.0.0" - "@csstools/postcss-normalize-display-values" "^1.0.1" - "@csstools/postcss-oklab-function" "^1.1.1" - "@csstools/postcss-progressive-custom-properties" "^1.3.0" - "@csstools/postcss-stepped-value-functions" "^1.0.1" - "@csstools/postcss-text-decoration-shorthand" "^1.0.0" - "@csstools/postcss-trigonometric-functions" "^1.0.2" - "@csstools/postcss-unset-value" "^1.0.2" - autoprefixer "^10.4.13" - browserslist "^4.21.4" - css-blank-pseudo "^3.0.3" - css-has-pseudo "^3.0.4" - css-prefers-color-scheme "^6.0.3" - cssdb "^7.1.0" - postcss-attribute-case-insensitive "^5.0.2" - postcss-clamp "^4.1.0" - postcss-color-functional-notation "^4.2.4" - postcss-color-hex-alpha "^8.0.4" - postcss-color-rebeccapurple "^7.1.1" - postcss-custom-media "^8.0.2" - postcss-custom-properties "^12.1.10" - postcss-custom-selectors "^6.0.3" - postcss-dir-pseudo-class "^6.0.5" - postcss-double-position-gradients "^3.1.2" - postcss-env-function "^4.0.6" - postcss-focus-visible "^6.0.4" - postcss-focus-within "^5.0.4" - postcss-font-variant "^5.0.0" - postcss-gap-properties "^3.0.5" - postcss-image-set-function "^4.0.7" - postcss-initial "^4.0.1" - postcss-lab-function "^4.2.1" - postcss-logical "^5.0.4" - postcss-media-minmax "^5.0.0" - postcss-nesting "^10.2.0" - postcss-opacity-percentage "^1.1.2" - postcss-overflow-shorthand "^3.0.4" - postcss-page-break "^3.0.4" - postcss-place "^7.0.5" - postcss-pseudo-class-any-link "^7.1.6" - postcss-replace-overflow-wrap "^4.0.0" - postcss-selector-not "^6.0.1" - postcss-value-parser "^4.2.0" - -postcss-pseudo-class-any-link@^7.1.6: - version "7.1.6" - resolved "https://registry.yarnpkg.com/postcss-pseudo-class-any-link/-/postcss-pseudo-class-any-link-7.1.6.tgz#2693b221902da772c278def85a4d9a64b6e617ab" - integrity sha512-9sCtZkO6f/5ML9WcTLcIyV1yz9D1rf0tWc+ulKcvV30s0iZKS/ONyETvoWsr6vnrmW+X+KmuK3gV/w5EWnT37w== - dependencies: - postcss-selector-parser "^6.0.10" - -postcss-reduce-initial@^5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/postcss-reduce-initial/-/postcss-reduce-initial-5.1.1.tgz#c18b7dfb88aee24b1f8e4936541c29adbd35224e" - integrity sha512-//jeDqWcHPuXGZLoolFrUXBDyuEGbr9S2rMo19bkTIjBQ4PqkaO+oI8wua5BOUxpfi97i3PCoInsiFIEBfkm9w== - dependencies: - browserslist "^4.21.4" - caniuse-api "^3.0.0" - -postcss-reduce-transforms@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/postcss-reduce-transforms/-/postcss-reduce-transforms-5.1.0.tgz#333b70e7758b802f3dd0ddfe98bb1ccfef96b6e9" - integrity sha512-2fbdbmgir5AvpW9RLtdONx1QoYG2/EtqpNQbFASDlixBbAYuTcJ0dECwlqNqH7VbaUnEnh8SrxOe2sRIn24XyQ== - dependencies: - postcss-value-parser "^4.2.0" - -postcss-replace-overflow-wrap@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/postcss-replace-overflow-wrap/-/postcss-replace-overflow-wrap-4.0.0.tgz#d2df6bed10b477bf9c52fab28c568b4b29ca4319" - integrity sha512-KmF7SBPphT4gPPcKZc7aDkweHiKEEO8cla/GjcBK+ckKxiZslIu3C4GCRW3DNfL0o7yW7kMQu9xlZ1kXRXLXtw== - -postcss-selector-not@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/postcss-selector-not/-/postcss-selector-not-6.0.1.tgz#8f0a709bf7d4b45222793fc34409be407537556d" - integrity sha512-1i9affjAe9xu/y9uqWH+tD4r6/hDaXJruk8xn2x1vzxC2U3J3LKO3zJW4CyxlNhA56pADJ/djpEwpH1RClI2rQ== - dependencies: - postcss-selector-parser "^6.0.10" - -postcss-selector-parser@^6.0.10, postcss-selector-parser@^6.0.2, postcss-selector-parser@^6.0.4, postcss-selector-parser@^6.0.5, postcss-selector-parser@^6.0.9: - version "6.0.11" - resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.0.11.tgz#2e41dc39b7ad74046e1615185185cd0b17d0c8dc" - integrity sha512-zbARubNdogI9j7WY4nQJBiNqQf3sLS3wCP4WfOidu+p28LofJqDH1tcXypGrcmMHhDk2t9wGhCsYe/+szLTy1g== - dependencies: - cssesc "^3.0.0" - util-deprecate "^1.0.2" - -postcss-svgo@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/postcss-svgo/-/postcss-svgo-5.1.0.tgz#0a317400ced789f233a28826e77523f15857d80d" - integrity sha512-D75KsH1zm5ZrHyxPakAxJWtkyXew5qwS70v56exwvw542d9CRtTo78K0WeFxZB4G7JXKKMbEZtZayTGdIky/eA== - dependencies: - postcss-value-parser "^4.2.0" - svgo "^2.7.0" + playwright-core "1.58.2" + optionalDependencies: + fsevents "2.3.2" -postcss-unique-selectors@^5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/postcss-unique-selectors/-/postcss-unique-selectors-5.1.1.tgz#a9f273d1eacd09e9aa6088f4b0507b18b1b541b6" - integrity sha512-5JiODlELrz8L2HwxfPnhOWZYWDxVHWL83ufOv84NrcgipI7TaeRsatAhK4Tr2/ZiYldpK/wBvw5BD3qfaK96GA== - dependencies: - postcss-selector-parser "^6.0.5" +possible-typed-array-names@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz#93e3582bc0e5426586d9d07b79ee40fc841de4ae" + integrity sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg== -postcss-value-parser@^4.0.0, postcss-value-parser@^4.0.2, postcss-value-parser@^4.1.0, postcss-value-parser@^4.2.0: +postcss-value-parser@^4.0.2: version "4.2.0" resolved "https://registry.yarnpkg.com/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz#723c09920836ba6d3e5af019f92bc0971c02e514" integrity sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ== -postcss@^7.0.35: - version "7.0.39" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-7.0.39.tgz#9624375d965630e2e1f2c02a935c82a59cb48309" - integrity sha512-yioayjNbHn6z1/Bywyb2Y4s3yvDAeXGOyxqD+LnVOinq6Mdmd++SW2wUNVzavyyHxd6+DxzWGIuosg6P1Rj8uA== +postcss@8.4.49: + version "8.4.49" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.49.tgz#4ea479048ab059ab3ae61d082190fabfd994fe19" + integrity sha512-OCVPnIObs4N29kxTjzLfUryOkvZEq+pf8jTF0lg8E7uETuWHA+v7j3c/xJmiqpX450191LlmZfUKkXxkTry7nA== dependencies: - picocolors "^0.2.1" - source-map "^0.6.1" + nanoid "^3.3.7" + picocolors "^1.1.1" + source-map-js "^1.2.1" -postcss@^8.3.5, postcss@^8.4.18, postcss@^8.4.19, postcss@^8.4.4: - version "8.4.20" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.20.tgz#64c52f509644cecad8567e949f4081d98349dc56" - integrity sha512-6Q04AXR1212bXr5fh03u8aAwbLxAQNGQ/Q1LNa0VfOI06ZAlhPHtQvE4OIdpj4kLThXilalPnmDSOD65DcHt+g== +postcss@^8.5.6: + version "8.5.6" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.5.6.tgz#2825006615a619b4f62a9e7426cc120b349a8f3c" + integrity sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg== dependencies: - nanoid "^3.3.4" - picocolors "^1.0.0" - source-map-js "^1.0.2" - -prebuild-install@^7.1.1: - version "7.1.1" - resolved "https://registry.yarnpkg.com/prebuild-install/-/prebuild-install-7.1.1.tgz#de97d5b34a70a0c81334fd24641f2a1702352e45" - integrity sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw== - dependencies: - detect-libc "^2.0.0" - expand-template "^2.0.3" - github-from-package "0.0.0" - minimist "^1.2.3" - mkdirp-classic "^0.5.3" - napi-build-utils "^1.0.1" - node-abi "^3.3.0" - pump "^3.0.0" - rc "^1.2.7" - simple-get "^4.0.0" - tar-fs "^2.0.0" - tunnel-agent "^0.6.0" - -prelude-ls@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396" - integrity sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g== - -prelude-ls@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54" - integrity sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w== - -prettier@2.8.1: - version "2.8.1" - resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.8.1.tgz#4e1fd11c34e2421bc1da9aea9bd8127cd0a35efc" - integrity sha512-lqGoSJBQNJidqCHE80vqZJHWHRFoNYsSpP9AjFhlhi9ODCJA541svILes/+/1GM3VaL/abZi7cpFzOpdR9UPKg== + nanoid "^3.3.11" + picocolors "^1.1.1" + source-map-js "^1.2.1" -pretty-bytes@^5.3.0, pretty-bytes@^5.4.1, pretty-bytes@^5.6.0: - version "5.6.0" - resolved "https://registry.yarnpkg.com/pretty-bytes/-/pretty-bytes-5.6.0.tgz#356256f643804773c82f64723fe78c92c62beaeb" - integrity sha512-FFw039TmrBqFK8ma/7OL3sDz/VytdtJr044/QUJtH0wK9lb9jLq9tJyIxUwtQJHwar2BqtiA4iCWSwo9JLkzFg== - -pretty-error@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/pretty-error/-/pretty-error-4.0.0.tgz#90a703f46dd7234adb46d0f84823e9d1cb8f10d6" - integrity sha512-AoJ5YMAcXKYxKhuJGdcvse+Voc6v1RgnsR3nWcYU7q4t6z0Q6T86sv5Zq8VIRbOWWFpvdGE83LtdSMNd+6Y0xw== +pretty-format@30.2.0, pretty-format@^30.0.0: + version "30.2.0" + resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-30.2.0.tgz#2d44fe6134529aed18506f6d11509d8a62775ebe" + integrity sha512-9uBdv/B4EefsuAL+pWqueZyZS2Ba+LxfFeQ9DN14HU4bN8bhaxKdkpjpB6fs9+pSjIBu+FXQHImEg8j/Lw0+vA== dependencies: - lodash "^4.17.20" - renderkid "^3.0.0" + "@jest/schemas" "30.0.5" + ansi-styles "^5.2.0" + react-is "^18.3.1" -pretty-format@^27.0.2, pretty-format@^27.5.1: +pretty-format@^27.0.2: version "27.5.1" resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-27.5.1.tgz#2181879fdea51a7a5851fb39d920faa63f01d88e" integrity sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ== @@ -10591,44 +6488,10 @@ pretty-format@^27.0.2, pretty-format@^27.5.1: ansi-styles "^5.0.0" react-is "^17.0.1" -pretty-format@^28.1.3: - version "28.1.3" - resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-28.1.3.tgz#c9fba8cedf99ce50963a11b27d982a9ae90970d5" - integrity sha512-8gFb/To0OmxHR9+ZTb14Df2vNxdGCX8g1xWGUTqUw5TiZvcQf5sHKObd5UcPyLLyowNwDAMTF3XWOG1B6mxl1Q== - dependencies: - "@jest/schemas" "^28.1.3" - ansi-regex "^5.0.1" - ansi-styles "^5.0.0" - react-is "^18.0.0" - -pretty-format@^29.0.0, pretty-format@^29.7.0: - version "29.7.0" - resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-29.7.0.tgz#ca42c758310f365bfa71a0bda0a807160b776812" - integrity sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ== - dependencies: - "@jest/schemas" "^29.6.3" - ansi-styles "^5.0.0" - react-is "^18.0.0" - -prismjs@^1.27.0: - version "1.29.0" - resolved "https://registry.yarnpkg.com/prismjs/-/prismjs-1.29.0.tgz#f113555a8fa9b57c35e637bba27509dcf802dd12" - integrity sha512-Kx/1w86q/epKcmte75LNrEoT+lX8pBpavuAbvJWRXar7Hz8jrtF+e3vY751p0R8H9HdArwaCTNDDzHg/ScJK1Q== - -prismjs@~1.27.0: - version "1.27.0" - resolved "https://registry.yarnpkg.com/prismjs/-/prismjs-1.27.0.tgz#bb6ee3138a0b438a3653dd4d6ce0cc6510a45057" - integrity sha512-t13BGPUlFDR7wRB5kQDG4jjl7XeuH6jbJGt11JHPL96qwsEHNX2+68tFXqc1/k+/jALsbSWJKUOT/hcYAZ5LkA== - -process-nextick-args@~2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" - integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== - -process@^0.11.10: - version "0.11.10" - resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182" - integrity sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A== +prismjs@^1.30.0: + version "1.30.0" + resolved "https://registry.yarnpkg.com/prismjs/-/prismjs-1.30.0.tgz#d9709969d9d4e16403f6f348c63553b19f0975a9" + integrity sha512-DEvV2ZF2r2/63V+tK8hQvrR2ZGn10srHbXviTlcv7Kpzw8jWiNTqbVgjO3IY8RxrrOUF8VPMQQFysYYYv0YZxw== promise@^8.1.0: version "8.3.0" @@ -10637,15 +6500,7 @@ promise@^8.1.0: dependencies: asap "~2.0.6" -prompts@^2.0.1, prompts@^2.4.2: - version "2.4.2" - resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.2.tgz#7b57e73b3a48029ad10ebd44f74b01722a4cb069" - integrity sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q== - dependencies: - kleur "^3.0.3" - sisteransi "^1.0.5" - -prop-types@^15.6.2, prop-types@^15.7.2, prop-types@^15.8.1: +prop-types@^15.6.2, prop-types@^15.8.1: version "15.8.1" resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.8.1.tgz#67d87bf1a694f48435cf332c24af10214a3140b5" integrity sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg== @@ -10654,102 +6509,31 @@ prop-types@^15.6.2, prop-types@^15.7.2, prop-types@^15.8.1: object-assign "^4.1.1" react-is "^16.13.1" -property-information@^5.0.0: - version "5.6.0" - resolved "https://registry.yarnpkg.com/property-information/-/property-information-5.6.0.tgz#61675545fb23002f245c6540ec46077d4da3ed69" - integrity sha512-YUHSPk+A30YPv+0Qf8i9Mbfe/C0hdPXk1s1jPVToV8pk8BQtpw10ct89Eo7OWkutrwqvT0eicAxlOg3dOAu8JA== - dependencies: - xtend "^4.0.0" - -proxy-addr@~2.0.7: - version "2.0.7" - resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025" - integrity sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg== - dependencies: - forwarded "0.2.0" - ipaddr.js "1.9.1" - -proxy-from-env@1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.0.0.tgz#33c50398f70ea7eb96d21f7b817630a55791c7ee" - integrity sha512-F2JHgJQ1iqwnHDcQjVBsq3n/uoaFL+iPW/eAeL7kVxy/2RrWaN4WroKjjvbsoRtv0ftelNyC01bjRhn/bhcf4A== +property-information@^7.0.0: + version "7.1.0" + resolved "https://registry.yarnpkg.com/property-information/-/property-information-7.1.0.tgz#b622e8646e02b580205415586b40804d3e8bfd5d" + integrity sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ== proxy-from-env@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== -psl@^1.1.33: - version "1.9.0" - resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" - integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== - -pump@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64" - integrity sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww== - dependencies: - end-of-stream "^1.1.0" - once "^1.3.1" - -punycode@^2.1.0, punycode@^2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" - integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== - -pure-rand@^6.0.0: - version "6.0.4" - resolved "https://registry.yarnpkg.com/pure-rand/-/pure-rand-6.0.4.tgz#50b737f6a925468679bff00ad20eade53f37d5c7" - integrity sha512-LA0Y9kxMYv47GIPJy6MI84fqTd2HmYZI83W/kM/SkKfDlajnZYfmXFTxkbY+xSBPkLJxltMa9hIkmdc29eguMA== - -q@^1.1.2: - version "1.5.1" - resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7" - integrity sha512-kV/CThkXo6xyFEZUugw/+pIOywXcDbFYgSct5cT3gqlbkBE1SJdwy6UQoZvodiWF/ckQLZyDE/Bu1M6gVu5lVw== - -qs@6.10.4: - version "6.10.4" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.10.4.tgz#6a3003755add91c0ec9eacdc5f878b034e73f9e7" - integrity sha512-OQiU+C+Ds5qiH91qh/mg0w+8nwQuLjM4F4M/PbmhDOoYehPh+Fb0bDjtR1sOvy7YKxvj28Y/M0PhP5uVX0kB+g== - dependencies: - side-channel "^1.0.4" - -qs@6.11.0: - version "6.11.0" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.0.tgz#fd0d963446f7a65e1367e01abd85429453f0c37a" - integrity sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q== - dependencies: - side-channel "^1.0.4" - -qs@6.13.0: - version "6.13.0" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.13.0.tgz#6ca3bd58439f7e245655798997787b0d88a51906" - integrity sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg== - dependencies: - side-channel "^1.0.6" +punycode@^2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.1.tgz#027422e2faec0b25e1549c3e1bd8309b9133b6e5" + integrity sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg== -qs@^6.10.2: - version "6.11.1" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.1.tgz#6c29dff97f0c0060765911ba65cbc9764186109f" - integrity sha512-0wsrzgTz/kAVIeuxSjnpGC56rzYtr6JT/2BwEvMaPhFIoYa1aGO8LbzuU1R0uUYQkLpWBTOj0l/CLAJB64J6nQ== - dependencies: - side-channel "^1.0.4" +pure-rand@^7.0.0: + version "7.0.1" + resolved "https://registry.yarnpkg.com/pure-rand/-/pure-rand-7.0.1.tgz#6f53a5a9e3e4a47445822af96821ca509ed37566" + integrity sha512-oTUZM/NAZS8p7ANR3SHh30kXB+zK2r2BPcEn/awJIbOvq82WoMN4p62AWWp3Hhw50G0xMsw1mhIBLqHw64EcNQ== querystringify@^2.1.1: version "2.2.0" resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-2.2.0.tgz#3345941b4153cb9d082d8eee4cda2016a9aef7f6" integrity sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ== -queue-microtask@^1.2.2: - version "1.2.3" - resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" - integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A== - -quick-lru@^5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-5.1.1.tgz#366493e6b3e42a3a6885e2e99d18f80fb7a8c932" - integrity sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA== - raf@^3.4.1: version "3.4.1" resolved "https://registry.yarnpkg.com/raf/-/raf-3.4.1.tgz#0742e99a4a6552f445d73e3ee0328af0ff1ede39" @@ -10757,15 +6541,15 @@ raf@^3.4.1: dependencies: performance-now "^2.1.0" -ramda-adjunct@^4.0.0, ramda-adjunct@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/ramda-adjunct/-/ramda-adjunct-4.1.1.tgz#085ca9a7bf19857378eff648f9852b15136dc66f" - integrity sha512-BnCGsZybQZMDGram9y7RiryoRHS5uwx8YeGuUeDKuZuvK38XO6JJfmK85BwRWAKFA6pZ5nZBO/HBFtExVaf31w== +ramda-adjunct@^5.0.0, ramda-adjunct@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/ramda-adjunct/-/ramda-adjunct-5.1.0.tgz#c1281100922b03e74b1535cb9c966628697c5cc1" + integrity sha512-8qCpl2vZBXEJyNbi4zqcgdfHtcdsWjOGbiNSEnEBrM6Y0OKOT8UxJbIVGm1TIcjaSu2MxaWcgtsNlKlCk7o7qg== -ramda@~0.29.0: - version "0.29.1" - resolved "https://registry.yarnpkg.com/ramda/-/ramda-0.29.1.tgz#408a6165b9555b7ba2fc62555804b6c5a2eca196" - integrity sha512-OfxIeWzd4xdUNxlWhgFazxsA/nl3mS4/jGZI5n00uWOoSSFRhC1b6gl6xvmzUamgmqELraWp0J/qqVlXYPDPyA== +ramda@^0.30.1, ramda@~0.30.0: + version "0.30.1" + resolved "https://registry.yarnpkg.com/ramda/-/ramda-0.30.1.tgz#7108ac95673062b060025052cd5143ae8fc605bf" + integrity sha512-tEF5I22zJnuclswcZMc8bDIrwRHRzf+NqVEmqg50ShAZMP7MWeR/RGDthfM/p+BlqvF2fXAzpn8i+SJcYD3alw== randexp@^0.5.3: version "0.5.3" @@ -10782,32 +6566,7 @@ randombytes@^2.1.0: dependencies: safe-buffer "^5.1.0" -range-parser@^1.2.1, range-parser@~1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" - integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== - -raw-body@2.5.2: - version "2.5.2" - resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.5.2.tgz#99febd83b90e08975087e8f1f9419a149366b68a" - integrity sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA== - dependencies: - bytes "3.1.2" - http-errors "2.0.0" - iconv-lite "0.4.24" - unpipe "1.0.0" - -rc@^1.2.7: - version "1.2.8" - resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed" - integrity sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw== - dependencies: - deep-extend "^0.6.0" - ini "~1.3.0" - minimist "^1.2.0" - strip-json-comments "~2.0.1" - -react-app-polyfill@^3.0.0: +react-app-polyfill@3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/react-app-polyfill/-/react-app-polyfill-3.0.0.tgz#95221e0a9bd259e5ca6b177c7bb1cb6768f68fd7" integrity sha512-sZ41cxiU5llIB003yxxQBYrARBqe0repqPTTYBTmMqTz9szeBbE37BehCE891NZsmdZqqP+xWKdT3eo3vOzN8w== @@ -10819,13 +6578,6 @@ react-app-polyfill@^3.0.0: regenerator-runtime "^0.13.9" whatwg-fetch "^3.6.2" -react-app-rewired@^2.2.1: - version "2.2.1" - resolved "https://registry.yarnpkg.com/react-app-rewired/-/react-app-rewired-2.2.1.tgz#84901ee1e3f26add0377ebec0b41bcdfce9fc211" - integrity sha512-uFQWTErXeLDrMzOJHKp0h8P1z0LV9HzPGsJ6adOtGlA/B9WfT6Shh4j2tLTTGlXOfiVx6w6iWpp7SOC5pvk+gA== - dependencies: - semver "^5.6.0" - react-copy-to-clipboard@5.1.0: version "5.1.0" resolved "https://registry.yarnpkg.com/react-copy-to-clipboard/-/react-copy-to-clipboard-5.1.0.tgz#09aae5ec4c62750ccb2e6421a58725eabc41255c" @@ -10842,49 +6594,12 @@ react-debounce-input@=3.3.0: lodash.debounce "^4" prop-types "^15.8.1" -react-dev-utils@^12.0.1: - version "12.0.1" - resolved "https://registry.yarnpkg.com/react-dev-utils/-/react-dev-utils-12.0.1.tgz#ba92edb4a1f379bd46ccd6bcd4e7bc398df33e73" - integrity sha512-84Ivxmr17KjUupyqzFode6xKhjwuEJDROWKJy/BthkL7Wn6NJ8h4WE6k/exAv6ImS+0oZLRRW5j/aINMHyeGeQ== - dependencies: - "@babel/code-frame" "^7.16.0" - address "^1.1.2" - browserslist "^4.18.1" - chalk "^4.1.2" - cross-spawn "^7.0.3" - detect-port-alt "^1.1.6" - escape-string-regexp "^4.0.0" - filesize "^8.0.6" - find-up "^5.0.0" - fork-ts-checker-webpack-plugin "^6.5.0" - global-modules "^2.0.0" - globby "^11.0.4" - gzip-size "^6.0.0" - immer "^9.0.7" - is-root "^2.1.0" - loader-utils "^3.2.0" - open "^8.4.0" - pkg-up "^3.1.0" - prompts "^2.4.2" - react-error-overlay "^6.0.11" - recursive-readdir "^2.2.2" - shell-quote "^1.7.3" - strip-ansi "^6.0.1" - text-table "^0.2.0" - -react-dom@^17.0.2: - version "17.0.2" - resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-17.0.2.tgz#ecffb6845e3ad8dbfcdc498f0d0a939736502c23" - integrity sha512-s4h96KtLDUQlsENhMn1ar8t2bEa+q/YAtj8pPPdIjPDGBDIVNsrD9aXNWqspUe6AzKCIG0C1HZZLqLV7qpOBGA== +react-dom@19.2.4: + version "19.2.4" + resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-19.2.4.tgz#6fac6bd96f7db477d966c7ec17c1a2b1ad8e6591" + integrity sha512-AXJdLo8kgMbimY95O2aKQqsz2iWi9jMgKJhRBAxECE4IFxfcazB2LmzloIoibJI3C12IlY20+KFaLv+71bUJeQ== dependencies: - loose-envify "^1.1.0" - object-assign "^4.1.1" - scheduler "^0.20.2" - -react-error-overlay@^6.0.11: - version "6.0.11" - resolved "https://registry.yarnpkg.com/react-error-overlay/-/react-error-overlay-6.0.11.tgz#92835de5841c5cf08ba00ddd2d677b6d17ff9adb" - integrity sha512-/6UZ2qgEyH2aqzYZgQPxEnz33NJ2gNsnHA2o5+o4wW9bLM/JYQitNP9xPhsXwC08hMMovfGe/8retsdDsczPRg== + scheduler "^0.27.0" react-immutable-proptypes@2.2.0: version "2.2.0" @@ -10913,122 +6628,53 @@ react-is@^17.0.1: resolved "https://registry.yarnpkg.com/react-is/-/react-is-17.0.2.tgz#e691d4a8e9c789365655539ab372762b0efb54f0" integrity sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w== -react-is@^18.0.0, react-is@^18.2.0: - version "18.2.0" - resolved "https://registry.yarnpkg.com/react-is/-/react-is-18.2.0.tgz#199431eeaaa2e09f86427efbb4f1473edb47609b" - integrity sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w== - -react-monaco-editor@^0.50.1: - version "0.50.1" - resolved "https://registry.yarnpkg.com/react-monaco-editor/-/react-monaco-editor-0.50.1.tgz#3b68ce03e4b316f435b9bcd8ec13ffd4c050c605" - integrity sha512-qvYdJhQdkPIrPDMxCrEl0T2x9TfBB+aUbrpFv69FwoTybeyfAjxgJ219MYSsgn3c/g06BgyLX4ENrXM4niZ9ag== - dependencies: - prop-types "^15.8.1" - -react-number-format@^4.9.3: - version "4.9.4" - resolved "https://registry.yarnpkg.com/react-number-format/-/react-number-format-4.9.4.tgz#013ae526000e676e491763ce14da66fb330a9bd8" - integrity sha512-Gq20Z3ugqPLFgeaidnx5on9cNpbQZntPN3QgNAL/WJrNNlQnNznY0LCx7g8xtssmRBw0/hw+SCqw6zAcajooiA== - dependencies: - prop-types "^15.7.2" +react-is@^18.3.1: + version "18.3.1" + resolved "https://registry.yarnpkg.com/react-is/-/react-is-18.3.1.tgz#e83557dc12eae63a99e003a46388b1dcbb44db7e" + integrity sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg== -react-redux@^8.1.2: - version "8.1.3" - resolved "https://registry.yarnpkg.com/react-redux/-/react-redux-8.1.3.tgz#4fdc0462d0acb59af29a13c27ffef6f49ab4df46" - integrity sha512-n0ZrutD7DaX/j9VscF+uTALI3oUPa/pO4Z3soOBIjuRn/FzVu6aehhysxZCLi6y7duMf52WNZGMl7CtuK5EnRw== - dependencies: - "@babel/runtime" "^7.12.1" - "@types/hoist-non-react-statics" "^3.3.1" - "@types/use-sync-external-store" "^0.0.3" - hoist-non-react-statics "^3.3.2" - react-is "^18.0.0" - use-sync-external-store "^1.0.0" +react-is@^19.2.0: + version "19.2.0" + resolved "https://registry.yarnpkg.com/react-is/-/react-is-19.2.0.tgz#ddc3b4a4e0f3336c3847f18b806506388d7b9973" + integrity sha512-x3Ax3kNSMIIkyVYhWPyO09bu0uttcAIoecO/um/rKGQ4EltYWVYtyiGkS/3xMynrbVQdS69Jhlv8FXUEZehlzA== -react-refresh@^0.11.0: - version "0.11.0" - resolved "https://registry.yarnpkg.com/react-refresh/-/react-refresh-0.11.0.tgz#77198b944733f0f1f1a90e791de4541f9f074046" - integrity sha512-F27qZr8uUqwhWZboondsPx8tnC3Ct3SxZA3V5WyEvujRyyNv0VYPhoBg1gZ8/MV5tubQp76Trw8lTv9hzRBa+A== +react-is@^19.2.3: + version "19.2.4" + resolved "https://registry.yarnpkg.com/react-is/-/react-is-19.2.4.tgz#a080758243c572ccd4a63386537654298c99d135" + integrity sha512-W+EWGn2v0ApPKgKKCy/7s7WHXkboGcsrXE+2joLyVxkbyVQfO3MUEaUQDHoSmb8TFFrSKYa9mw64WZHNHSDzYA== -react-router-dom@6: - version "6.6.1" - resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.6.1.tgz#1b96ec0b2cefa7319f1251383ea5b41295ee260d" - integrity sha512-u+8BKUtelStKbZD5UcY0NY90WOzktrkJJhyhNg7L0APn9t1qJNLowzrM9CHdpB6+rcPt6qQrlkIXsTvhuXP68g== - dependencies: - "@remix-run/router" "1.2.1" - react-router "6.6.1" +react-number-format@5.4.4: + version "5.4.4" + resolved "https://registry.yarnpkg.com/react-number-format/-/react-number-format-5.4.4.tgz#d31f0e260609431500c8d3f81bbd3ae1fb7cacad" + integrity sha512-wOmoNZoOpvMminhifQYiYSTCLUDOiUbBunrMrMjA+dV52sY+vck1S4UhR6PkgnoCquvvMSeJjErXZ4qSaWCliA== -react-router@6.6.1: - version "6.6.1" - resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.6.1.tgz#17de6cf285f2d1c9721a3afca999c984e5558854" - integrity sha512-YkvlYRusnI/IN0kDtosUCgxqHeulN5je+ew8W+iA1VvFhf86kA+JEI/X/8NqYcr11hCDDp906S+SGMpBheNeYQ== +react-redux@^9.2.0: + version "9.2.0" + resolved "https://registry.yarnpkg.com/react-redux/-/react-redux-9.2.0.tgz#96c3ab23fb9a3af2cb4654be4b51c989e32366f5" + integrity sha512-ROY9fvHhwOD9ySfrF0wmvu//bKCQ6AeZZq1nJNtbDC+kk5DuSuNX/n6YWYF/SYy7bSba4D4FSz8DJeKY/S/r+g== dependencies: - "@remix-run/router" "1.2.1" + "@types/use-sync-external-store" "^0.0.6" + use-sync-external-store "^1.4.0" -react-scripts@~5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/react-scripts/-/react-scripts-5.0.1.tgz#6285dbd65a8ba6e49ca8d651ce30645a6d980003" - integrity sha512-8VAmEm/ZAwQzJ+GOMLbBsTdDKOpuZh7RPs0UymvBR2vRk4iZWCskjbFnxqjrzoIvlNNRZ3QJFx6/qDSi6zSnaQ== +react-router@7.13.1: + version "7.13.1" + resolved "https://registry.yarnpkg.com/react-router/-/react-router-7.13.1.tgz#5e2b3ebafd6c78d9775e135474bf5060645077f7" + integrity sha512-td+xP4X2/6BJvZoX6xw++A2DdEi++YypA69bJUV5oVvqf6/9/9nNlD70YO1e9d3MyamJEBQFEzk6mbfDYbqrSA== dependencies: - "@babel/core" "^7.16.0" - "@pmmmwh/react-refresh-webpack-plugin" "^0.5.3" - "@svgr/webpack" "^5.5.0" - babel-jest "^27.4.2" - babel-loader "^8.2.3" - babel-plugin-named-asset-import "^0.3.8" - babel-preset-react-app "^10.0.1" - bfj "^7.0.2" - browserslist "^4.18.1" - camelcase "^6.2.1" - case-sensitive-paths-webpack-plugin "^2.4.0" - css-loader "^6.5.1" - css-minimizer-webpack-plugin "^3.2.0" - dotenv "^10.0.0" - dotenv-expand "^5.1.0" - eslint "^8.3.0" - eslint-config-react-app "^7.0.1" - eslint-webpack-plugin "^3.1.1" - file-loader "^6.2.0" - fs-extra "^10.0.0" - html-webpack-plugin "^5.5.0" - identity-obj-proxy "^3.0.0" - jest "^27.4.3" - jest-resolve "^27.4.2" - jest-watch-typeahead "^1.0.0" - mini-css-extract-plugin "^2.4.5" - postcss "^8.4.4" - postcss-flexbugs-fixes "^5.0.2" - postcss-loader "^6.2.1" - postcss-normalize "^10.0.1" - postcss-preset-env "^7.0.1" - prompts "^2.4.2" - react-app-polyfill "^3.0.0" - react-dev-utils "^12.0.1" - react-refresh "^0.11.0" - resolve "^1.20.0" - resolve-url-loader "^4.0.0" - sass-loader "^12.3.0" - semver "^7.3.5" - source-map-loader "^3.0.0" - style-loader "^3.3.1" - tailwindcss "^3.0.2" - terser-webpack-plugin "^5.2.5" - webpack "^5.64.4" - webpack-dev-server "^4.6.0" - webpack-manifest-plugin "^4.0.2" - workbox-webpack-plugin "^6.4.1" - optionalDependencies: - fsevents "^2.3.2" + cookie "^1.0.1" + set-cookie-parser "^2.6.0" -react-syntax-highlighter@^15.5.0: - version "15.5.0" - resolved "https://registry.yarnpkg.com/react-syntax-highlighter/-/react-syntax-highlighter-15.5.0.tgz#4b3eccc2325fa2ec8eff1e2d6c18fa4a9e07ab20" - integrity sha512-+zq2myprEnQmH5yw6Gqc8lD55QHnpKaU8TOcFeC/Lg/MQSs8UknEA0JC4nTZGFAXC2J2Hyj/ijJ7NlabyPi2gg== +react-syntax-highlighter@^16.0.0: + version "16.1.0" + resolved "https://registry.yarnpkg.com/react-syntax-highlighter/-/react-syntax-highlighter-16.1.0.tgz#ebe0bb5ae7a3540859212cedafd767f0189c516c" + integrity sha512-E40/hBiP5rCNwkeBN1vRP+xow1X0pndinO+z3h7HLsHyjztbyjfzNWNKuAsJj+7DLam9iT4AaaOZnueCU+Nplg== dependencies: - "@babel/runtime" "^7.3.1" + "@babel/runtime" "^7.28.4" highlight.js "^10.4.1" + highlightjs-vue "^1.0.0" lowlight "^1.17.0" - prismjs "^1.27.0" - refractor "^3.6.0" + prismjs "^1.30.0" + refractor "^5.0.0" react-transition-group@^4.4.5: version "4.4.5" @@ -11040,65 +6686,10 @@ react-transition-group@^4.4.5: loose-envify "^1.4.0" prop-types "^15.6.2" -react@^17.0.2: - version "17.0.2" - resolved "https://registry.yarnpkg.com/react/-/react-17.0.2.tgz#d0b5cc516d29eb3eee383f75b62864cfb6800037" - integrity sha512-gnhPt75i/dq/z3/6q/0asP78D0u592D5L1pd7M8P+dck6Fu/jJeL6iVVK23fptSUZj8Vjf++7wXA8UNclGQcbA== - dependencies: - loose-envify "^1.1.0" - object-assign "^4.1.1" - -read-cache@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/read-cache/-/read-cache-1.0.0.tgz#e664ef31161166c9751cdbe8dbcf86b5fb58f774" - integrity sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA== - dependencies: - pify "^2.3.0" - -readable-stream@^2.0.1: - version "2.3.7" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" - integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== - dependencies: - core-util-is "~1.0.0" - inherits "~2.0.3" - isarray "~1.0.0" - process-nextick-args "~2.0.0" - safe-buffer "~5.1.1" - string_decoder "~1.1.1" - util-deprecate "~1.0.1" - -readable-stream@^3.0.6: - version "3.6.0" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" - integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== - dependencies: - inherits "^2.0.3" - string_decoder "^1.1.1" - util-deprecate "^1.0.1" - -readable-stream@^3.1.1, readable-stream@^3.4.0: - version "3.6.2" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.2.tgz#56a9b36ea965c00c5a93ef31eb111a0f11056967" - integrity sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA== - dependencies: - inherits "^2.0.3" - string_decoder "^1.1.1" - util-deprecate "^1.0.1" - -readdirp@~3.6.0: - version "3.6.0" - resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.6.0.tgz#74a370bd857116e245b29cc97340cd431a02a6c7" - integrity sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA== - dependencies: - picomatch "^2.2.1" - -recursive-readdir@^2.2.2: - version "2.2.3" - resolved "https://registry.yarnpkg.com/recursive-readdir/-/recursive-readdir-2.2.3.tgz#e726f328c0d69153bcabd5c322d3195252379372" - integrity sha512-8HrF5ZsXk5FAH9dgsx3BlUer73nIhuj+9OrQwEbLTPOBzGkL1lsFCR01am+v+0m2Cmbs1nP12hLDl5FA7EszKA== - dependencies: - minimatch "^3.0.5" +react@19.2.4: + version "19.2.4" + resolved "https://registry.yarnpkg.com/react/-/react-19.2.4.tgz#438e57baa19b77cb23aab516cf635cd0579ee09a" + integrity sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ== redent@^3.0.0: version "3.0.0" @@ -11113,21 +6704,20 @@ redux-immutable@^4.0.0: resolved "https://registry.yarnpkg.com/redux-immutable/-/redux-immutable-4.0.0.tgz#3a1a32df66366462b63691f0e1dc35e472bbc9f3" integrity sha512-SchSn/DWfGb3oAejd+1hhHx01xUoxY+V7TeK0BKqpkLKiQPVFf7DYzEaKmrEVxsWxielKfSK9/Xq66YyxgR1cg== -redux@^4.1.2: - version "4.2.1" - resolved "https://registry.yarnpkg.com/redux/-/redux-4.2.1.tgz#c08f4306826c49b5e9dc901dee0452ea8fce6197" - integrity sha512-LAUYz4lc+Do8/g7aeRa8JkyDErK6ekstQaqWQrNRW//MY1TvCEpMtpTWvlQ+FPbWCx+Xixu/6SHt5N0HR+SB4w== - dependencies: - "@babel/runtime" "^7.9.2" +redux@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/redux/-/redux-5.0.1.tgz#97fa26881ce5746500125585d5642c77b6e9447b" + integrity sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w== -refractor@^3.6.0: - version "3.6.0" - resolved "https://registry.yarnpkg.com/refractor/-/refractor-3.6.0.tgz#ac318f5a0715ead790fcfb0c71f4dd83d977935a" - integrity sha512-MY9W41IOWxxk31o+YvFCNyNzdkc9M20NoZK5vq6jkv4I/uh2zkWcfudj0Q1fovjUQJrNewS9NMzeTtqPf+n5EA== +refractor@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/refractor/-/refractor-5.0.0.tgz#85daf0448a6d947f5361796eb22c31733d61d904" + integrity sha512-QXOrHQF5jOpjjLfiNk5GFnWhRXvxjUVnlFxkeDmewR5sXkr3iM46Zo+CnRR8B+MDVqkULW4EcLVcRBNOPXHosw== dependencies: - hastscript "^6.0.0" - parse-entities "^2.0.0" - prismjs "~1.27.0" + "@types/hast" "^3.0.0" + "@types/prismjs" "^1.0.0" + hastscript "^9.0.0" + parse-entities "^4.0.0" regenerate-unicode-properties@^10.1.0: version "10.1.0" @@ -11141,16 +6731,11 @@ regenerate@^1.4.2: resolved "https://registry.yarnpkg.com/regenerate/-/regenerate-1.4.2.tgz#b9346d8827e8f5a32f7ba29637d398b69014848a" integrity sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A== -regenerator-runtime@^0.13.11, regenerator-runtime@^0.13.9: +regenerator-runtime@^0.13.9: version "0.13.11" resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz#f6dca3e7ceec20590d07ada785636a90cdca17f9" integrity sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg== -regenerator-runtime@^0.14.0: - version "0.14.0" - resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.14.0.tgz#5e19d68eb12d486f797e15a3c6a918f7cec5eb45" - integrity sha512-srw17NI0TUWHuGa5CFGGmhfNIeja30WMBfbslPNhf6JrqQlLN5gcrvig1oqPxiVaXb0oW0XRKtH6Nngs5lKCIA== - regenerator-transform@^0.15.1: version "0.15.1" resolved "https://registry.yarnpkg.com/regenerator-transform/-/regenerator-transform-0.15.1.tgz#f6c4e99fc1b4591f780db2586328e4d9a9d8dc56" @@ -11158,11 +6743,6 @@ regenerator-transform@^0.15.1: dependencies: "@babel/runtime" "^7.8.4" -regex-parser@^2.2.11: - version "2.2.11" - resolved "https://registry.yarnpkg.com/regex-parser/-/regex-parser-2.2.11.tgz#3b37ec9049e19479806e878cabe7c1ca83ccfe58" - integrity sha512-jbD/FT0+9MBU2XAZluI7w2OBs1RBi6p9M83nkoZayQXXU9e8Robt69FcZc7wU4eJD/YFTjn1JdCk3rbMJajz8Q== - regexp.prototype.flags@^1.4.3: version "1.4.3" resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.4.3.tgz#87cab30f80f66660181a3bb7bf5981a872b367ac" @@ -11172,11 +6752,6 @@ regexp.prototype.flags@^1.4.3: define-properties "^1.1.3" functions-have-names "^1.2.2" -regexpp@^3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-3.2.0.tgz#0425a2768d8f23bad70ca4b90461fa2f1213e1b2" - integrity sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg== - regexpu-core@^5.2.1: version "5.2.2" resolved "https://registry.yarnpkg.com/regexpu-core/-/regexpu-core-5.2.2.tgz#3e4e5d12103b64748711c3aad69934d7718e75fc" @@ -11201,11 +6776,6 @@ regjsparser@^0.9.1: dependencies: jsesc "~0.5.0" -relateurl@^0.2.7: - version "0.2.7" - resolved "https://registry.yarnpkg.com/relateurl/-/relateurl-0.2.7.tgz#54dbf377e51440aca90a4cd274600d3ff2d888a9" - integrity sha512-G08Dxvm4iDN3MLM0EsP62EDV9IuhXPR6blNz6Utcp7zyV3tr4HVNINt6MpaRWbxoOHT3Q7YN2P+jaHX8vUbgog== - remarkable@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/remarkable/-/remarkable-2.0.1.tgz#280ae6627384dfb13d98ee3995627ca550a12f31" @@ -11214,48 +6784,25 @@ remarkable@^2.0.1: argparse "^1.0.10" autolinker "^3.11.0" -renderkid@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/renderkid/-/renderkid-3.0.0.tgz#5fd823e4d6951d37358ecc9a58b1f06836b6268a" - integrity sha512-q/7VIQA8lmM1hF+jn+sFSPWGlMkSAeNYcPLmDQx2zzuiDfaLrOmumR8iaUKlenFgh0XRPIUeSPlH3A+AW3Z5pg== - dependencies: - css-select "^4.1.3" - dom-converter "^0.2.0" - htmlparser2 "^6.1.0" - lodash "^4.17.21" - strip-ansi "^6.0.1" - repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" integrity sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w== -request-progress@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/request-progress/-/request-progress-3.0.0.tgz#4ca754081c7fec63f505e4faa825aa06cd669dbe" - integrity sha512-MnWzEHHaxHO2iWiQuHrUPBi/1WeBf5PkxQqNyNvLl9VAYSdXkP8tQ3pBSeCPD+yw0v0Aq1zosWLz0BdeXpWwZg== - dependencies: - throttleit "^1.0.0" - require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" integrity sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q== -require-from-string@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909" - integrity sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw== - requires-port@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff" integrity sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ== -reselect@^4.1.8: - version "4.1.8" - resolved "https://registry.yarnpkg.com/reselect/-/reselect-4.1.8.tgz#3f5dc671ea168dccdeb3e141236f69f02eaec524" - integrity sha512-ab9EmR80F/zQTMNeneUr4cv+jSwPJgIlvEmVwLerwrWVbpLlBuls9XHzIeTFy4cegU2NHBp3va0LKOzU5qFEYQ== +reselect@^5.1.1: + version "5.1.1" + resolved "https://registry.yarnpkg.com/reselect/-/reselect-5.1.1.tgz#c766b1eb5d558291e5e550298adb0becc24bb72e" + integrity sha512-K/BG6eIky/SBpzfHZv/dd+9JBFiS4SWV7FIujVyJRux6e45+73RaUHXLmIR1f7WOMaQ0U1km6qwklRQxpJJY0w== resolve-cwd@^3.0.0: version "3.0.0" @@ -11274,28 +6821,7 @@ resolve-from@^5.0.0: resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-5.0.0.tgz#c35225843df8f776df21c57557bc087e9dfdfc69" integrity sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw== -resolve-url-loader@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/resolve-url-loader/-/resolve-url-loader-4.0.0.tgz#d50d4ddc746bb10468443167acf800dcd6c3ad57" - integrity sha512-05VEMczVREcbtT7Bz+C+96eUO5HDNvdthIiMB34t7FcF8ehcu4wC0sSgPUubs3XW2Q3CNLJk/BJrCU9wVRymiA== - dependencies: - adjust-sourcemap-loader "^4.0.0" - convert-source-map "^1.7.0" - loader-utils "^2.0.0" - postcss "^7.0.35" - source-map "0.6.1" - -resolve.exports@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/resolve.exports/-/resolve.exports-1.1.0.tgz#5ce842b94b05146c0e03076985d1d0e7e48c90c9" - integrity sha512-J1l+Zxxp4XK3LUDZ9m60LRJF/mAe4z6a4xyabPHk7pvK5t35dACV32iIjJDFeWZFfZlO29w6SZ67knR0tHzJtQ== - -resolve.exports@^2.0.0: - version "2.0.2" - resolved "https://registry.yarnpkg.com/resolve.exports/-/resolve.exports-2.0.2.tgz#f8c934b8e6a13f539e38b7098e2e36134f01e800" - integrity sha512-X2UW6Nw3n/aMgDVy+0rSqgHlv39WZAlZrXCdnbyEiKm17DSqHX4MmQMaST3FbeWR5FTuRcUwYAziZajji0Y7mg== - -resolve@^1.1.7, resolve@^1.14.2, resolve@^1.19.0, resolve@^1.20.0, resolve@^1.22.0, resolve@^1.22.1: +resolve@^1.14.2, resolve@^1.19.0: version "1.22.1" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.1.tgz#27cb2ebb53f91abb49470a928bba7558066ac177" integrity sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw== @@ -11304,267 +6830,88 @@ resolve@^1.1.7, resolve@^1.14.2, resolve@^1.19.0, resolve@^1.20.0, resolve@^1.22 path-parse "^1.0.7" supports-preserve-symlinks-flag "^1.0.0" -resolve@^2.0.0-next.3: - version "2.0.0-next.4" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-2.0.0-next.4.tgz#3d37a113d6429f496ec4752d2a2e58efb1fd4660" - integrity sha512-iMDbmAWtfU+MHpxt/I5iWI7cY6YVEZUQ3MBgPQ++XD1PELuJHIl82xBmObyP2KyQmkNB2dsqF7seoQQiAn5yDQ== - dependencies: - is-core-module "^2.9.0" - path-parse "^1.0.7" - supports-preserve-symlinks-flag "^1.0.0" - -restore-cursor@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-3.1.0.tgz#39f67c54b3a7a58cea5236d95cf0034239631f7e" - integrity sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA== - dependencies: - onetime "^5.1.0" - signal-exit "^3.0.2" - ret@^0.2.0: version "0.2.2" resolved "https://registry.yarnpkg.com/ret/-/ret-0.2.2.tgz#b6861782a1f4762dce43402a71eb7a283f44573c" integrity sha512-M0b3YWQs7R3Z917WRQy1HHA7Ba7D8hvZg6UE5mLykJxQVE2ju0IXbGlaHPPlkY+WN7wFP+wUMXmBFA0aV6vYGQ== -retry@^0.13.1: - version "0.13.1" - resolved "https://registry.yarnpkg.com/retry/-/retry-0.13.1.tgz#185b1587acf67919d63b357349e03537b2484658" - integrity sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg== - -reusify@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76" - integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw== - -rfdc@^1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/rfdc/-/rfdc-1.3.0.tgz#d0b7c441ab2720d05dc4cf26e01c89631d9da08b" - integrity sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA== - -rifm@^0.12.1: - version "0.12.1" - resolved "https://registry.yarnpkg.com/rifm/-/rifm-0.12.1.tgz#8fa77f45b7f1cda2a0068787ac821f0593967ac4" - integrity sha512-OGA1Bitg/dSJtI/c4dh90svzaUPt228kzFsUkJbtA2c964IqEAwWXeL9ZJi86xWv3j5SMqRvGULl7bA6cK0Bvg== - -rimraf@^2.6.3: - version "2.7.1" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec" - integrity sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w== - dependencies: - glob "^7.1.3" - -rimraf@^3.0.0, rimraf@^3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" - integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== - dependencies: - glob "^7.1.3" - -robust-predicates@^3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/robust-predicates/-/robust-predicates-3.0.2.tgz#d5b28528c4824d20fc48df1928d41d9efa1ad771" - integrity sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg== - -rollup-plugin-terser@^7.0.0: - version "7.0.2" - resolved "https://registry.yarnpkg.com/rollup-plugin-terser/-/rollup-plugin-terser-7.0.2.tgz#e8fbba4869981b2dc35ae7e8a502d5c6c04d324d" - integrity sha512-w3iIaU4OxcF52UUXiZNsNeuXIMDvFrr+ZXK6bFZ0Q60qyVfq4uLptoS4bbq3paG3x216eQllFZX7zt6TIImguQ== +rollup@^4.43.0: + version "4.60.0" + resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.60.0.tgz#d7d68c8cda873e96e08b2443505609b7e7be9eb8" + integrity sha512-yqjxruMGBQJ2gG4HtjZtAfXArHomazDHoFwFFmZZl0r7Pdo7qCIXKqKHZc8yeoMgzJJ+pO6pEEHa+V7uzWlrAQ== dependencies: - "@babel/code-frame" "^7.10.4" - jest-worker "^26.2.1" - serialize-javascript "^4.0.0" - terser "^5.0.0" - -rollup@^2.43.1: - version "2.79.2" - resolved "https://registry.yarnpkg.com/rollup/-/rollup-2.79.2.tgz#f150e4a5db4b121a21a747d762f701e5e9f49090" - integrity sha512-fS6iqSPZDs3dr/y7Od6y5nha8dW1YnbgtsyotCVvoFGKbERG++CVRFv1meyGDE1SNItQA8BrnCw7ScdAhRJ3XQ== + "@types/estree" "1.0.8" optionalDependencies: + "@rollup/rollup-android-arm-eabi" "4.60.0" + "@rollup/rollup-android-arm64" "4.60.0" + "@rollup/rollup-darwin-arm64" "4.60.0" + "@rollup/rollup-darwin-x64" "4.60.0" + "@rollup/rollup-freebsd-arm64" "4.60.0" + "@rollup/rollup-freebsd-x64" "4.60.0" + "@rollup/rollup-linux-arm-gnueabihf" "4.60.0" + "@rollup/rollup-linux-arm-musleabihf" "4.60.0" + "@rollup/rollup-linux-arm64-gnu" "4.60.0" + "@rollup/rollup-linux-arm64-musl" "4.60.0" + "@rollup/rollup-linux-loong64-gnu" "4.60.0" + "@rollup/rollup-linux-loong64-musl" "4.60.0" + "@rollup/rollup-linux-ppc64-gnu" "4.60.0" + "@rollup/rollup-linux-ppc64-musl" "4.60.0" + "@rollup/rollup-linux-riscv64-gnu" "4.60.0" + "@rollup/rollup-linux-riscv64-musl" "4.60.0" + "@rollup/rollup-linux-s390x-gnu" "4.60.0" + "@rollup/rollup-linux-x64-gnu" "4.60.0" + "@rollup/rollup-linux-x64-musl" "4.60.0" + "@rollup/rollup-openbsd-x64" "4.60.0" + "@rollup/rollup-openharmony-arm64" "4.60.0" + "@rollup/rollup-win32-arm64-msvc" "4.60.0" + "@rollup/rollup-win32-ia32-msvc" "4.60.0" + "@rollup/rollup-win32-x64-gnu" "4.60.0" + "@rollup/rollup-win32-x64-msvc" "4.60.0" fsevents "~2.3.2" -run-parallel@^1.1.9: - version "1.2.0" - resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.2.0.tgz#66d1368da7bdf921eb9d95bd1a9229e7f21a43ee" - integrity sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA== - dependencies: - queue-microtask "^1.2.2" - -rxjs@^7.5.1: - version "7.8.0" - resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-7.8.0.tgz#90a938862a82888ff4c7359811a595e14e1e09a4" - integrity sha512-F2+gxDshqmIub1KdvZkaEfGDwLNpPvk9Fs6LD/MyQxNgMds/WH9OdDDXOmxUZpME+iSK3rQCctkL0DYyytUqMg== - dependencies: - tslib "^2.1.0" - -safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: - version "5.1.2" - resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" - integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== +rrweb-cssom@^0.8.0: + version "0.8.0" + resolved "https://registry.yarnpkg.com/rrweb-cssom/-/rrweb-cssom-0.8.0.tgz#3021d1b4352fbf3b614aaeed0bc0d5739abe0bc2" + integrity sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw== -safe-buffer@5.2.1, safe-buffer@>=5.1.0, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.2, safe-buffer@~5.2.0: +safe-buffer@^5.1.0, safe-buffer@^5.2.1: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== -safe-regex-test@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/safe-regex-test/-/safe-regex-test-1.0.0.tgz#793b874d524eb3640d1873aad03596db2d4f2295" - integrity sha512-JBUUzyOgEwXQY1NuPtvcj/qcBDbDmEvWufhlnXZIm75DEHp+afM1r1ujJpJsV/gSM4t59tpDyPi1sd6ZaPFfsA== - dependencies: - call-bind "^1.0.2" - get-intrinsic "^1.1.3" - is-regex "^1.1.4" - -"safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: +"safer-buffer@>= 2.1.2 < 3.0.0": version "2.1.2" resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== -sanitize.css@*: - version "13.0.0" - resolved "https://registry.yarnpkg.com/sanitize.css/-/sanitize.css-13.0.0.tgz#2675553974b27964c75562ade3bd85d79879f173" - integrity sha512-ZRwKbh/eQ6w9vmTjkuG0Ioi3HBwPFce0O+v//ve+aOq1oeCy7jMV2qzzAlpsNuqpqCBjjriM1lbtZbF/Q8jVyA== - -sass-loader@^12.3.0: - version "12.6.0" - resolved "https://registry.yarnpkg.com/sass-loader/-/sass-loader-12.6.0.tgz#5148362c8e2cdd4b950f3c63ac5d16dbfed37bcb" - integrity sha512-oLTaH0YCtX4cfnJZxKSLAyglED0naiYfNG1iXfU5w1LNZ+ukoA5DtyDIN5zmKVZwYNJP4KRc5Y3hkWga+7tYfA== - dependencies: - klona "^2.0.4" - neo-async "^2.6.2" - -sax@~1.2.4: - version "1.2.4" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" - integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== - -saxes@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/saxes/-/saxes-5.0.1.tgz#eebab953fa3b7608dbe94e5dadb15c888fa6696d" - integrity sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw== +saxes@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/saxes/-/saxes-6.0.0.tgz#fe5b4a4768df4f14a201b1ba6a65c1f3d9988cc5" + integrity sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA== dependencies: xmlchars "^2.2.0" -scheduler@^0.20.2: - version "0.20.2" - resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.20.2.tgz#4baee39436e34aa93b4874bddcbf0fe8b8b50e91" - integrity sha512-2eWfGgAqqWFGqtdMmcL5zCMK1U8KlXv8SQFGglL3CEtd0aDVDWgeF/YoCmvln55m5zSk3J/20hTaSBeSObsQDQ== - dependencies: - loose-envify "^1.1.0" - object-assign "^4.1.1" - -schema-utils@2.7.0: - version "2.7.0" - resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-2.7.0.tgz#17151f76d8eae67fbbf77960c33c676ad9f4efc7" - integrity sha512-0ilKFI6QQF5nxDZLFn2dMjvc4hjg/Wkg7rHd3jK6/A4a1Hl9VFdQWvgB1UMGoU94pad1P/8N7fMcEnLnSiju8A== - dependencies: - "@types/json-schema" "^7.0.4" - ajv "^6.12.2" - ajv-keywords "^3.4.1" - -schema-utils@^2.6.5: - version "2.7.1" - resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-2.7.1.tgz#1ca4f32d1b24c590c203b8e7a50bf0ea4cd394d7" - integrity sha512-SHiNtMOUGWBQJwzISiVYKu82GiV4QYGePp3odlY1tuKO7gPtphAT5R/py0fA6xtbgLL/RvtJZnU9b8s0F1q0Xg== - dependencies: - "@types/json-schema" "^7.0.5" - ajv "^6.12.4" - ajv-keywords "^3.5.2" - -schema-utils@^3.0.0, schema-utils@^3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-3.1.1.tgz#bc74c4b6b6995c1d88f76a8b77bea7219e0c8281" - integrity sha512-Y5PQxS4ITlC+EahLuXaY86TXfR7Dc5lw294alXOq86JAHCihAIZfqv8nNCWvaEJvaC51uN9hbLGeV0cFBdH+Fw== - dependencies: - "@types/json-schema" "^7.0.8" - ajv "^6.12.5" - ajv-keywords "^3.5.2" - -schema-utils@^3.2.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-3.3.0.tgz#f50a88877c3c01652a15b622ae9e9795df7a60fe" - integrity sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg== - dependencies: - "@types/json-schema" "^7.0.8" - ajv "^6.12.5" - ajv-keywords "^3.5.2" - -schema-utils@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-4.0.0.tgz#60331e9e3ae78ec5d16353c467c34b3a0a1d3df7" - integrity sha512-1edyXKgh6XnJsJSQ8mKWXnN/BVaIbFMLpouRUrXgVq7WYne5kw3MW7UPhO44uRXQSIpTSXoJbmrR2X0w9kUTyg== - dependencies: - "@types/json-schema" "^7.0.9" - ajv "^8.8.0" - ajv-formats "^2.1.1" - ajv-keywords "^5.0.0" - -select-hose@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/select-hose/-/select-hose-2.0.0.tgz#625d8658f865af43ec962bfc376a37359a4994ca" - integrity sha512-mEugaLK+YfkijB4fx0e6kImuJdCIt2LxCRcbEYPqRGCs4F2ogyfZU5IAZRdjCP8JPq2AtdNoC/Dux63d9Kiryg== - -selfsigned@^2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/selfsigned/-/selfsigned-2.1.1.tgz#18a7613d714c0cd3385c48af0075abf3f266af61" - integrity sha512-GSL3aowiF7wa/WtSFwnUrludWFoNhftq8bUkH9pkzjpN2XSPOAYEgg6e0sS9s0rZwgJzJiQRPU18A6clnoW5wQ== - dependencies: - node-forge "^1" - -semver@^5.6.0: - version "5.7.2" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" - integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== +scheduler@^0.27.0: + version "0.27.0" + resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.27.0.tgz#0c4ef82d67d1e5c1e359e8fc76d3a87f045fe5bd" + integrity sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q== semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.3.0, semver@^6.3.1: version "6.3.1" resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== -semver@^7.3.2, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4: +semver@^7.5.4: version "7.5.4" resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.4.tgz#483986ec4ed38e1c6c48c34894a9182dbff68a6e" integrity sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA== dependencies: lru-cache "^6.0.0" -send@0.18.0: - version "0.18.0" - resolved "https://registry.yarnpkg.com/send/-/send-0.18.0.tgz#670167cc654b05f5aa4a767f9113bb371bc706be" - integrity sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg== - dependencies: - debug "2.6.9" - depd "2.0.0" - destroy "1.2.0" - encodeurl "~1.0.2" - escape-html "~1.0.3" - etag "~1.8.1" - fresh "0.5.2" - http-errors "2.0.0" - mime "1.6.0" - ms "2.1.3" - on-finished "2.4.1" - range-parser "~1.2.1" - statuses "2.0.1" - -send@0.19.0: - version "0.19.0" - resolved "https://registry.yarnpkg.com/send/-/send-0.19.0.tgz#bbc5a388c8ea6c048967049dbeac0e4a3f09d7f8" - integrity sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw== - dependencies: - debug "2.6.9" - depd "2.0.0" - destroy "1.2.0" - encodeurl "~1.0.2" - escape-html "~1.0.3" - etag "~1.8.1" - fresh "0.5.2" - http-errors "2.0.0" - mime "1.6.0" - ms "2.1.3" - on-finished "2.4.1" - range-parser "~1.2.1" - statuses "2.0.1" +semver@^7.7.2: + version "7.7.3" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.7.3.tgz#4b5f4143d007633a8dc671cd0a6ef9147b8bb946" + integrity sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q== serialize-error@^8.1.0: version "8.1.0" @@ -11573,51 +6920,12 @@ serialize-error@^8.1.0: dependencies: type-fest "^0.20.2" -serialize-javascript@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-4.0.0.tgz#b525e1238489a5ecfc42afacc3fe99e666f4b1aa" - integrity sha512-GaNA54380uFefWghODBWEGisLZFj00nS5ACs6yHa9nLqlLpVLO8ChDGeKRjZnV4Nh4n0Qi7nhYZD/9fCPzEqkw== - dependencies: - randombytes "^2.1.0" - -serialize-javascript@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-6.0.0.tgz#efae5d88f45d7924141da8b5c3a7a7e663fefeb8" - integrity sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag== - dependencies: - randombytes "^2.1.0" - -serialize-javascript@^6.0.1: - version "6.0.2" - resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-6.0.2.tgz#defa1e055c83bf6d59ea805d8da862254eb6a6c2" - integrity sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g== - dependencies: - randombytes "^2.1.0" +set-cookie-parser@^2.6.0: + version "2.7.2" + resolved "https://registry.yarnpkg.com/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz#ccd08673a9ae5d2e44ea2a2de25089e67c7edf68" + integrity sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw== -serve-index@^1.9.1: - version "1.9.1" - resolved "https://registry.yarnpkg.com/serve-index/-/serve-index-1.9.1.tgz#d3768d69b1e7d82e5ce050fff5b453bea12a9239" - integrity sha512-pXHfKNP4qujrtteMrSBb0rc8HJ9Ms/GrXwcUtUtD5s4ewDJI8bT3Cz2zTVRMKtri49pLx2e0Ya8ziP5Ya2pZZw== - dependencies: - accepts "~1.3.4" - batch "0.6.1" - debug "2.6.9" - escape-html "~1.0.3" - http-errors "~1.6.2" - mime-types "~2.1.17" - parseurl "~1.3.2" - -serve-static@1.16.0: - version "1.16.0" - resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.16.0.tgz#2bf4ed49f8af311b519c46f272bf6ac3baf38a92" - integrity sha512-pDLK8zwl2eKaYrs8mrPZBJua4hMplRWJ1tIFksVC3FtBEBnl8dxgeHtsaMS8DhS9i4fLObaon6ABoc4/hQGdPA== - dependencies: - encodeurl "~1.0.2" - escape-html "~1.0.3" - parseurl "~1.3.3" - send "0.18.0" - -set-function-length@^1.2.1: +set-function-length@^1.2.2: version "1.2.2" resolved "https://registry.yarnpkg.com/set-function-length/-/set-function-length-1.2.2.tgz#aac72314198eaed975cf77b2c3b6b880695e5449" integrity sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg== @@ -11629,25 +6937,16 @@ set-function-length@^1.2.1: gopd "^1.0.1" has-property-descriptors "^1.0.2" -setprototypeof@1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.0.tgz#d0bd85536887b6fe7c0d818cb962d9d91c54e656" - integrity sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ== - -setprototypeof@1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.2.0.tgz#66c9a24a73f9fc28cbe66b09fed3d33dcaf1b424" - integrity sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw== - -sha.js@^2.4.11: - version "2.4.11" - resolved "https://registry.yarnpkg.com/sha.js/-/sha.js-2.4.11.tgz#37a5cf0b81ecbc6943de109ba2960d1b26584ae7" - integrity sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ== +sha.js@^2.4.12: + version "2.4.12" + resolved "https://registry.yarnpkg.com/sha.js/-/sha.js-2.4.12.tgz#eb8b568bf383dfd1867a32c3f2b74eb52bdbf23f" + integrity sha512-8LzC5+bvI45BjpfXU8V5fdU2mfeKiQe1D1gIMn7XUlF3OTUrpdJpPPH4EMAnF0DsHHdSZqCdSss5qCmJKuiO3w== dependencies: - inherits "^2.0.1" - safe-buffer "^5.0.1" + inherits "^2.0.4" + safe-buffer "^5.2.1" + to-buffer "^1.2.0" -shallowequal@^1.1.0: +shallowequal@1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/shallowequal/-/shallowequal-1.1.0.tgz#188d521de95b9087404fd4dcb68b13df0ae4e7f8" integrity sha512-y0m1JoUZSlPAjXVtPPW70aZWfIL/dSP7AFkRnniLCrK/8MDKog3TySTBmckD+RObVxH0v4Tox67+F14PdED2oQ== @@ -11664,15 +6963,10 @@ shebang-regex@^3.0.0: resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== -shell-quote@^1.7.3: - version "1.7.4" - resolved "https://registry.yarnpkg.com/shell-quote/-/shell-quote-1.7.4.tgz#33fe15dee71ab2a81fcbd3a52106c5cfb9fb75d8" - integrity sha512-8o/QEhSSRb1a5i7TFR0iM4G16Z0vYB2OQVs4G3aAFXjn3T6yEx8AZxy1PgDF7I00LZHYA3WxaSYIf5e5sAX8Rw== - -short-unique-id@^5.0.2: - version "5.0.3" - resolved "https://registry.yarnpkg.com/short-unique-id/-/short-unique-id-5.0.3.tgz#bc6975dc5e8b296960ff5ac91ddabbc7ddb693d9" - integrity sha512-yhniEILouC0s4lpH0h7rJsfylZdca10W9mDJRAFh3EpcSUanCHGb0R7kcFOIUCZYSAPo0PUD5ZxWQdW0T4xaug== +short-unique-id@^5.3.2: + version "5.3.2" + resolved "https://registry.yarnpkg.com/short-unique-id/-/short-unique-id-5.3.2.tgz#6fea1b8fcaac44455abd1fa25840ebdb9c0a8ae7" + integrity sha512-KRT/hufMSxXKEDSQujfVE0Faa/kZ51ihUcZQAcmP04t00DvPj7Ox5anHke1sJYUtzSuiT/Y5uyzg/W7bBEGhCg== side-channel@^1.0.4: version "1.0.4" @@ -11683,100 +6977,25 @@ side-channel@^1.0.4: get-intrinsic "^1.0.2" object-inspect "^1.9.0" -side-channel@^1.0.6: - version "1.0.6" - resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.6.tgz#abd25fb7cd24baf45466406b1096b7831c9215f2" - integrity sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA== - dependencies: - call-bind "^1.0.7" - es-errors "^1.3.0" - get-intrinsic "^1.2.4" - object-inspect "^1.13.1" - -signal-exit@^3.0.2, signal-exit@^3.0.3, signal-exit@^3.0.7: +signal-exit@^3.0.3: version "3.0.7" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9" integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ== -simple-concat@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/simple-concat/-/simple-concat-1.0.1.tgz#f46976082ba35c2263f1c8ab5edfe26c41c9552f" - integrity sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q== - -simple-get@^4.0.0: - version "4.0.1" - resolved "https://registry.yarnpkg.com/simple-get/-/simple-get-4.0.1.tgz#4a39db549287c979d352112fa03fd99fd6bc3543" - integrity sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA== - dependencies: - decompress-response "^6.0.0" - once "^1.3.1" - simple-concat "^1.0.0" - -sisteransi@^1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/sisteransi/-/sisteransi-1.0.5.tgz#134d681297756437cc05ca01370d3a7a571075ed" - integrity sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg== - -slash@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/slash/-/slash-2.0.0.tgz#de552851a1759df3a8f206535442f5ec4ddeab44" - integrity sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A== +signal-exit@^4.0.1: + version "4.1.0" + resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-4.1.0.tgz#952188c1cbd546070e2dd20d0f41c0ae0530cb04" + integrity sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw== slash@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634" integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q== -slash@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/slash/-/slash-4.0.0.tgz#2422372176c4c6c5addb5e2ada885af984b396a7" - integrity sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew== - -slice-ansi@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-3.0.0.tgz#31ddc10930a1b7e0b67b08c96c2f49b77a789787" - integrity sha512-pSyv7bSTC7ig9Dcgbw9AuRNUb5k5V6oDudjZoMBSr13qpLBG7tB+zgCkARjq7xIUgdz5P1Qe8u+rSGdouOOIyQ== - dependencies: - ansi-styles "^4.0.0" - astral-regex "^2.0.0" - is-fullwidth-code-point "^3.0.0" - -slice-ansi@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-4.0.0.tgz#500e8dd0fd55b05815086255b3195adf2a45fe6b" - integrity sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ== - dependencies: - ansi-styles "^4.0.0" - astral-regex "^2.0.0" - is-fullwidth-code-point "^3.0.0" - -sockjs@^0.3.24: - version "0.3.24" - resolved "https://registry.yarnpkg.com/sockjs/-/sockjs-0.3.24.tgz#c9bc8995f33a111bea0395ec30aa3206bdb5ccce" - integrity sha512-GJgLTZ7vYb/JtPSSZ10hsOYIvEYsjbNU+zPdIHcUaWVNUEPivzxku31865sSSud0Da0W4lEeOPlmw93zLQchuQ== - dependencies: - faye-websocket "^0.11.3" - uuid "^8.3.2" - websocket-driver "^0.7.4" - -source-list-map@^2.0.0, source-list-map@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/source-list-map/-/source-list-map-2.0.1.tgz#3993bd873bfc48479cca9ea3a547835c7c154b34" - integrity sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw== - -source-map-js@^1.0.1, source-map-js@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.0.2.tgz#adbc361d9c62df380125e7f161f71c826f1e490c" - integrity sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw== - -source-map-loader@^3.0.0: - version "3.0.2" - resolved "https://registry.yarnpkg.com/source-map-loader/-/source-map-loader-3.0.2.tgz#af23192f9b344daa729f6772933194cc5fa54fee" - integrity sha512-BokxPoLjyl3iOrgkWaakaxqnelAJSS+0V+De0kKIq6lyWrXuiPgYTGp6z3iHmqljKAaLXwZa+ctD8GccRJeVvg== - dependencies: - abab "^2.0.5" - iconv-lite "^0.6.3" - source-map-js "^1.0.1" +source-map-js@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.2.1.tgz#1ce5650fddd87abc099eda37dcff024c2667ae46" + integrity sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA== source-map-support@0.5.13: version "0.5.13" @@ -11786,205 +7005,79 @@ source-map-support@0.5.13: buffer-from "^1.0.0" source-map "^0.6.0" -source-map-support@^0.5.6, source-map-support@~0.5.20: - version "0.5.21" - resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.21.tgz#04fe7c7f9e1ed2d662233c28cb2b35b9f63f6e4f" - integrity sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w== - dependencies: - buffer-from "^1.0.0" - source-map "^0.6.0" - -source-map@0.6.1, source-map@^0.6.0, source-map@^0.6.1, source-map@~0.6.0, source-map@~0.6.1: - version "0.6.1" - resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" - integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== - source-map@^0.5.7: version "0.5.7" resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc" integrity sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ== -source-map@^0.7.3: - version "0.7.4" - resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.7.4.tgz#a9bbe705c9d8846f4e08ff6765acf0f1b0898656" - integrity sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA== - -source-map@^0.8.0-beta.0: - version "0.8.0-beta.0" - resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.8.0-beta.0.tgz#d4c1bb42c3f7ee925f005927ba10709e0d1d1f11" - integrity sha512-2ymg6oRBpebeZi9UUNsgQ89bhx01TcTkmNTGnNO88imTmbSgy4nfujrgVEFKWpMTEGA11EDkTt7mqObTPdigIA== - dependencies: - whatwg-url "^7.0.0" - -sourcemap-codec@^1.4.8: - version "1.4.8" - resolved "https://registry.yarnpkg.com/sourcemap-codec/-/sourcemap-codec-1.4.8.tgz#ea804bd94857402e6992d05a38ef1ae35a9ab4c4" - integrity sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA== - -space-separated-tokens@^1.0.0: - version "1.1.5" - resolved "https://registry.yarnpkg.com/space-separated-tokens/-/space-separated-tokens-1.1.5.tgz#85f32c3d10d9682007e917414ddc5c26d1aa6899" - integrity sha512-q/JSVd1Lptzhf5bkYm4ob4iWPjx0KiRe3sRFBNrVqbJkFaBm5vbbowy1mymoPNLRa52+oadOhJ+K49wsSeSjTA== - -spdy-transport@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/spdy-transport/-/spdy-transport-3.0.0.tgz#00d4863a6400ad75df93361a1608605e5dcdcf31" - integrity sha512-hsLVFE5SjA6TCisWeJXFKniGGOpBgMLmerfO2aCyCU5s7nJ/rpAepqmFifv/GCbSbueEeAJJnmSQ2rKC/g8Fcw== - dependencies: - debug "^4.1.0" - detect-node "^2.0.4" - hpack.js "^2.1.6" - obuf "^1.1.2" - readable-stream "^3.0.6" - wbuf "^1.7.3" +source-map@^0.6.0: + version "0.6.1" + resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" + integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== -spdy@^4.0.2: - version "4.0.2" - resolved "https://registry.yarnpkg.com/spdy/-/spdy-4.0.2.tgz#b74f466203a3eda452c02492b91fb9e84a27677b" - integrity sha512-r46gZQZQV+Kl9oItvl1JZZqJKGr+oEkB08A6BzkiR7593/7IbtuncXHd2YoYeTsG4157ZssMu9KYvUHLcjcDoA== - dependencies: - debug "^4.1.0" - handle-thing "^2.0.0" - http-deceiver "^1.2.7" - select-hose "^2.0.0" - spdy-transport "^3.0.0" +space-separated-tokens@^2.0.0: + version "2.0.2" + resolved "https://registry.yarnpkg.com/space-separated-tokens/-/space-separated-tokens-2.0.2.tgz#1ecd9d2350a3844572c3f4a312bceb018348859f" + integrity sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q== sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" integrity sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g== -sshpk@^1.14.1: - version "1.17.0" - resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.17.0.tgz#578082d92d4fe612b13007496e543fa0fbcbe4c5" - integrity sha512-/9HIEs1ZXGhSPE8X6Ccm7Nam1z8KcoCqPdI7ecm1N33EzAetWahvQWVqLZtaZQ+IDKX4IyA2o0gBzqIMkAagHQ== - dependencies: - asn1 "~0.2.3" - assert-plus "^1.0.0" - bcrypt-pbkdf "^1.0.0" - dashdash "^1.12.0" - ecc-jsbn "~0.1.1" - getpass "^0.1.1" - jsbn "~0.1.0" - safer-buffer "^2.0.2" - tweetnacl "~0.14.0" - -stable@^0.1.8: - version "0.1.8" - resolved "https://registry.yarnpkg.com/stable/-/stable-0.1.8.tgz#836eb3c8382fe2936feaf544631017ce7d47a3cf" - integrity sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w== - -stack-utils@^2.0.3: +stack-utils@^2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/stack-utils/-/stack-utils-2.0.6.tgz#aaf0748169c02fc33c8232abccf933f54a1cc34f" integrity sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ== dependencies: escape-string-regexp "^2.0.0" -stackframe@^1.3.4: - version "1.3.4" - resolved "https://registry.yarnpkg.com/stackframe/-/stackframe-1.3.4.tgz#b881a004c8c149a5e8efef37d51b16e412943310" - integrity sha512-oeVtt7eWQS+Na6F//S4kJ2K2VbRlS9D43mAlMyVpVWovy9o+jfgH8O9agzANzaiLjclA0oYzUXEM4PurhSUChw== - -stampit@^4.3.2: - version "4.3.2" - resolved "https://registry.yarnpkg.com/stampit/-/stampit-4.3.2.tgz#cfd3f607dd628a161ce6305621597994b4d56573" - integrity sha512-pE2org1+ZWQBnIxRPrBM2gVupkuDD0TTNIo1H6GdT/vO82NXli2z8lRE8cu/nBIHrcOCXFBAHpb9ZldrB2/qOA== - -statuses@2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/statuses/-/statuses-2.0.1.tgz#55cb000ccf1d48728bd23c685a063998cf1a1b63" - integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ== - -"statuses@>= 1.4.0 < 2": - version "1.5.0" - resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c" - integrity sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA== +state-local@^1.0.6: + version "1.0.7" + resolved "https://registry.yarnpkg.com/state-local/-/state-local-1.0.7.tgz#da50211d07f05748d53009bee46307a37db386d5" + integrity sha512-HTEHMNieakEnoe33shBYcZ7NX83ACUjCu8c40iOGEZsngj9zRnkqS9j1pqQPXwobB0ZcVTk27REb7COQ0UR59w== -string-length@^4.0.1: +string-length@^4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/string-length/-/string-length-4.0.2.tgz#a8a8dc7bd5c1a82b9b3c8b87e125f66871b6e57a" integrity sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ== dependencies: - char-regex "^1.0.2" - strip-ansi "^6.0.0" - -string-length@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/string-length/-/string-length-5.0.1.tgz#3d647f497b6e8e8d41e422f7e0b23bc536c8381e" - integrity sha512-9Ep08KAMUn0OadnVaBuRdE2l615CQ508kr0XMadjClfYpdCyvrbFp6Taebo8yyxokQ4viUd/xPPUA4FGgUa0ow== - dependencies: - char-regex "^2.0.0" - strip-ansi "^7.0.1" - -string-natural-compare@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/string-natural-compare/-/string-natural-compare-3.0.1.tgz#7a42d58474454963759e8e8b7ae63d71c1e7fdf4" - integrity sha512-n3sPwynL1nwKi3WJ6AIsClwBMa0zTi54fn2oLU6ndfTSIO05xaznjSf15PcBZU6FNWbmN5Q6cxT4V5hGvB4taw== - -string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -string.prototype.matchall@^4.0.6, string.prototype.matchall@^4.0.8: - version "4.0.8" - resolved "https://registry.yarnpkg.com/string.prototype.matchall/-/string.prototype.matchall-4.0.8.tgz#3bf85722021816dcd1bf38bb714915887ca79fd3" - integrity sha512-6zOCOcJ+RJAQshcTvXPHoxoQGONa3e/Lqx90wUA+wEzX78sg5Bo+1tQo4N0pohS0erG9qtCqJDjNCQBjeWVxyg== - dependencies: - call-bind "^1.0.2" - define-properties "^1.1.4" - es-abstract "^1.20.4" - get-intrinsic "^1.1.3" - has-symbols "^1.0.3" - internal-slot "^1.0.3" - regexp.prototype.flags "^1.4.3" - side-channel "^1.0.4" - -string.prototype.trimend@^1.0.6: - version "1.0.6" - resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.6.tgz#c4a27fa026d979d79c04f17397f250a462944533" - integrity sha512-JySq+4mrPf9EsDBEDYMOb/lM7XQLulwg5R/m1r0PXEFqrV0qHvl58sdTilSXtKOflCsK2E8jxf+GKC0T07RWwQ== - dependencies: - call-bind "^1.0.2" - define-properties "^1.1.4" - es-abstract "^1.20.4" + char-regex "^1.0.2" + strip-ansi "^6.0.0" -string.prototype.trimstart@^1.0.6: - version "1.0.6" - resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.6.tgz#e90ab66aa8e4007d92ef591bbf3cd422c56bdcf4" - integrity sha512-omqjMDaY92pbn5HOX7f9IccLA+U1tA9GvtU4JrodiXFfYB7jPzzHpRzpglLAjtUV6bB557zwClJezTqnAiYnQA== +"string-width-cjs@npm:string-width@^4.2.0": + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== dependencies: - call-bind "^1.0.2" - define-properties "^1.1.4" - es-abstract "^1.20.4" + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" -string_decoder@^1.1.1: - version "1.3.0" - resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e" - integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== +string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== dependencies: - safe-buffer "~5.2.0" + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" -string_decoder@~1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" - integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== +string-width@^5.0.1, string-width@^5.1.2: + version "5.1.2" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-5.1.2.tgz#14f8daec6d81e7221d2a357e668cab73bdbca794" + integrity sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA== dependencies: - safe-buffer "~5.1.0" + eastasianwidth "^0.2.0" + emoji-regex "^9.2.2" + strip-ansi "^7.0.1" -stringify-object@^3.3.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/stringify-object/-/stringify-object-3.3.0.tgz#703065aefca19300d3ce88af4f5b3956d7556629" - integrity sha512-rHqiFh1elqCQ9WPLIC8I0Q/g/wj5J1eMkyoiD6eoQApWHP0FtlK7rqnhmabL5VUY9JQCcqwwvlOaSuutekgyrw== +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== dependencies: - get-own-enumerable-property-symbols "^3.0.0" - is-obj "^1.0.1" - is-regexp "^1.0.0" + ansi-regex "^5.0.1" strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" @@ -12000,21 +7093,11 @@ strip-ansi@^7.0.1: dependencies: ansi-regex "^6.0.1" -strip-bom@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3" - integrity sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA== - strip-bom@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-4.0.0.tgz#9c3505c1db45bcedca3d9cf7a16f5c5aa3901878" integrity sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w== -strip-comments@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/strip-comments/-/strip-comments-2.0.1.tgz#4ad11c3fbcac177a67a40ac224ca339ca1c1ba9b" - integrity sha512-ZprKx+bBLXv067WTCALv8SSz5l2+XhpYCsVtSqlMnkAXMWDq+/ekVbl1ghqP9rUHTzv6sm/DwCOiYutU/yp1fw== - strip-final-newline@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/strip-final-newline/-/strip-final-newline-2.0.0.tgz#89b852fb2fcbe936f6f4b3187afb0a12c1ab58ad" @@ -12027,41 +7110,30 @@ strip-indent@^3.0.0: dependencies: min-indent "^1.0.0" -strip-json-comments@^3.1.0, strip-json-comments@^3.1.1: +strip-json-comments@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006" integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig== -strip-json-comments@~2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" - integrity sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ== - -style-loader@^3.3.1: - version "3.3.1" - resolved "https://registry.yarnpkg.com/style-loader/-/style-loader-3.3.1.tgz#057dfa6b3d4d7c7064462830f9113ed417d38575" - integrity sha512-GPcQ+LDJbrcxHORTRes6Jy2sfvK2kS6hpSfI/fXhPt+spVzxF6LJ1dHLN9zIGmVaaP044YKaIatFaufENRiDoQ== - -styled-components@^5.3.6: - version "5.3.6" - resolved "https://registry.yarnpkg.com/styled-components/-/styled-components-5.3.6.tgz#27753c8c27c650bee9358e343fc927966bfd00d1" - integrity sha512-hGTZquGAaTqhGWldX7hhfzjnIYBZ0IXQXkCYdvF1Sq3DsUaLx6+NTHC5Jj1ooM2F68sBiVz3lvhfwQs/S3l6qg== - dependencies: - "@babel/helper-module-imports" "^7.0.0" - "@babel/traverse" "^7.4.5" - "@emotion/is-prop-valid" "^1.1.0" - "@emotion/stylis" "^0.8.4" - "@emotion/unitless" "^0.7.4" - babel-plugin-styled-components ">= 1.12.0" - css-to-react-native "^3.0.0" - hoist-non-react-statics "^3.0.0" - shallowequal "^1.1.0" - supports-color "^5.5.0" - -styled-icons@^10.47.0: - version "10.47.0" - resolved "https://registry.yarnpkg.com/styled-icons/-/styled-icons-10.47.0.tgz#79765d950db088d6cde52bdcb6bdb0f543823d11" - integrity sha512-tMdhM1kB8qJEf1TLrZrPiVqfODEe4JjbqTJ5mGAyJOCnqC3BvqGwL5q30xuWeU2gqGPwyFfgrQM9nupka55IGw== +styled-components@6.1.19: + version "6.1.19" + resolved "https://registry.yarnpkg.com/styled-components/-/styled-components-6.1.19.tgz#9a41b4db79a3b7a2477daecabe8dd917235263d6" + integrity sha512-1v/e3Dl1BknC37cXMhwGomhO8AkYmN41CqyX9xhUDxry1ns3BFQy2lLDRQXJRdVVWB9OHemv/53xaStimvWyuA== + dependencies: + "@emotion/is-prop-valid" "1.2.2" + "@emotion/unitless" "0.8.1" + "@types/stylis" "4.2.5" + css-to-react-native "3.2.0" + csstype "3.1.3" + postcss "8.4.49" + shallowequal "1.1.0" + stylis "4.3.2" + tslib "2.6.2" + +styled-icons@10.47.1: + version "10.47.1" + resolved "https://registry.yarnpkg.com/styled-icons/-/styled-icons-10.47.1.tgz#74511ca779147f482eba7434b0e5ca1fa8c8f8a6" + integrity sha512-O6SguU8Z8Qk+rWJkiPmoLOxCQ3x8Emabue2r4NPqX+CK1uyaNahzFtKaGq/0d13RTWXqfVOBUSIJZaIlu4G/CA== dependencies: "@babel/runtime" "^7.20.7" "@styled-icons/bootstrap" "10.47.0" @@ -12098,138 +7170,89 @@ styled-icons@^10.47.0: "@styled-icons/remix-fill" "10.46.0" "@styled-icons/remix-line" "10.46.0" "@styled-icons/simple-icons" "10.46.0" - "@styled-icons/styled-icon" "10.7.0" + "@styled-icons/styled-icon" "10.7.1" "@styled-icons/typicons" "10.46.0" "@styled-icons/zondicons" "10.46.0" -stylehacks@^5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/stylehacks/-/stylehacks-5.1.1.tgz#7934a34eb59d7152149fa69d6e9e56f2fc34bcc9" - integrity sha512-sBpcd5Hx7G6seo7b1LkpttvTz7ikD0LlH5RmdcBNb6fFR0Fl7LQwHDFr300q4cwUqi+IYrFGmsIHieMBfnN/Bw== - dependencies: - browserslist "^4.21.4" - postcss-selector-parser "^6.0.4" - -stylis@4.1.3: - version "4.1.3" - resolved "https://registry.yarnpkg.com/stylis/-/stylis-4.1.3.tgz#fd2fbe79f5fed17c55269e16ed8da14c84d069f7" - integrity sha512-GP6WDNWf+o403jrEp9c5jibKavrtLW+/qYGhFxFrG8maXhwTBI7gLLhiBb0o7uFccWN+EOS9aMO6cGHWAO07OA== - stylis@4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/stylis/-/stylis-4.2.0.tgz#79daee0208964c8fe695a42fcffcac633a211a51" integrity sha512-Orov6g6BB1sDfYgzWfTHDOxamtX1bE/zo104Dh9e6fqJ3PooipYyfJ0pUmrZO2wAvO8YbEyeFrkV91XTsGMSrw== -supports-color@^5.3.0, supports-color@^5.5.0: +stylis@4.3.2: + version "4.3.2" + resolved "https://registry.yarnpkg.com/stylis/-/stylis-4.3.2.tgz#8f76b70777dd53eb669c6f58c997bf0a9972e444" + integrity sha512-bhtUjWd/z6ltJiQwg0dUfxEJ+W+jdqQd8TbWLWyeIJHlnsqmGLRFFd8e5mA0AZi/zx90smXRlN66YMTcaSFifg== + +supports-color@^5.3.0: version "5.5.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f" integrity sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow== dependencies: has-flag "^3.0.0" -supports-color@^7.0.0, supports-color@^7.1.0: +supports-color@^7.1.0: version "7.2.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.2.0.tgz#1b7dcdcb32b8138801b3e478ba6a51caa89648da" integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw== dependencies: has-flag "^4.0.0" -supports-color@^8.0.0, supports-color@^8.1.1: +supports-color@^8.1.1: version "8.1.1" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-8.1.1.tgz#cd6fc17e28500cff56c1b86c0a7fd4a54a73005c" integrity sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q== dependencies: has-flag "^4.0.0" -supports-hyperlinks@^2.0.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/supports-hyperlinks/-/supports-hyperlinks-2.3.0.tgz#3943544347c1ff90b15effb03fc14ae45ec10624" - integrity sha512-RpsAZlpWcDwOPQA22aCH4J0t7L8JmAvsCxfOSEwm7cQs3LshN36QaTkwd70DnBOXDWGssw2eUoc8CaRWT0XunA== - dependencies: - has-flag "^4.0.0" - supports-color "^7.0.0" - supports-preserve-symlinks-flag@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09" integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== -svg-parser@^2.0.2: - version "2.0.4" - resolved "https://registry.yarnpkg.com/svg-parser/-/svg-parser-2.0.4.tgz#fdc2e29e13951736140b76cb122c8ee6630eb6b5" - integrity sha512-e4hG1hRwoOdRb37cIMSgzNsxyzKfayW6VOflrwvR+/bzrkyxY/31WkbgnQpgtrNp1SdpJvpUAGTa/ZoiPNDuRQ== - -svgo@^1.2.2: - version "1.3.2" - resolved "https://registry.yarnpkg.com/svgo/-/svgo-1.3.2.tgz#b6dc511c063346c9e415b81e43401145b96d4167" - integrity sha512-yhy/sQYxR5BkC98CY7o31VGsg014AKLEPxdfhora76l36hD9Rdy5NZA/Ocn6yayNPgSamYdtX2rFJdcv07AYVw== - dependencies: - chalk "^2.4.1" - coa "^2.0.2" - css-select "^2.0.0" - css-select-base-adapter "^0.1.1" - css-tree "1.0.0-alpha.37" - csso "^4.0.2" - js-yaml "^3.13.1" - mkdirp "~0.5.1" - object.values "^1.1.0" - sax "~1.2.4" - stable "^0.1.8" - unquote "~1.1.1" - util.promisify "~1.0.0" - -svgo@^2.7.0: - version "2.8.0" - resolved "https://registry.yarnpkg.com/svgo/-/svgo-2.8.0.tgz#4ff80cce6710dc2795f0c7c74101e6764cfccd24" - integrity sha512-+N/Q9kV1+F+UeWYoSiULYo4xYSDQlTgb+ayMobAXPwMnLvop7oxKMo9OzIrX5x3eS4L4f2UHhc9axXwY8DpChg== - dependencies: - "@trysound/sax" "0.2.0" - commander "^7.2.0" - css-select "^4.1.3" - css-tree "^1.1.3" - csso "^4.2.0" - picocolors "^1.0.0" - stable "^0.1.8" - -swagger-client@^3.22.3: - version "3.23.1" - resolved "https://registry.yarnpkg.com/swagger-client/-/swagger-client-3.23.1.tgz#22364b76b6f61b7c69f8846563fad59f28891380" - integrity sha512-ecRJsoGozhGvEUmim2kIc/pH9BllnPVuajuEXVm49EDbwbwbp7P+i5EW+8w5FLaqmGrx9eio51G9bvJV/XC+YQ== +swagger-client@^3.37.0: + version "3.37.0" + resolved "https://registry.yarnpkg.com/swagger-client/-/swagger-client-3.37.0.tgz#1ac57955d650cf92105e11900ca7b640f5604cc7" + integrity sha512-pzU+B+DkUbrSwlj4/E8sGeP1w84/CFgDJAt80fHu650TxnOHbqFLGQjiE6luvpRxTPdfK2zRHJP7I6CgUkI8yA== dependencies: "@babel/runtime-corejs3" "^7.22.15" - "@swagger-api/apidom-core" ">=0.77.0 <1.0.0" - "@swagger-api/apidom-json-pointer" ">=0.77.0 <1.0.0" - "@swagger-api/apidom-ns-openapi-3-1" ">=0.77.0 <1.0.0" - "@swagger-api/apidom-reference" ">=0.77.0 <1.0.0" - cookie "~0.5.0" + "@scarf/scarf" "=1.4.0" + "@swagger-api/apidom-core" "^1.6.0" + "@swagger-api/apidom-error" "^1.6.0" + "@swagger-api/apidom-json-pointer" "^1.6.0" + "@swagger-api/apidom-ns-openapi-3-1" "^1.6.0" + "@swagger-api/apidom-ns-openapi-3-2" "^1.6.0" + "@swagger-api/apidom-reference" "^1.6.0" + "@swaggerexpert/cookie" "^2.0.2" deepmerge "~4.3.0" fast-json-patch "^3.0.0-1" - is-plain-object "^5.0.0" js-yaml "^4.1.0" + neotraverse "=0.6.18" node-abort-controller "^3.1.1" - node-fetch-commonjs "^3.3.1" - qs "^6.10.2" - traverse "~0.6.6" - undici "^5.24.0" - -swagger-ui-react@^5.9.0: - version "5.9.0" - resolved "https://registry.yarnpkg.com/swagger-ui-react/-/swagger-ui-react-5.9.0.tgz#574411a191f454d2451d2f533caa397b503934fd" - integrity sha512-j45ceuGHMRmI8nhOaG71VeQwrPutFHDq6QhgrxOmf4BRMOdOQgVY1POQY9ksnXZtskbD9J2NHURs4BLEDIs8gA== - dependencies: - "@babel/runtime-corejs3" "^7.23.1" - "@braintree/sanitize-url" "=6.0.4" + node-fetch-commonjs "^3.3.2" + openapi-path-templating "^2.2.1" + openapi-server-url-templating "^1.3.0" + ramda "^0.30.1" + ramda-adjunct "^5.1.0" + +swagger-ui-react@5.32.0: + version "5.32.0" + resolved "https://registry.yarnpkg.com/swagger-ui-react/-/swagger-ui-react-5.32.0.tgz#fd764a7be746b5fdbff748fee6ef723eb05b4846" + integrity sha512-2mmrtvfp0EA90pdT8qXTMu26ex03TG2bsjvDAwXhdfCm+9foyadYJN+nEvDHM6/c6/xtXbdAsb6cVxBvbltnpw== + dependencies: + "@babel/runtime-corejs3" "^7.27.1" + "@scarf/scarf" "=1.4.0" base64-js "^1.5.1" - classnames "^2.3.1" + buffer "^6.0.3" + classnames "^2.5.1" css.escape "1.5.1" deep-extend "0.6.0" - dompurify "=3.0.6" + dompurify "=3.2.6" ieee754 "^1.2.1" immutable "^3.x.x" js-file-download "^0.4.12" - js-yaml "=4.1.0" + js-yaml "=4.1.1" lodash "^4.17.21" - patch-package "^8.0.0" prop-types "^15.8.1" randexp "^0.5.3" randombytes "^2.1.0" @@ -12238,15 +7261,15 @@ swagger-ui-react@^5.9.0: react-immutable-proptypes "2.2.0" react-immutable-pure-component "^2.2.0" react-inspector "^6.0.1" - react-redux "^8.1.2" - react-syntax-highlighter "^15.5.0" - redux "^4.1.2" + react-redux "^9.2.0" + react-syntax-highlighter "^16.0.0" + redux "^5.0.1" redux-immutable "^4.0.0" remarkable "^2.0.1" - reselect "^4.1.8" + reselect "^5.1.1" serialize-error "^8.1.0" - sha.js "^2.4.11" - swagger-client "^3.22.3" + sha.js "^2.4.12" + swagger-client "^3.37.0" url-parse "^1.5.10" xml "=1.0.1" xml-but-prettier "^1.0.1" @@ -12257,130 +7280,17 @@ symbol-tree@^3.2.4: resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.4.tgz#430637d248ba77e078883951fb9aa0eed7c63fa2" integrity sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw== -tailwindcss@^3.0.2: - version "3.2.4" - resolved "https://registry.yarnpkg.com/tailwindcss/-/tailwindcss-3.2.4.tgz#afe3477e7a19f3ceafb48e4b083e292ce0dc0250" - integrity sha512-AhwtHCKMtR71JgeYDaswmZXhPcW9iuI9Sp2LvZPo9upDZ7231ZJ7eA9RaURbhpXGVlrjX4cFNlB4ieTetEb7hQ== - dependencies: - arg "^5.0.2" - chokidar "^3.5.3" - color-name "^1.1.4" - detective "^5.2.1" - didyoumean "^1.2.2" - dlv "^1.1.3" - fast-glob "^3.2.12" - glob-parent "^6.0.2" - is-glob "^4.0.3" - lilconfig "^2.0.6" - micromatch "^4.0.5" - normalize-path "^3.0.0" - object-hash "^3.0.0" - picocolors "^1.0.0" - postcss "^8.4.18" - postcss-import "^14.1.0" - postcss-js "^4.0.0" - postcss-load-config "^3.1.4" - postcss-nested "6.0.0" - postcss-selector-parser "^6.0.10" - postcss-value-parser "^4.2.0" - quick-lru "^5.1.1" - resolve "^1.22.1" - -tapable@^1.0.0: - version "1.1.3" - resolved "https://registry.yarnpkg.com/tapable/-/tapable-1.1.3.tgz#a1fccc06b58db61fd7a45da2da44f5f3a3e67ba2" - integrity sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA== - -tapable@^2.0.0, tapable@^2.1.1, tapable@^2.2.0: - version "2.2.1" - resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.2.1.tgz#1967a73ef4060a82f12ab96af86d52fdb76eeca0" - integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ== - -tar-fs@^2.0.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784" - integrity sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng== - dependencies: - chownr "^1.1.1" - mkdirp-classic "^0.5.2" - pump "^3.0.0" - tar-stream "^2.1.4" - -tar-stream@^2.1.4: - version "2.2.0" - resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287" - integrity sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ== - dependencies: - bl "^4.0.3" - end-of-stream "^1.4.1" - fs-constants "^1.0.0" - inherits "^2.0.3" - readable-stream "^3.1.1" - -temp-dir@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/temp-dir/-/temp-dir-2.0.0.tgz#bde92b05bdfeb1516e804c9c00ad45177f31321e" - integrity sha512-aoBAniQmmwtcKp/7BzsH8Cxzv8OL736p7v1ihGb5e9DJ9kTwGWHrQrVB5+lfVDzfGrdRzXch+ig7LHaY1JTOrg== - -tempy@^0.6.0: - version "0.6.0" - resolved "https://registry.yarnpkg.com/tempy/-/tempy-0.6.0.tgz#65e2c35abc06f1124a97f387b08303442bde59f3" - integrity sha512-G13vtMYPT/J8A4X2SjdtBTphZlrp1gKv6hZiOjw14RCWg6GbHuQBGtjlx75xLbYV/wEc0D7G5K4rxKP/cXk8Bw== +synckit@^0.11.8: + version "0.11.11" + resolved "https://registry.yarnpkg.com/synckit/-/synckit-0.11.11.tgz#c0b619cf258a97faa209155d9cd1699b5c998cb0" + integrity sha512-MeQTA1r0litLUf0Rp/iisCaL8761lKAZHaimlbGK4j0HysC4PLfqygQj9srcs0m2RdtDYnF8UuYyKpbjHYp7Jw== dependencies: - is-stream "^2.0.0" - temp-dir "^2.0.0" - type-fest "^0.16.0" - unique-string "^2.0.0" + "@pkgr/core" "^0.2.9" -terminal-link@^2.0.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/terminal-link/-/terminal-link-2.1.1.tgz#14a64a27ab3c0df933ea546fba55f2d078edc994" - integrity sha512-un0FmiRUQNr5PJqy9kP7c40F5BOfpGlYTrxonDChEZB7pzZxRNp/bt+ymiy9/npwXya9KH99nJ/GXFIiUkYGFQ== - dependencies: - ansi-escapes "^4.2.1" - supports-hyperlinks "^2.0.0" - -terser-webpack-plugin@^5.2.5: - version "5.3.6" - resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-5.3.6.tgz#5590aec31aa3c6f771ce1b1acca60639eab3195c" - integrity sha512-kfLFk+PoLUQIbLmB1+PZDMRSZS99Mp+/MHqDNmMA6tOItzRt+Npe3E+fsMs5mfcM0wCtrrdU387UnV+vnSffXQ== - dependencies: - "@jridgewell/trace-mapping" "^0.3.14" - jest-worker "^27.4.5" - schema-utils "^3.1.1" - serialize-javascript "^6.0.0" - terser "^5.14.1" - -terser-webpack-plugin@^5.3.10: - version "5.3.10" - resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-5.3.10.tgz#904f4c9193c6fd2a03f693a2150c62a92f40d199" - integrity sha512-BKFPWlPDndPs+NGGCr1U59t0XScL5317Y0UReNrHaw9/FwhPENlq6bfgs+4yPfyP51vqC1bQ4rp1EfXW5ZSH9w== - dependencies: - "@jridgewell/trace-mapping" "^0.3.20" - jest-worker "^27.4.5" - schema-utils "^3.1.1" - serialize-javascript "^6.0.1" - terser "^5.26.0" - -terser@^5.0.0, terser@^5.10.0, terser@^5.14.1: - version "5.16.1" - resolved "https://registry.yarnpkg.com/terser/-/terser-5.16.1.tgz#5af3bc3d0f24241c7fb2024199d5c461a1075880" - integrity sha512-xvQfyfA1ayT0qdK47zskQgRZeWLoOQ8JQ6mIgRGVNwZKdQMU+5FkCBjmv4QjcrTzyZquRw2FVtlJSRUmMKQslw== - dependencies: - "@jridgewell/source-map" "^0.3.2" - acorn "^8.5.0" - commander "^2.20.0" - source-map-support "~0.5.20" - -terser@^5.26.0: - version "5.31.6" - resolved "https://registry.yarnpkg.com/terser/-/terser-5.31.6.tgz#c63858a0f0703988d0266a82fcbf2d7ba76422b1" - integrity sha512-PQ4DAriWzKj+qgehQ7LK5bQqCFNMmlhjR2PFFLuqGCpuCAauxemVBWwWOxo3UIwWQx8+Pr61Df++r76wDmkQBg== - dependencies: - "@jridgewell/source-map" "^0.3.3" - acorn "^8.8.2" - commander "^2.20.0" - source-map-support "~0.5.20" +tapable@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.3.0.tgz#7e3ea6d5ca31ba8e078b560f0d83ce9a14aa8be6" + integrity sha512-g9ljZiwki/LfxmQADO3dEY1CbpmXT5Hm2fJ+QaGKwSXUylMybePR7/67YW7jOrrvjEgL1Fmz5kzyAjWVWLlucg== test-exclude@^6.0.0: version "6.0.0" @@ -12391,50 +7301,40 @@ test-exclude@^6.0.0: glob "^7.1.4" minimatch "^3.0.4" -text-table@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" - integrity sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw== - -throat@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/throat/-/throat-6.0.1.tgz#d514fedad95740c12c2d7fc70ea863eb51ade375" - integrity sha512-8hmiGIJMDlwjg7dlJ4yKGLK8EsYqKgPWbG3b4wjJddKNwc7N7Dpn08Df4szr/sZdMVeOstrdYSsqzX6BYbcB+w== - -throttleit@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/throttleit/-/throttleit-1.0.0.tgz#9e785836daf46743145a5984b6268d828528ac6c" - integrity sha512-rkTVqu6IjfQ/6+uNuuc3sZek4CEYxTJom3IktzgdSxcZqdARuebbA/f4QmAxMQIxqq9ZLEUkSYqvuk1I6VKq4g== - -through@^2.3.8: - version "2.3.8" - resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" - integrity sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg== - -thunky@^1.0.2: - version "1.1.0" - resolved "https://registry.yarnpkg.com/thunky/-/thunky-1.1.0.tgz#5abaf714a9405db0504732bbccd2cedd9ef9537d" - integrity sha512-eHY7nBftgThBqOyHGVN+l8gF0BucP09fMo0oO/Lb0w1OF80dJv+lDVpXG60WMQvkcxAkNybKsrEIE3ZtKGmPrA== - -tmp@^0.0.33: - version "0.0.33" - resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9" - integrity sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw== +tinyglobby@^0.2.15: + version "0.2.15" + resolved "https://registry.yarnpkg.com/tinyglobby/-/tinyglobby-0.2.15.tgz#e228dd1e638cea993d2fdb4fcd2d4602a79951c2" + integrity sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ== dependencies: - os-tmpdir "~1.0.2" + fdir "^6.5.0" + picomatch "^4.0.3" -tmp@~0.2.1: - version "0.2.1" - resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.2.1.tgz#8457fc3037dcf4719c251367a1af6500ee1ccf14" - integrity sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ== +tldts-core@^6.1.86: + version "6.1.86" + resolved "https://registry.yarnpkg.com/tldts-core/-/tldts-core-6.1.86.tgz#a93e6ed9d505cb54c542ce43feb14c73913265d8" + integrity sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA== + +tldts@^6.1.32: + version "6.1.86" + resolved "https://registry.yarnpkg.com/tldts/-/tldts-6.1.86.tgz#087e0555b31b9725ee48ca7e77edc56115cd82f7" + integrity sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ== dependencies: - rimraf "^3.0.0" + tldts-core "^6.1.86" tmpl@1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.5.tgz#8683e0b902bb9c20c4f726e3c0b69f36518c07cc" integrity sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw== +to-buffer@^1.2.0: + version "1.2.2" + resolved "https://registry.yarnpkg.com/to-buffer/-/to-buffer-1.2.2.tgz#ffe59ef7522ada0a2d1cb5dfe03bb8abc3cdc133" + integrity sha512-db0E3UJjcFhpDhAF4tLo03oli3pwl3dbnzXOUIlRKrp+ldk/VUxzpWYZENsw2SZiuBjHAk7DfB0VU7NKdpb6sw== + dependencies: + isarray "^2.0.5" + safe-buffer "^5.2.1" + typed-array-buffer "^1.0.3" + to-fast-properties@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e" @@ -12452,159 +7352,74 @@ toggle-selection@^1.0.6: resolved "https://registry.yarnpkg.com/toggle-selection/-/toggle-selection-1.0.6.tgz#6e45b1263f2017fa0acc7d89d78b15b8bf77da32" integrity sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ== -toidentifier@1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35" - integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA== - -tough-cookie@^4.0.0, tough-cookie@^4.1.3: - version "4.1.3" - resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf" - integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw== - dependencies: - psl "^1.1.33" - punycode "^2.1.1" - universalify "^0.2.0" - url-parse "^1.5.3" - -tr46@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/tr46/-/tr46-1.0.1.tgz#a8b13fd6bfd2489519674ccde55ba3693b706d09" - integrity sha512-dTpowEjclQ7Kgx5SdBkqRzVhERQXov8/l9Ft9dVM9fmg0W0KQSVaXX9T4i6twCPNtYiZM53lpSSUAwJbFPOHxA== +tough-cookie@^5.1.1: + version "5.1.2" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-5.1.2.tgz#66d774b4a1d9e12dc75089725af3ac75ec31bed7" + integrity sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A== dependencies: - punycode "^2.1.0" + tldts "^6.1.32" -tr46@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" - integrity sha512-15Ih7phfcdP5YxqiB+iDtLoaTz4Nd35+IiAv0kQ5FNKHzXgdWqPoTIqEDDJmXceQt4JZk6lVPT8lnDlPpGDppw== +tr46@^5.1.0: + version "5.1.1" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-5.1.1.tgz#96ae867cddb8fdb64a49cc3059a8d428bcf238ca" + integrity sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw== dependencies: - punycode "^2.1.1" - -traverse@~0.6.6: - version "0.6.7" - resolved "https://registry.yarnpkg.com/traverse/-/traverse-0.6.7.tgz#46961cd2d57dd8706c36664acde06a248f1173fe" - integrity sha512-/y956gpUo9ZNCb99YjxG7OaslxZWHfCHAUUfshwqOXmxUIvqLjVO581BT+gM59+QV9tFe6/CGG53tsA1Y7RSdg== + punycode "^2.3.1" -tree-sitter-json@=0.20.1: - version "0.20.1" - resolved "https://registry.yarnpkg.com/tree-sitter-json/-/tree-sitter-json-0.20.1.tgz#d1fe6c59571dd3a987ebb3f5aeef404f37b3a453" - integrity sha512-482hf7J+aBwhksSw8yWaqI8nyP1DrSwnS4IMBShsnkFWD3SE8oalHnsEik59fEVi3orcTCUtMzSjZx+0Tpa6Vw== +tree-sitter-json@=0.24.8: + version "0.24.8" + resolved "https://registry.yarnpkg.com/tree-sitter-json/-/tree-sitter-json-0.24.8.tgz#72bfa26942691f2bf59d973b6794923c033f04c2" + integrity sha512-Tc9ZZYwHyWZ3Tt1VEw7Pa2scu1YO7/d2BCBbKTx5hXwig3UfdQjsOPkPyLpDJOn/m1UBEWYAtSdGAwCSyagBqQ== dependencies: - nan "^2.18.0" + node-addon-api "^8.2.2" + node-gyp-build "^4.8.2" -tree-sitter-yaml@=0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/tree-sitter-yaml/-/tree-sitter-yaml-0.5.0.tgz#c617ba72837399d8105ec10cdb4c360e1ed76076" - integrity sha512-POJ4ZNXXSWIG/W4Rjuyg36MkUD4d769YRUGKRqN+sVaj/VCo6Dh6Pkssn1Rtewd5kybx+jT1BWMyWN0CijXnMA== +tree-sitter@=0.21.1: + version "0.21.1" + resolved "https://registry.yarnpkg.com/tree-sitter/-/tree-sitter-0.21.1.tgz#fbb34c09056700814af0e1e37688e06463ba04c4" + integrity sha512-7dxoA6kYvtgWw80265MyqJlkRl4yawIjO7S5MigytjELkX43fV2WsAXzsNfO7sBpPPCF5Gp0+XzHk0DwLCq3xQ== dependencies: - nan "^2.14.0" + node-addon-api "^8.0.0" + node-gyp-build "^4.8.0" -tree-sitter@=0.20.4: - version "0.20.4" - resolved "https://registry.yarnpkg.com/tree-sitter/-/tree-sitter-0.20.4.tgz#7d9d4f769fc05342ef43e5559f7ff34b0fc48327" - integrity sha512-rjfR5dc4knG3jnJNN/giJ9WOoN1zL/kZyrS0ILh+eqq8RNcIbiXA63JsMEgluug0aNvfQvK4BfCErN1vIzvKog== +tree-sitter@=0.22.4: + version "0.22.4" + resolved "https://registry.yarnpkg.com/tree-sitter/-/tree-sitter-0.22.4.tgz#7d29547f663ff6f49ee8d1ac444e6fa169c72b77" + integrity sha512-usbHZP9/oxNsUY65MQUsduGRqDHQOou1cagUSwjhoSYAmSahjQDAVsh9s+SlZkn8X8+O1FULRGwHu7AFP3kjzg== dependencies: - nan "^2.17.0" - prebuild-install "^7.1.1" - -tryer@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/tryer/-/tryer-1.0.1.tgz#f2c85406800b9b0f74c9f7465b81eaad241252f8" - integrity sha512-c3zayb8/kWWpycWYg87P71E1S1ZL6b6IJxfb5fvsUgsf0S2MVGaDhDXXjDMpdCpfWXqptc+4mXwmiy1ypXqRAA== - -ts-api-utils@^1.0.1: - version "1.0.3" - resolved "https://registry.yarnpkg.com/ts-api-utils/-/ts-api-utils-1.0.3.tgz#f12c1c781d04427313dbac808f453f050e54a331" - integrity sha512-wNMeqtMz5NtwpT/UZGY5alT+VoKdSsOOP/kqHFcUW1P/VRhH2wJ48+DN2WwUliNbQ976ETwDL0Ifd2VVvgonvg== + node-addon-api "^8.3.0" + node-gyp-build "^4.8.4" -ts-jest@^29.1.1: - version "29.1.1" - resolved "https://registry.yarnpkg.com/ts-jest/-/ts-jest-29.1.1.tgz#f58fe62c63caf7bfcc5cc6472082f79180f0815b" - integrity sha512-D6xjnnbP17cC85nliwGiL+tpoKN0StpgE0TeOjXQTU6MVCfsB4v7aW05CgQ/1OywGb0x/oy9hHFnN+sczTiRaA== - dependencies: - bs-logger "0.x" - fast-json-stable-stringify "2.x" - jest-util "^29.0.0" - json5 "^2.2.3" - lodash.memoize "4.x" - make-error "1.x" - semver "^7.5.3" - yargs-parser "^21.0.1" +ts-mixer@^6.0.3, ts-mixer@^6.0.4: + version "6.0.4" + resolved "https://registry.yarnpkg.com/ts-mixer/-/ts-mixer-6.0.4.tgz#1da39ceabc09d947a82140d9f09db0f84919ca28" + integrity sha512-ufKpbmrugz5Aou4wcr5Wc1UUFWOLhq+Fm6qa6P0w0K5Qw2yhaUoiWszhCVuNQyNwrlGiscHOmqYoAox1PtvgjA== ts-toolbelt@^9.6.0: version "9.6.0" resolved "https://registry.yarnpkg.com/ts-toolbelt/-/ts-toolbelt-9.6.0.tgz#50a25426cfed500d4a09bd1b3afb6f28879edfd5" integrity sha512-nsZd8ZeNUzukXPlJmTBwUAuABDe/9qtVDelJeT/qW0ow3ZS3BsQJtNkan1802aM9Uf68/Y8ljw86Hu0h5IUW3w== -tsconfig-paths@^3.14.1: - version "3.14.1" - resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.14.1.tgz#ba0734599e8ea36c862798e920bcf163277b137a" - integrity sha512-fxDhWnFSLt3VuTwtvJt5fpwxBHg5AdKWMsgcPOOIilyjymcYVZoCQF8fvFRezCNfblEXmi+PcM1eYHeOAgXCOQ== - dependencies: - "@types/json5" "^0.0.29" - json5 "^1.0.1" - minimist "^1.2.6" - strip-bom "^3.0.0" - -tslib@^1.8.1: - version "1.14.1" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" - integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== - -tslib@^2.0.3, tslib@^2.1.0: - version "2.4.1" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.1.tgz#0d0bfbaac2880b91e22df0768e55be9753a5b17e" - integrity sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA== +tslib@2.6.2: + version "2.6.2" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" + integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== tslib@^2.3.0: version "2.5.0" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.5.0.tgz#42bfed86f5787aeb41d031866c8f402429e0fddf" integrity sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg== -tsutils@^3.21.0: - version "3.21.0" - resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623" - integrity sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA== - dependencies: - tslib "^1.8.1" - -tunnel-agent@^0.6.0: - version "0.6.0" - resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" - integrity sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w== - dependencies: - safe-buffer "^5.0.1" - -tweetnacl@^0.14.3, tweetnacl@~0.14.0: - version "0.14.5" - resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" - integrity sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA== - -type-check@^0.4.0, type-check@~0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1" - integrity sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew== - dependencies: - prelude-ls "^1.2.1" - -type-check@~0.3.2: - version "0.3.2" - resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.3.2.tgz#5884cab512cf1d355e3fb784f30804b2b520db72" - integrity sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg== - dependencies: - prelude-ls "~1.1.2" +tslib@^2.4.0: + version "2.8.1" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.8.1.tgz#612efe4ed235d567e8aba5f2a5fab70280ade83f" + integrity sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w== type-detect@4.0.8: version "4.0.8" resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.0.8.tgz#7646fb5f18871cfbb7749e69bd39a6388eb7450c" integrity sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g== -type-fest@^0.16.0: - version "0.16.0" - resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.16.0.tgz#3240b891a78b0deae910dbeb86553e552a148860" - integrity sha512-eaBzG6MxNzEn9kiwvtre90cXaNLkmadMWa1zQMs3XORCXNbsH/OewwbxC5ia9dCxIxnTAsSxXJaa/p5y8DlvJg== - type-fest@^0.20.2: version "0.20.2" resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4" @@ -12615,54 +7430,31 @@ type-fest@^0.21.3: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.21.3.tgz#d260a24b0198436e133fa26a524a6d65fa3b2e37" integrity sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w== -type-is@~1.6.18: - version "1.6.18" - resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131" - integrity sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g== - dependencies: - media-typer "0.3.0" - mime-types "~2.1.24" - -typedarray-to-buffer@^3.1.5: - version "3.1.5" - resolved "https://registry.yarnpkg.com/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz#a97ee7a9ff42691b9f783ff1bc5112fe3fca9080" - integrity sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q== +typed-array-buffer@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/typed-array-buffer/-/typed-array-buffer-1.0.3.tgz#a72395450a4869ec033fd549371b47af3a2ee536" + integrity sha512-nAYYwfY3qnzX30IkA6AQZjVbtK6duGontcQm1WSG1MD94YLqK0515GNApXkoxKOWMusVssAHWLh9SeaoefYFGw== dependencies: - is-typedarray "^1.0.0" + call-bound "^1.0.3" + es-errors "^1.3.0" + is-typed-array "^1.1.14" -types-ramda@^0.29.5: - version "0.29.5" - resolved "https://registry.yarnpkg.com/types-ramda/-/types-ramda-0.29.5.tgz#1cb0488d39eb72723a8f95af9b6dfe483e4f34a7" - integrity sha512-u+bAYXHDPJR+amB0qMrMU/NXRB2PG8QqpO2v6j7yK/0mPZhlaaZj++ynYjnVpkPEpCkZEGxNpWY3X7qyLCGE3w== +types-ramda@^0.30.1: + version "0.30.1" + resolved "https://registry.yarnpkg.com/types-ramda/-/types-ramda-0.30.1.tgz#03d255128e3696aeaac76281ca19949e01dddc78" + integrity sha512-1HTsf5/QVRmLzcGfldPFvkVsAdi1db1BBKzi7iW3KBUlOICg/nKnFS+jGqDJS3YD8VsWbAh7JiHeBvbsw8RPxA== dependencies: ts-toolbelt "^9.6.0" -typescript@^4.4.2: - version "4.9.5" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.5.tgz#095979f9bcc0d09da324d58d03ce8f8374cbe65a" - integrity sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g== - -unbox-primitive@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/unbox-primitive/-/unbox-primitive-1.0.2.tgz#29032021057d5e6cdbd08c5129c226dff8ed6f9e" - integrity sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw== - dependencies: - call-bind "^1.0.2" - has-bigints "^1.0.2" - has-symbols "^1.0.3" - which-boxed-primitive "^1.0.2" - -undici-types@~5.25.1: - version "5.25.3" - resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.25.3.tgz#e044115914c85f0bcbb229f346ab739f064998c3" - integrity sha512-Ga1jfYwRn7+cP9v8auvEXN1rX3sWqlayd4HP7OKk4mZWylEmu3KzXDUGrQUN6Ol7qo1gPvB2e5gX6udnyEPgdA== +typescript@5.9.3: + version "5.9.3" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.9.3.tgz#5b4f59e15310ab17a216f5d6cf53ee476ede670f" + integrity sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw== -undici@^5.24.0: - version "5.28.5" - resolved "https://registry.yarnpkg.com/undici/-/undici-5.28.5.tgz#b2b94b6bf8f1d919bc5a6f31f2c01deb02e54d4b" - integrity sha512-zICwjrDrcrUE0pyyJc1I2QzBkLM8FINsgOrt6WjA+BgajVq9Nxu2PbFFXUrAggLfDXlZGZBVZYw7WNV5KiBiBA== - dependencies: - "@fastify/busboy" "^2.0.0" +undici-types@~7.16.0: + version "7.16.0" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-7.16.0.tgz#ffccdff36aea4884cbfce9a750a0580224f58a46" + integrity sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw== unicode-canonical-property-names-ecmascript@^2.0.0: version "2.0.0" @@ -12687,55 +7479,37 @@ unicode-property-aliases-ecmascript@^2.0.0: resolved "https://registry.yarnpkg.com/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-2.1.0.tgz#43d41e3be698bd493ef911077c9b131f827e8ccd" integrity sha512-6t3foTQI9qne+OZoVQB/8x8rk2k1eVy1gRXhV3oFQ5T6R1dqQ1xtin3XqSlx3+ATBkliTaR/hHyJBm+LVPNM8w== -unique-string@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/unique-string/-/unique-string-2.0.0.tgz#39c6451f81afb2749de2b233e3f7c5e8843bd89d" - integrity sha512-uNaeirEPvpZWSgzwsPGtU2zVSTrn/8L5q/IexZmH0eH6SA73CmAA5U4GwORTxQAZs95TAXLNqeLoPPNO5gZfWg== - dependencies: - crypto-random-string "^2.0.0" - -universalify@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.2.0.tgz#6451760566fa857534745ab1dde952d1b1761be0" - integrity sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg== - -universalify@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.0.tgz#75a4984efedc4b08975c5aeb73f530d02df25717" - integrity sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ== - -unpipe@1.0.0, unpipe@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" - integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== - -unquote@~1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/unquote/-/unquote-1.1.1.tgz#8fded7324ec6e88a0ff8b905e7c098cdc086d544" - integrity sha512-vRCqFv6UhXpWxZPyGDh/F3ZpNv8/qo7w6iufLpQg9aKnQ71qM4B5KiI7Mia9COcjEhrO9LueHpMYjYzsWH3OIg== - unraw@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/unraw/-/unraw-3.0.0.tgz#73443ed70d2ab09ccbac2b00525602d5991fbbe3" integrity sha512-08/DA66UF65OlpUDIQtbJyrqTR0jTAlJ+jsnkQ4jxR7+K5g5YG1APZKQSMCE1vqqmD+2pv6+IdEjmopFatacvg== -untildify@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" - integrity sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw== - -upath@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/upath/-/upath-1.2.0.tgz#8f66dbcd55a883acdae4408af8b035a5044c1894" - integrity sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg== - -update-browserslist-db@^1.0.13: - version "1.0.13" - resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz#3c5e4f5c083661bd38ef64b6328c26ed6c8248c4" - integrity sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg== +unrs-resolver@^1.7.11: + version "1.11.1" + resolved "https://registry.yarnpkg.com/unrs-resolver/-/unrs-resolver-1.11.1.tgz#be9cd8686c99ef53ecb96df2a473c64d304048a9" + integrity sha512-bSjt9pjaEBnNiGgc9rUiHGKv5l4/TGzDmYw3RhnkJGtLhbnnA/5qJj7x3dNDCRx/PJxu774LlH8lCOlB4hEfKg== dependencies: - escalade "^3.1.1" - picocolors "^1.0.0" + napi-postinstall "^0.3.0" + optionalDependencies: + "@unrs/resolver-binding-android-arm-eabi" "1.11.1" + "@unrs/resolver-binding-android-arm64" "1.11.1" + "@unrs/resolver-binding-darwin-arm64" "1.11.1" + "@unrs/resolver-binding-darwin-x64" "1.11.1" + "@unrs/resolver-binding-freebsd-x64" "1.11.1" + "@unrs/resolver-binding-linux-arm-gnueabihf" "1.11.1" + "@unrs/resolver-binding-linux-arm-musleabihf" "1.11.1" + "@unrs/resolver-binding-linux-arm64-gnu" "1.11.1" + "@unrs/resolver-binding-linux-arm64-musl" "1.11.1" + "@unrs/resolver-binding-linux-ppc64-gnu" "1.11.1" + "@unrs/resolver-binding-linux-riscv64-gnu" "1.11.1" + "@unrs/resolver-binding-linux-riscv64-musl" "1.11.1" + "@unrs/resolver-binding-linux-s390x-gnu" "1.11.1" + "@unrs/resolver-binding-linux-x64-gnu" "1.11.1" + "@unrs/resolver-binding-linux-x64-musl" "1.11.1" + "@unrs/resolver-binding-wasm32-wasi" "1.11.1" + "@unrs/resolver-binding-win32-arm64-msvc" "1.11.1" + "@unrs/resolver-binding-win32-ia32-msvc" "1.11.1" + "@unrs/resolver-binding-win32-x64-msvc" "1.11.1" update-browserslist-db@^1.0.9: version "1.0.10" @@ -12745,22 +7519,15 @@ update-browserslist-db@^1.0.9: escalade "^3.1.1" picocolors "^1.0.0" -update-browserslist-db@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.1.0.tgz#7ca61c0d8650766090728046e416a8cde682859e" - integrity sha512-EdRAaAyk2cUE1wOf2DkEhzxqOQvFOoRJFNS6NeyJ01Gp2beMRpBAINjM2iDXE3KCuKhwnvHIQCJm6ThL2Z+HzQ== - dependencies: - escalade "^3.1.2" - picocolors "^1.0.1" - -uri-js@^4.2.2: - version "4.4.1" - resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.1.tgz#9b1a52595225859e55f669d928f88c6c57f2a77e" - integrity sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg== +update-browserslist-db@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.1.4.tgz#7802aa2ae91477f255b86e0e46dbc787a206ad4a" + integrity sha512-q0SPT4xyU84saUX+tomz1WLkxUbuaJnR1xWt17M7fJtEJigJeWUNGUqrauFXsHnqev9y9JTRGwk13tFBuKby4A== dependencies: - punycode "^2.1.0" + escalade "^3.2.0" + picocolors "^1.1.1" -url-parse@^1.5.10, url-parse@^1.5.3: +url-parse@^1.5.10: version "1.5.10" resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.5.10.tgz#9d3c2f736c1d75dd3bd2be507dcc111f1e2ea9c1" integrity sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ== @@ -12768,49 +7535,10 @@ url-parse@^1.5.10, url-parse@^1.5.3: querystringify "^2.1.1" requires-port "^1.0.0" -use-sync-external-store@^1.0.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz#7dbefd6ef3fe4e767a0cf5d7287aacfb5846928a" - integrity sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA== - -util-deprecate@^1.0.1, util-deprecate@^1.0.2, util-deprecate@~1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" - integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw== - -util.promisify@~1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/util.promisify/-/util.promisify-1.0.1.tgz#6baf7774b80eeb0f7520d8b81d07982a59abbaee" - integrity sha512-g9JpC/3He3bm38zsLupWryXHoEcS22YHthuPQSJdMy6KNrzIRzWqcsHzD/WUnqe45whVou4VIsPew37DoXWNrA== - dependencies: - define-properties "^1.1.3" - es-abstract "^1.17.2" - has-symbols "^1.0.1" - object.getownpropertydescriptors "^2.1.0" - -utila@~0.4: - version "0.4.0" - resolved "https://registry.yarnpkg.com/utila/-/utila-0.4.0.tgz#8a16a05d445657a3aea5eecc5b12a4fa5379772c" - integrity sha512-Z0DbgELS9/L/75wZbro8xAnT50pBVFQZ+hUEueGDU5FN51YSCYM+jdxsfCiHjwNP/4LCDD0i/graKpeBnOXKRA== - -utils-merge@1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" - integrity sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA== - -uuid@^8.3.2: - version "8.3.2" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" - integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== - -v8-to-istanbul@^8.1.0: - version "8.1.1" - resolved "https://registry.yarnpkg.com/v8-to-istanbul/-/v8-to-istanbul-8.1.1.tgz#77b752fd3975e31bbcef938f85e9bd1c7a8d60ed" - integrity sha512-FGtKtv3xIpR6BYhvgH8MI/y78oT7d8Au3ww4QIxymrCtZEh5b8gCw2siywE+puhEmuWKDtmfrvF5UlB298ut3w== - dependencies: - "@types/istanbul-lib-coverage" "^2.0.1" - convert-source-map "^1.6.0" - source-map "^0.7.3" +use-sync-external-store@^1.4.0, use-sync-external-store@^1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/use-sync-external-store/-/use-sync-external-store-1.6.0.tgz#b174bfa65cb2b526732d9f2ac0a408027876f32d" + integrity sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w== v8-to-istanbul@^9.0.1: version "9.1.3" @@ -12821,238 +7549,73 @@ v8-to-istanbul@^9.0.1: "@types/istanbul-lib-coverage" "^2.0.1" convert-source-map "^2.0.0" -vary@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" - integrity sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg== - -verror@1.10.0: - version "1.10.0" - resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" - integrity sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw== - dependencies: - assert-plus "^1.0.0" - core-util-is "1.0.2" - extsprintf "^1.2.0" - -w3c-hr-time@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz#0a89cdf5cc15822df9c360543676963e0cc308cd" - integrity sha512-z8P5DvDNjKDoFIHK7q8r8lackT6l+jo/Ye3HOle7l9nICP9lf1Ci25fy9vHd0JOWewkIFzXIEig3TdKT7JQ5fQ== - dependencies: - browser-process-hrtime "^1.0.0" +vite@7.3.1: + version "7.3.1" + resolved "https://registry.yarnpkg.com/vite/-/vite-7.3.1.tgz#7f6cfe8fb9074138605e822a75d9d30b814d6507" + integrity sha512-w+N7Hifpc3gRjZ63vYBXA56dvvRlNWRczTdmCBBa+CotUzAPf5b7YMdMR/8CQoeYE5LX3W4wj6RYTgonm1b9DA== + dependencies: + esbuild "^0.27.0" + fdir "^6.5.0" + picomatch "^4.0.3" + postcss "^8.5.6" + rollup "^4.43.0" + tinyglobby "^0.2.15" + optionalDependencies: + fsevents "~2.3.3" -w3c-xmlserializer@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/w3c-xmlserializer/-/w3c-xmlserializer-2.0.0.tgz#3e7104a05b75146cc60f564380b7f683acf1020a" - integrity sha512-4tzD0mF8iSiMiNs30BiLO3EpfGLZUT2MSX/G+o7ZywDzliWQ3OPtTZ0PTC3B3ca1UAf4cJMHB+2Bf56EriJuRA== +w3c-xmlserializer@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz#f925ba26855158594d907313cedd1476c5967f6c" + integrity sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA== dependencies: - xml-name-validator "^3.0.0" + xml-name-validator "^5.0.0" -walker@^1.0.7, walker@^1.0.8: +walker@^1.0.8: version "1.0.8" resolved "https://registry.yarnpkg.com/walker/-/walker-1.0.8.tgz#bd498db477afe573dc04185f011d3ab8a8d7653f" integrity sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ== dependencies: makeerror "1.0.12" -watchpack@^2.4.1: - version "2.4.2" - resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-2.4.2.tgz#2feeaed67412e7c33184e5a79ca738fbd38564da" - integrity sha512-TnbFSbcOCcDgjZ4piURLCbJ3nJhznVh9kw6F6iokjiFPl8ONxe9A6nMDVXDiNbrSfLILs6vB07F7wLBrwPYzJw== - dependencies: - glob-to-regexp "^0.4.1" - graceful-fs "^4.1.2" - -wbuf@^1.1.0, wbuf@^1.7.3: - version "1.7.3" - resolved "https://registry.yarnpkg.com/wbuf/-/wbuf-1.7.3.tgz#c1d8d149316d3ea852848895cb6a0bfe887b87df" - integrity sha512-O84QOnr0icsbFGLS0O3bI5FswxzRr8/gHwWkDlQFskhSPryQXvrTMxjxGP4+iWYoauLoBvfDpkrOauZ+0iZpDA== - dependencies: - minimalistic-assert "^1.0.0" - web-streams-polyfill@^3.0.3: version "3.2.1" resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz#71c2718c52b45fd49dbeee88634b3a60ceab42a6" integrity sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q== -web-tree-sitter@=0.20.3: - version "0.20.3" - resolved "https://registry.yarnpkg.com/web-tree-sitter/-/web-tree-sitter-0.20.3.tgz#3dd17b283ad63b1d8c07c5ea814f0fefb2b1f776" - integrity sha512-zKGJW9r23y3BcJusbgvnOH2OYAW40MXAOi9bi3Gcc7T4Gms9WWgXF8m6adsJWpGJEhgOzCrfiz1IzKowJWrtYw== - -web-vitals@^2.1.0: - version "2.1.4" - resolved "https://registry.yarnpkg.com/web-vitals/-/web-vitals-2.1.4.tgz#76563175a475a5e835264d373704f9dde718290c" - integrity sha512-sVWcwhU5mX6crfI5Vd2dC4qchyTqxV8URinzt25XqVh+bHEPGH4C3NPrNionCP7Obx59wrYEbNlw4Z8sjALzZg== - -webidl-conversions@^4.0.2: - version "4.0.2" - resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-4.0.2.tgz#a855980b1f0b6b359ba1d5d9fb39ae941faa63ad" - integrity sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg== - -webidl-conversions@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-5.0.0.tgz#ae59c8a00b121543a2acc65c0434f57b0fc11aff" - integrity sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA== - -webidl-conversions@^6.1.0: - version "6.1.0" - resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-6.1.0.tgz#9111b4d7ea80acd40f5270d666621afa78b69514" - integrity sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w== - -webpack-dev-middleware@^5.3.1: - version "5.3.4" - resolved "https://registry.yarnpkg.com/webpack-dev-middleware/-/webpack-dev-middleware-5.3.4.tgz#eb7b39281cbce10e104eb2b8bf2b63fce49a3517" - integrity sha512-BVdTqhhs+0IfoeAf7EoH5WE+exCmqGerHfDM0IL096Px60Tq2Mn9MAbnaGUe6HiMa41KMCYF19gyzZmBcq/o4Q== - dependencies: - colorette "^2.0.10" - memfs "^3.4.3" - mime-types "^2.1.31" - range-parser "^1.2.1" - schema-utils "^4.0.0" - -webpack-dev-server@^4.6.0: - version "4.11.1" - resolved "https://registry.yarnpkg.com/webpack-dev-server/-/webpack-dev-server-4.11.1.tgz#ae07f0d71ca0438cf88446f09029b92ce81380b5" - integrity sha512-lILVz9tAUy1zGFwieuaQtYiadImb5M3d+H+L1zDYalYoDl0cksAB1UNyuE5MMWJrG6zR1tXkCP2fitl7yoUJiw== - dependencies: - "@types/bonjour" "^3.5.9" - "@types/connect-history-api-fallback" "^1.3.5" - "@types/express" "^4.17.13" - "@types/serve-index" "^1.9.1" - "@types/serve-static" "^1.13.10" - "@types/sockjs" "^0.3.33" - "@types/ws" "^8.5.1" - ansi-html-community "^0.0.8" - bonjour-service "^1.0.11" - chokidar "^3.5.3" - colorette "^2.0.10" - compression "^1.7.4" - connect-history-api-fallback "^2.0.0" - default-gateway "^6.0.3" - express "^4.17.3" - graceful-fs "^4.2.6" - html-entities "^2.3.2" - http-proxy-middleware "^2.0.3" - ipaddr.js "^2.0.1" - open "^8.0.9" - p-retry "^4.5.0" - rimraf "^3.0.2" - schema-utils "^4.0.0" - selfsigned "^2.1.1" - serve-index "^1.9.1" - sockjs "^0.3.24" - spdy "^4.0.2" - webpack-dev-middleware "^5.3.1" - ws "^8.4.2" - -webpack-manifest-plugin@^4.0.2: - version "4.1.1" - resolved "https://registry.yarnpkg.com/webpack-manifest-plugin/-/webpack-manifest-plugin-4.1.1.tgz#10f8dbf4714ff93a215d5a45bcc416d80506f94f" - integrity sha512-YXUAwxtfKIJIKkhg03MKuiFAD72PlrqCiwdwO4VEXdRO5V0ORCNwaOwAZawPZalCbmH9kBDmXnNeQOw+BIEiow== - dependencies: - tapable "^2.0.0" - webpack-sources "^2.2.0" - -webpack-sources@^1.4.3: - version "1.4.3" - resolved "https://registry.yarnpkg.com/webpack-sources/-/webpack-sources-1.4.3.tgz#eedd8ec0b928fbf1cbfe994e22d2d890f330a933" - integrity sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ== - dependencies: - source-list-map "^2.0.0" - source-map "~0.6.1" - -webpack-sources@^2.2.0: - version "2.3.1" - resolved "https://registry.yarnpkg.com/webpack-sources/-/webpack-sources-2.3.1.tgz#570de0af163949fe272233c2cefe1b56f74511fd" - integrity sha512-y9EI9AO42JjEcrTJFOYmVywVZdKVUfOvDUPsJea5GIr1JOEGFVqwlY2K098fFoIjOkDzHn2AjRvM8dsBZu+gCA== - dependencies: - source-list-map "^2.0.1" - source-map "^0.6.1" +web-tree-sitter@=0.24.5: + version "0.24.5" + resolved "https://registry.yarnpkg.com/web-tree-sitter/-/web-tree-sitter-0.24.5.tgz#16cea449da63012f23ca7b83bd32817dd0520400" + integrity sha512-+J/2VSHN8J47gQUAvF8KDadrfz6uFYVjxoxbKWDoXVsH2u7yLdarCnIURnrMA6uSRkgX3SdmqM5BOoQjPdSh5w== -webpack-sources@^3.2.3: - version "3.2.3" - resolved "https://registry.yarnpkg.com/webpack-sources/-/webpack-sources-3.2.3.tgz#2d4daab8451fd4b240cc27055ff6a0c2ccea0cde" - integrity sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w== - -webpack@^5.64.4: - version "5.94.0" - resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.94.0.tgz#77a6089c716e7ab90c1c67574a28da518a20970f" - integrity sha512-KcsGn50VT+06JH/iunZJedYGUJS5FGjow8wb9c0v5n1Om8O1g4L6LjtfxwlXIATopoQu+vOXXa7gYisWxCoPyg== - dependencies: - "@types/estree" "^1.0.5" - "@webassemblyjs/ast" "^1.12.1" - "@webassemblyjs/wasm-edit" "^1.12.1" - "@webassemblyjs/wasm-parser" "^1.12.1" - acorn "^8.7.1" - acorn-import-attributes "^1.9.5" - browserslist "^4.21.10" - chrome-trace-event "^1.0.2" - enhanced-resolve "^5.17.1" - es-module-lexer "^1.2.1" - eslint-scope "5.1.1" - events "^3.2.0" - glob-to-regexp "^0.4.1" - graceful-fs "^4.2.11" - json-parse-even-better-errors "^2.3.1" - loader-runner "^4.2.0" - mime-types "^2.1.27" - neo-async "^2.6.2" - schema-utils "^3.2.0" - tapable "^2.1.1" - terser-webpack-plugin "^5.3.10" - watchpack "^2.4.1" - webpack-sources "^3.2.3" - -websocket-driver@>=0.5.1, websocket-driver@^0.7.4: - version "0.7.4" - resolved "https://registry.yarnpkg.com/websocket-driver/-/websocket-driver-0.7.4.tgz#89ad5295bbf64b480abcba31e4953aca706f5760" - integrity sha512-b17KeDIQVjvb0ssuSDF2cYXSg2iztliJ4B9WdsuB6J952qCPKmnVq4DyW5motImXHDC1cBT/1UezrJVsKw5zjg== - dependencies: - http-parser-js ">=0.5.1" - safe-buffer ">=5.1.0" - websocket-extensions ">=0.1.1" - -websocket-extensions@>=0.1.1: - version "0.1.4" - resolved "https://registry.yarnpkg.com/websocket-extensions/-/websocket-extensions-0.1.4.tgz#7f8473bc839dfd87608adb95d7eb075211578a42" - integrity sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg== +webidl-conversions@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-7.0.0.tgz#256b4e1882be7debbf01d05f0aa2039778ea080a" + integrity sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g== -whatwg-encoding@^1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz#5abacf777c32166a51d085d6b4f3e7d27113ddb0" - integrity sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw== +whatwg-encoding@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz#d0f4ef769905d426e1688f3e34381a99b60b76e5" + integrity sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ== dependencies: - iconv-lite "0.4.24" + iconv-lite "0.6.3" whatwg-fetch@^3.6.2: version "3.6.2" resolved "https://registry.yarnpkg.com/whatwg-fetch/-/whatwg-fetch-3.6.2.tgz#dced24f37f2624ed0281725d51d0e2e3fe677f8c" integrity sha512-bJlen0FcuU/0EMLrdbJ7zOnW6ITZLrZMIarMUVmdKtsGvZna8vxKYaexICWPfZ8qwf9fzNq+UEIZrnSaApt6RA== -whatwg-mimetype@^2.3.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz#3d4b1e0312d2079879f826aff18dbeeca5960fbf" - integrity sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g== - -whatwg-url@^7.0.0: - version "7.1.0" - resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-7.1.0.tgz#c2c492f1eca612988efd3d2266be1b9fc6170d06" - integrity sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg== - dependencies: - lodash.sortby "^4.7.0" - tr46 "^1.0.1" - webidl-conversions "^4.0.2" +whatwg-mimetype@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz#bc1bf94a985dc50388d54a9258ac405c3ca2fc0a" + integrity sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg== -whatwg-url@^8.0.0, whatwg-url@^8.5.0: - version "8.7.0" - resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-8.7.0.tgz#656a78e510ff8f3937bc0bcbe9f5c0ac35941b77" - integrity sha512-gAojqb/m9Q8a5IV96E3fHJM70AzCkgt4uXYX2O7EmuyOnLrViCQlsEBmF9UQIu3/aeAIp2U17rtbpZWNntQqdg== +whatwg-url@^14.0.0, whatwg-url@^14.1.1: + version "14.2.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-14.2.0.tgz#4ee02d5d725155dae004f6ae95c73e7ef5d95663" + integrity sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw== dependencies: - lodash "^4.7.0" - tr46 "^2.1.0" - webidl-conversions "^6.1.0" + tr46 "^5.1.0" + webidl-conversions "^7.0.0" which-boxed-primitive@^1.0.2: version "1.0.2" @@ -13075,6 +7638,19 @@ which-collection@^1.0.1: is-weakmap "^2.0.1" is-weakset "^2.0.1" +which-typed-array@^1.1.16: + version "1.1.19" + resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.19.tgz#df03842e870b6b88e117524a4b364b6fc689f956" + integrity sha512-rEvr90Bck4WZt9HHFC4DJMsjvu7x+r6bImz0/BrbWb7A2djJ8hnZMrWnHo9F8ssv0OMErasDhftrfROTyqSDrw== + dependencies: + available-typed-arrays "^1.0.7" + call-bind "^1.0.8" + call-bound "^1.0.4" + for-each "^0.3.5" + get-proto "^1.0.1" + gopd "^1.2.0" + has-tostringtag "^1.0.2" + which-typed-array@^1.1.8: version "1.1.9" resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.9.tgz#307cf898025848cf995e795e8423c7f337efbde6" @@ -13087,13 +7663,6 @@ which-typed-array@^1.1.8: has-tostringtag "^1.0.0" is-typed-array "^1.1.10" -which@^1.3.1: - version "1.3.1" - resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a" - integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ== - dependencies: - isexe "^2.0.0" - which@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1" @@ -13101,184 +7670,10 @@ which@^2.0.1: dependencies: isexe "^2.0.0" -word-wrap@^1.2.3, word-wrap@~1.2.3: - version "1.2.4" - resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.4.tgz#cb4b50ec9aca570abd1f52f33cd45b6c61739a9f" - integrity sha512-2V81OA4ugVo5pRo46hAoD2ivUJx8jXmWXfUkY4KFNw0hEptvN0QfH3K4nHiwzGeKl5rFKedV48QVoqYavy4YpA== - -workbox-background-sync@6.5.4: - version "6.5.4" - resolved "https://registry.yarnpkg.com/workbox-background-sync/-/workbox-background-sync-6.5.4.tgz#3141afba3cc8aa2ae14c24d0f6811374ba8ff6a9" - integrity sha512-0r4INQZMyPky/lj4Ou98qxcThrETucOde+7mRGJl13MPJugQNKeZQOdIJe/1AchOP23cTqHcN/YVpD6r8E6I8g== - dependencies: - idb "^7.0.1" - workbox-core "6.5.4" - -workbox-broadcast-update@6.5.4: - version "6.5.4" - resolved "https://registry.yarnpkg.com/workbox-broadcast-update/-/workbox-broadcast-update-6.5.4.tgz#8441cff5417cd41f384ba7633ca960a7ffe40f66" - integrity sha512-I/lBERoH1u3zyBosnpPEtcAVe5lwykx9Yg1k6f8/BGEPGaMMgZrwVrqL1uA9QZ1NGGFoyE6t9i7lBjOlDhFEEw== - dependencies: - workbox-core "6.5.4" - -workbox-build@6.5.4: - version "6.5.4" - resolved "https://registry.yarnpkg.com/workbox-build/-/workbox-build-6.5.4.tgz#7d06d31eb28a878817e1c991c05c5b93409f0389" - integrity sha512-kgRevLXEYvUW9WS4XoziYqZ8Q9j/2ziJYEtTrjdz5/L/cTUa2XfyMP2i7c3p34lgqJ03+mTiz13SdFef2POwbA== - dependencies: - "@apideck/better-ajv-errors" "^0.3.1" - "@babel/core" "^7.11.1" - "@babel/preset-env" "^7.11.0" - "@babel/runtime" "^7.11.2" - "@rollup/plugin-babel" "^5.2.0" - "@rollup/plugin-node-resolve" "^11.2.1" - "@rollup/plugin-replace" "^2.4.1" - "@surma/rollup-plugin-off-main-thread" "^2.2.3" - ajv "^8.6.0" - common-tags "^1.8.0" - fast-json-stable-stringify "^2.1.0" - fs-extra "^9.0.1" - glob "^7.1.6" - lodash "^4.17.20" - pretty-bytes "^5.3.0" - rollup "^2.43.1" - rollup-plugin-terser "^7.0.0" - source-map "^0.8.0-beta.0" - stringify-object "^3.3.0" - strip-comments "^2.0.1" - tempy "^0.6.0" - upath "^1.2.0" - workbox-background-sync "6.5.4" - workbox-broadcast-update "6.5.4" - workbox-cacheable-response "6.5.4" - workbox-core "6.5.4" - workbox-expiration "6.5.4" - workbox-google-analytics "6.5.4" - workbox-navigation-preload "6.5.4" - workbox-precaching "6.5.4" - workbox-range-requests "6.5.4" - workbox-recipes "6.5.4" - workbox-routing "6.5.4" - workbox-strategies "6.5.4" - workbox-streams "6.5.4" - workbox-sw "6.5.4" - workbox-window "6.5.4" - -workbox-cacheable-response@6.5.4: - version "6.5.4" - resolved "https://registry.yarnpkg.com/workbox-cacheable-response/-/workbox-cacheable-response-6.5.4.tgz#a5c6ec0c6e2b6f037379198d4ef07d098f7cf137" - integrity sha512-DCR9uD0Fqj8oB2TSWQEm1hbFs/85hXXoayVwFKLVuIuxwJaihBsLsp4y7J9bvZbqtPJ1KlCkmYVGQKrBU4KAug== - dependencies: - workbox-core "6.5.4" - -workbox-core@6.5.4: - version "6.5.4" - resolved "https://registry.yarnpkg.com/workbox-core/-/workbox-core-6.5.4.tgz#df48bf44cd58bb1d1726c49b883fb1dffa24c9ba" - integrity sha512-OXYb+m9wZm8GrORlV2vBbE5EC1FKu71GGp0H4rjmxmF4/HLbMCoTFws87M3dFwgpmg0v00K++PImpNQ6J5NQ6Q== - -workbox-expiration@6.5.4: - version "6.5.4" - resolved "https://registry.yarnpkg.com/workbox-expiration/-/workbox-expiration-6.5.4.tgz#501056f81e87e1d296c76570bb483ce5e29b4539" - integrity sha512-jUP5qPOpH1nXtjGGh1fRBa1wJL2QlIb5mGpct3NzepjGG2uFFBn4iiEBiI9GUmfAFR2ApuRhDydjcRmYXddiEQ== - dependencies: - idb "^7.0.1" - workbox-core "6.5.4" - -workbox-google-analytics@6.5.4: - version "6.5.4" - resolved "https://registry.yarnpkg.com/workbox-google-analytics/-/workbox-google-analytics-6.5.4.tgz#c74327f80dfa4c1954cbba93cd7ea640fe7ece7d" - integrity sha512-8AU1WuaXsD49249Wq0B2zn4a/vvFfHkpcFfqAFHNHwln3jK9QUYmzdkKXGIZl9wyKNP+RRX30vcgcyWMcZ9VAg== - dependencies: - workbox-background-sync "6.5.4" - workbox-core "6.5.4" - workbox-routing "6.5.4" - workbox-strategies "6.5.4" - -workbox-navigation-preload@6.5.4: - version "6.5.4" - resolved "https://registry.yarnpkg.com/workbox-navigation-preload/-/workbox-navigation-preload-6.5.4.tgz#ede56dd5f6fc9e860a7e45b2c1a8f87c1c793212" - integrity sha512-IIwf80eO3cr8h6XSQJF+Hxj26rg2RPFVUmJLUlM0+A2GzB4HFbQyKkrgD5y2d84g2IbJzP4B4j5dPBRzamHrng== - dependencies: - workbox-core "6.5.4" - -workbox-precaching@6.5.4: - version "6.5.4" - resolved "https://registry.yarnpkg.com/workbox-precaching/-/workbox-precaching-6.5.4.tgz#740e3561df92c6726ab5f7471e6aac89582cab72" - integrity sha512-hSMezMsW6btKnxHB4bFy2Qfwey/8SYdGWvVIKFaUm8vJ4E53JAY+U2JwLTRD8wbLWoP6OVUdFlXsTdKu9yoLTg== - dependencies: - workbox-core "6.5.4" - workbox-routing "6.5.4" - workbox-strategies "6.5.4" - -workbox-range-requests@6.5.4: - version "6.5.4" - resolved "https://registry.yarnpkg.com/workbox-range-requests/-/workbox-range-requests-6.5.4.tgz#86b3d482e090433dab38d36ae031b2bb0bd74399" - integrity sha512-Je2qR1NXCFC8xVJ/Lux6saH6IrQGhMpDrPXWZWWS8n/RD+WZfKa6dSZwU+/QksfEadJEr/NfY+aP/CXFFK5JFg== - dependencies: - workbox-core "6.5.4" - -workbox-recipes@6.5.4: - version "6.5.4" - resolved "https://registry.yarnpkg.com/workbox-recipes/-/workbox-recipes-6.5.4.tgz#cca809ee63b98b158b2702dcfb741b5cc3e24acb" - integrity sha512-QZNO8Ez708NNwzLNEXTG4QYSKQ1ochzEtRLGaq+mr2PyoEIC1xFW7MrWxrONUxBFOByksds9Z4//lKAX8tHyUA== - dependencies: - workbox-cacheable-response "6.5.4" - workbox-core "6.5.4" - workbox-expiration "6.5.4" - workbox-precaching "6.5.4" - workbox-routing "6.5.4" - workbox-strategies "6.5.4" - -workbox-routing@6.5.4: - version "6.5.4" - resolved "https://registry.yarnpkg.com/workbox-routing/-/workbox-routing-6.5.4.tgz#6a7fbbd23f4ac801038d9a0298bc907ee26fe3da" - integrity sha512-apQswLsbrrOsBUWtr9Lf80F+P1sHnQdYodRo32SjiByYi36IDyL2r7BH1lJtFX8fwNHDa1QOVY74WKLLS6o5Pg== - dependencies: - workbox-core "6.5.4" - -workbox-strategies@6.5.4: - version "6.5.4" - resolved "https://registry.yarnpkg.com/workbox-strategies/-/workbox-strategies-6.5.4.tgz#4edda035b3c010fc7f6152918370699334cd204d" - integrity sha512-DEtsxhx0LIYWkJBTQolRxG4EI0setTJkqR4m7r4YpBdxtWJH1Mbg01Cj8ZjNOO8etqfA3IZaOPHUxCs8cBsKLw== - dependencies: - workbox-core "6.5.4" - -workbox-streams@6.5.4: - version "6.5.4" - resolved "https://registry.yarnpkg.com/workbox-streams/-/workbox-streams-6.5.4.tgz#1cb3c168a6101df7b5269d0353c19e36668d7d69" - integrity sha512-FXKVh87d2RFXkliAIheBojBELIPnWbQdyDvsH3t74Cwhg0fDheL1T8BqSM86hZvC0ZESLsznSYWw+Va+KVbUzg== - dependencies: - workbox-core "6.5.4" - workbox-routing "6.5.4" - -workbox-sw@6.5.4: - version "6.5.4" - resolved "https://registry.yarnpkg.com/workbox-sw/-/workbox-sw-6.5.4.tgz#d93e9c67924dd153a61367a4656ff4d2ae2ed736" - integrity sha512-vo2RQo7DILVRoH5LjGqw3nphavEjK4Qk+FenXeUsknKn14eCNedHOXWbmnvP4ipKhlE35pvJ4yl4YYf6YsJArA== - -workbox-webpack-plugin@^6.4.1: - version "6.5.4" - resolved "https://registry.yarnpkg.com/workbox-webpack-plugin/-/workbox-webpack-plugin-6.5.4.tgz#baf2d3f4b8f435f3469887cf4fba2b7fac3d0fd7" - integrity sha512-LmWm/zoaahe0EGmMTrSLUi+BjyR3cdGEfU3fS6PN1zKFYbqAKuQ+Oy/27e4VSXsyIwAw8+QDfk1XHNGtZu9nQg== - dependencies: - fast-json-stable-stringify "^2.1.0" - pretty-bytes "^5.4.1" - upath "^1.2.0" - webpack-sources "^1.4.3" - workbox-build "6.5.4" - -workbox-window@6.5.4: - version "6.5.4" - resolved "https://registry.yarnpkg.com/workbox-window/-/workbox-window-6.5.4.tgz#d991bc0a94dff3c2dbb6b84558cff155ca878e91" - integrity sha512-HnLZJDwYBE+hpG25AQBO8RUWBJRaCsI9ksQJEp3aCOFCaG5kqaToAYXFRAHxzRluM2cQbGzdQF5rjKPWPA1fug== - dependencies: - "@types/trusted-types" "^2.0.2" - workbox-core "6.5.4" - -wrap-ansi@^6.2.0: - version "6.2.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-6.2.0.tgz#e9393ba07102e6c91a3b221478f0257cd2856e53" - integrity sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA== +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== dependencies: ansi-styles "^4.0.0" string-width "^4.1.0" @@ -13293,38 +7688,27 @@ wrap-ansi@^7.0.0: string-width "^4.1.0" strip-ansi "^6.0.0" -wrappy@1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" - integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ== - -write-file-atomic@^3.0.0: - version "3.0.3" - resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-3.0.3.tgz#56bd5c5a5c70481cd19c571bd39ab965a5de56e8" - integrity sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q== +wrap-ansi@^8.1.0: + version "8.1.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" + integrity sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ== dependencies: - imurmurhash "^0.1.4" - is-typedarray "^1.0.0" - signal-exit "^3.0.2" - typedarray-to-buffer "^3.1.5" + ansi-styles "^6.1.0" + string-width "^5.0.1" + strip-ansi "^7.0.1" -write-file-atomic@^4.0.2: - version "4.0.2" - resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-4.0.2.tgz#a9df01ae5b77858a027fd2e80768ee433555fcfd" - integrity sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg== +write-file-atomic@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-5.0.1.tgz#68df4717c55c6fa4281a7860b4c2ba0a6d2b11e7" + integrity sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw== dependencies: imurmurhash "^0.1.4" - signal-exit "^3.0.7" - -ws@^7.4.6: - version "7.5.10" - resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.10.tgz#58b5c20dc281633f6c19113f39b349bd8bd558d9" - integrity sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ== + signal-exit "^4.0.1" -ws@^8.4.2: - version "8.17.1" - resolved "https://registry.yarnpkg.com/ws/-/ws-8.17.1.tgz#9293da530bb548febc95371d90f9c878727d919b" - integrity sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ== +ws@^8.18.0: + version "8.18.3" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.3.tgz#b56b88abffde62791c639170400c93dcb0c95472" + integrity sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg== xml-but-prettier@^1.0.1: version "1.0.1" @@ -13333,10 +7717,10 @@ xml-but-prettier@^1.0.1: dependencies: repeat-string "^1.5.2" -xml-name-validator@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" - integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== +xml-name-validator@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-5.0.0.tgz#82be9b957f7afdacf961e5980f1bf227c0bf7673" + integrity sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg== xml@=1.0.1: version "1.0.1" @@ -13348,11 +7732,6 @@ xmlchars@^2.2.0: resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" integrity sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw== -xtend@^4.0.0, xtend@^4.0.2: - version "4.0.2" - resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" - integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== - y18n@^5.0.5: version "5.0.8" resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" @@ -13368,40 +7747,17 @@ yallist@^4.0.0: resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== -yaml@^1.10.0, yaml@^1.10.2, yaml@^1.7.2: - version "1.10.2" - resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b" - integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== +yaml@^1.10.0: + version "1.10.3" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.3.tgz#76e407ed95c42684fb8e14641e5de62fe65bbcb3" + integrity sha512-vIYeF1u3CjlhAFekPPAk2h/Kv4T3mAkMox5OymRiJQB0spDP10LHvt+K7G9Ny6NuuMAb25/6n1qyUjAcGNf/AA== -yaml@^2.2.2: - version "2.3.3" - resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.3.3.tgz#01f6d18ef036446340007db8e016810e5d64aad9" - integrity sha512-zw0VAJxgeZ6+++/su5AFoqBbZbrEakwu+X0M5HmcwUiBL7AzcuPKjj5we4xfQLp78LkEMpD0cOnUhmgOVy3KdQ== - -yargs-parser@^20.2.2: - version "20.2.9" - resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.9.tgz#2eb7dc3b0289718fc295f362753845c41a0c94ee" - integrity sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w== - -yargs-parser@^21.0.1, yargs-parser@^21.1.1: +yargs-parser@^21.1.1: version "21.1.1" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-21.1.1.tgz#9096bceebf990d21bb31fa9516e0ede294a77d35" integrity sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw== -yargs@^16.2.0: - version "16.2.0" - resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.2.0.tgz#1c82bf0f6b6a66eafce7ef30e376f49a12477f66" - integrity sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw== - dependencies: - cliui "^7.0.2" - escalade "^3.1.1" - get-caller-file "^2.0.5" - require-directory "^2.1.1" - string-width "^4.2.0" - y18n "^5.0.5" - yargs-parser "^20.2.2" - -yargs@^17.3.1: +yargs@^17.7.2: version "17.7.2" resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.7.2.tgz#991df39aca675a192b816e1e0363f9d75d2aa269" integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w== @@ -13414,14 +7770,6 @@ yargs@^17.3.1: y18n "^5.0.5" yargs-parser "^21.1.1" -yauzl@^2.10.0: - version "2.10.0" - resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9" - integrity sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g== - dependencies: - buffer-crc32 "~0.2.3" - fd-slicer "~1.1.0" - yocto-queue@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" diff --git a/quickwit/rest-api-tests/Pipfile.lock b/quickwit/rest-api-tests/Pipfile.lock index 86684391fd4..a922df23664 100644 --- a/quickwit/rest-api-tests/Pipfile.lock +++ b/quickwit/rest-api-tests/Pipfile.lock @@ -206,12 +206,12 @@ }, "urllib3": { "hashes": [ - "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", - "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc" + "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", + "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4" ], "index": "pypi", "markers": "python_version >= '3.9'", - "version": "==2.5.0" + "version": "==2.6.3" } }, "develop": {} diff --git a/quickwit/rest-api-tests/run_tests.py b/quickwit/rest-api-tests/run_tests.py index 905945768a5..c1ad6c3b950 100755 --- a/quickwit/rest-api-tests/run_tests.py +++ b/quickwit/rest-api-tests/run_tests.py @@ -18,6 +18,31 @@ from os import mkdir from os import path as osp +# Simple !include constructor for YAML to allow reusing fragments across files. +# Usage examples: +# - !include path/to/file.yaml -> includes full file content +# - !include path/to/file.yaml::doc_mapping -> includes the 'doc_mapping' key +# - !include path/to/file.yaml::a.b.c -> includes nested key a -> b -> c +def _yaml_include(loader, node): + value = loader.construct_scalar(node) + if "::" in value: + filepath, subpath = value.split("::", 1) + else: + filepath, subpath = value, None + with open(filepath, "r") as f: + included = yaml.load(f, Loader=yaml.Loader) + if subpath: + cur = included + for seg in filter(None, subpath.split(".")): + if not isinstance(cur, dict) or seg not in cur: + raise KeyError(f"!include path '{subpath}' not found in {filepath}") + cur = cur[seg] + return cur + return included + +# Register the constructor on the default Loader used by this script. +yaml.Loader.add_constructor("!include", _yaml_include) + def debug_http(): old_send = http.client.HTTPConnection.send def new_send(self, data): diff --git a/quickwit/rest-api-tests/scenarii/aggregations/0001-aggregations.yaml b/quickwit/rest-api-tests/scenarii/aggregations/0001-aggregations.yaml index f81c2215f40..d6b48893a0a 100644 --- a/quickwit/rest-api-tests/scenarii/aggregations/0001-aggregations.yaml +++ b/quickwit/rest-api-tests/scenarii/aggregations/0001-aggregations.yaml @@ -42,7 +42,7 @@ expected: - { "doc_count": 2, "key": 1422662400000.0, "key_as_string": "2015-01-31T00:00:00Z" } - { "doc_count": 0, "key": 1425254400000.0, "key_as_string": "2015-03-02T00:00:00Z" } --- -# Test date histogram aggregation and sub-aggregation +# Test date histogram aggregation and sub-aggregation method: [GET] engines: - quickwit @@ -50,8 +50,8 @@ endpoint: _elastic/aggregations/_search json: query: { match_all: {} } aggs: - date_histo: - date_histogram: + date_histo: + date_histogram: field: "date" fixed_interval: "30d" offset: "-4d" @@ -65,8 +65,8 @@ expected: buckets: - { "doc_count": 5, "key": 1420070400000.0, "key_as_string": "2015-01-01T00:00:00Z", "response": { "avg": 85.0, "count": 4, "max": 120.0, "min": 20.0, "sum": 340.0 } } - { "doc_count": 2, "key": 1422662400000.0, "key_as_string": "2015-01-31T00:00:00Z", "response": { "avg": 80.0, "count": 2, "max": 130.0, "min": 30.0, "sum": 160.0 } } ---- -# Test date histogram aggregation + exists and sub-aggregation +--- +# Test date histogram aggregation + exists and sub-aggregation method: [GET] engines: - quickwit @@ -78,8 +78,8 @@ json: - exists: field: response aggs: - date_histo: - date_histogram: + date_histo: + date_histogram: field: "date" fixed_interval: "30d" offset: "-4d" @@ -93,7 +93,7 @@ expected: buckets: - { "doc_count": 4, "key": 1420070400000.0, "key_as_string": "2015-01-01T00:00:00Z", "response": { "avg": 85.0, "count": 4, "max": 120.0, "min": 20.0, "sum": 340.0 } } - { "doc_count": 2, "key": 1422662400000.0, "key_as_string": "2015-01-31T00:00:00Z", "response": { "avg": 80.0, "count": 2, "max": 130.0, "min": 30.0, "sum": 160.0 } } ---- +--- # Test range aggregation method: [GET] engines: @@ -103,7 +103,7 @@ json: query: { match_all: {} } aggs: my_range: - range: + range: field: response ranges: - { to: 50, key: fast } @@ -116,7 +116,7 @@ expected: - { "doc_count": 5, "key": "fast", "to": 50.0 } - { "doc_count": 0, "from": 50.0, "key": "medium", "to": 80.0 } - { "doc_count": 4, "from": 80.0, "key": "slow" } ---- +--- # Test term aggs method: [GET] engines: @@ -125,11 +125,11 @@ endpoint: _elastic/aggregations/_search json: query: { match_all: {} } aggs: - hosts: - terms: + hosts: + terms: field: "host" - tags: - terms: + tags: + terms: field: "tags" expected: aggregations: @@ -151,7 +151,7 @@ expected: key: cool doc_count_error_upper_bound: 0 sum_other_doc_count: 0 ---- +--- # Test term aggs with split_size # We set split_size to 1, so one document with name "Fritz" will be missing from one split. method: [GET] @@ -161,8 +161,8 @@ endpoint: _elastic/aggregations/_search json: query: { match_all: {} } aggs: - names: - terms: + names: + terms: field: "name" size: 1 split_size: 1 @@ -170,14 +170,14 @@ expected: aggregations: names: buckets: - # There are 3 documents with name "Fritz" but we only get 2. One does not get passed to the + # There are 3 documents with name "Fritz" but we only get 2. One does not get passed to the # root node, because it gets cut off due to the split_size parameter set to 1. # We also get doc_count_error_upper_bound: 2, which signals that the result is approximate. - - doc_count: 2 + - doc_count: 2 key: "Fritz" sum_other_doc_count: 8 doc_count_error_upper_bound: 2 ---- +--- # Test term aggs with shard_size # segment_size is an alias to shard_size method: [GET] @@ -187,8 +187,8 @@ endpoint: _elastic/aggregations/_search json: query: { match_all: {} } aggs: - names: - terms: + names: + terms: field: "name" size: 1 segment_size: 1 @@ -196,14 +196,14 @@ expected: aggregations: names: buckets: - # There are 3 documents with name "Fritz" but we only get 2. One does not get passed to the + # There are 3 documents with name "Fritz" but we only get 2. One does not get passed to the # root node, because it gets cut off due to the split_size parameter set to 1. # We also get doc_count_error_upper_bound: 2, which signals that the result is approximate. - - doc_count: 2 + - doc_count: 2 key: "Fritz" sum_other_doc_count: 8 doc_count_error_upper_bound: 2 ---- +--- # Test term aggs with shard_size method: [GET] engines: @@ -212,8 +212,8 @@ endpoint: _elastic/aggregations/_search json: query: { match_all: {} } aggs: - names: - terms: + names: + terms: field: "name" size: 1 shard_size: 1 @@ -221,10 +221,10 @@ expected: aggregations: names: buckets: - # There are 3 documents with name "Fritz" but we only get 2. One does not get passed to the + # There are 3 documents with name "Fritz" but we only get 2. One does not get passed to the # root node, because it gets cut off due to the split_size parameter set to 1. # We also get doc_count_error_upper_bound: 2, which signals that the result is approximate. - - doc_count: 2 + - doc_count: 2 key: "Fritz" sum_other_doc_count: 8 doc_count_error_upper_bound: 2 @@ -238,8 +238,8 @@ endpoint: _elastic/aggregations/_search json: query: { match_all: {} } aggs: - names: - terms: + names: + terms: field: "name" size: 1 split_size: 5 @@ -249,11 +249,11 @@ expected: buckets: # We get all 3 documents with name "Fritz" # We also get doc_count_error_upper_bound: 0, to the result is exact. - - doc_count: 3 + - doc_count: 3 key: "Fritz" sum_other_doc_count: 7 doc_count_error_upper_bound: 0 ---- +--- # Test date histogram + percentiles sub-aggregation method: [GET] engines: @@ -284,15 +284,17 @@ expected: response: values: - key: 85.0 - value: 100.49456770856702 + value: + $expect: 'abs(val - 100.49456770856702) < 0.1' - doc_count: 2 key: 1422662400000.0 key_as_string: '2015-01-31T00:00:00Z' response: values: - key: 85.0 - value: 30.26717133872237 ---- + value: + $expect: 'abs(val - 30.2617) < 0.1' +--- # Test histogram method: [GET] engines: @@ -316,7 +318,7 @@ expected: - doc_count: 4 key: 100.0 ---- +--- # Test histogram empty result on empty index method: [GET] engines: @@ -354,11 +356,14 @@ json: expected: aggregations: unique_names: - value: 8.0 + value: + $expect: 'abs(val - 8) <= 2' unique_response: - value: 5.0 # TODO: Check. The correct number is 6 + value: + $expect: 'abs(val - 6) <= 2' # TODO: Check. The correct number is 6 unique_dates: - value: 6.0 + value: + $expect: 'abs(val - 6) <= 3' --- # Test extended stats aggregation method: [GET] @@ -375,6 +380,7 @@ expected: aggregations: response_stats: sum_of_squares: 55300.0 +--- # Test term aggs number precision method: [GET] engines: @@ -384,12 +390,100 @@ json: query: { match_all: {} } size: 0 aggs: - names: - terms: + names: + terms: field: "high_prec_test" expected: aggregations: names: buckets: - - doc_count: 1 + - doc_count: 1 key: 1769070189829214200 +--- +# Test composite aggregation +method: [GET] +engines: + - quickwit +endpoint: _elastic/aggregations/_search +json: + size: 0 + aggs: + host_name_composite: + composite: + size: 5 + sources: + - host: + terms: + field: "host" + missing_bucket: true + - name: + terms: + field: "name" + - response: + histogram: + field: "response" + interval: 50 +expected: + aggregations: + host_name_composite: + buckets: + - key: { "host": null, "name": "Bernhard", "response": 100.0 } + doc_count: 1 + - key: { "host": null, "name": "Fritz", "response": 0.0 } + doc_count: 2 + - key: { "host": "192.168.0.1", "name": "Fred", "response": 100.0 } + doc_count: 1 + - key: { "host": "192.168.0.1", "name": "Fritz", "response": 0.0 } + doc_count: 1 + - key: { "host": "192.168.0.10", "name": "Albert", "response": 100.0 } + doc_count: 1 + after_key: + host: + # tantivy does not coerce. + # For this reason, we can have 2 fields with the different types, and tantivy + # needs to prepend after key with its type. std: + $expect: "val in ['192.168.0.10', 'str:192.168.0.10']" + name: + $expect: "val in ['Albert', 'str:Albert']" + response: + $expect: val in [100.0, 'f64:100'] + +--- +# Test composite aggregation paging +method: [GET] +engines: + - quickwit +endpoint: _elastic/aggregations/_search +json: + size: 0 + aggs: + host_name_composite: + composite: + size: 5 + sources: + - host: + terms: + field: "host" + missing_bucket: true + - name: + terms: + field: "name" + - response: + histogram: + field: "response" + interval: 50 + after: + host: "str:192.168.0.10" + name: "str:Albert" + response: "f64:100" +expected: + aggregations: + host_name_composite: + buckets: + - key: { "host": "192.168.0.10", "name": "Holger", "response": 0.0 } + doc_count: 1 + # Horst is missing because his response field is missing + - key: { "host": "192.168.0.10", "name": "Werner", "response": 0.0 } + doc_count: 1 + - key: { "host": "192.168.0.11", "name": "Manfred", "response": 100.0 } + doc_count: 1 diff --git a/quickwit/rest-api-tests/scenarii/concat_fields/0001_concat_field.yaml b/quickwit/rest-api-tests/scenarii/concat_fields/0001_concat_field.yaml index c90794cca4b..2fa894ec26c 100644 --- a/quickwit/rest-api-tests/scenarii/concat_fields/0001_concat_field.yaml +++ b/quickwit/rest-api-tests/scenarii/concat_fields/0001_concat_field.yaml @@ -115,3 +115,59 @@ params: expected: num_hits: 1 --- +endpoint: concat/search +params: + query: "concat_default:1.5" +expected: + num_hits: 1 +--- +endpoint: concat/search +params: + query: "concat_default:2.5" +expected: + num_hits: 1 +--- +endpoint: concat/search +params: + query: "concat_default:3.5" +expected: + num_hits: 0 +--- +endpoint: concat/search +params: + query: "concat_raw:1.5" +expected: + num_hits: 1 +--- +endpoint: concat/search +params: + query: "concat_raw:2.5" +expected: + num_hits: 1 +--- +endpoint: concat/search +params: + query: "concat_raw:3.5" +expected: + num_hits: 1 +--- +endpoint: concat/search +params: + query: "concat_raw:9223372036854775808" +expected: + num_hits: 1 +--- +endpoint: concat/search +params: + query: "concat_raw:-5" +expected: + num_hits: 1 +--- +endpoint: concat/search +params: + # concat date values are stored as strings to enable some level of range + # querying even though they don't support fast fields + query: "concat_raw:\"2024-01-01\"*" +expected: + num_hits: 1 +--- diff --git a/quickwit/rest-api-tests/scenarii/concat_fields/_setup.quickwit.yaml b/quickwit/rest-api-tests/scenarii/concat_fields/_setup.quickwit.yaml index 877fce489f7..911f8572c68 100644 --- a/quickwit/rest-api-tests/scenarii/concat_fields/_setup.quickwit.yaml +++ b/quickwit/rest-api-tests/scenarii/concat_fields/_setup.quickwit.yaml @@ -22,6 +22,8 @@ json: type: bool - name: int type: u64 + - name: float + type: f64 - name: json type: json - name: concat_raw @@ -32,6 +34,7 @@ json: - boolean - int - json + - float tokenizer: raw include_dynamic_fields: true - name: concat_default @@ -42,6 +45,7 @@ json: - boolean - int - json + - float tokenizer: default dynamic_mapping: tokenizer: default @@ -59,6 +63,12 @@ ndjson: - {"text1": "true"} - {"boolean": true} - {"text2": "i like 42"} - - { "int": 42} + - {"int": 42} - {"other-field": "otherfieldvalue", "other-field-number": 9, "other-field-bool": false} - {"json": {"some_bool": false, "some_int": 10, "nested": {"some_string": "nestedstring"}}} + - {"float": 1.5} + - {"json": {"val:": 2.5, "date": "2024-01-01T00:13:00Z"}} + - {"other": 3.5} + # too big to be a i64, parsed as a u64 + - {"big": 9223372036854775808} + - {"neg": -5} diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/0006-term_query.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/0006-term_query.yaml index 14599566959..3fe75d61973 100644 --- a/quickwit/rest-api-tests/scenarii/es_compatibility/0006-term_query.yaml +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/0006-term_query.yaml @@ -1,9 +1,8 @@ -# case_insensitive not supported. -engines: ["elasticsearch"] params: # this overrides the query sent in body apparently size: 3 json: + track_total_hits: true query: term: type: @@ -21,6 +20,7 @@ expected: params: size: 0 json: + track_total_hits: true query: term: type: @@ -35,6 +35,7 @@ expected: params: size: 0 json: + track_total_hits: true query: term: type: @@ -49,6 +50,7 @@ params: size: 0 # Testing the format without the "value" object json: + track_total_hits: true query: term: type: "pushevent" @@ -63,6 +65,7 @@ engines: ["elasticsearch"] params: size: 0 json: + track_total_hits: true query: term: actor.id: 1762355 @@ -75,6 +78,7 @@ expected: params: size: 0 json: + track_total_hits: true query: term: actor.id: "1762355" @@ -87,6 +91,7 @@ expected: params: size: 0 json: + track_total_hits: true query: term: actor.id: @@ -100,6 +105,7 @@ expected: params: size: 0 json: + track_total_hits: true query: term: actor.id: @@ -114,6 +120,7 @@ expected: params: size: 0 json: + track_total_hits: true query: term: id: @@ -127,6 +134,7 @@ expected: params: size: 0 json: + track_total_hits: true query: term: id: @@ -140,6 +148,7 @@ expected: params: size: 0 json: + track_total_hits: true query: term: id: 2549961272 @@ -152,6 +161,7 @@ expected: params: size: 0 json: + track_total_hits: true query: term: id: "2549961272" diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/0020-stats.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/0020-stats.yaml index 461911f73c9..0d4d82b5f2f 100644 --- a/quickwit/rest-api-tests/scenarii/es_compatibility/0020-stats.yaml +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/0020-stats.yaml @@ -64,12 +64,12 @@ expected: _all: primaries: docs: - count: 100 + count: 104 total: segments: - count: 1 + count: 3 docs: - count: 100 + count: 104 indices: gharchive: primaries: @@ -80,6 +80,15 @@ expected: count: 1 docs: count: 100 + fast_only: + primaries: + docs: + count: 2 + total: + segments: + count: 1 + docs: + count: 2 empty_index: primaries: docs: diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/0021-cat-indices.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/0021-cat-indices.yaml index 23a5c00192a..42a879c01c0 100644 --- a/quickwit/rest-api-tests/scenarii/es_compatibility/0021-cat-indices.yaml +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/0021-cat-indices.yaml @@ -5,11 +5,13 @@ endpoint: "_cat/indices?format=json" expected: - index: empty_index docs.count: '0' -- dataset.size: 222.8kb +- index: fast_only + docs.count: '2' +- index: gharchive + dataset.size: 222.8kb docs.count: '100' docs.deleted: '0' health: green - index: gharchive pri: '1' pri.store.size: $expect: 270 < float(val[:-2]) < 280 @@ -24,6 +26,8 @@ expected: docs.count: '0' - index: otel-traces-v0_9 docs.count: '0' +- index: simple_es_compat + docs.count: '2' --- method: [GET] engines: diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/0025-msearch.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/0025-msearch.yaml index 1a7fbf631ed..e0a5c1505c4 100644 --- a/quickwit/rest-api-tests/scenarii/es_compatibility/0025-msearch.yaml +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/0025-msearch.yaml @@ -2,7 +2,7 @@ endpoint: "_msearch" method: POST ndjson: - {"index":"gharchive"} - - {"query" : {"match" : { "type": "PushEvent"}}, "size": 0} + - {"query" : {"match" : { "type": "PushEvent"}}, "size": 0, "track_total_hits": true} expected: responses: - hits: @@ -16,7 +16,7 @@ params: extra_filters: "type:PushEvent,actor.login:jadonk" ndjson: - {"index":"gharchive"} - - {"query" : {"match" : { "type": "PushEvent"}}, "size": 0} + - {"query" : {"match" : { "type": "PushEvent"}}, "size": 0, "track_total_hits": true} expected: responses: - hits: @@ -104,3 +104,25 @@ expected: $expect: "len(val) == 1" # Contains only 'actor' actor: id: 5688 +--- +# test missing index +endpoint: "_msearch" +method: POST +ndjson: + - {"index":"idontexist"} + - {"query" : {"match" : { "type": "PushEvent"}}, "size": 0, "track_total_hits": true} +expected: + responses: + - status: 404 +--- +endpoint: "_msearch" +method: POST +ndjson: + - {"index":"idontexist", "ignore_unavailable": true} + - {"query" : {"match" : { "type": "PushEvent"}}, "size": 0} +expected: + responses: + - hits: + total: + value: 0 + status: 200 diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/0028-fast_only_field_query.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/0028-fast_only_field_query.yaml new file mode 100644 index 00000000000..8e55d54b471 --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/0028-fast_only_field_query.yaml @@ -0,0 +1,141 @@ +# Search for a term in a field that is not indexed but is a fast field +engines: + - quickwit +endpoint: "fast_only/_search" +params: + size: 0 +json: + track_total_hits: true + query: + term: + fast_text: "abc-123" +expected: + hits: + total: + value: 1 + relation: "eq" +--- # term query with no matches +engines: + - quickwit +endpoint: "fast_only/_search" +params: + size: 0 +json: + track_total_hits: true + query: + term: + fast_text: "zzz" +expected: + hits: + total: + value: 0 + relation: "eq" + +--- # term set query with partial match +engines: + - quickwit +endpoint: "fast_only/_search" +params: + size: 0 +json: + track_total_hits: true + query: + terms: + fast_text: + - "abc-123" + - "zzz" +expected: + hits: + total: + value: 1 + relation: "eq" + +--- # term set query with multiple matches +engines: + - quickwit +endpoint: "fast_only/_search" +params: + size: 0 +json: + track_total_hits: true + query: + terms: + fast_text: + - "abc-123" + - "def-456" +expected: + hits: + total: + value: 2 + relation: "eq" + +--- # term query on nested JSON field +engines: + - quickwit +endpoint: "fast_only/_search" +params: + size: 0 +json: + track_total_hits: true + query: + term: + obj.nested_text: "abc-123" +expected: + hits: + total: + value: 1 + relation: "eq" + +--- # term query with no matches +engines: + - quickwit +endpoint: "fast_only/_search" +params: + size: 0 +json: + track_total_hits: true + query: + term: + obj.nested_text: "zzz" +expected: + hits: + total: + value: 0 + relation: "eq" + +--- # term set query +engines: + - quickwit +endpoint: "fast_only/_search" +params: + size: 0 +json: + track_total_hits: true + query: + terms: + obj.nested_text: + - "abc-123" + - "ghi-789" +expected: + hits: + total: + value: 2 + relation: "eq" + +--- # term set query with no matches +engines: + - quickwit +endpoint: "fast_only/_search" +params: + size: 0 +json: + track_total_hits: true + query: + terms: + obj.nested_text: + - "zzz" +expected: + hits: + total: + value: 0 + relation: "eq" diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/0029-wildcard.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/0029-wildcard.yaml new file mode 100644 index 00000000000..1dd8d9c5658 --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/0029-wildcard.yaml @@ -0,0 +1,50 @@ +json: + query: + wildcard: + actor.login: + value: jad?nk +expected: + hits: + total: + value: 2 +--- +json: + query: + wildcard: + actor.login: + value: j*nk +expected: + hits: + total: + value: 2 +--- +json: + query: + wildcard: + actor.login: jad?nk +expected: + hits: + total: + value: 2 +--- +json: + query: + wildcard: + repo.name: + value: RUS* + case_insensitive: true +expected: + hits: + total: + value: 1 +--- +json: + query: + wildcard: + repo.name: + value: RUS* + case_insensitive: false +expected: + hits: + total: + value: 0 diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/0030-prefix.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/0030-prefix.yaml new file mode 100644 index 00000000000..9f5d9c80420 --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/0030-prefix.yaml @@ -0,0 +1,50 @@ +json: + query: + prefix: + actor.login: + value: jado +expected: + hits: + total: + value: 2 +--- +json: + query: + prefix: + actor.login: + value: j +expected: + hits: + total: + value: 10 +--- +json: + query: + prefix: + actor.login: jado +expected: + hits: + total: + value: 2 +--- +json: + query: + prefix: + repo.name: + value: RUST + case_insensitive: true +expected: + hits: + total: + value: 1 +--- +json: + query: + prefix: + repo.name: + value: RUST + case_insensitive: false +expected: + hits: + total: + value: 0 diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/0031-regex.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/0031-regex.yaml new file mode 100644 index 00000000000..6c7b2721a33 --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/0031-regex.yaml @@ -0,0 +1,113 @@ +# Basic regex match +params: + size: 0 +json: + track_total_hits: true + query: + regexp: + type: + value: ".*event" +expected: + hits: + total: + value: 100 + relation: "eq" +--- +# Regex always match from start to end (`(re)` equivalent to `^(re)$`) +params: + size: 3 +json: + track_total_hits: true + query: + regexp: + type: + value: "event" +expected: + hits: + total: + value: 0 + relation: "eq" +--- +# Regex with case_insensitive flag +params: + size: 3 +json: + track_total_hits: true + query: + regexp: + repo.name: + # lowercased by the tokenizer + value: "RUST.*" + case_insensitive: true +expected: + hits: + total: + value: 1 + relation: "eq" +--- +params: + size: 3 +json: + track_total_hits: true + query: + regexp: + type: + # lowercased by the tokenizer + value: "RUST.*" + case_insensitive: false +expected: + hits: + total: + value: 0 + relation: "eq" +--- +# In Elasticsearch, ^ and $ are escaped when they are used as anchors, so +# ^pushevent$ only matches if the original term is "^pushevent$". In Quickwit +# this fails (for now) because tantivy-fst returns an error on all zero width +# assertions. +engines: + - elasticsearch +endpoint: "simple_es_compat/_search" +params: + size: 3 +json: + track_total_hits: true + query: + regexp: + keyword_text: + value: "red$" +expected: + hits: + total: + value: 0 + relation: "eq" +--- +engines: + - elasticsearch +endpoint: "simple_es_compat/_search" +params: + size: 3 +json: + track_total_hits: true + query: + regexp: + keyword_text: + value: "gold$" +expected: + hits: + total: + value: 1 + relation: "eq" +--- +# regex in query_string +params: + size: 10 +json: + query: + query_string: + query: "type:/pushevent/" +expected: + hits: + total: + value: 60 + relation: "eq" diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/0032-mappings.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/0032-mappings.yaml new file mode 100644 index 00000000000..5f241c81dfc --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/0032-mappings.yaml @@ -0,0 +1,100 @@ +method: [GET] +engines: + - elasticsearch +endpoint: "gharchive/_mappings" +expected: + gharchive: {} + # _all: + # primaries: + # docs: + # count: 100 + # store: + # size_in_bytes: + # $expect: "val > 278300" + # total: + # segments: + # count: 1 + # docs: + # count: 100 + # indices: + # gharchive: + # primaries: + # docs: + # count: 100 + # store: + # size_in_bytes: + # $expect: "val > 278300" + # total: + # segments: + # count: 1 + # docs: + # count: 100 +--- +# method: [GET] +# engines: +# - quickwit +# - elasticsearch +# endpoint: "ghar*/_stats" +# expected: +# _all: +# primaries: +# docs: +# count: 100 +# total: +# segments: +# count: 1 +# docs: +# count: 100 +# indices: +# gharchive: +# primaries: +# docs: +# count: 100 +# total: +# segments: +# count: 1 +# docs: +# count: 100 +# --- +# method: [GET] +# engines: +# - quickwit +# endpoint: "_stats" +# expected: +# _all: +# primaries: +# docs: +# count: 102 +# total: +# segments: +# count: 2 +# docs: +# count: 102 +# indices: +# gharchive: +# primaries: +# docs: +# count: 100 +# total: +# segments: +# count: 1 +# docs: +# count: 100 +# fast_only: +# primaries: +# docs: +# count: 2 +# total: +# segments: +# count: 1 +# docs: +# count: 2 +# empty_index: +# primaries: +# docs: +# count: 0 +# total: +# segments: +# count: 0 +# docs: +# count: 0 diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/_setup.elasticsearch.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/_setup.elasticsearch.yaml index f21d5c9598e..22d847c8679 100644 --- a/quickwit/rest-api-tests/scenarii/es_compatibility/_setup.elasticsearch.yaml +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/_setup.elasticsearch.yaml @@ -7,6 +7,10 @@ method: DELETE endpoint: empty_index status_code: null --- +method: DELETE +endpoint: simple_es_compat +status_code: null +--- # empty index method: PUT endpoint: empty_index @@ -118,6 +122,23 @@ method: PUT endpoint: gharchive/_settings json: { "number_of_replicas": 0 } --- +# Create index +method: PUT +endpoint: simple_es_compat +json: { + "mappings": { + "properties": { + "keyword_text": { + "type": "keyword", + } + } + } +} +--- +method: PUT +endpoint: simple_es_compat/_settings +json: { "number_of_replicas": 0 } +--- # Ingest documents method: POST endpoint: _bulk @@ -125,3 +146,14 @@ params: refresh: "true" headers: {"Content-Type": "application/json", "content-encoding": "gzip"} body_from_file: gharchive-bulk.json.gz +--- +method: POST +endpoint: _bulk +params: + refresh: "true" +headers: {"Content-Type": "application/json"} +ndjson: + - {"index":{"_index":"simple_es_compat"}} + - {"keyword_text": "red"} + - {"index":{"_index":"simple_es_compat"}} + - {"keyword_text": "gold$"} diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/_setup.quickwit.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/_setup.quickwit.yaml index fad920d1b38..f84d3587fed 100644 --- a/quickwit/rest-api-tests/scenarii/es_compatibility/_setup.quickwit.yaml +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/_setup.quickwit.yaml @@ -10,6 +10,12 @@ api_root: http://localhost:7280/api/v1/ endpoint: indexes/empty_index status_code: null --- +# Delete possibly remaining index +method: DELETE +api_root: http://localhost:7280/api/v1/ +endpoint: indexes/simple_es_compat +status_code: null +--- # Create index method: POST api_root: http://localhost:7280/api/v1/ @@ -43,6 +49,7 @@ json: type: text fast: true indexed: true + tokenizer: raw - name: public type: bool fast: false @@ -65,3 +72,68 @@ params: refresh: "true" headers: {"Content-Type": "application/json", "content-encoding": "gzip"} body_from_file: gharchive-bulk.json.gz +--- +# Delete possibly remaining index +method: DELETE +api_root: http://localhost:7280/api/v1/ +endpoint: indexes/fast_only +status_code: null +--- +# Create a dedicated index with a root fast-only field +method: POST +api_root: http://localhost:7280/api/v1/ +endpoint: indexes/ +json: + version: "0.7" + index_id: fast_only + doc_mapping: + field_mappings: + - name: fast_text + type: text + fast: true + indexed: false + - name: obj + type: object + field_mappings: + - name: nested_text + type: text + fast: true + indexed: false +sleep_after: 1 +--- +# Ingest a couple documents into fast_only +method: POST +api_root: http://localhost:7280/api/v1/ +endpoint: fast_only/ingest +params: + commit: force +ndjson: + - {"fast_text": "abc-123", "obj": {"nested_text": "abc-123"}} + - {"fast_text": "def-456", "obj": {"nested_text": "ghi-789"}} + +--- +# Create simple_es_compat index +method: POST +api_root: http://localhost:7280/api/v1/ +endpoint: indexes/ +json: + version: "0.7" + index_id: simple_es_compat + doc_mapping: + field_mappings: + - name: keyword_text + type: text + fast: true + indexed: true + tokenizer: raw +sleep_after: 1 +--- +# Ingest documents into simple_es_compat +method: POST +api_root: http://localhost:7280/api/v1/ +endpoint: simple_es_compat/ingest +params: + commit: force +ndjson: + - {"keyword_text": "red"} + - {"keyword_text": "gold$"} diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/_teardown.quickwit.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/_teardown.quickwit.yaml index b9612bfad96..d82b2092091 100644 --- a/quickwit/rest-api-tests/scenarii/es_compatibility/_teardown.quickwit.yaml +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/_teardown.quickwit.yaml @@ -10,6 +10,11 @@ endpoint: indexes/empty_index --- method: DELETE api_root: http://localhost:7280/api/v1/ +endpoint: indexes/fast_only +status_code: null +--- +method: DELETE +api_root: http://localhost:7280/api/v1/ endpoint: indexes/test_index1 status_code: null --- # Cleanup diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/0007-illegal-index-name.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/0007-illegal-index-name.yaml index 89d611f9562..ab1481fbd9a 100644 --- a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/0007-illegal-index-name.yaml +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/0007-illegal-index-name.yaml @@ -1,3 +1,6 @@ +# allowed characters are different between ES and Quickwit +engines: + - quickwit ndjson: - index: { "_index": "test-index" } - message: Hola, Mundo! diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_setup.elasticsearch.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_setup.elasticsearch.yaml index 3911140cb97..27804f82848 100644 --- a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_setup.elasticsearch.yaml +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_setup.elasticsearch.yaml @@ -1,10 +1,6 @@ # Delete possibly remaining index method: DELETE -endpoint: test-index -status_code: null ---- -method: DELETE -endpoint: test-index-pattern-777 +endpoint: test-index* status_code: null --- method: PUT diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_setup.quickwit.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_setup.quickwit.yaml index 82febfc2517..cfa9d6e8fae 100644 --- a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_setup.quickwit.yaml +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_setup.quickwit.yaml @@ -6,6 +6,11 @@ status_code: null --- method: DELETE api_root: http://localhost:7280/api/v1/ +endpoint: indexes/test-index-pattern-11 +status_code: null +--- +method: DELETE +api_root: http://localhost:7280/api/v1/ endpoint: indexes/test-index-pattern-777 status_code: null --- diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_teardown.quickwit.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_teardown.quickwit.yaml index 4accabf684c..3e2fa4b9e55 100644 --- a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_teardown.quickwit.yaml +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_teardown.quickwit.yaml @@ -4,4 +4,12 @@ endpoint: indexes/test-index --- method: DELETE api_root: http://localhost:7280/api/v1/ +endpoint: indexes/test-index-pattern-11 +--- +method: DELETE +api_root: http://localhost:7280/api/v1/ +endpoint: indexes/test-index-pattern-777 +--- +method: DELETE +api_root: http://localhost:7280/api/v1/ endpoint: templates/test-index-template diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/multi-indices/0004-missing_index_query.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/multi-indices/0004-missing_index_query.yaml new file mode 100644 index 00000000000..096f3a8f8d3 --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/multi-indices/0004-missing_index_query.yaml @@ -0,0 +1,27 @@ +endpoint: "idontexist/_search" +params: + q: "*" +status_code: 404 +--- +endpoint: "idontexist/_search" +params: + q: "*" + ignore_unavailable: "true" +expected: + hits: + total: + value: 0 +--- +endpoint: "gharchive-*,idontexist/_search" +params: + q: "*" +status_code: 404 +--- +endpoint: "gharchive-*,idontexist/_search" +params: + q: "*" + ignore_unavailable: "true" +expected: + hits: + total: + value: 4 diff --git a/quickwit/rest-api-tests/scenarii/es_field_capabilities/0001-field-capabilities.yaml b/quickwit/rest-api-tests/scenarii/es_field_capabilities/0001-field-capabilities.yaml index bd3cd917acd..a3c5041926d 100644 --- a/quickwit/rest-api-tests/scenarii/es_field_capabilities/0001-field-capabilities.yaml +++ b/quickwit/rest-api-tests/scenarii/es_field_capabilities/0001-field-capabilities.yaml @@ -31,10 +31,10 @@ expected: searchable: true aggregatable: true mixed: # This is a little weird case (values [5, -5.5]), since coercion happens only on the columnar side. That's why `long` is not aggregatable. - long: + long: metadata_field: false searchable: true - aggregatable: false + aggregatable: false double: metadata_field: false searchable: true @@ -88,10 +88,10 @@ expected: fields: $expect: "not 'id' in val" # Filtered by start_timestamp mixed: # This is a little weird case (values [5, -5.5]), since coercion happens only on the columnar side. That's why `long` is not aggregatable. - long: + long: metadata_field: false searchable: true - aggregatable: false + aggregatable: false double: metadata_field: false searchable: true @@ -103,8 +103,6 @@ expected: aggregatable: true --- # Test fields parameter with `.dynamic` suffix -engines: - - quickwit method: [GET] engines: - quickwit @@ -193,9 +191,6 @@ expected: --- # Compare with elastic search method: [GET] -engines: - - quickwit - - elasticsearch endpoint: fieldcaps/_field_caps?fields=nested.*ponse expected: indices: @@ -210,9 +205,6 @@ expected: --- # Compare ip field with elastic search method: [GET] -engines: - - quickwit - - elasticsearch endpoint: fieldcaps*/_field_caps?fields=host expected: indices: @@ -295,9 +287,6 @@ expected: --- # Wildcard on index name + Wildcard without match method: [GET] -engines: - - quickwit - - elasticsearch endpoint: fieldca*,blub*/_field_caps?fields=date expected: indices: @@ -313,24 +302,140 @@ expected: --- # Exact match index + Non matching exact index method: [GET] -engines: - - quickwit - - elasticsearch endpoint: fieldcaps,blub/_field_caps?fields=date status_code: 404 --- # Compare ip field with elastic search method: [GET] -engines: - - quickwit - - elasticsearch endpoint: doesnotexist/_field_caps?fields=date status_code: 404 --- # Compare ip field with elastic search method: [GET] +endpoint: doesno*texist/_field_caps?fields=date +status_code: 200 +--- +# Test _field_caps API with index_filter (term query) +# Note: term queries require exact token match; 'fritz' is lowercase due to default tokenizer +method: [POST] +endpoint: fieldcaps/_field_caps?fields=* +json: + index_filter: + term: + name: "fritz" +expected: + indices: + - fieldcaps + fields: + name: + keyword: + type: keyword + metadata_field: false + searchable: true + aggregatable: true + text: + type: text + metadata_field: false + searchable: true + aggregatable: true +--- +# Test _field_caps API with index_filter (match_all query) +method: [POST] +endpoint: fieldcaps/_field_caps?fields=name +json: + index_filter: + match_all: {} +expected: + indices: + - fieldcaps + fields: + name: + keyword: + type: keyword + metadata_field: false + searchable: true + aggregatable: true + text: + type: text + metadata_field: false + searchable: true + aggregatable: true +--- +# Test _field_caps API with index_filter (bool query) +method: [POST] +endpoint: fieldcaps/_field_caps?fields=response,name +json: + index_filter: + bool: + must: + - term: + name: "fritz" + filter: + - range: + response: + gte: 30 +expected: + indices: + - fieldcaps + fields: + response: + long: + type: long + metadata_field: false + searchable: true + aggregatable: true + name: + keyword: + type: keyword + metadata_field: false + searchable: true + aggregatable: true + text: + type: text + metadata_field: false + searchable: true + aggregatable: true +--- +# Test _field_caps API with invalid index_filter +method: [POST] +endpoint: fieldcaps/_field_caps?fields=* +json: + index_filter: + invalid_query_type: + field: "value" +status_code: 400 +--- +# Test _field_caps API with empty index_filter (should return 400 like ES) +method: [POST] engines: - quickwit - elasticsearch -endpoint: doesno*texist/_field_caps?fields=date -status_code: 200 +endpoint: fieldcaps/_field_caps?fields=name +json: + index_filter: {} +status_code: 400 +--- +# Test _field_caps API with index_filter using tag field for split pruning (QW-only) +method: [POST] +engines: + - quickwit +endpoint: fieldcaps/_field_caps?fields=name +json: + index_filter: + term: + tags: "nice" +expected: + indices: + - fieldcaps + fields: + name: + keyword: + type: keyword + metadata_field: false + searchable: true + aggregatable: true + text: + type: text + metadata_field: false + searchable: true + aggregatable: true diff --git a/quickwit/rest-api-tests/scenarii/es_field_capabilities/_setup.quickwit.yaml b/quickwit/rest-api-tests/scenarii/es_field_capabilities/_setup.quickwit.yaml index 8b02ee01882..5576e6cec28 100644 --- a/quickwit/rest-api-tests/scenarii/es_field_capabilities/_setup.quickwit.yaml +++ b/quickwit/rest-api-tests/scenarii/es_field_capabilities/_setup.quickwit.yaml @@ -22,6 +22,7 @@ json: tokenizer: default fast: true timestamp_field: date + tag_fields: ["tags"] field_mappings: - name: date type: datetime @@ -32,6 +33,10 @@ json: - name: host type: ip fast: true + - name: tags + type: array + tokenizer: raw + fast: true --- # Create index method: POST diff --git a/quickwit/rest-api-tests/scenarii/qw_search_api/0001_ts_range.yaml b/quickwit/rest-api-tests/scenarii/qw_search_api/0001_ts_range.yaml index 8c42620ca55..dc9765b634e 100644 --- a/quickwit/rest-api-tests/scenarii/qw_search_api/0001_ts_range.yaml +++ b/quickwit/rest-api-tests/scenarii/qw_search_api/0001_ts_range.yaml @@ -34,3 +34,9 @@ params: query: "ts:>=1684993002 AND ts:<1684993004" expected: num_hits: 2 +--- +endpoint: simple/search +params: + query: "auto_date:>=2023-05-25T00:00:00Z AND auto_date:<2023-05-26T00:00:00Z" +expected: + num_hits: 2 diff --git a/quickwit/rest-api-tests/scenarii/qw_search_api/0004_exact_string.yaml b/quickwit/rest-api-tests/scenarii/qw_search_api/0004_exact_string.yaml index 0a877d1447a..a5782be7e19 100644 --- a/quickwit/rest-api-tests/scenarii/qw_search_api/0004_exact_string.yaml +++ b/quickwit/rest-api-tests/scenarii/qw_search_api/0004_exact_string.yaml @@ -39,9 +39,8 @@ endpoint: nested/search method: POST json: query: "text_fast:fast-text-value-dashes" -status_code: 400 expected: - message: "invalid query: query is incompatible with schema. field text_fast is not full-text searchable)" + num_hits: 1 --- endpoint: nested/search method: POST diff --git a/quickwit/rest-api-tests/scenarii/qw_search_api/0005_fast_field_search.yaml b/quickwit/rest-api-tests/scenarii/qw_search_api/0005_fast_field_search.yaml new file mode 100644 index 00000000000..bfa19c6f978 --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/qw_search_api/0005_fast_field_search.yaml @@ -0,0 +1,15 @@ +# Validate searching on a fast-only (non-indexed) field +endpoint: nested/search +method: POST +json: + query: "text_fast:fast-text-value-dashes" +expected: + num_hits: 1 +--- +# Non-matching exact value should return no hits +endpoint: nested/search +method: POST +json: + query: "text_fast:fast_text_value_dashes" +expected: + num_hits: 0 diff --git a/quickwit/rest-api-tests/scenarii/qw_search_api/_setup.quickwit.yaml b/quickwit/rest-api-tests/scenarii/qw_search_api/_setup.quickwit.yaml index 51bb6bf2b7a..b333ed3c86a 100644 --- a/quickwit/rest-api-tests/scenarii/qw_search_api/_setup.quickwit.yaml +++ b/quickwit/rest-api-tests/scenarii/qw_search_api/_setup.quickwit.yaml @@ -30,8 +30,8 @@ endpoint: simple/ingest params: commit: force ndjson: - - {"ts": 1684993001, "not_fast": 1684993001} - - {"ts": 1684993002, "not_fast": 1684993002} + - {"ts": 1684993001, "not_fast": 1684993001, "auto_date": "2023-05-25T10:00:00Z"} + - {"ts": 1684993002, "not_fast": 1684993002, "auto_date": "2023-05-25T11:00:00Z"} --- # Ingest documents split #2 method: POST diff --git a/quickwit/rust-toolchain.toml b/quickwit/rust-toolchain.toml index b81d21958e0..2a30998f14b 100644 --- a/quickwit/rust-toolchain.toml +++ b/quickwit/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "1.88" +channel = "1.93" components = ["cargo", "clippy", "rustfmt", "rust-docs"]

{ Publisher { publisher_type, metastore, @@ -77,8 +97,35 @@ impl Publisher { } } +fn serialize_checkpoint_delta( + checkpoint_delta_opt: &Option, +) -> anyhow::Result> { + checkpoint_delta_opt + .as_ref() + .map(serde_json::to_string) + .transpose() + .context("failed to serialize `IndexCheckpointDelta`") +} + +async fn suggest_truncate( + ctx: &ActorContext, + source_mailbox_opt: &Option>>, + checkpoint_delta_opt: Option, +) { + if let Some(source_mailbox) = source_mailbox_opt.as_ref() + && let Some(checkpoint) = checkpoint_delta_opt + { + let _ = ctx + .send_message( + source_mailbox, + SuggestTruncate(checkpoint.source_delta.get_source_checkpoint()), + ) + .await; + } +} + #[async_trait] -impl Actor for Publisher { +impl Actor for Publisher