diff --git a/.github/scripts/run-bazel-ci.sh b/.github/scripts/run-bazel-ci.sh index cf2d4ce3400f..f49abccfca19 100755 --- a/.github/scripts/run-bazel-ci.sh +++ b/.github/scripts/run-bazel-ci.sh @@ -380,14 +380,15 @@ else # --noexperimental_remote_repo_contents_cache: # disable remote repo contents cache enabled in .bazelrc startup options. # https://bazel.build/reference/command-line-reference#startup_options-flag--experimental_remote_repo_contents_cache - # --remote_cache= and --remote_executor=: - # clear remote cache/execution endpoints configured in .bazelrc. + # --remote_cache=, --remote_executor=, and --experimental_remote_downloader=: + # clear remote cache/execution/downloader endpoints configured in .bazelrc. # https://bazel.build/reference/command-line-reference#common_options-flag--remote_cache # https://bazel.build/reference/command-line-reference#common_options-flag--remote_executor bazel_run_args=( "${bazel_args[@]}" --remote_cache= --remote_executor= + --experimental_remote_downloader= ) if (( ${#post_config_bazel_args[@]} > 0 )); then bazel_run_args+=("${post_config_bazel_args[@]}") diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index f4501440c91d..c26c366b1a91 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -18,14 +18,16 @@ concurrency: jobs: test: timeout-minutes: 30 + env: + BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} strategy: fail-fast: false matrix: include: # macOS - - os: macos-15-xlarge + - os: macos-15 target: aarch64-apple-darwin - - os: macos-15-xlarge + - os: macos-15-intel target: x86_64-apple-darwin # Linux @@ -49,16 +51,24 @@ jobs: name: Local Bazel build on ${{ matrix.os }} for ${{ matrix.target }} steps: + - name: Skip macOS Bazel without BuildBuddy + if: runner.os == 'macOS' && env.BUILDBUDDY_API_KEY == '' + shell: bash + run: | + echo "::notice::Skipping macOS Bazel in this fork because the hermetic Apple SDK is only available through the authenticated BuildBuddy downloader/cache path." + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + if: runner.os != 'macOS' || env.BUILDBUDDY_API_KEY != '' - name: Check rusty_v8 MODULE.bazel checksums - if: matrix.os == 'ubuntu-24.04' && matrix.target == 'x86_64-unknown-linux-gnu' + if: (runner.os != 'macOS' || env.BUILDBUDDY_API_KEY != '') && matrix.os == 'ubuntu-24.04' && matrix.target == 'x86_64-unknown-linux-gnu' shell: bash run: | python3 .github/scripts/rusty_v8_bazel.py check-module-bazel python3 -m unittest discover -s .github/scripts -p test_rusty_v8_bazel.py - name: Prepare Bazel CI + if: runner.os != 'macOS' || env.BUILDBUDDY_API_KEY != '' id: prepare_bazel uses: ./.github/actions/prepare-bazel-ci with: @@ -66,11 +76,12 @@ jobs: cache-scope: bazel-${{ github.job }} install-test-prereqs: "true" - name: Check MODULE.bazel.lock is up to date - if: matrix.os == 'ubuntu-24.04' && matrix.target == 'x86_64-unknown-linux-gnu' + if: (runner.os != 'macOS' || env.BUILDBUDDY_API_KEY != '') && matrix.os == 'ubuntu-24.04' && matrix.target == 'x86_64-unknown-linux-gnu' shell: bash run: ./scripts/check-module-bazel-lock.sh - name: bazel test //... + if: runner.os != 'macOS' || env.BUILDBUDDY_API_KEY != '' env: BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} shell: bash @@ -106,7 +117,7 @@ jobs: "${bazel_targets[@]}" - name: Upload Bazel execution logs - if: always() && !cancelled() + if: always() && !cancelled() && (runner.os != 'macOS' || env.BUILDBUDDY_API_KEY != '') continue-on-error: true uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 with: @@ -117,7 +128,7 @@ jobs: # Save the job-scoped Bazel repository cache after cache misses. Keep the # upload non-fatal so cache service issues never fail the job itself. - name: Save bazel repository cache - if: always() && !cancelled() && steps.prepare_bazel.outputs.repository-cache-hit != 'true' + if: always() && !cancelled() && (runner.os != 'macOS' || env.BUILDBUDDY_API_KEY != '') && steps.prepare_bazel.outputs.repository-cache-hit != 'true' continue-on-error: true uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 with: @@ -126,6 +137,8 @@ jobs: clippy: timeout-minutes: 30 + env: + BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} strategy: fail-fast: false matrix: @@ -136,7 +149,7 @@ jobs: # that the Bazel test job uses. - os: ubuntu-24.04 target: x86_64-unknown-linux-gnu - - os: macos-15-xlarge + - os: macos-15 target: aarch64-apple-darwin - os: windows-latest target: x86_64-pc-windows-gnullvm @@ -144,9 +157,17 @@ jobs: name: Bazel clippy on ${{ matrix.os }} for ${{ matrix.target }} steps: + - name: Skip macOS Bazel without BuildBuddy + if: runner.os == 'macOS' && env.BUILDBUDDY_API_KEY == '' + shell: bash + run: | + echo "::notice::Skipping macOS Bazel in this fork because the hermetic Apple SDK is only available through the authenticated BuildBuddy downloader/cache path." + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + if: runner.os != 'macOS' || env.BUILDBUDDY_API_KEY != '' - name: Prepare Bazel CI + if: runner.os != 'macOS' || env.BUILDBUDDY_API_KEY != '' id: prepare_bazel uses: ./.github/actions/prepare-bazel-ci with: @@ -154,6 +175,7 @@ jobs: cache-scope: bazel-${{ github.job }} - name: bazel build --config=clippy lint targets + if: runner.os != 'macOS' || env.BUILDBUDDY_API_KEY != '' env: BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} shell: bash @@ -190,7 +212,7 @@ jobs: "${bazel_targets[@]}" - name: Upload Bazel execution logs - if: always() && !cancelled() + if: always() && !cancelled() && (runner.os != 'macOS' || env.BUILDBUDDY_API_KEY != '') continue-on-error: true uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 with: @@ -201,7 +223,7 @@ jobs: # Save the job-scoped Bazel repository cache after cache misses. Keep the # upload non-fatal so cache service issues never fail the job itself. - name: Save bazel repository cache - if: always() && !cancelled() && steps.prepare_bazel.outputs.repository-cache-hit != 'true' + if: always() && !cancelled() && (runner.os != 'macOS' || env.BUILDBUDDY_API_KEY != '') && steps.prepare_bazel.outputs.repository-cache-hit != 'true' continue-on-error: true uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 with: @@ -210,13 +232,15 @@ jobs: verify-release-build: timeout-minutes: 30 + env: + BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} strategy: fail-fast: false matrix: include: - os: ubuntu-24.04 target: x86_64-unknown-linux-gnu - - os: macos-15-xlarge + - os: macos-15 target: aarch64-apple-darwin - os: windows-latest target: x86_64-pc-windows-gnullvm @@ -224,9 +248,17 @@ jobs: name: Verify release build on ${{ matrix.os }} for ${{ matrix.target }} steps: + - name: Skip macOS Bazel without BuildBuddy + if: runner.os == 'macOS' && env.BUILDBUDDY_API_KEY == '' + shell: bash + run: | + echo "::notice::Skipping macOS Bazel in this fork because the hermetic Apple SDK is only available through the authenticated BuildBuddy downloader/cache path." + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + if: runner.os != 'macOS' || env.BUILDBUDDY_API_KEY != '' - name: Prepare Bazel CI + if: runner.os != 'macOS' || env.BUILDBUDDY_API_KEY != '' id: prepare_bazel uses: ./.github/actions/prepare-bazel-ci with: @@ -234,6 +266,7 @@ jobs: cache-scope: bazel-${{ github.job }} - name: bazel build verify-release-build targets + if: runner.os != 'macOS' || env.BUILDBUDDY_API_KEY != '' env: BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} shell: bash @@ -273,7 +306,7 @@ jobs: "${bazel_targets[@]}" - name: Upload Bazel execution logs - if: always() && !cancelled() + if: always() && !cancelled() && (runner.os != 'macOS' || env.BUILDBUDDY_API_KEY != '') continue-on-error: true uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 with: @@ -284,7 +317,7 @@ jobs: # Save the job-scoped Bazel repository cache after cache misses. Keep the # upload non-fatal so cache service issues never fail the job itself. - name: Save bazel repository cache - if: always() && !cancelled() && steps.prepare_bazel.outputs.repository-cache-hit != 'true' + if: always() && !cancelled() && (runner.os != 'macOS' || env.BUILDBUDDY_API_KEY != '') && steps.prepare_bazel.outputs.repository-cache-hit != 'true' continue-on-error: true uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 with: diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 42a3ca876412..9727dc747fed 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -139,6 +139,8 @@ jobs: runs-on: ${{ matrix.runs_on || matrix.runner }} timeout-minutes: ${{ matrix.timeout_minutes }} needs: changed + env: + BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} strategy: fail-fast: false matrix: @@ -147,14 +149,11 @@ jobs: runner: ubuntu-24.04 timeout_minutes: 30 - name: macOS - runner: macos-15-xlarge + runner: macos-15 timeout_minutes: 30 - name: Windows - runner: windows-x64 + runner: windows-latest timeout_minutes: 30 - runs_on: - group: codex-runners - labels: codex-windows-x64 steps: - name: Check whether argument comment lint should run id: argument_comment_lint_gate @@ -170,10 +169,15 @@ jobs: echo "No argument-comment-lint relevant changes." echo "run=false" >> "$GITHUB_OUTPUT" + - name: Skip macOS argument comment lint without BuildBuddy + if: steps.argument_comment_lint_gate.outputs.run == 'true' && runner.os == 'macOS' && env.BUILDBUDDY_API_KEY == '' + shell: bash + run: | + echo "::notice::Skipping macOS argument comment lint in this fork because the hermetic Apple SDK is only available through the authenticated BuildBuddy downloader/cache path." - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - if: ${{ steps.argument_comment_lint_gate.outputs.run == 'true' }} + if: steps.argument_comment_lint_gate.outputs.run == 'true' && (runner.os != 'macOS' || env.BUILDBUDDY_API_KEY != '') - name: Run argument comment lint on codex-rs via Bazel - if: ${{ steps.argument_comment_lint_gate.outputs.run == 'true' }} + if: steps.argument_comment_lint_gate.outputs.run == 'true' && (runner.os != 'macOS' || env.BUILDBUDDY_API_KEY != '') uses: ./.github/actions/run-argument-comment-lint with: target: ${{ runner.os }} diff --git a/.github/workflows/sdk.yml b/.github/workflows/sdk.yml index 45c983ac1ee8..0e742077f428 100644 --- a/.github/workflows/sdk.yml +++ b/.github/workflows/sdk.yml @@ -7,9 +7,7 @@ on: jobs: sdks: - runs-on: - group: codex-runners - labels: codex-linux-x64 + runs-on: ubuntu-24.04 timeout-minutes: 10 steps: - name: Checkout repository diff --git a/AGENTS.md b/AGENTS.md index ef184bd3d6b2..f59d9fe988ed 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,9 +1,11 @@ # rawr-ai/codex Router ## Scope + - Applies to this repo root (`./**`) unless a deeper `AGENTS.md` exists in the working directory subtree. ## Operating Invariants (Non-Negotiable) + - MUST use Graphite (`gt`) for all work in this repo. - MUST start any new stack from `codex/integration-upstream-main` via `gt create`. - Exception: only stack on an existing PR branch when the change is tightly coupled and intended to merge together. @@ -11,6 +13,7 @@ - MUST keep the repo clean (no dirty worktree) when finishing a task/turn. ## Process Rules + - Entry (always): - `git status --porcelain` - `gt branch info --no-interactive` @@ -21,11 +24,13 @@ - Use `gt modify` / `gt submit` for commits and PR updates. ## Routing + - Human-friendly workflows and setup pathways: `docs/agents.md` (root `agents.md` collides with `AGENTS.md` on case-insensitive filesystems) - rawr fork workflows and runbooks: `rawr/AGENTS.md` - Rust/Codex workspace rules (clippy/tests/docs/schema): `codex-rs/AGENTS.md` - AGENTS mechanics/precedence reference: `docs/agents_md.md` ## Ownership + - Scope owner: rawr maintainers - Update cadence: when workflows or safety invariants change (avoid temporal status text) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index b1ef43e3e503..e920a7d5e41c 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -402,7 +402,7 @@ checksum = "5f0e0fee31ef5ed1ba1316088939cea399010ed7731dba877ed44aeb407a75ea" [[package]] name = "app_test_support" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "base64 0.22.1", @@ -1750,7 +1750,7 @@ dependencies = [ [[package]] name = "codex-agent-identity" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "base64 0.22.1", @@ -1768,7 +1768,7 @@ dependencies = [ [[package]] name = "codex-analytics" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "codex-app-server-protocol", "codex-git-utils", @@ -1788,7 +1788,7 @@ dependencies = [ [[package]] name = "codex-ansi-escape" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "ansi-to-tui", "ratatui", @@ -1797,7 +1797,7 @@ dependencies = [ [[package]] name = "codex-api" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "assert_matches", @@ -1831,7 +1831,7 @@ dependencies = [ [[package]] name = "codex-app-server" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "app_test_support", @@ -1912,7 +1912,7 @@ dependencies = [ [[package]] name = "codex-app-server-client" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "codex-app-server", "codex-app-server-protocol", @@ -1936,7 +1936,7 @@ dependencies = [ [[package]] name = "codex-app-server-protocol" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "clap", @@ -1963,7 +1963,7 @@ dependencies = [ [[package]] name = "codex-app-server-test-client" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "clap", @@ -1984,7 +1984,7 @@ dependencies = [ [[package]] name = "codex-apply-patch" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "assert_cmd", @@ -2003,7 +2003,7 @@ dependencies = [ [[package]] name = "codex-arg0" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "codex-apply-patch", @@ -2020,7 +2020,7 @@ dependencies = [ [[package]] name = "codex-async-utils" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "async-trait", "pretty_assertions", @@ -2030,7 +2030,7 @@ dependencies = [ [[package]] name = "codex-aws-auth" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "aws-config", "aws-credential-types", @@ -2045,7 +2045,7 @@ dependencies = [ [[package]] name = "codex-backend-client" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "codex-api", @@ -2062,7 +2062,7 @@ dependencies = [ [[package]] name = "codex-backend-openapi-models" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "serde", "serde_json", @@ -2071,7 +2071,7 @@ dependencies = [ [[package]] name = "codex-chatgpt" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "clap", @@ -2092,7 +2092,7 @@ dependencies = [ [[package]] name = "codex-cli" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "assert_cmd", @@ -2150,7 +2150,7 @@ dependencies = [ [[package]] name = "codex-client" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "async-trait", "bytes", @@ -2180,7 +2180,7 @@ dependencies = [ [[package]] name = "codex-cloud-requirements" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "async-trait", "base64 0.22.1", @@ -2205,7 +2205,7 @@ dependencies = [ [[package]] name = "codex-cloud-tasks" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "async-trait", @@ -2237,7 +2237,7 @@ dependencies = [ [[package]] name = "codex-cloud-tasks-client" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "async-trait", @@ -2252,7 +2252,7 @@ dependencies = [ [[package]] name = "codex-cloud-tasks-mock-client" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "async-trait", "chrono", @@ -2262,7 +2262,7 @@ dependencies = [ [[package]] name = "codex-code-mode" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "async-channel", "async-trait", @@ -2279,11 +2279,11 @@ dependencies = [ [[package]] name = "codex-collaboration-mode-templates" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" [[package]] name = "codex-config" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "async-trait", @@ -2323,7 +2323,7 @@ dependencies = [ [[package]] name = "codex-connectors" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "codex-app-server-protocol", @@ -2335,7 +2335,7 @@ dependencies = [ [[package]] name = "codex-core" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "arc-swap", @@ -2457,7 +2457,7 @@ dependencies = [ [[package]] name = "codex-core-plugins" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "chrono", @@ -2491,7 +2491,7 @@ dependencies = [ [[package]] name = "codex-core-skills" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "codex-analytics", @@ -2522,7 +2522,7 @@ dependencies = [ [[package]] name = "codex-debug-client" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "clap", @@ -2534,7 +2534,7 @@ dependencies = [ [[package]] name = "codex-device-key" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "async-trait", "base64 0.22.1", @@ -2550,7 +2550,7 @@ dependencies = [ [[package]] name = "codex-exec" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "assert_cmd", @@ -2594,7 +2594,7 @@ dependencies = [ [[package]] name = "codex-exec-server" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "arc-swap", @@ -2628,7 +2628,7 @@ dependencies = [ [[package]] name = "codex-execpolicy" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "clap", @@ -2645,7 +2645,7 @@ dependencies = [ [[package]] name = "codex-execpolicy-legacy" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "allocative", "anyhow", @@ -2665,7 +2665,7 @@ dependencies = [ [[package]] name = "codex-experimental-api-macros" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "proc-macro2", "quote", @@ -2674,7 +2674,7 @@ dependencies = [ [[package]] name = "codex-features" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "codex-otel", "codex-protocol", @@ -2687,7 +2687,7 @@ dependencies = [ [[package]] name = "codex-feedback" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "codex-login", @@ -2700,7 +2700,7 @@ dependencies = [ [[package]] name = "codex-file-search" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "clap", @@ -2716,7 +2716,7 @@ dependencies = [ [[package]] name = "codex-git-utils" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "assert_matches", @@ -2741,7 +2741,7 @@ dependencies = [ [[package]] name = "codex-hooks" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "chrono", @@ -2760,7 +2760,7 @@ dependencies = [ [[package]] name = "codex-install-context" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "codex-utils-home-dir", "pretty_assertions", @@ -2769,7 +2769,7 @@ dependencies = [ [[package]] name = "codex-keyring-store" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "keyring", "tracing", @@ -2777,7 +2777,7 @@ dependencies = [ [[package]] name = "codex-linux-sandbox" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "cc", "clap", @@ -2801,7 +2801,7 @@ dependencies = [ [[package]] name = "codex-lmstudio" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "codex-core", "codex-model-provider-info", @@ -2815,7 +2815,7 @@ dependencies = [ [[package]] name = "codex-login" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "async-trait", @@ -2856,7 +2856,7 @@ dependencies = [ [[package]] name = "codex-mcp" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "async-channel", @@ -2889,7 +2889,7 @@ dependencies = [ [[package]] name = "codex-mcp-server" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "codex-arg0", @@ -2922,7 +2922,7 @@ dependencies = [ [[package]] name = "codex-model-provider" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "async-trait", "codex-agent-identity", @@ -2946,7 +2946,7 @@ dependencies = [ [[package]] name = "codex-model-provider-info" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "codex-api", "codex-app-server-protocol", @@ -2963,7 +2963,7 @@ dependencies = [ [[package]] name = "codex-models-manager" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "async-trait", "chrono", @@ -2984,7 +2984,7 @@ dependencies = [ [[package]] name = "codex-network-proxy" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "async-trait", @@ -3015,7 +3015,7 @@ dependencies = [ [[package]] name = "codex-ollama" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "assert_matches", "async-stream", @@ -3034,7 +3034,7 @@ dependencies = [ [[package]] name = "codex-otel" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "chrono", "codex-api", @@ -3066,7 +3066,7 @@ dependencies = [ [[package]] name = "codex-plugin" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "codex-utils-absolute-path", "codex-utils-plugins", @@ -3075,7 +3075,7 @@ dependencies = [ [[package]] name = "codex-process-hardening" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "libc", "pretty_assertions", @@ -3083,7 +3083,7 @@ dependencies = [ [[package]] name = "codex-protocol" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "chardetng", @@ -3123,7 +3123,7 @@ dependencies = [ [[package]] name = "codex-realtime-webrtc" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "libwebrtc", "thiserror 2.0.18", @@ -3132,7 +3132,7 @@ dependencies = [ [[package]] name = "codex-response-debug-context" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "base64 0.22.1", "codex-api", @@ -3143,7 +3143,7 @@ dependencies = [ [[package]] name = "codex-responses-api-proxy" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "clap", @@ -3160,7 +3160,7 @@ dependencies = [ [[package]] name = "codex-rmcp-client" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "axum", @@ -3197,7 +3197,7 @@ dependencies = [ [[package]] name = "codex-rollout" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "async-trait", @@ -3222,7 +3222,7 @@ dependencies = [ [[package]] name = "codex-rollout-trace" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "codex-code-mode", @@ -3237,7 +3237,7 @@ dependencies = [ [[package]] name = "codex-sandboxing" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "async-trait", @@ -3258,7 +3258,7 @@ dependencies = [ [[package]] name = "codex-secrets" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "age", "anyhow", @@ -3279,7 +3279,7 @@ dependencies = [ [[package]] name = "codex-shell-command" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "base64 0.22.1", @@ -3299,7 +3299,7 @@ dependencies = [ [[package]] name = "codex-shell-escalation" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "async-trait", @@ -3320,7 +3320,7 @@ dependencies = [ [[package]] name = "codex-skills" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "codex-utils-absolute-path", "include_dir", @@ -3329,7 +3329,7 @@ dependencies = [ [[package]] name = "codex-state" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "chrono", @@ -3352,7 +3352,7 @@ dependencies = [ [[package]] name = "codex-stdio-to-uds" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "codex-uds", @@ -3364,7 +3364,7 @@ dependencies = [ [[package]] name = "codex-terminal-detection" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "pretty_assertions", "tracing", @@ -3372,7 +3372,7 @@ dependencies = [ [[package]] name = "codex-test-binary-support" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "codex-arg0", "tempfile", @@ -3380,7 +3380,7 @@ dependencies = [ [[package]] name = "codex-thread-store" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "async-trait", "chrono", @@ -3405,7 +3405,7 @@ dependencies = [ [[package]] name = "codex-tools" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "codex-app-server-protocol", "codex-code-mode", @@ -3422,7 +3422,7 @@ dependencies = [ [[package]] name = "codex-tui" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "arboard", @@ -3527,7 +3527,7 @@ dependencies = [ [[package]] name = "codex-uds" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "async-io", "pretty_assertions", @@ -3539,7 +3539,7 @@ dependencies = [ [[package]] name = "codex-utils-absolute-path" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "dirs", "dunce", @@ -3553,14 +3553,14 @@ dependencies = [ [[package]] name = "codex-utils-approval-presets" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "codex-protocol", ] [[package]] name = "codex-utils-cache" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "lru 0.16.3", "sha1", @@ -3569,7 +3569,7 @@ dependencies = [ [[package]] name = "codex-utils-cargo-bin" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "assert_cmd", "runfiles", @@ -3578,7 +3578,7 @@ dependencies = [ [[package]] name = "codex-utils-cli" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "clap", "codex-protocol", @@ -3589,15 +3589,15 @@ dependencies = [ [[package]] name = "codex-utils-elapsed" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" [[package]] name = "codex-utils-fuzzy-match" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" [[package]] name = "codex-utils-home-dir" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "codex-utils-absolute-path", "dirs", @@ -3607,7 +3607,7 @@ dependencies = [ [[package]] name = "codex-utils-image" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "base64 0.22.1", "codex-utils-cache", @@ -3619,7 +3619,7 @@ dependencies = [ [[package]] name = "codex-utils-json-to-toml" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "pretty_assertions", "serde_json", @@ -3628,7 +3628,7 @@ dependencies = [ [[package]] name = "codex-utils-oss" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "codex-core", "codex-lmstudio", @@ -3638,7 +3638,7 @@ dependencies = [ [[package]] name = "codex-utils-output-truncation" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "codex-protocol", "codex-utils-string", @@ -3647,7 +3647,7 @@ dependencies = [ [[package]] name = "codex-utils-path" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "codex-utils-absolute-path", "dunce", @@ -3657,7 +3657,7 @@ dependencies = [ [[package]] name = "codex-utils-plugins" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "codex-exec-server", "codex-login", @@ -3670,7 +3670,7 @@ dependencies = [ [[package]] name = "codex-utils-pty" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "filedescriptor", @@ -3686,7 +3686,7 @@ dependencies = [ [[package]] name = "codex-utils-readiness" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "assert_matches", "async-trait", @@ -3697,14 +3697,14 @@ dependencies = [ [[package]] name = "codex-utils-rustls-provider" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "rustls", ] [[package]] name = "codex-utils-sandbox-summary" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "codex-core", "codex-model-provider-info", @@ -3715,7 +3715,7 @@ dependencies = [ [[package]] name = "codex-utils-sleep-inhibitor" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "core-foundation 0.9.4", "libc", @@ -3725,14 +3725,14 @@ dependencies = [ [[package]] name = "codex-utils-stream-parser" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "pretty_assertions", ] [[package]] name = "codex-utils-string" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "pretty_assertions", "regex-lite", @@ -3740,14 +3740,14 @@ dependencies = [ [[package]] name = "codex-utils-template" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "pretty_assertions", ] [[package]] name = "codex-v8-poc" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "pretty_assertions", "v8", @@ -3755,7 +3755,7 @@ dependencies = [ [[package]] name = "codex-windows-sandbox" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "base64 0.22.1", @@ -4005,7 +4005,7 @@ dependencies = [ [[package]] name = "core_test_support" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "assert_cmd", @@ -8184,7 +8184,7 @@ dependencies = [ [[package]] name = "mcp_test_support" -version = "0.125.0-alpha.3" +version = "0.126.0-alpha.3" dependencies = [ "anyhow", "codex-login", diff --git a/codex-rs/app-server/tests/suite/v2/compaction.rs b/codex-rs/app-server/tests/suite/v2/compaction.rs index 44b5dd6dc6df..300e0aa96613 100644 --- a/codex-rs/app-server/tests/suite/v2/compaction.rs +++ b/codex-rs/app-server/tests/suite/v2/compaction.rs @@ -28,7 +28,9 @@ use codex_app_server_protocol::TurnCompletedNotification; use codex_app_server_protocol::TurnStartParams; use codex_app_server_protocol::TurnStartResponse; use codex_app_server_protocol::UserInput as V2UserInput; +use codex_app_server_protocol::WarningNotification; use codex_config::types::AuthCredentialsStoreMode; +use codex_features::Feature; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; use core_test_support::responses; @@ -48,6 +50,265 @@ const AUTO_COMPACT_LIMIT: i64 = 1_000; const COMPACT_PROMPT: &str = "Summarize the conversation."; const INVALID_REQUEST_ERROR_CODE: i64 = -32600; +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn rawr_turn_complete_auto_compaction_emits_started_and_completed_items() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let sse1 = responses::sse(vec![ + responses::ev_assistant_message( + "m1", + "Completed the first step. Final thoughts: continue with the next task.", + ), + responses::ev_completed_with_tokens("r1", /*total_tokens*/ 70_000), + ]); + let sse2 = responses::sse(vec![ + responses::ev_assistant_message("m2", "RAWR_SUMMARY"), + responses::ev_completed_with_tokens("r2", /*total_tokens*/ 200), + ]); + let responses_log = responses::mount_sse_sequence(&server, vec![sse1, sse2]).await; + + let codex_home = TempDir::new()?; + write_mock_responses_config_toml( + codex_home.path(), + &server.uri(), + &BTreeMap::from([(Feature::RawrAutoCompaction, true)]), + /*auto_compact_limit*/ i64::MAX, + /*requires_openai_auth*/ None, + "mock_provider", + COMPACT_PROMPT, + )?; + let config_path = codex_home.path().join("config.toml"); + let config = std::fs::read_to_string(&config_path)?; + std::fs::write( + &config_path, + config.replacen( + "model_auto_compact_token_limit", + "model_context_window = 100000\nmodel_auto_compact_token_limit", + 1, + ), + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_id = start_thread(&mut mcp).await?; + let turn_id = send_turn(&mut mcp, &thread_id, "first").await?; + wait_for_turn_completed_before_context_compaction_started(&mut mcp, &turn_id).await?; + + let started = wait_for_context_compaction_started(&mut mcp).await?; + let completed = wait_for_context_compaction_completed(&mut mcp).await?; + + let ThreadItem::ContextCompaction { id: started_id } = started.item else { + unreachable!("started item should be context compaction"); + }; + let ThreadItem::ContextCompaction { id: completed_id } = completed.item else { + unreachable!("completed item should be context compaction"); + }; + + assert_eq!(started.thread_id, thread_id); + assert_eq!(completed.thread_id, thread_id); + assert_eq!(started_id, completed_id); + assert_eq!(responses_log.requests().len(), 2); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn rawr_judgment_denial_emits_warning_and_skips_compaction() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let sse1 = responses::sse(vec![ + responses::ev_assistant_message("m1", "Completed the first step. Final thoughts follow."), + responses::ev_completed_with_tokens("r1", /*total_tokens*/ 70_000), + ]); + let sse2 = responses::sse(vec![ + responses::ev_assistant_message( + "m2", + r#"{"should_compact":false,"reason":"critical handoff is incomplete"}"#, + ), + responses::ev_completed_with_tokens("r2", /*total_tokens*/ 200), + ]); + let responses_log = responses::mount_sse_sequence(&server, vec![sse1, sse2]).await; + + let codex_home = TempDir::new()?; + write_rawr_config(codex_home.path(), &server.uri())?; + write_judgment_prompt(codex_home.path(), "judgment.md")?; + append_config_toml( + codex_home.path(), + r#" + +[rawr_auto_compaction.policy.asap] +decision_prompt_path = "judgment.md" +"#, + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_id = start_thread(&mut mcp).await?; + let turn_id = send_turn(&mut mcp, &thread_id, "first").await?; + wait_for_turn_completed(&mut mcp, &turn_id).await?; + + let warning = wait_for_warning(&mut mcp, "rawr auto-compaction skipped by judgment").await?; + assert_eq!(warning.thread_id, Some(thread_id)); + assert!(warning.message.contains("critical handoff is incomplete")); + assert_eq!(responses_log.requests().len(), 2); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn rawr_judgment_failure_emits_warning_and_compacts() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let sse1 = responses::sse(vec![ + responses::ev_assistant_message("m1", "Completed the first step. Final thoughts follow."), + responses::ev_completed_with_tokens("r1", /*total_tokens*/ 70_000), + ]); + let sse2 = responses::sse(vec![ + responses::ev_assistant_message("m2", "RAWR_SUMMARY"), + responses::ev_completed_with_tokens("r2", /*total_tokens*/ 200), + ]); + let responses_log = responses::mount_sse_sequence(&server, vec![sse1, sse2]).await; + + let codex_home = TempDir::new()?; + write_rawr_config(codex_home.path(), &server.uri())?; + append_config_toml( + codex_home.path(), + r#" + +[rawr_auto_compaction.policy.asap] +decision_prompt_path = "missing-judgment.md" +"#, + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_id = start_thread(&mut mcp).await?; + let turn_id = send_turn(&mut mcp, &thread_id, "first").await?; + wait_for_turn_completed(&mut mcp, &turn_id).await?; + + let warning = wait_for_warning( + &mut mcp, + "rawr auto-compaction judgment failed; continuing with static policy", + ) + .await?; + assert_eq!(warning.thread_id, Some(thread_id.clone())); + + let started = wait_for_context_compaction_started(&mut mcp).await?; + let completed = wait_for_context_compaction_completed(&mut mcp).await?; + assert_eq!(started.thread_id, thread_id); + assert_eq!(completed.thread_id, started.thread_id); + assert_eq!(responses_log.requests().len(), 2); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn rawr_artifact_generation_failure_emits_warning_and_compacts() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let sse1 = responses::sse(vec![ + responses::ev_assistant_message("m1", "Completed the first step. Final thoughts follow."), + responses::ev_completed_with_tokens("r1", /*total_tokens*/ 70_000), + ]); + let sse2 = responses::sse(vec![ + responses::ev_assistant_message("m2", "this is not json"), + responses::ev_completed_with_tokens("r2", /*total_tokens*/ 200), + ]); + let sse3 = responses::sse(vec![ + responses::ev_assistant_message("m3", "RAWR_SUMMARY"), + responses::ev_completed_with_tokens("r3", /*total_tokens*/ 200), + ]); + let responses_log = responses::mount_sse_sequence(&server, vec![sse1, sse2, sse3]).await; + + let codex_home = TempDir::new()?; + write_rawr_config(codex_home.path(), &server.uri())?; + append_config_toml( + codex_home.path(), + r#" + +[rawr_auto_compaction] +packet_author = "agent" +"#, + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_id = start_thread(&mut mcp).await?; + let turn_id = send_turn(&mut mcp, &thread_id, "first").await?; + wait_for_turn_completed(&mut mcp, &turn_id).await?; + + let warning = wait_for_warning(&mut mcp, "rawr pre-compact artifact generation failed").await?; + assert_eq!(warning.thread_id, Some(thread_id.clone())); + + let started = wait_for_context_compaction_started(&mut mcp).await?; + let completed = wait_for_context_compaction_completed(&mut mcp).await?; + assert_eq!(started.thread_id, thread_id); + assert_eq!(completed.thread_id, started.thread_id); + assert_eq!(responses_log.requests().len(), 3); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn rawr_scratch_write_failure_emits_warning_and_compacts() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let sse1 = responses::sse(vec![ + responses::ev_assistant_message("m1", "Completed the first step. Final thoughts follow."), + responses::ev_completed_with_tokens("r1", /*total_tokens*/ 70_000), + ]); + let sse2 = responses::sse(vec![ + responses::ev_assistant_message("m2", r#"{"scratchpad_contents":"verbatim notes"}"#), + responses::ev_completed_with_tokens("r2", /*total_tokens*/ 200), + ]); + let sse3 = responses::sse(vec![ + responses::ev_assistant_message("m3", "RAWR_SUMMARY"), + responses::ev_completed_with_tokens("r3", /*total_tokens*/ 200), + ]); + let responses_log = responses::mount_sse_sequence(&server, vec![sse1, sse2, sse3]).await; + + let codex_home = TempDir::new()?; + let workspace = TempDir::new()?; + std::fs::write(workspace.path().join(".scratch"), "not a directory")?; + write_rawr_config(codex_home.path(), &server.uri())?; + append_config_toml( + codex_home.path(), + r#" + +[rawr_auto_compaction] +scratch_write_enabled = true +"#, + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_id = + start_thread_with_cwd(&mut mcp, Some(workspace.path().display().to_string())).await?; + let turn_id = send_turn(&mut mcp, &thread_id, "first").await?; + wait_for_turn_completed(&mut mcp, &turn_id).await?; + + let warning = wait_for_warning(&mut mcp, "rawr scratch write failed").await?; + assert_eq!(warning.thread_id, Some(thread_id.clone())); + + let started = wait_for_context_compaction_started(&mut mcp).await?; + let completed = wait_for_context_compaction_completed(&mut mcp).await?; + assert_eq!(started.thread_id, thread_id); + assert_eq!(completed.thread_id, started.thread_id); + assert_eq!(responses_log.requests().len(), 3); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn auto_compaction_local_emits_started_and_completed_items() -> Result<()> { skip_if_no_network!(Ok(())); @@ -324,9 +585,14 @@ async fn thread_compact_start_rejects_unknown_thread_id() -> Result<()> { } async fn start_thread(mcp: &mut McpProcess) -> Result { + start_thread_with_cwd(mcp, /*cwd*/ None).await +} + +async fn start_thread_with_cwd(mcp: &mut McpProcess, cwd: Option) -> Result { let thread_id = mcp .send_thread_start_request(ThreadStartParams { model: Some("mock-model".to_string()), + cwd, ..Default::default() }) .await?; @@ -339,7 +605,54 @@ async fn start_thread(mcp: &mut McpProcess) -> Result { Ok(thread.id) } +fn write_rawr_config(codex_home: &std::path::Path, server_uri: &str) -> Result<()> { + write_mock_responses_config_toml( + codex_home, + server_uri, + &BTreeMap::from([(Feature::RawrAutoCompaction, true)]), + /*auto_compact_limit*/ i64::MAX, + /*requires_openai_auth*/ None, + "mock_provider", + COMPACT_PROMPT, + )?; + let config_path = codex_home.join("config.toml"); + let config = std::fs::read_to_string(&config_path)?; + std::fs::write( + &config_path, + config.replacen( + "model_auto_compact_token_limit", + "model_context_window = 100000\nmodel_auto_compact_token_limit", + 1, + ), + )?; + Ok(()) +} + +fn append_config_toml(codex_home: &std::path::Path, contents: &str) -> Result<()> { + let config_path = codex_home.join("config.toml"); + let mut config = std::fs::read_to_string(&config_path)?; + config.push_str(contents); + std::fs::write(config_path, config)?; + Ok(()) +} + +fn write_judgment_prompt(codex_home: &std::path::Path, file_name: &str) -> Result<()> { + let prompt_dir = codex_home.join("auto-compact"); + std::fs::create_dir_all(&prompt_dir)?; + std::fs::write( + prompt_dir.join(file_name), + "Return JSON with should_compact and reason.", + )?; + Ok(()) +} + async fn send_turn_and_wait(mcp: &mut McpProcess, thread_id: &str, text: &str) -> Result { + let turn_id = send_turn(mcp, thread_id, text).await?; + wait_for_turn_completed(mcp, &turn_id).await?; + Ok(turn_id) +} + +async fn send_turn(mcp: &mut McpProcess, thread_id: &str, text: &str) -> Result { let turn_id = mcp .send_turn_start_request(TurnStartParams { thread_id: thread_id.to_string(), @@ -356,10 +669,42 @@ async fn send_turn_and_wait(mcp: &mut McpProcess, thread_id: &str, text: &str) - ) .await??; let TurnStartResponse { turn } = to_response::(turn_resp)?; - wait_for_turn_completed(mcp, &turn.id).await?; Ok(turn.id) } +async fn wait_for_turn_completed_before_context_compaction_started( + mcp: &mut McpProcess, + turn_id: &str, +) -> Result<()> { + let notification: JSONRPCNotification = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_matching_notification( + "turn/completed or context compaction item/started", + |notification| { + if notification.method == "turn/completed" { + return true; + } + notification.method == "item/started" + && notification + .params + .as_ref() + .and_then(|params| { + serde_json::from_value::(params.clone()).ok() + }) + .is_some_and(|started| { + matches!(started.item, ThreadItem::ContextCompaction { .. }) + }) + }, + ), + ) + .await??; + assert_eq!(notification.method, "turn/completed"); + let completed: TurnCompletedNotification = + serde_json::from_value(notification.params.expect("turn/completed params"))?; + assert_eq!(completed.turn.id, turn_id); + Ok(()) +} + async fn wait_for_turn_completed(mcp: &mut McpProcess, turn_id: &str) -> Result<()> { loop { let notification: JSONRPCNotification = timeout( @@ -408,3 +753,21 @@ async fn wait_for_context_compaction_completed( } } } + +async fn wait_for_warning( + mcp: &mut McpProcess, + message_prefix: &str, +) -> Result { + loop { + let notification: JSONRPCNotification = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("warning"), + ) + .await??; + let warning: WarningNotification = + serde_json::from_value(notification.params.expect("warning params"))?; + if warning.message.starts_with(message_prefix) { + return Ok(warning); + } + } +} diff --git a/codex-rs/config/src/config_toml.rs b/codex-rs/config/src/config_toml.rs index 9ee784117358..2bb72de12faf 100644 --- a/codex-rs/config/src/config_toml.rs +++ b/codex-rs/config/src/config_toml.rs @@ -20,6 +20,7 @@ use crate::types::Notice; use crate::types::OAuthCredentialsStoreMode; use crate::types::OtelConfigToml; use crate::types::PluginConfig; +use crate::types::RawrAutoCompactionToml; use crate::types::SandboxWorkspaceWrite; use crate::types::ShellEnvironmentPolicyToml; use crate::types::SkillsConfig; @@ -354,6 +355,9 @@ pub struct ConfigToml { #[schemars(schema_with = "crate::schema::features_schema")] pub features: Option, + /// RAWR fork automatic compaction policy. + pub rawr_auto_compaction: Option, + /// Suppress warnings about unstable (under development) features. pub suppress_unstable_features_warning: Option, @@ -872,3 +876,92 @@ pub fn validate_oss_provider(provider: &str) -> std::io::Result<()> { )), } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::RawrAutoCompactionMode; + use crate::types::RawrAutoCompactionPacketAuthor; + use crate::types::RawrAutoCompactionToml; + use pretty_assertions::assert_eq; + + #[test] + fn rawr_auto_compaction_accepts_legacy_bool() { + let config: ConfigToml = + toml::from_str("rawr_auto_compaction = true\n").expect("config should parse"); + assert_eq!( + config.rawr_auto_compaction, + Some(RawrAutoCompactionToml::Enabled(true)) + ); + } + + #[test] + fn rawr_auto_compaction_accepts_structured_policy() { + let config: ConfigToml = toml::from_str( + r#" +[rawr_auto_compaction] +mode = "auto" +packet_author = "agent" +scratch_write_enabled = true +packet_max_tail_chars = 1200 +auto_compact_prompt_path = "auto-compact.md" +scratch_write_prompt_path = "scratch-write.md" +watcher_packet_prompt_path = "watcher-packet.md" +judgment_context_prompt_path = "judgment-context.md" +scratch_file_template = ".scratch/agent-{agentName}.scratch.md" + +[rawr_auto_compaction.semantic_signals] +agent_done_phrases = ["wrapped"] +agent_done_negative_phrases = ["not wrapped"] +topic_shift_phrases = ["moving next"] +concluding_thought_phrases = ["carry forward"] + +[rawr_auto_compaction.policy.early] +percent_remaining_lt = 90 +requires_any_boundary = ["turn_complete"] +"#, + ) + .expect("config should parse"); + let Some(RawrAutoCompactionToml::Config(rawr)) = config.rawr_auto_compaction else { + panic!("expected structured rawr config"); + }; + assert_eq!(rawr.mode, Some(RawrAutoCompactionMode::Auto)); + assert_eq!( + rawr.packet_author, + Some(RawrAutoCompactionPacketAuthor::Agent) + ); + assert_eq!(rawr.scratch_write_enabled, Some(true)); + assert_eq!(rawr.packet_max_tail_chars, Some(1200)); + assert_eq!( + rawr.auto_compact_prompt_path.as_deref(), + Some("auto-compact.md") + ); + assert_eq!( + rawr.scratch_write_prompt_path.as_deref(), + Some("scratch-write.md") + ); + assert_eq!( + rawr.watcher_packet_prompt_path.as_deref(), + Some("watcher-packet.md") + ); + assert_eq!( + rawr.judgment_context_prompt_path.as_deref(), + Some("judgment-context.md") + ); + assert_eq!( + rawr.scratch_file_template.as_deref(), + Some(".scratch/agent-{agentName}.scratch.md") + ); + let semantic_signals = rawr.semantic_signals.as_ref().expect("semantic signals"); + assert_eq!( + semantic_signals.agent_done_phrases.as_deref(), + Some(&["wrapped".to_string()][..]) + ); + let early = rawr + .policy + .as_ref() + .and_then(|policy| policy.early.as_ref()) + .expect("early policy"); + assert_eq!(early.percent_remaining_lt, Some(90)); + } +} diff --git a/codex-rs/config/src/types.rs b/codex-rs/config/src/types.rs index 7413686a77b2..da3ee47be3a9 100644 --- a/codex-rs/config/src/types.rs +++ b/codex-rs/config/src/types.rs @@ -179,6 +179,127 @@ pub struct ToolSuggestConfig { pub discoverables: Vec, } +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum RawrAutoCompactionMode { + Tag, + Suggest, + Auto, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum RawrAutoCompactionPacketAuthor { + Watcher, + Agent, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum RawrAutoCompactionBoundary { + Commit, + PlanCheckpoint, + PlanUpdate, + PrCheckpoint, + AgentDone, + TopicShift, + ConcludingThought, + TurnComplete, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, JsonSchema)] +#[serde(deny_unknown_fields)] +#[schemars(deny_unknown_fields)] +pub struct RawrAutoCompactionPolicyToml { + pub early: Option, + pub ready: Option, + pub asap: Option, + pub emergency: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, JsonSchema)] +#[serde(deny_unknown_fields)] +#[schemars(deny_unknown_fields)] +pub struct RawrAutoCompactionPolicyTierToml { + /// Trigger threshold for this tier. When `percent_remaining < percent_remaining_lt`, + /// this tier is eligible. + pub percent_remaining_lt: Option, + /// Boundaries that may trigger compaction for this tier. If unset, defaults are used. + pub requires_any_boundary: Option>, + /// When enabled, plan boundaries only count when there is a semantic break unless + /// a non-plan boundary is also present. + pub plan_boundaries_require_semantic_break: Option, + /// Optional prompt file used to make a judgment call before compaction triggers. + pub decision_prompt_path: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, JsonSchema)] +#[serde(deny_unknown_fields)] +#[schemars(deny_unknown_fields)] +pub struct RawrAutoCompactionSemanticSignalsToml { + /// Phrases that mark finalized assistant output as an agent-done boundary. + pub agent_done_phrases: Option>, + /// Phrases that prevent finalized assistant output from being treated as agent-done. + pub agent_done_negative_phrases: Option>, + /// Phrases that mark finalized assistant output as a topic-shift boundary. + pub topic_shift_phrases: Option>, + /// Phrases that mark finalized assistant output as a concluding-thought boundary. + pub concluding_thought_phrases: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, JsonSchema)] +#[serde(deny_unknown_fields)] +#[schemars(deny_unknown_fields)] +pub struct RawrAutoCompactionSettingsToml { + pub enabled: Option, + pub mode: Option, + pub packet_author: Option, + /// When enabled, the watcher may ask the in-session agent to write a scratchpad file + /// before compaction so verbatim research notes survive history rewrite. + pub scratch_write_enabled: Option, + /// Max chars of tail context included in watcher-side packet prompts. + pub packet_max_tail_chars: Option, + /// Optional model override used for watcher-triggered compactions. + pub compaction_model: Option, + /// Optional override for the agent-authored packet prompt path. + pub auto_compact_prompt_path: Option, + /// Optional override for the scratch-write prompt path. + pub scratch_write_prompt_path: Option, + /// Optional override for the watcher-authored packet prompt path. + pub watcher_packet_prompt_path: Option, + /// Optional override for the judgment context template path. + pub judgment_context_prompt_path: Option, + /// Relative scratch path template. Supports `{agentName}`, `{agent_name}`, and `{threadId}`. + pub scratch_file_template: Option, + /// Phrase lists used to derive semantic natural-boundary signals from finalized output. + pub semantic_signals: Option, + /// Per-tier thresholds, boundary requirements, and optional judgment prompts. + pub policy: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema)] +#[serde(untagged)] +pub enum RawrAutoCompactionToml { + Enabled(bool), + Config(Box), +} + +impl RawrAutoCompactionToml { + pub fn enabled(&self) -> bool { + match self { + Self::Enabled(enabled) => *enabled, + Self::Config(settings) => settings.enabled.unwrap_or(true), + } + } + + pub fn settings(&self) -> Option<&RawrAutoCompactionSettingsToml> { + match self { + Self::Enabled(_) => None, + Self::Config(settings) => Some(settings), + } + } +} + /// Memories settings loaded from config.toml. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default, JsonSchema)] #[schemars(deny_unknown_fields)] diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index 6cac42908372..87ebdb607740 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -466,6 +466,9 @@ "prevent_idle_sleep": { "type": "boolean" }, + "rawr_auto_compaction": { + "type": "boolean" + }, "realtime_conversation": { "type": "boolean" }, @@ -1842,6 +1845,187 @@ }, "type": "object" }, + "RawrAutoCompactionBoundary": { + "enum": [ + "commit", + "plan_checkpoint", + "plan_update", + "pr_checkpoint", + "agent_done", + "topic_shift", + "concluding_thought", + "turn_complete" + ], + "type": "string" + }, + "RawrAutoCompactionMode": { + "enum": [ + "tag", + "suggest", + "auto" + ], + "type": "string" + }, + "RawrAutoCompactionPacketAuthor": { + "enum": [ + "watcher", + "agent" + ], + "type": "string" + }, + "RawrAutoCompactionPolicyTierToml": { + "additionalProperties": false, + "properties": { + "decision_prompt_path": { + "description": "Optional prompt file used to make a judgment call before compaction triggers.", + "type": "string" + }, + "percent_remaining_lt": { + "description": "Trigger threshold for this tier. When `percent_remaining < percent_remaining_lt`, this tier is eligible.", + "format": "int64", + "type": "integer" + }, + "plan_boundaries_require_semantic_break": { + "description": "When enabled, plan boundaries only count when there is a semantic break unless a non-plan boundary is also present.", + "type": "boolean" + }, + "requires_any_boundary": { + "description": "Boundaries that may trigger compaction for this tier. If unset, defaults are used.", + "items": { + "$ref": "#/definitions/RawrAutoCompactionBoundary" + }, + "type": "array" + } + }, + "type": "object" + }, + "RawrAutoCompactionPolicyToml": { + "additionalProperties": false, + "properties": { + "asap": { + "$ref": "#/definitions/RawrAutoCompactionPolicyTierToml" + }, + "early": { + "$ref": "#/definitions/RawrAutoCompactionPolicyTierToml" + }, + "emergency": { + "$ref": "#/definitions/RawrAutoCompactionPolicyTierToml" + }, + "ready": { + "$ref": "#/definitions/RawrAutoCompactionPolicyTierToml" + } + }, + "type": "object" + }, + "RawrAutoCompactionSemanticSignalsToml": { + "additionalProperties": false, + "properties": { + "agent_done_negative_phrases": { + "description": "Phrases that prevent finalized assistant output from being treated as agent-done.", + "items": { + "type": "string" + }, + "type": "array" + }, + "agent_done_phrases": { + "description": "Phrases that mark finalized assistant output as an agent-done boundary.", + "items": { + "type": "string" + }, + "type": "array" + }, + "concluding_thought_phrases": { + "description": "Phrases that mark finalized assistant output as a concluding-thought boundary.", + "items": { + "type": "string" + }, + "type": "array" + }, + "topic_shift_phrases": { + "description": "Phrases that mark finalized assistant output as a topic-shift boundary.", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "RawrAutoCompactionSettingsToml": { + "additionalProperties": false, + "properties": { + "auto_compact_prompt_path": { + "description": "Optional override for the agent-authored packet prompt path.", + "type": "string" + }, + "compaction_model": { + "description": "Optional model override used for watcher-triggered compactions.", + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "judgment_context_prompt_path": { + "description": "Optional override for the judgment context template path.", + "type": "string" + }, + "mode": { + "$ref": "#/definitions/RawrAutoCompactionMode" + }, + "packet_author": { + "$ref": "#/definitions/RawrAutoCompactionPacketAuthor" + }, + "packet_max_tail_chars": { + "description": "Max chars of tail context included in watcher-side packet prompts.", + "format": "uint", + "minimum": 0.0, + "type": "integer" + }, + "policy": { + "allOf": [ + { + "$ref": "#/definitions/RawrAutoCompactionPolicyToml" + } + ], + "description": "Per-tier thresholds, boundary requirements, and optional judgment prompts." + }, + "scratch_file_template": { + "description": "Relative scratch path template. Supports `{agentName}`, `{agent_name}`, and `{threadId}`.", + "type": "string" + }, + "scratch_write_enabled": { + "description": "When enabled, the watcher may ask the in-session agent to write a scratchpad file before compaction so verbatim research notes survive history rewrite.", + "type": "boolean" + }, + "scratch_write_prompt_path": { + "description": "Optional override for the scratch-write prompt path.", + "type": "string" + }, + "semantic_signals": { + "allOf": [ + { + "$ref": "#/definitions/RawrAutoCompactionSemanticSignalsToml" + } + ], + "description": "Phrase lists used to derive semantic natural-boundary signals from finalized output." + }, + "watcher_packet_prompt_path": { + "description": "Optional override for the watcher-authored packet prompt path.", + "type": "string" + } + }, + "type": "object" + }, + "RawrAutoCompactionToml": { + "anyOf": [ + { + "type": "boolean" + }, + { + "$ref": "#/definitions/RawrAutoCompactionSettingsToml" + } + ] + }, "RealtimeAudioToml": { "additionalProperties": false, "properties": { @@ -2624,6 +2808,9 @@ "prevent_idle_sleep": { "type": "boolean" }, + "rawr_auto_compaction": { + "type": "boolean" + }, "realtime_conversation": { "type": "boolean" }, @@ -3030,6 +3217,14 @@ }, "type": "object" }, + "rawr_auto_compaction": { + "allOf": [ + { + "$ref": "#/definitions/RawrAutoCompactionToml" + } + ], + "description": "RAWR fork automatic compaction policy." + }, "realtime": { "allOf": [ { diff --git a/codex-rs/core/src/agent/control_tests.rs b/codex-rs/core/src/agent/control_tests.rs index 6018c3747411..26f2491b5a99 100644 --- a/codex-rs/core/src/agent/control_tests.rs +++ b/codex-rs/core/src/agent/control_tests.rs @@ -34,6 +34,8 @@ use tokio::time::sleep; use tokio::time::timeout; use toml::Value as TomlValue; +const EVENTUAL_PERSISTENCE_TIMEOUT: Duration = Duration::from_secs(15); + async fn test_config_with_cli_overrides( cli_overrides: Vec<(String, TomlValue)>, ) -> (TempDir, Config) { @@ -213,7 +215,7 @@ async fn wait_for_live_thread_spawn_children( let mut expected_children = expected_children.to_vec(); expected_children.sort_by_key(std::string::ToString::to_string); - timeout(Duration::from_secs(5), async { + timeout(EVENTUAL_PERSISTENCE_TIMEOUT, async { loop { let mut child_ids = control .open_thread_spawn_children(parent_thread_id) @@ -478,7 +480,7 @@ async fn send_inter_agent_communication_without_turn_queues_message_without_trig .find(|entry| *entry == expected); assert_eq!(captured, Some(expected)); - timeout(Duration::from_secs(5), async { + timeout(EVENTUAL_PERSISTENCE_TIMEOUT, async { loop { if thread.codex.session.has_pending_input().await { break; @@ -527,7 +529,7 @@ async fn append_message_records_assistant_message() { .expect("append_message should succeed"); assert!(!submission_id.is_empty()); - timeout(Duration::from_secs(5), async { + timeout(EVENTUAL_PERSISTENCE_TIMEOUT, async { loop { let history_items = thread .codex @@ -1334,7 +1336,7 @@ async fn multi_agent_v2_completion_queues_message_for_direct_parent() { }, ); - timeout(Duration::from_secs(5), async { + timeout(EVENTUAL_PERSISTENCE_TIMEOUT, async { loop { let captured = harness .manager @@ -1549,7 +1551,7 @@ async fn resume_thread_subagent_restores_stored_nickname_and_role() { .await .expect("status subscription should succeed"); if matches!(status_rx.borrow().clone(), AgentStatus::PendingInit) { - timeout(Duration::from_secs(5), async { + timeout(EVENTUAL_PERSISTENCE_TIMEOUT, async { loop { status_rx .changed() @@ -1571,7 +1573,7 @@ async fn resume_thread_subagent_restores_stored_nickname_and_role() { let state_db = child_thread .state_db() .expect("sqlite state db should be available for nickname resume test"); - timeout(Duration::from_secs(5), async { + timeout(EVENTUAL_PERSISTENCE_TIMEOUT, async { loop { if let Ok(Some(metadata)) = state_db.get_thread(child_thread_id).await && metadata.agent_nickname.is_some() diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rs index 481e6a5580c7..fc8711f0f5de 100644 --- a/codex-rs/core/src/config/config_tests.rs +++ b/codex-rs/core/src/config/config_tests.rs @@ -5315,6 +5315,7 @@ async fn test_precedence_fixture_with_o3_profile() -> std::io::Result<()> { ghost_snapshot: GhostSnapshotConfig::default(), multi_agent_v2: MultiAgentV2Config::default(), features: Features::with_defaults().into(), + rawr_auto_compaction: None, suppress_unstable_features_warning: false, active_profile: Some("o3".to_string()), active_project: ProjectConfig { trust_level: None }, @@ -5513,6 +5514,7 @@ async fn test_precedence_fixture_with_gpt3_profile() -> std::io::Result<()> { ghost_snapshot: GhostSnapshotConfig::default(), multi_agent_v2: MultiAgentV2Config::default(), features: Features::with_defaults().into(), + rawr_auto_compaction: None, suppress_unstable_features_warning: false, active_profile: Some("gpt3".to_string()), active_project: ProjectConfig { trust_level: None }, @@ -5665,6 +5667,7 @@ async fn test_precedence_fixture_with_zdr_profile() -> std::io::Result<()> { ghost_snapshot: GhostSnapshotConfig::default(), multi_agent_v2: MultiAgentV2Config::default(), features: Features::with_defaults().into(), + rawr_auto_compaction: None, suppress_unstable_features_warning: false, active_profile: Some("zdr".to_string()), active_project: ProjectConfig { trust_level: None }, @@ -5802,6 +5805,7 @@ async fn test_precedence_fixture_with_gpt5_profile() -> std::io::Result<()> { ghost_snapshot: GhostSnapshotConfig::default(), multi_agent_v2: MultiAgentV2Config::default(), features: Features::with_defaults().into(), + rawr_auto_compaction: None, suppress_unstable_features_warning: false, active_profile: Some("gpt5".to_string()), active_project: ProjectConfig { trust_level: None }, diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index cfd31675368c..a9bef0cc6d04 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -43,6 +43,7 @@ use codex_config::types::OAuthCredentialsStoreMode; use codex_config::types::OtelConfig; use codex_config::types::OtelConfigToml; use codex_config::types::OtelExporterKind; +use codex_config::types::RawrAutoCompactionToml; use codex_config::types::ShellEnvironmentPolicy; use codex_config::types::ToolSuggestConfig; use codex_config::types::ToolSuggestDiscoverable; @@ -585,6 +586,9 @@ pub struct Config { /// Centralized feature flags; source of truth for feature gating. pub features: ManagedFeatures, + /// RAWR fork automatic compaction policy. + pub rawr_auto_compaction: Option, + /// When `true`, suppress warnings about unstable (under development) features. pub suppress_unstable_features_warning: bool, @@ -1658,7 +1662,7 @@ impl Config { web_search_request: override_tools_web_search_request, }; - let configured_features = Features::from_sources( + let mut configured_features = Features::from_sources( FeatureConfigSource { features: cfg.features.as_ref(), include_apply_patch_tool: None, @@ -1675,6 +1679,13 @@ impl Config { }, feature_overrides, ); + if cfg + .rawr_auto_compaction + .as_ref() + .is_some_and(RawrAutoCompactionToml::enabled) + { + configured_features.enable(Feature::RawrAutoCompaction); + } let features = ManagedFeatures::from_configured_with_warnings( configured_features, feature_requirements, @@ -2445,6 +2456,7 @@ impl Config { ghost_snapshot, multi_agent_v2, features, + rawr_auto_compaction: cfg.rawr_auto_compaction, suppress_unstable_features_warning: cfg .suppress_unstable_features_warning .unwrap_or(false), diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 54fadc6fd371..8246467efbd1 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -10,6 +10,9 @@ mod apps; mod arc_monitor; mod client; mod client_common; +mod rawr_auto_compaction; +mod rawr_auto_compaction_model; +mod rawr_prompts; mod realtime_context; mod realtime_conversation; mod realtime_prompt; diff --git a/codex-rs/core/src/rawr_auto_compaction.rs b/codex-rs/core/src/rawr_auto_compaction.rs new file mode 100644 index 000000000000..a9fe0a455a92 --- /dev/null +++ b/codex-rs/core/src/rawr_auto_compaction.rs @@ -0,0 +1,1227 @@ +use std::collections::hash_map::DefaultHasher; +use std::hash::Hash; +use std::hash::Hasher; +use std::path::Component; +use std::path::Path; + +use crate::config::Config; +use crate::rawr_prompts; +use codex_config::types::RawrAutoCompactionBoundary; +use codex_config::types::RawrAutoCompactionMode; +use codex_config::types::RawrAutoCompactionPacketAuthor; +use codex_config::types::RawrAutoCompactionPolicyTierToml; +use codex_config::types::RawrAutoCompactionSemanticSignalsToml; +use codex_features::Feature; +use codex_protocol::ThreadId; +use codex_protocol::parse_command::ParsedCommand; +use codex_protocol::plan_tool::StepStatus; +use codex_protocol::plan_tool::UpdatePlanArgs; +use codex_protocol::protocol::ExecCommandEndEvent; +use codex_protocol::protocol::ExecCommandStatus; +use codex_protocol::protocol::SessionSource; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RawrAutoCompactionTier { + Early, + Ready, + Asap, + Emergency, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct RawrAutoCompactionThresholds { + pub early_percent_remaining_lt: i64, + pub ready_percent_remaining_lt: i64, + pub asap_percent_remaining_lt: i64, + pub emergency_percent_remaining_lt: i64, +} + +impl RawrAutoCompactionThresholds { + pub(crate) fn from_config(config: &Config) -> Self { + let defaults = Self { + early_percent_remaining_lt: 85, + ready_percent_remaining_lt: 75, + asap_percent_remaining_lt: 65, + emergency_percent_remaining_lt: 15, + }; + + let Some(policy) = config + .rawr_auto_compaction + .as_ref() + .and_then(|rawr| rawr.settings()) + .and_then(|settings| settings.policy.as_ref()) + else { + return defaults; + }; + + Self { + early_percent_remaining_lt: policy + .early + .as_ref() + .and_then(|tier| tier.percent_remaining_lt) + .unwrap_or(defaults.early_percent_remaining_lt), + ready_percent_remaining_lt: policy + .ready + .as_ref() + .and_then(|tier| tier.percent_remaining_lt) + .unwrap_or(defaults.ready_percent_remaining_lt), + asap_percent_remaining_lt: policy + .asap + .as_ref() + .and_then(|tier| tier.percent_remaining_lt) + .unwrap_or(defaults.asap_percent_remaining_lt), + emergency_percent_remaining_lt: policy + .emergency + .as_ref() + .and_then(|tier| tier.percent_remaining_lt) + .unwrap_or(defaults.emergency_percent_remaining_lt), + } + } +} + +#[derive(Default, Debug, Clone, PartialEq, Eq)] +pub(crate) struct RawrAutoCompactionSignals { + pub saw_commit: bool, + pub saw_plan_checkpoint: bool, + pub saw_plan_update: bool, + pub saw_pr_checkpoint: bool, + pub saw_agent_done: bool, + pub saw_topic_shift: bool, + pub saw_concluding_thought: bool, +} + +pub(crate) fn rawr_note_plan_update( + signals: &mut RawrAutoCompactionSignals, + completed_steps_seen: &mut usize, + update: &UpdatePlanArgs, +) { + signals.saw_plan_update = true; + let completed_steps = rawr_completed_plan_steps(update); + if completed_steps > *completed_steps_seen { + signals.saw_plan_checkpoint = true; + *completed_steps_seen = completed_steps; + } +} + +pub(crate) fn rawr_note_exec_command_end( + signals: &mut RawrAutoCompactionSignals, + event: &ExecCommandEndEvent, +) { + if event.status != ExecCommandStatus::Completed { + return; + } + + if rawr_command_looks_like_git_commit(&event.command, &event.parsed_cmd) { + signals.saw_commit = true; + } + if rawr_command_looks_like_pr_checkpoint(&event.command) { + signals.saw_pr_checkpoint = true; + } +} + +pub(crate) fn rawr_note_completion_message( + signals: &mut RawrAutoCompactionSignals, + config: &Config, + last_agent_message: Option<&str>, +) { + let Some(last_agent_message) = last_agent_message else { + return; + }; + if rawr_agent_message_looks_done(config, last_agent_message) { + signals.saw_agent_done = true; + } + if rawr_agent_message_looks_like_topic_shift(config, last_agent_message) { + signals.saw_topic_shift = true; + } + if rawr_agent_message_looks_like_concluding_thought(config, last_agent_message) { + signals.saw_concluding_thought = true; + } +} + +const RAWR_SCRATCH_FALLBACK_AGENT_NAMES: [&str; 24] = [ + "Aria", "Atlas", "Beau", "Cleo", "Ezra", "Jade", "Juno", "Luna", "Milo", "Nova", "Orion", + "Pax", "Quinn", "Reid", "Remy", "Rhea", "Rory", "Sage", "Skye", "Toby", "Vera", "Wren", "Zane", + "Zoe", +]; +const DEFAULT_AGENT_DONE_PHRASES: &[&str] = &["done", "completed", "finished", "shipped", "pushed"]; +const DEFAULT_AGENT_DONE_NEGATIVE_PHRASES: &[&str] = &["not done", "not completed", "not finished"]; +const DEFAULT_TOPIC_SHIFT_PHRASES: &[&str] = &[ + "moving on", + "switching to", + "next,", + "next:", + "next up", + "now, let's", + "now let's", + "we'll now", +]; +const DEFAULT_CONCLUDING_THOUGHT_PHRASES: &[&str] = &[ + "in summary", + "to summarize", + "to wrap up", + "wrapping up", + "conclusion", + "concluding", + "final thoughts", + "next steps", +]; +const DEFAULT_SCRATCH_FILE_TEMPLATE: &str = ".scratch/agent-{agentName}.scratch.md"; + +pub(crate) fn rawr_pick_tier( + thresholds: RawrAutoCompactionThresholds, + percent_remaining: i64, +) -> Option { + if percent_remaining < thresholds.emergency_percent_remaining_lt { + return Some(RawrAutoCompactionTier::Emergency); + } + if percent_remaining < thresholds.asap_percent_remaining_lt { + return Some(RawrAutoCompactionTier::Asap); + } + if percent_remaining < thresholds.ready_percent_remaining_lt { + return Some(RawrAutoCompactionTier::Ready); + } + if percent_remaining < thresholds.early_percent_remaining_lt { + return Some(RawrAutoCompactionTier::Early); + } + None +} + +pub(crate) fn rawr_compaction_tier( + config: &Config, + percent_remaining: i64, +) -> Option { + rawr_pick_tier( + RawrAutoCompactionThresholds::from_config(config), + percent_remaining, + ) +} + +pub(crate) fn rawr_auto_compaction_mode(config: &Config) -> RawrAutoCompactionMode { + config + .rawr_auto_compaction + .as_ref() + .and_then(|rawr| rawr.settings()) + .and_then(|settings| settings.mode) + .unwrap_or(RawrAutoCompactionMode::Auto) +} + +pub(crate) fn rawr_auto_compaction_packet_author( + config: &Config, +) -> RawrAutoCompactionPacketAuthor { + config + .rawr_auto_compaction + .as_ref() + .and_then(|rawr| rawr.settings()) + .and_then(|settings| settings.packet_author) + .unwrap_or(RawrAutoCompactionPacketAuthor::Watcher) +} + +pub(crate) fn rawr_packet_max_tail_chars(config: &Config) -> usize { + config + .rawr_auto_compaction + .as_ref() + .and_then(|rawr| rawr.settings()) + .and_then(|settings| settings.packet_max_tail_chars) + .unwrap_or(2_000) +} + +pub(crate) fn rawr_compaction_model(config: &Config) -> Option { + config + .rawr_auto_compaction + .as_ref() + .and_then(|rawr| rawr.settings()) + .and_then(|settings| settings.compaction_model.clone()) +} + +pub(crate) fn rawr_scratch_write_enabled(config: &Config) -> bool { + config + .rawr_auto_compaction + .as_ref() + .and_then(|rawr| rawr.settings()) + .and_then(|settings| settings.scratch_write_enabled) + .unwrap_or(false) +} + +fn rawr_auto_compact_prompt_path(config: &Config) -> Option<&str> { + config + .rawr_auto_compaction + .as_ref() + .and_then(|rawr| rawr.settings()) + .and_then(|settings| settings.auto_compact_prompt_path.as_deref()) +} + +fn rawr_scratch_write_prompt_path(config: &Config) -> Option<&str> { + config + .rawr_auto_compaction + .as_ref() + .and_then(|rawr| rawr.settings()) + .and_then(|settings| settings.scratch_write_prompt_path.as_deref()) +} + +pub(crate) fn rawr_watcher_packet_prompt_path(config: &Config) -> Option<&str> { + config + .rawr_auto_compaction + .as_ref() + .and_then(|rawr| rawr.settings()) + .and_then(|settings| settings.watcher_packet_prompt_path.as_deref()) +} + +pub(crate) fn rawr_judgment_context_prompt_path(config: &Config) -> Option<&str> { + config + .rawr_auto_compaction + .as_ref() + .and_then(|rawr| rawr.settings()) + .and_then(|settings| settings.judgment_context_prompt_path.as_deref()) +} + +fn rawr_semantic_signals_config(config: &Config) -> Option<&RawrAutoCompactionSemanticSignalsToml> { + config + .rawr_auto_compaction + .as_ref() + .and_then(|rawr| rawr.settings()) + .and_then(|settings| settings.semantic_signals.as_ref()) +} + +pub(crate) fn rawr_should_compact_at_turn_complete( + config: &Config, + percent_remaining: i64, + signals: &RawrAutoCompactionSignals, +) -> bool { + rawr_should_compact_with_boundary(config, percent_remaining, signals, true) +} + +fn rawr_should_compact_with_boundary( + config: &Config, + percent_remaining: i64, + signals: &RawrAutoCompactionSignals, + turn_complete: bool, +) -> bool { + if !config.features.enabled(Feature::RawrAutoCompaction) { + return false; + } + + let tier = match rawr_compaction_tier(config, percent_remaining) { + Some(tier) => tier, + None => return false, + }; + + if tier == RawrAutoCompactionTier::Emergency { + return true; + } + + let default_allowed = match tier { + RawrAutoCompactionTier::Early => &[ + RawrAutoCompactionBoundary::PlanCheckpoint, + RawrAutoCompactionBoundary::PlanUpdate, + RawrAutoCompactionBoundary::PrCheckpoint, + RawrAutoCompactionBoundary::TopicShift, + ][..], + RawrAutoCompactionTier::Ready => &[ + RawrAutoCompactionBoundary::Commit, + RawrAutoCompactionBoundary::PlanCheckpoint, + RawrAutoCompactionBoundary::PlanUpdate, + RawrAutoCompactionBoundary::PrCheckpoint, + RawrAutoCompactionBoundary::TopicShift, + ][..], + RawrAutoCompactionTier::Asap => &[ + RawrAutoCompactionBoundary::Commit, + RawrAutoCompactionBoundary::PlanCheckpoint, + RawrAutoCompactionBoundary::PlanUpdate, + RawrAutoCompactionBoundary::PrCheckpoint, + RawrAutoCompactionBoundary::AgentDone, + RawrAutoCompactionBoundary::TopicShift, + RawrAutoCompactionBoundary::ConcludingThought, + ][..], + RawrAutoCompactionTier::Emergency => unreachable!(), + }; + + let policy_tier = rawr_policy_tier(config, tier); + let required = policy_tier + .and_then(|tier| tier.requires_any_boundary.as_deref()) + .unwrap_or(default_allowed); + + let has_semantic_boundary = + signals.saw_agent_done || signals.saw_topic_shift || signals.saw_concluding_thought; + let requires_semantic_boundary_for_plan = policy_tier + .and_then(|tier| tier.plan_boundaries_require_semantic_break) + .unwrap_or(matches!( + tier, + RawrAutoCompactionTier::Early | RawrAutoCompactionTier::Ready + )); + let mut satisfied_any_required_boundary = false; + let mut satisfied_plan_boundary = false; + let mut satisfied_non_plan_boundary = false; + + for boundary in required { + let satisfied = match boundary { + RawrAutoCompactionBoundary::Commit => signals.saw_commit, + RawrAutoCompactionBoundary::PlanCheckpoint => signals.saw_plan_checkpoint, + RawrAutoCompactionBoundary::PlanUpdate => signals.saw_plan_update, + RawrAutoCompactionBoundary::PrCheckpoint => signals.saw_pr_checkpoint, + RawrAutoCompactionBoundary::AgentDone => signals.saw_agent_done, + RawrAutoCompactionBoundary::TopicShift => signals.saw_topic_shift, + RawrAutoCompactionBoundary::ConcludingThought => signals.saw_concluding_thought, + RawrAutoCompactionBoundary::TurnComplete => turn_complete, + }; + if !satisfied { + continue; + } + + satisfied_any_required_boundary = true; + match boundary { + RawrAutoCompactionBoundary::PlanCheckpoint | RawrAutoCompactionBoundary::PlanUpdate => { + satisfied_plan_boundary = true; + } + RawrAutoCompactionBoundary::Commit | RawrAutoCompactionBoundary::PrCheckpoint => { + satisfied_non_plan_boundary = true; + } + RawrAutoCompactionBoundary::AgentDone + | RawrAutoCompactionBoundary::TopicShift + | RawrAutoCompactionBoundary::ConcludingThought + | RawrAutoCompactionBoundary::TurnComplete => {} + } + } + + satisfied_any_required_boundary + && (!requires_semantic_boundary_for_plan + || !satisfied_plan_boundary + || satisfied_non_plan_boundary + || has_semantic_boundary) +} + +fn rawr_policy_tier( + config: &Config, + tier: RawrAutoCompactionTier, +) -> Option<&RawrAutoCompactionPolicyTierToml> { + let policy = config + .rawr_auto_compaction + .as_ref() + .and_then(|rawr| rawr.settings()) + .and_then(|settings| settings.policy.as_ref())?; + + match tier { + RawrAutoCompactionTier::Early => policy.early.as_ref(), + RawrAutoCompactionTier::Ready => policy.ready.as_ref(), + RawrAutoCompactionTier::Asap => policy.asap.as_ref(), + RawrAutoCompactionTier::Emergency => policy.emergency.as_ref(), + } +} + +pub(crate) fn rawr_policy_decision_prompt_path( + config: &Config, + tier: RawrAutoCompactionTier, +) -> Option { + rawr_policy_tier(config, tier).and_then(|policy| policy.decision_prompt_path.clone()) +} + +pub(crate) fn rawr_boundaries_present( + signals: &RawrAutoCompactionSignals, + turn_complete: bool, +) -> Vec { + let mut boundaries = Vec::new(); + if signals.saw_commit { + boundaries.push("commit".to_string()); + } + if signals.saw_plan_checkpoint { + boundaries.push("plan_checkpoint".to_string()); + } + if signals.saw_plan_update { + boundaries.push("plan_update".to_string()); + } + if signals.saw_pr_checkpoint { + boundaries.push("pr_checkpoint".to_string()); + } + if signals.saw_agent_done { + boundaries.push("agent_done".to_string()); + } + if signals.saw_topic_shift { + boundaries.push("topic_shift".to_string()); + } + if signals.saw_concluding_thought { + boundaries.push("concluding_thought".to_string()); + } + if turn_complete { + boundaries.push("turn_complete".to_string()); + } + boundaries +} + +fn rawr_completed_plan_steps(update: &UpdatePlanArgs) -> usize { + update + .plan + .iter() + .filter(|item| matches!(item.status, StepStatus::Completed)) + .count() +} + +pub(crate) fn rawr_command_looks_like_git_commit( + command: &[String], + parsed_cmd: &[ParsedCommand], +) -> bool { + if command.is_empty() { + return false; + } + if parsed_cmd.iter().any(|parsed| match parsed { + ParsedCommand::Unknown { cmd } => cmd.to_ascii_lowercase().contains("git commit"), + _ => false, + }) { + return true; + } + + let joined = command.join(" ").to_ascii_lowercase(); + if joined.contains("git commit") { + return true; + } + + fn basename(s: &str) -> &str { + std::path::Path::new(s) + .file_name() + .and_then(std::ffi::OsStr::to_str) + .unwrap_or(s) + } + + command + .windows(2) + .any(|pair| basename(pair[0].as_str()) == "git" && pair[1].eq_ignore_ascii_case("commit")) +} + +pub(crate) fn rawr_command_looks_like_pr_checkpoint(command: &[String]) -> bool { + if command.is_empty() { + return false; + } + let joined = command.join(" ").to_ascii_lowercase(); + + if joined.contains("gt submit") || joined.contains("gt ss") { + return true; + } + if joined.contains("gt create") || joined.contains("gt review") || joined.contains("gt land") { + return true; + } + if joined.contains("gh pr create") + || joined.contains("gh pr close") + || joined.contains("gh pr merge") + || joined.contains("gh pr reopen") + || joined.contains("gh pr review") + { + return true; + } + + false +} + +pub(crate) fn rawr_agent_message_looks_done(config: &Config, message: &str) -> bool { + let lower = message.trim().to_ascii_lowercase(); + if lower.is_empty() { + return false; + } + let semantic_config = rawr_semantic_signals_config(config); + let negative_phrases = + semantic_config.and_then(|signals| signals.agent_done_negative_phrases.as_ref()); + if rawr_message_contains_any( + &lower, + negative_phrases, + DEFAULT_AGENT_DONE_NEGATIVE_PHRASES, + ) { + return false; + } + rawr_message_contains_any( + &lower, + semantic_config.and_then(|signals| signals.agent_done_phrases.as_ref()), + DEFAULT_AGENT_DONE_PHRASES, + ) +} + +pub(crate) fn rawr_agent_message_looks_like_topic_shift(config: &Config, message: &str) -> bool { + let lower = message.trim().to_ascii_lowercase(); + if lower.is_empty() { + return false; + } + rawr_message_contains_any( + &lower, + rawr_semantic_signals_config(config) + .and_then(|signals| signals.topic_shift_phrases.as_ref()), + DEFAULT_TOPIC_SHIFT_PHRASES, + ) +} + +pub(crate) fn rawr_agent_message_looks_like_concluding_thought( + config: &Config, + message: &str, +) -> bool { + let lower = message.trim().to_ascii_lowercase(); + if lower.is_empty() { + return false; + } + rawr_message_contains_any( + &lower, + rawr_semantic_signals_config(config) + .and_then(|signals| signals.concluding_thought_phrases.as_ref()), + DEFAULT_CONCLUDING_THOUGHT_PHRASES, + ) +} + +fn rawr_message_contains_any( + lower_message: &str, + configured_phrases: Option<&Vec>, + default_phrases: &[&str], +) -> bool { + if let Some(configured_phrases) = configured_phrases { + return configured_phrases.iter().any(|phrase| { + let phrase = phrase.trim().to_ascii_lowercase(); + !phrase.is_empty() && lower_message.contains(&phrase) + }); + } + default_phrases + .iter() + .any(|phrase| lower_message.contains(phrase)) +} + +pub(crate) fn rawr_load_agent_packet_prompt(config: &Config) -> String { + let prompt = rawr_prompts::read_prompt_path_or_default( + &config.codex_home, + rawr_auto_compact_prompt_path(config), + rawr_prompts::RawrPromptKind::AutoCompact, + ); + let prompt = strip_yaml_frontmatter(&prompt).trim(); + if prompt.is_empty() { + return default_rawr_agent_packet_prompt(); + } + prompt.to_string() +} + +pub(crate) fn rawr_load_scratch_write_prompt(config: &Config) -> String { + let prompt = rawr_prompts::read_prompt_path_or_default( + &config.codex_home, + rawr_scratch_write_prompt_path(config), + rawr_prompts::RawrPromptKind::ScratchWrite, + ); + let prompt = strip_yaml_frontmatter(&prompt).trim(); + if prompt.is_empty() { + return default_rawr_scratch_write_prompt(); + } + prompt.to_string() +} + +pub(crate) fn rawr_load_watcher_packet_prompt(config: &Config) -> String { + let prompt = rawr_prompts::read_prompt_path_or_default( + &config.codex_home, + rawr_watcher_packet_prompt_path(config), + rawr_prompts::RawrPromptKind::WatcherPacket, + ); + let prompt = strip_yaml_frontmatter(&prompt).trim(); + if prompt.is_empty() { + return default_rawr_watcher_packet_prompt(); + } + prompt.to_string() +} + +pub(crate) fn rawr_build_scratch_write_prompt( + prompt: &str, + scratch_file: &str, + thread_id: Option, +) -> String { + let expanded = rawr_expand_prompt_template(prompt, Some(scratch_file), thread_id); + if prompt.contains("{scratch_file}") || prompt.contains("{scratchFile}") { + expanded + } else { + format!("{expanded}\n\nTarget file: `{scratch_file}`") + } +} + +pub(crate) fn rawr_build_agent_continuation_packet_prompt( + packet_prompt: &str, + scratch_prompt: &str, + do_scratch: bool, + scratch_file: Option<&str>, + thread_id: Option, +) -> String { + if !do_scratch { + return rawr_expand_prompt_template(packet_prompt, scratch_file, thread_id); + } + + let scratch_prompt = if let Some(scratch_file) = scratch_file { + rawr_build_scratch_write_prompt(scratch_prompt, scratch_file, thread_id) + } else { + rawr_expand_prompt_template(scratch_prompt, None, thread_id) + }; + let packet_prompt = rawr_expand_prompt_template(packet_prompt, scratch_file, thread_id); + format!("{scratch_prompt}\n\n---\n\n{packet_prompt}") +} + +pub(crate) fn rawr_build_watcher_post_compact_packet( + prompt_template: &str, + trigger_percent_remaining: i64, + signals: &RawrAutoCompactionSignals, + last_agent_message: Option<&str>, + max_tail_chars: usize, +) -> String { + let tail = truncate_chars(last_agent_message.unwrap_or("").trim(), max_tail_chars); + let tail = if tail.is_empty() { + "(none)".to_string() + } else { + tail + }; + + let boundary_signals = format!( + "commit={}, plan_checkpoint={}, plan_update={}, pr_checkpoint={}, agent_done={}, topic_shift={}, concluding_thought={}", + signals.saw_commit, + signals.saw_plan_checkpoint, + signals.saw_plan_update, + signals.saw_pr_checkpoint, + signals.saw_agent_done, + signals.saw_topic_shift, + signals.saw_concluding_thought, + ); + let template = if prompt_template.trim().is_empty() { + default_rawr_watcher_packet_prompt() + } else { + prompt_template.trim().to_string() + }; + + rawr_prompts::expand_placeholders( + &template, + &[ + ( + "triggerPercentRemaining", + trigger_percent_remaining.to_string(), + ), + ( + "trigger_percent_remaining", + trigger_percent_remaining.to_string(), + ), + ("boundarySignals", boundary_signals.clone()), + ("boundary_signals", boundary_signals), + ("lastAgentMessage", tail.clone()), + ("last_agent_message", tail), + ], + ) + .trim() + .to_string() +} + +pub(crate) fn rawr_should_schedule_scratch_write( + scratch_write_enabled: bool, + is_emergency: bool, + signals: &RawrAutoCompactionSignals, +) -> bool { + if !scratch_write_enabled || is_emergency { + return false; + } + signals.saw_commit + || signals.saw_plan_checkpoint + || signals.saw_plan_update + || signals.saw_pr_checkpoint + || signals.saw_agent_done +} + +pub(crate) fn rawr_build_post_compact_handoff_message( + packet: String, + scratch_file: Option<&str>, +) -> String { + if let Some(scratch_file) = scratch_file { + format!("Scratchpad: `{scratch_file}`\n\n{packet}") + } else { + packet + } +} + +pub(crate) fn rawr_scratch_file_rel_path( + config: &Config, + session_source: &SessionSource, + thread_id: &ThreadId, +) -> String { + let agent_name = rawr_scratch_agent_name(session_source, thread_id); + let template = config + .rawr_auto_compaction + .as_ref() + .and_then(|rawr| rawr.settings()) + .and_then(|settings| settings.scratch_file_template.as_deref()) + .unwrap_or(DEFAULT_SCRATCH_FILE_TEMPLATE); + rawr_expand_scratch_file_template(template, &agent_name, thread_id) +} + +fn rawr_scratch_agent_name(session_source: &SessionSource, thread_id: &ThreadId) -> String { + rawr_agent_identity_from_session_source(session_source) + .unwrap_or_else(|| rawr_random_agent_name(thread_id)) +} + +fn rawr_agent_identity_from_session_source(source: &SessionSource) -> Option { + let identity = source.to_string(); + let identity = identity.strip_prefix("subagent_")?; + let sanitized = rawr_sanitize_agent_name(identity); + (!sanitized.is_empty()).then_some(sanitized) +} + +fn rawr_random_agent_name(thread_id: &ThreadId) -> String { + let mut hasher = DefaultHasher::new(); + thread_id.hash(&mut hasher); + let seed = hasher.finish() as usize; + RAWR_SCRATCH_FALLBACK_AGENT_NAMES[seed % RAWR_SCRATCH_FALLBACK_AGENT_NAMES.len()].to_string() +} + +fn rawr_sanitize_agent_name(name: &str) -> String { + let mut out = String::with_capacity(name.len()); + let mut last_dash = false; + for ch in name.chars() { + let ch = ch.to_ascii_lowercase(); + if ch.is_ascii_alphanumeric() { + out.push(ch); + last_dash = false; + } else if !last_dash { + out.push('-'); + last_dash = true; + } + } + out.trim_matches('-').to_string() +} + +fn rawr_expand_scratch_file_template( + template: &str, + agent_name: &str, + thread_id: &ThreadId, +) -> String { + let expanded = rawr_prompts::expand_placeholders( + template, + &[ + ("agentName", agent_name.to_string()), + ("agent_name", agent_name.to_string()), + ("threadId", thread_id.to_string()), + ], + ); + if rawr_is_safe_relative_path(&expanded) { + return expanded; + } + + rawr_prompts::expand_placeholders( + DEFAULT_SCRATCH_FILE_TEMPLATE, + &[("agentName", agent_name.to_string())], + ) +} + +fn rawr_is_safe_relative_path(path: &str) -> bool { + let path = Path::new(path); + !path.is_absolute() + && path + .components() + .all(|component| matches!(component, Component::Normal(_))) +} + +fn strip_yaml_frontmatter(contents: &str) -> &str { + let mut iter = contents.split_inclusive('\n'); + let Some(first_line) = iter.next() else { + return contents; + }; + if first_line.trim_end_matches(['\r', '\n']) != "---" { + return contents; + } + + let mut cursor = first_line.len(); + for piece in iter { + let piece_start = cursor; + let line = piece.trim_end_matches(['\r', '\n']); + if line == "---" { + let body_start = piece_start.saturating_add(piece.len()); + return contents.get(body_start..).unwrap_or(""); + } + cursor = cursor.saturating_add(piece.len()); + } + + contents +} + +fn truncate_chars(text: &str, max_chars: usize) -> String { + let mut chars = text.chars(); + let mut out = chars.by_ref().take(max_chars).collect::(); + if chars.next().is_some() { + out.push_str("..."); + } + out +} + +fn rawr_expand_prompt_template( + prompt: &str, + scratch_file: Option<&str>, + thread_id: Option, +) -> String { + let mut values = Vec::new(); + if let Some(scratch_file) = scratch_file { + values.push(("scratchFile", scratch_file.to_string())); + values.push(("scratch_file", scratch_file.to_string())); + } + if let Some(thread_id) = thread_id { + values.push(("threadId", thread_id.to_string())); + } + rawr_prompts::expand_placeholders(prompt, &values) + .trim() + .to_string() +} + +fn default_rawr_agent_packet_prompt() -> String { + [ + "[rawr] Agent: before we compact this thread, you must self-reflect and write a continuation context packet for yourself.", + "", + "Keep it short and structured. Do not include secrets; redact tokens/keys.", + ] + .join("\n") +} + +fn default_rawr_watcher_packet_prompt() -> String { + [ + "**Continuation context packet (post-compaction injection)**", + "", + "Overarching goal", + "- Continue the work you were doing immediately before compaction.", + "", + "Why compaction happened", + "- Triggered by rawr auto-compaction watcher at {triggerPercentRemaining}% context remaining.", + "- Natural boundary signals: {boundarySignals}", + "", + "Last agent output (memory trigger)", + "- {lastAgentMessage}", + "", + "Directive", + "- Continue with the remaining work now; do not restart from scratch.", + ] + .join("\n") +} + +fn default_rawr_scratch_write_prompt() -> String { + [ + "[rawr] Before auto-compaction, write a verbatim scratchpad of the work you just completed so it survives compaction.", + "", + "Target file: `{scratch_file}`", + "", + "Requirements:", + "- Create the `.scratch/` directory if it doesn't exist.", + "- Append a new section; do not delete prior scratch content.", + "- Prefer verbatim notes/drafts over summaries.", + "- Include links/paths to any important files you edited or created.", + ] + .join("\n") +} + +#[cfg(test)] +mod tests { + use super::*; + use codex_config::types::RawrAutoCompactionPolicyToml; + use codex_config::types::RawrAutoCompactionSemanticSignalsToml; + use codex_config::types::RawrAutoCompactionSettingsToml; + use codex_config::types::RawrAutoCompactionToml; + use codex_protocol::plan_tool::PlanItemArg; + use codex_protocol::plan_tool::StepStatus; + use codex_protocol::plan_tool::UpdatePlanArgs; + use codex_protocol::protocol::ExecCommandEndEvent; + use codex_protocol::protocol::ExecCommandSource; + use codex_protocol::protocol::ExecCommandStatus; + use codex_utils_absolute_path::AbsolutePathBuf; + use pretty_assertions::assert_eq; + use std::time::Duration; + + #[tokio::test] + async fn rawr_turn_complete_boundary_only_matches_turn_complete_path() { + let mut config = crate::config::test_config().await; + config + .features + .enable(Feature::RawrAutoCompaction) + .expect("enable feature"); + config.rawr_auto_compaction = Some(RawrAutoCompactionToml::Config(Box::new( + RawrAutoCompactionSettingsToml { + policy: Some(RawrAutoCompactionPolicyToml { + early: Some(RawrAutoCompactionPolicyTierToml { + requires_any_boundary: Some(vec![RawrAutoCompactionBoundary::TurnComplete]), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }, + ))); + + let signals = RawrAutoCompactionSignals::default(); + + assert_eq!( + rawr_should_compact_with_boundary(&config, 80, &signals, false), + false + ); + assert_eq!( + rawr_should_compact_at_turn_complete(&config, 80, &signals), + true + ); + } + + #[tokio::test] + async fn default_turn_complete_alone_is_not_a_boundary() { + let mut config = crate::config::test_config().await; + config + .features + .enable(Feature::RawrAutoCompaction) + .expect("enable feature"); + + let signals = RawrAutoCompactionSignals::default(); + + assert_eq!( + rawr_should_compact_at_turn_complete(&config, 80, &signals), + false + ); + } + + #[test] + fn watcher_packet_tail_truncation_is_char_safe() { + assert_eq!(truncate_chars("alpha café beta", 10), "alpha café..."); + + let signals = RawrAutoCompactionSignals::default(); + let packet = rawr_build_watcher_post_compact_packet( + default_rawr_watcher_packet_prompt().as_str(), + 42, + &signals, + Some("alpha café beta"), + /*max_tail_chars*/ 10, + ); + assert!(packet.contains("- alpha café...")); + } + + #[test] + fn agent_packet_prompt_expands_scratch_placeholders() { + let prompt = rawr_build_agent_continuation_packet_prompt( + "thread={threadId} scratch={scratchFile}", + "write {scratch_file}", + true, + Some(".scratch/agent-codex.scratch.md"), + Some(ThreadId::new()), + ); + assert!(prompt.contains("write .scratch/agent-codex.scratch.md")); + assert!(prompt.contains("scratch=.scratch/agent-codex.scratch.md")); + } + + #[tokio::test] + async fn scratch_handoff_uses_agent_specific_path_when_enabled() { + let config = crate::config::test_config().await; + let signals = RawrAutoCompactionSignals { + saw_commit: true, + ..Default::default() + }; + + assert!(rawr_should_schedule_scratch_write( + true, /*is_emergency*/ false, &signals + )); + assert!(!rawr_should_schedule_scratch_write( + true, /*is_emergency*/ true, &signals + )); + + let thread_id = ThreadId::new(); + let scratch_file = rawr_scratch_file_rel_path(&config, &SessionSource::Cli, &thread_id); + let packet = rawr_build_agent_continuation_packet_prompt( + "continue using {scratchFile}", + "write notes to {scratch_file}", + true, + Some(scratch_file.as_str()), + Some(thread_id), + ); + let handoff = rawr_build_post_compact_handoff_message(packet, Some(scratch_file.as_str())); + + assert!(scratch_file.starts_with(".scratch/agent-")); + assert!(handoff.starts_with(&format!("Scratchpad: `{scratch_file}`"))); + assert!(handoff.contains(&format!("write notes to {scratch_file}"))); + assert!(handoff.contains(&format!("continue using {scratch_file}"))); + } + + #[tokio::test] + async fn scratch_file_template_is_configurable_and_stays_relative() { + let mut config = crate::config::test_config().await; + let thread_id = ThreadId::new(); + config.rawr_auto_compaction = Some(RawrAutoCompactionToml::Config(Box::new( + RawrAutoCompactionSettingsToml { + scratch_file_template: Some(".rawr/{agent_name}/{threadId}.md".to_string()), + ..Default::default() + }, + ))); + + let configured = rawr_scratch_file_rel_path(&config, &SessionSource::Cli, &thread_id); + assert!(configured.starts_with(".rawr/")); + assert!(configured.ends_with(&format!("/{thread_id}.md"))); + + config.rawr_auto_compaction = Some(RawrAutoCompactionToml::Config(Box::new( + RawrAutoCompactionSettingsToml { + scratch_file_template: Some("../outside.md".to_string()), + ..Default::default() + }, + ))); + let fallback = rawr_scratch_file_rel_path(&config, &SessionSource::Cli, &thread_id); + assert!(fallback.starts_with(".scratch/agent-")); + } + + #[test] + fn plan_update_checkpoint_sets_plan_signals() { + let mut signals = RawrAutoCompactionSignals::default(); + let mut completed_steps_seen = 0; + rawr_note_plan_update( + &mut signals, + &mut completed_steps_seen, + &UpdatePlanArgs { + explanation: None, + plan: vec![PlanItemArg { + step: "done".to_string(), + status: StepStatus::Completed, + }], + }, + ); + + rawr_note_plan_update( + &mut signals, + &mut completed_steps_seen, + &UpdatePlanArgs { + explanation: None, + plan: vec![ + PlanItemArg { + step: "done".to_string(), + status: StepStatus::Completed, + }, + PlanItemArg { + step: "pending".to_string(), + status: StepStatus::Pending, + }, + ], + }, + ); + + assert_eq!(completed_steps_seen, 1); + assert!(signals.saw_plan_checkpoint); + assert!(signals.saw_plan_update); + } + + #[test] + fn completed_exec_command_sets_commit_and_pr_signals() { + let mut commit_signals = RawrAutoCompactionSignals::default(); + rawr_note_exec_command_end( + &mut commit_signals, + &ExecCommandEndEvent { + call_id: "call-1".to_string(), + process_id: None, + turn_id: "turn-1".to_string(), + command: vec![ + "git".to_string(), + "commit".to_string(), + "-m".to_string(), + "x".to_string(), + ], + cwd: AbsolutePathBuf::try_from(std::path::PathBuf::from("/tmp")) + .expect("absolute path"), + parsed_cmd: Vec::new(), + source: ExecCommandSource::Agent, + interaction_input: None, + stdout: String::new(), + stderr: String::new(), + aggregated_output: String::new(), + exit_code: 0, + duration: Duration::from_secs(1), + formatted_output: String::new(), + status: ExecCommandStatus::Completed, + }, + ); + assert!(commit_signals.saw_commit); + assert!(!commit_signals.saw_pr_checkpoint); + + let mut pr_signals = RawrAutoCompactionSignals::default(); + rawr_note_exec_command_end( + &mut pr_signals, + &ExecCommandEndEvent { + call_id: "call-2".to_string(), + process_id: None, + turn_id: "turn-1".to_string(), + command: vec!["gh".to_string(), "pr".to_string(), "create".to_string()], + cwd: AbsolutePathBuf::try_from(std::path::PathBuf::from("/tmp")) + .expect("absolute path"), + parsed_cmd: Vec::new(), + source: ExecCommandSource::Agent, + interaction_input: None, + stdout: String::new(), + stderr: String::new(), + aggregated_output: String::new(), + exit_code: 0, + duration: Duration::from_secs(1), + formatted_output: String::new(), + status: ExecCommandStatus::Completed, + }, + ); + assert!(!pr_signals.saw_commit); + assert!(pr_signals.saw_pr_checkpoint); + + let mut push_signals = RawrAutoCompactionSignals::default(); + rawr_note_exec_command_end( + &mut push_signals, + &ExecCommandEndEvent { + call_id: "call-3".to_string(), + process_id: None, + turn_id: "turn-1".to_string(), + command: vec!["git".to_string(), "push".to_string()], + cwd: AbsolutePathBuf::try_from(std::path::PathBuf::from("/tmp")) + .expect("absolute path"), + parsed_cmd: Vec::new(), + source: ExecCommandSource::Agent, + interaction_input: None, + stdout: String::new(), + stderr: String::new(), + aggregated_output: String::new(), + exit_code: 0, + duration: Duration::from_secs(1), + formatted_output: String::new(), + status: ExecCommandStatus::Completed, + }, + ); + assert!(!push_signals.saw_pr_checkpoint); + } + + #[tokio::test] + async fn completion_message_sets_semantic_signals() { + let config = crate::config::test_config().await; + let mut signals = RawrAutoCompactionSignals::default(); + rawr_note_completion_message( + &mut signals, + &config, + Some( + "Completed the implementation. Next, let's update the docs. Final thoughts: keep the hook in tasks.", + ), + ); + + assert!(signals.saw_agent_done); + assert!(signals.saw_topic_shift); + assert!(signals.saw_concluding_thought); + } + + #[tokio::test] + async fn completion_message_uses_configured_semantic_signal_phrases() { + let mut config = crate::config::test_config().await; + config.rawr_auto_compaction = Some(RawrAutoCompactionToml::Config(Box::new( + RawrAutoCompactionSettingsToml { + semantic_signals: Some(RawrAutoCompactionSemanticSignalsToml { + agent_done_phrases: Some(vec!["wrapped the slice".to_string()]), + agent_done_negative_phrases: Some(vec!["still wrapping".to_string()]), + topic_shift_phrases: Some(vec!["handoff next".to_string()]), + concluding_thought_phrases: Some(vec!["carry forward".to_string()]), + }), + ..Default::default() + }, + ))); + let mut signals = RawrAutoCompactionSignals::default(); + rawr_note_completion_message( + &mut signals, + &config, + Some("Wrapped the slice. Handoff next. Carry forward the branch state."), + ); + + assert!(signals.saw_agent_done); + assert!(signals.saw_topic_shift); + assert!(signals.saw_concluding_thought); + + let mut negative_signals = RawrAutoCompactionSignals::default(); + rawr_note_completion_message( + &mut negative_signals, + &config, + Some("Still wrapping the slice. Handoff next. Carry forward the branch state."), + ); + + assert!(!negative_signals.saw_agent_done); + assert!(negative_signals.saw_topic_shift); + assert!(negative_signals.saw_concluding_thought); + } +} diff --git a/codex-rs/core/src/rawr_auto_compaction_model.rs b/codex-rs/core/src/rawr_auto_compaction_model.rs new file mode 100644 index 000000000000..003b9ed794e7 --- /dev/null +++ b/codex-rs/core/src/rawr_auto_compaction_model.rs @@ -0,0 +1,452 @@ +use std::path::Path; +use std::path::PathBuf; + +use crate::client_common::Prompt; +use crate::rawr_prompts; +use crate::session::session::Session; +use crate::session::turn_context::TurnContext; +use crate::stream_events_utils::last_assistant_message_from_item; +use codex_protocol::ThreadId; +use codex_protocol::error::CodexErr; +use codex_protocol::error::Result as CodexResult; +use codex_protocol::models::BaseInstructions; +use codex_protocol::models::ContentItem; +use codex_protocol::models::ResponseItem; +use codex_rollout_trace::InferenceTraceContext; +use futures::StreamExt; +use serde::Deserialize; +use serde::de::DeserializeOwned; +use serde_json::json; +use tokio::fs; + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +pub(crate) struct RawrAutoCompactionJudgment { + pub should_compact: bool, + pub reason: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +pub(crate) struct RawrAutoCompactionAgentArtifacts { + pub continuation_packet: Option, + pub scratchpad_contents: Option, +} + +pub(crate) async fn request_rawr_auto_compaction_judgment( + sess: &Session, + turn_context: &TurnContext, + decision_prompt_path: &str, + tier: &str, + percent_remaining: i64, + boundaries_present: &[String], + last_agent_message: &str, +) -> CodexResult { + let codex_home = turn_context.config.codex_home.clone(); + if let Err(err) = rawr_prompts::ensure_rawr_prompt_files(&codex_home) { + tracing::warn!("failed to ensure rawr prompt directory: {err}"); + } + + let prompt_path = resolve_prompt_path(&turn_context.cwd, &codex_home, decision_prompt_path); + let prompt_contents = fs::read_to_string(&prompt_path) + .await + .map_err(CodexErr::from)?; + let instructions = strip_yaml_frontmatter(&prompt_contents).trim().to_string(); + + let excerpt = build_recent_transcript_excerpt(sess, 12, 800).await; + let context = build_decision_context( + &rawr_prompts::read_prompt_path_or_default( + &codex_home, + crate::rawr_auto_compaction::rawr_judgment_context_prompt_path( + turn_context.config.as_ref(), + ), + rawr_prompts::RawrPromptKind::JudgmentContext, + ), + tier, + percent_remaining, + boundaries_present, + last_agent_message, + &excerpt, + sess.conversation_id, + &turn_context.sub_id, + sess.get_total_token_usage().await, + turn_context.model_context_window(), + ); + + run_structured_prompt( + sess, + turn_context, + instructions, + context, + judgment_output_schema(), + ) + .await +} + +#[expect( + clippy::too_many_arguments, + reason = "focused internal RAWR request surface" +)] +pub(crate) async fn request_rawr_auto_compaction_agent_artifacts( + sess: &Session, + turn_context: &TurnContext, + packet_prompt: Option<&str>, + scratch_prompt: Option<&str>, + tier: &str, + percent_remaining: i64, + boundaries_present: &[String], + last_agent_message: &str, + scratch_file: Option<&str>, +) -> CodexResult { + let instructions = build_artifact_instructions(packet_prompt, scratch_prompt); + let excerpt = build_recent_transcript_excerpt(sess, 12, 800).await; + let context = build_artifact_context( + tier, + percent_remaining, + boundaries_present, + last_agent_message, + &excerpt, + sess.conversation_id, + &turn_context.sub_id, + sess.get_total_token_usage().await, + turn_context.model_context_window(), + scratch_file, + packet_prompt.is_some(), + scratch_prompt.is_some(), + ); + + run_structured_prompt( + sess, + turn_context, + instructions, + context, + artifact_output_schema(packet_prompt.is_some(), scratch_prompt.is_some()), + ) + .await +} + +async fn run_structured_prompt( + sess: &Session, + turn_context: &TurnContext, + instructions: String, + context: String, + output_schema: serde_json::Value, +) -> CodexResult { + let prompt = Prompt { + input: vec![ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { text: context }], + end_turn: None, + phase: None, + }], + tools: Vec::new(), + parallel_tool_calls: false, + base_instructions: BaseInstructions { text: instructions }, + personality: None, + output_schema: Some(output_schema), + output_schema_strict: true, + }; + + let mut client_session = sess.services.model_client.new_session(); + let turn_metadata_header = turn_context.turn_metadata_state.current_header_value(); + let mut stream = client_session + .stream( + &prompt, + &turn_context.model_info, + &turn_context.session_telemetry, + turn_context.reasoning_effort, + turn_context.reasoning_summary, + turn_context.config.service_tier, + turn_metadata_header.as_deref(), + &InferenceTraceContext::disabled(), + ) + .await?; + + let mut last_message: Option = None; + + loop { + let Some(event) = stream.next().await else { + return Err(CodexErr::Stream( + "stream closed before response.completed".into(), + None, + )); + }; + match event { + Ok(codex_api::ResponseEvent::OutputItemDone(item)) => { + if let Some(text) = last_assistant_message_from_item(&item, false) { + last_message = Some(text); + } + } + Ok(codex_api::ResponseEvent::ServerReasoningIncluded(included)) => { + sess.set_server_reasoning_included(included).await; + } + Ok(codex_api::ResponseEvent::RateLimits(snapshot)) => { + sess.update_rate_limits(turn_context, snapshot).await; + } + Ok(codex_api::ResponseEvent::Completed { token_usage, .. }) => { + sess.update_token_usage_info(turn_context, token_usage.as_ref()) + .await; + break; + } + Ok(_) => continue, + Err(err) => return Err(err), + } + } + + let raw = + last_message.ok_or_else(|| CodexErr::Stream("missing assistant output".into(), None))?; + parse_json_from_text(&raw) +} + +fn parse_json_from_text(text: &str) -> CodexResult { + if let Ok(parsed) = serde_json::from_str::(text) { + return Ok(parsed); + } + + let trimmed = text.trim(); + let trimmed = trimmed + .strip_prefix("```json") + .or_else(|| trimmed.strip_prefix("```")) + .unwrap_or(trimmed); + let trimmed = trimmed.strip_suffix("```").unwrap_or(trimmed).trim(); + if let Ok(parsed) = serde_json::from_str::(trimmed) { + return Ok(parsed); + } + + let Some(start) = trimmed.find('{') else { + return Err(CodexErr::Stream( + "structured output was not JSON".into(), + None, + )); + }; + let Some(end) = trimmed.rfind('}') else { + return Err(CodexErr::Stream( + "structured output was not JSON".into(), + None, + )); + }; + let candidate = &trimmed[start..=end]; + serde_json::from_str::(candidate) + .map_err(|err| CodexErr::Stream(format!("failed to parse structured JSON: {err}"), None)) +} + +fn judgment_output_schema() -> serde_json::Value { + json!({ + "type": "object", + "additionalProperties": false, + "properties": { + "should_compact": { "type": "boolean" }, + "reason": { "type": "string" } + }, + "required": ["should_compact", "reason"] + }) +} + +fn artifact_output_schema(include_packet: bool, include_scratch: bool) -> serde_json::Value { + let mut properties = serde_json::Map::new(); + let mut required = Vec::new(); + + if include_packet { + properties.insert( + "continuation_packet".to_string(), + json!({ "type": "string" }), + ); + required.push("continuation_packet"); + } + if include_scratch { + properties.insert( + "scratchpad_contents".to_string(), + json!({ "type": "string" }), + ); + required.push("scratchpad_contents"); + } + + json!({ + "type": "object", + "additionalProperties": false, + "properties": properties, + "required": required + }) +} + +fn build_artifact_instructions( + packet_prompt: Option<&str>, + scratch_prompt: Option<&str>, +) -> String { + let mut sections = Vec::new(); + if let Some(packet_prompt) = packet_prompt { + sections.push(packet_prompt.trim().to_string()); + } + if let Some(scratch_prompt) = scratch_prompt { + sections.push(scratch_prompt.trim().to_string()); + } + sections.push( + "Return strict JSON matching the requested schema. Do not wrap the JSON in prose." + .to_string(), + ); + sections.join("\n\n---\n\n") +} + +#[expect(clippy::too_many_arguments, reason = "small prompt context builder")] +fn build_decision_context( + template: &str, + tier: &str, + percent_remaining: i64, + boundaries_present: &[String], + last_agent_message: &str, + transcript_excerpt: &str, + thread_id: ThreadId, + turn_id: &str, + total_usage_tokens: i64, + model_context_window: Option, +) -> String { + let boundaries_json = serde_json::to_string(boundaries_present).unwrap_or_else(|_| "[]".into()); + let model_context_window = model_context_window + .map(|value| value.to_string()) + .unwrap_or_else(|| "unknown".to_string()); + let values = [ + ("tier", tier.to_string()), + ("percentRemaining", percent_remaining.to_string()), + ("boundariesJson", boundaries_json), + ("lastAgentMessage", last_agent_message.trim().to_string()), + ("transcriptExcerpt", transcript_excerpt.trim().to_string()), + ("threadId", thread_id.to_string()), + ("turnId", turn_id.to_string()), + ("totalUsageTokens", total_usage_tokens.to_string()), + ("modelContextWindow", model_context_window), + ]; + + rawr_prompts::expand_placeholders(template, &values) +} + +#[expect(clippy::too_many_arguments, reason = "small focused prompt builder")] +fn build_artifact_context( + tier: &str, + percent_remaining: i64, + boundaries_present: &[String], + last_agent_message: &str, + transcript_excerpt: &str, + thread_id: ThreadId, + turn_id: &str, + total_usage_tokens: i64, + model_context_window: Option, + scratch_file: Option<&str>, + include_packet: bool, + include_scratch: bool, +) -> String { + let mut sections = vec![ + format!("Tier: {tier}"), + format!("Percent remaining: {percent_remaining}"), + format!( + "Boundaries present: {}", + serde_json::to_string(boundaries_present).unwrap_or_else(|_| "[]".to_string()) + ), + format!("Thread: {thread_id}"), + format!("Turn: {turn_id}"), + format!("Total usage tokens: {total_usage_tokens}"), + format!( + "Model context window: {}", + model_context_window + .map(|value| value.to_string()) + .unwrap_or_else(|| "unknown".to_string()) + ), + "Last agent message:".to_string(), + last_agent_message.trim().to_string(), + String::new(), + "Recent transcript excerpt:".to_string(), + transcript_excerpt.trim().to_string(), + ]; + + if let Some(scratch_file) = scratch_file { + sections.push(String::new()); + sections.push(format!("Scratch file: {scratch_file}")); + } + + sections.push(String::new()); + sections.push("Return only JSON.".to_string()); + sections.push(format!("Include continuation_packet: {include_packet}")); + sections.push(format!("Include scratchpad_contents: {include_scratch}")); + + sections.join("\n") +} + +fn resolve_prompt_path(cwd: &Path, codex_home: &Path, raw: &str) -> PathBuf { + let path = Path::new(raw); + if path.is_absolute() { + return path.to_path_buf(); + } + let prompt_dir = rawr_prompts::rawr_prompt_dir(codex_home); + let candidate = prompt_dir.join(path); + if candidate.exists() { + return candidate; + } + cwd.join(path) +} + +fn strip_yaml_frontmatter(contents: &str) -> &str { + let mut iter = contents.split_inclusive('\n'); + let Some(first_line) = iter.next() else { + return contents; + }; + if first_line.trim_end_matches(['\r', '\n']) != "---" { + return contents; + } + + let mut cursor = first_line.len(); + for piece in iter { + let piece_start = cursor; + let line = piece.trim_end_matches(['\r', '\n']); + if line == "---" { + let body_start = piece_start.saturating_add(piece.len()); + return contents.get(body_start..).unwrap_or(""); + } + cursor = cursor.saturating_add(piece.len()); + } + + contents +} + +async fn build_recent_transcript_excerpt( + sess: &Session, + max_messages: usize, + max_chars_per_message: usize, +) -> String { + let history = sess.clone_history().await; + let mut out: Vec = Vec::new(); + + for item in history.raw_items().iter().rev() { + if out.len() >= max_messages { + break; + } + let ResponseItem::Message { role, content, .. } = item else { + continue; + }; + + let Some(text) = content + .iter() + .rev() + .find_map(|content_item| match content_item { + ContentItem::InputText { text } | ContentItem::OutputText { text } => { + Some(text.as_str()) + } + _ => None, + }) + else { + continue; + }; + + let text = text.trim(); + if text.is_empty() { + continue; + } + + let mut snippet = text.to_string(); + if snippet.len() > max_chars_per_message { + snippet.truncate(max_chars_per_message); + snippet.push('…'); + } + out.push(format!("{role}: {snippet}")); + } + + out.reverse(); + out.join("\n") +} diff --git a/codex-rs/core/src/rawr_prompts.rs b/codex-rs/core/src/rawr_prompts.rs new file mode 100644 index 000000000000..08362a0891cf --- /dev/null +++ b/codex-rs/core/src/rawr_prompts.rs @@ -0,0 +1,302 @@ +use std::fs; +use std::io; +use std::path::Path; +use std::path::PathBuf; + +use tracing::warn; + +pub const RAWR_PROMPT_DIR_NAME: &str = "auto-compact"; +pub const RAWR_AUTO_COMPACT_PROMPT_FILE: &str = "auto-compact.md"; +pub const RAWR_SCRATCH_WRITE_PROMPT_FILE: &str = "scratch-write.md"; +pub const RAWR_JUDGMENT_PROMPT_FILE: &str = "judgment.md"; +pub const RAWR_JUDGMENT_CONTEXT_PROMPT_FILE: &str = "judgment-context.md"; +pub const RAWR_WATCHER_PACKET_PROMPT_FILE: &str = "watcher-packet.md"; + +const DEFAULT_AUTO_COMPACT_PROMPT: &str = "\ +[rawr] Agent: before we compact this thread, you must self-reflect and write a continuation context packet for yourself. + +This is not a generic compact. This is your tight, intra-turn handoff: you are responsible for capturing the minimum, precise context you will need to resume smoothly after compaction and continue exactly where you left off (no drift, no restart). + +Precedence (important): +- This continuation context packet is the authoritative source of what to do next after compaction. +- The generic compacted context is background only and must not override or supersede this packet. + +Accountability: +- You own what gets carried forward. Be deliberate: reflect on your actual goal, state, decisions, and immediate next action. +- If something is uncertain, name the assumption you are carrying forward rather than hand-waving it. + +Write the packet in my voice, as if I (the user) am speaking directly to you (the in-session agent). But the content must come from your self-reflection on this conversation and your work so far. + +Keep it short and structured. Do not include secrets; redact tokens/keys. + +Include exactly these sections: + +1) Overarching goal +- Briefly restate the overall objective you are trying to accomplish (higher-level than the last message, but still concise). + +2) Current state / progress snapshot +- State the very last thing that just happened (commit, PR checkpoint, plan step completion, etc.). +- Explain how that action relates to the overarching goal and where it leaves you right now. + +3) Invariants and decisions (for this continuation) +- Enumerate the rules/choices that must continue to hold when you resume (specific to this ongoing effort). + +4) Next step / immediate continuation +- Specify the single next thing to do when you resume. +- Tie it explicitly to the overarching goal and the just-completed action. + +5) Verbatim continuation snippet (programmatically inserted) +- Include a literal placeholder for a verbatim memory trigger snippet to be inserted later from your most recent messages: + - {{RAWR_VERBATIM_CONTINUATION_SNIPPET}} + +Final directive: +- End with one clear directive to immediately continue from Next step / immediate continuation after compaction (do not restart or re-plan from scratch). + +Heuristic notes (for auditing) +- commit: a successful git commit occurred in this turn. +- pr_checkpoint: a PR lifecycle checkpoint occurred (publish/review/open/close heuristics). +- plan_checkpoint: the plan was updated and at least one step was marked completed. +- agent_done: the assistant explicitly indicates completion (for example done, completed, finished). +"; + +const DEFAULT_SCRATCH_WRITE_PROMPT: &str = "\ +[rawr] Before auto-compaction, write a verbatim scratchpad of the work you just completed so it survives compaction. + +Target file: `{scratch_file}` + +Requirements: +- Create the `.scratch/` directory if it doesn't exist. +- Create your scratch document if it doesn't exist, in whatever working space you're already keep scratch documents and transient context. +- Append a new section (do not delete prior scratch content). +- Prefer verbatim notes/drafts over summaries; include raw details that are useful later. +- Include links/paths to any important files you edited or created. +- After writing, confirm in your next message that the scratch file was written and include the exact path. + +Overall goal: +Create (or rewrite) the scratch document to have two sections, optimized for continuity + fast reorientation, not as a status report. + +1) Current frame / objective / vision (high level) +- the user’s communicated frame, objective, vision +- the overall process/workflow as a short but complete relevant narration +- the current state of things: what we’re working on and what we’re driving toward + +2) Precision references (ground truth) +- the specific links, file paths, and precise references needed to act + +Style constraints: +- Start high-level, then progressively zoom in until you reach precise links/paths. +- Don’t be overly verbose; be directional and clear. +- Preserve the most important invariants. +- Write it to yourself, with the goal of surviving compaction and making reorientation immediate. +"; + +const DEFAULT_JUDGMENT_PROMPT: &str = "\ +[rawr] Decide whether automatic post-turn compaction should run now. + +Requirements: +- Return strict JSON matching the requested schema. +- Approve compaction only when the supplied context shows a real boundary and compaction would help. +- Deny compaction when the boundary is weak, the turn is still in the middle of a cohesive thread, or compaction would likely lose important short-term working context. +- Keep the reason concise and specific. +"; + +const DEFAULT_JUDGMENT_CONTEXT_PROMPT: &str = "\ +Tier: {tier} +Percent remaining: {percentRemaining} +Boundaries present: {boundariesJson} +Last agent message: +{lastAgentMessage} + +Recent transcript excerpt: +{transcriptExcerpt} + +Thread: {threadId} +Turn: {turnId} +Total usage tokens: {totalUsageTokens} +Model context window: {modelContextWindow} +"; + +const DEFAULT_WATCHER_PACKET_PROMPT: &str = "\ +**Continuation context packet (post-compaction injection)** + +Overarching goal +- Continue the work you were doing immediately before compaction. + +Why compaction happened +- Triggered by rawr auto-compaction watcher at {triggerPercentRemaining}% context remaining. +- Natural boundary signals: {boundarySignals} + +Last agent output (memory trigger) +- {lastAgentMessage} + +Directive +- Continue with the remaining work now; do not restart from scratch. +"; + +#[derive(Debug, Clone, Copy)] +pub enum RawrPromptKind { + AutoCompact, + ScratchWrite, + JudgmentContext, + WatcherPacket, +} + +#[derive(Debug, Clone)] +pub struct RawrPromptPaths { + pub auto_compact: PathBuf, + pub scratch_write: PathBuf, + pub judgment_context: PathBuf, + pub watcher_packet: PathBuf, +} + +pub fn rawr_prompt_dir(codex_home: &Path) -> PathBuf { + codex_home.join(RAWR_PROMPT_DIR_NAME) +} + +pub fn ensure_rawr_prompt_files(codex_home: &Path) -> io::Result { + let dir = rawr_prompt_dir(codex_home); + fs::create_dir_all(&dir)?; + + let auto_compact = dir.join(RAWR_AUTO_COMPACT_PROMPT_FILE); + let scratch_write = dir.join(RAWR_SCRATCH_WRITE_PROMPT_FILE); + let judgment = dir.join(RAWR_JUDGMENT_PROMPT_FILE); + let judgment_context = dir.join(RAWR_JUDGMENT_CONTEXT_PROMPT_FILE); + let watcher_packet = dir.join(RAWR_WATCHER_PACKET_PROMPT_FILE); + + write_default_if_missing(&auto_compact, DEFAULT_AUTO_COMPACT_PROMPT)?; + write_default_if_missing(&scratch_write, DEFAULT_SCRATCH_WRITE_PROMPT)?; + write_default_if_missing(&judgment, DEFAULT_JUDGMENT_PROMPT)?; + write_default_if_missing(&judgment_context, DEFAULT_JUDGMENT_CONTEXT_PROMPT)?; + write_default_if_missing(&watcher_packet, DEFAULT_WATCHER_PACKET_PROMPT)?; + + Ok(RawrPromptPaths { + auto_compact, + scratch_write, + judgment_context, + watcher_packet, + }) +} + +pub fn read_prompt_path_or_default( + codex_home: &Path, + path_override: Option<&str>, + kind: RawrPromptKind, +) -> String { + let paths = match ensure_rawr_prompt_files(codex_home) { + Ok(paths) => paths, + Err(err) => { + warn!("failed to ensure rawr prompt directory: {err}"); + return default_prompt(kind).to_string(); + } + }; + + let path = path_override + .map(|raw| resolve_prompt_path(codex_home, raw)) + .unwrap_or_else(|| match kind { + RawrPromptKind::AutoCompact => paths.auto_compact, + RawrPromptKind::ScratchWrite => paths.scratch_write, + RawrPromptKind::JudgmentContext => paths.judgment_context, + RawrPromptKind::WatcherPacket => paths.watcher_packet, + }); + + match fs::read_to_string(&path) { + Ok(contents) => contents, + Err(err) => { + warn!("failed to read rawr prompt {}: {err}", path.display()); + default_prompt(kind).to_string() + } + } +} + +pub fn expand_placeholders(template: &str, values: &[(&str, String)]) -> String { + let mut out = template.to_string(); + for (key, value) in values { + out = out.replace(&format!("{{{key}}}"), value); + } + out +} + +fn default_prompt(kind: RawrPromptKind) -> &'static str { + match kind { + RawrPromptKind::AutoCompact => DEFAULT_AUTO_COMPACT_PROMPT, + RawrPromptKind::ScratchWrite => DEFAULT_SCRATCH_WRITE_PROMPT, + RawrPromptKind::JudgmentContext => DEFAULT_JUDGMENT_CONTEXT_PROMPT, + RawrPromptKind::WatcherPacket => DEFAULT_WATCHER_PACKET_PROMPT, + } +} + +fn resolve_prompt_path(codex_home: &Path, raw: &str) -> PathBuf { + let path = Path::new(raw); + if path.is_absolute() { + return path.to_path_buf(); + } + rawr_prompt_dir(codex_home).join(path) +} + +fn write_default_if_missing(path: &Path, contents: &str) -> io::Result<()> { + if path.exists() { + return Ok(()); + } + fs::write(path, contents) +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use tempfile::tempdir; + + #[test] + fn ensure_rawr_prompt_files_creates_defaults() { + let dir = tempdir().expect("create temp dir"); + let paths = ensure_rawr_prompt_files(dir.path()).expect("create prompt files"); + assert!(paths.auto_compact.exists()); + assert!(paths.scratch_write.exists()); + assert!(paths.watcher_packet.exists()); + } + + #[test] + fn ensure_rawr_prompt_files_preserves_user_edits() { + let dir = tempdir().expect("create temp dir"); + let prompt_dir = rawr_prompt_dir(dir.path()); + fs::create_dir_all(&prompt_dir).expect("create prompt dir"); + let auto_compact = prompt_dir.join(RAWR_AUTO_COMPACT_PROMPT_FILE); + fs::write(&auto_compact, "custom packet prompt").expect("write custom prompt"); + + ensure_rawr_prompt_files(dir.path()).expect("ensure prompt files"); + + assert_eq!( + fs::read_to_string(auto_compact).expect("read prompt"), + "custom packet prompt" + ); + } + + #[test] + fn expand_placeholders_replaces_values() { + let template = "tier={tier} percent={percentRemaining}"; + let output = expand_placeholders( + template, + &[ + ("tier", "ready".to_string()), + ("percentRemaining", "42".to_string()), + ], + ); + assert_eq!(output, "tier=ready percent=42"); + } + + #[test] + fn prompt_path_override_reads_relative_to_prompt_dir() { + let dir = tempdir().expect("create temp dir"); + let prompt_dir = rawr_prompt_dir(dir.path()); + fs::create_dir_all(&prompt_dir).expect("create prompt dir"); + fs::write(prompt_dir.join("custom.md"), "custom").expect("write prompt"); + + let output = read_prompt_path_or_default( + dir.path(), + Some("custom.md"), + RawrPromptKind::WatcherPacket, + ); + + assert_eq!(output, "custom"); + } +} diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index ca865300a384..e3b8ec8d743e 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -1462,6 +1462,8 @@ impl Session { self.services .rollout_thread_trace .record_tool_call_event(turn_context.sub_id.clone(), &legacy_source); + self.record_rawr_auto_compaction_signal(turn_context, &legacy_source) + .await; let event = Event { id: turn_context.sub_id.clone(), msg, @@ -1484,6 +1486,43 @@ impl Session { } } + async fn record_rawr_auto_compaction_signal(&self, turn_context: &TurnContext, msg: &EventMsg) { + if !turn_context.features.enabled(Feature::RawrAutoCompaction) { + return; + } + + if !matches!( + msg, + EventMsg::PlanUpdate(_) | EventMsg::ExecCommandEnd(_) | EventMsg::TurnComplete(_) + ) { + return; + } + + let Some(turn_state) = self.turn_state_for_sub_id(&turn_context.sub_id).await else { + return; + }; + let mut turn_state = turn_state.lock().await; + match msg { + EventMsg::PlanUpdate(update) => { + turn_state.note_rawr_plan_update(update); + } + EventMsg::ExecCommandEnd(event) => { + crate::rawr_auto_compaction::rawr_note_exec_command_end( + turn_state.rawr_auto_compaction_signals_mut(), + event, + ); + } + EventMsg::TurnComplete(event) => { + crate::rawr_auto_compaction::rawr_note_completion_message( + turn_state.rawr_auto_compaction_signals_mut(), + turn_context.config.as_ref(), + event.last_agent_message.as_deref(), + ); + } + _ => {} + } + } + /// Forwards terminal turn events from spawned MultiAgentV2 children to their direct parent. async fn maybe_notify_parent_of_terminal_turn( &self, @@ -3091,6 +3130,17 @@ impl Session { turn_state.lock().await.has_memory_citation = true; } + pub(crate) async fn rawr_auto_compaction_signals_for_turn( + &self, + sub_id: &str, + ) -> crate::rawr_auto_compaction::RawrAutoCompactionSignals { + let turn_state = self.turn_state_for_sub_id(sub_id).await; + let Some(turn_state) = turn_state else { + return crate::rawr_auto_compaction::RawrAutoCompactionSignals::default(); + }; + turn_state.lock().await.rawr_auto_compaction_signals() + } + async fn turn_state_for_sub_id( &self, sub_id: &str, @@ -3186,6 +3236,18 @@ impl Session { idle_pending_input.extend(items); } + /// Queue response items ahead of any pending next-turn input. + pub(crate) async fn prepend_response_items_for_next_turn(&self, items: Vec) { + if items.is_empty() { + return; + } + + let mut idle_pending_input = self.idle_pending_input.lock().await; + let mut next_items = items; + next_items.extend(std::mem::take(&mut *idle_pending_input)); + *idle_pending_input = next_items; + } + pub(crate) async fn take_queued_response_items_for_next_turn(&self) -> Vec { std::mem::take(&mut *self.idle_pending_input.lock().await) } diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index 508eadfba038..c079f02a5dda 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -6060,8 +6060,12 @@ async fn task_finish_emits_turn_item_lifecycle_for_leftover_pending_user_input() .await .expect("inject pending input into active turn"); - sess.on_task_finished(Arc::clone(&tc), /*last_agent_message*/ None) - .await; + sess.on_task_finished( + Arc::clone(&tc), + /*last_agent_message*/ None, + TaskKind::Regular, + ) + .await; let history = sess.clone_history().await; let expected = ResponseItem::Message { @@ -6769,7 +6773,7 @@ async fn sample_rollout( std::iter::once(&user1), reconstruction_turn.truncation_policy, ); - rollout_items.push(RolloutItem::ResponseItem(user1.clone())); + rollout_items.push(RolloutItem::ResponseItem(user1)); let assistant1 = ResponseItem::Message { id: None, @@ -6784,7 +6788,7 @@ async fn sample_rollout( std::iter::once(&assistant1), reconstruction_turn.truncation_policy, ); - rollout_items.push(RolloutItem::ResponseItem(assistant1.clone())); + rollout_items.push(RolloutItem::ResponseItem(assistant1)); let summary1 = "summary one"; let snapshot1 = live_history @@ -6811,7 +6815,7 @@ async fn sample_rollout( std::iter::once(&user2), reconstruction_turn.truncation_policy, ); - rollout_items.push(RolloutItem::ResponseItem(user2.clone())); + rollout_items.push(RolloutItem::ResponseItem(user2)); let assistant2 = ResponseItem::Message { id: None, @@ -6826,7 +6830,7 @@ async fn sample_rollout( std::iter::once(&assistant2), reconstruction_turn.truncation_policy, ); - rollout_items.push(RolloutItem::ResponseItem(assistant2.clone())); + rollout_items.push(RolloutItem::ResponseItem(assistant2)); let summary2 = "summary two"; let snapshot2 = live_history diff --git a/codex-rs/core/src/state/turn.rs b/codex-rs/core/src/state/turn.rs index 48b7a26ccb53..822d2d82642f 100644 --- a/codex-rs/core/src/state/turn.rs +++ b/codex-rs/core/src/state/turn.rs @@ -19,9 +19,11 @@ use codex_utils_absolute_path::AbsolutePathBuf; use rmcp::model::RequestId; use tokio::sync::oneshot; +use crate::rawr_auto_compaction::RawrAutoCompactionSignals; use crate::session::turn_context::TurnContext; use crate::tasks::AnySessionTask; use codex_protocol::models::AdditionalPermissionProfile; +use codex_protocol::plan_tool::UpdatePlanArgs; use codex_protocol::protocol::ReviewDecision; use codex_protocol::protocol::TokenUsage; @@ -85,9 +87,9 @@ impl ActiveTurn { self.tasks.insert(sub_id, task); } - pub(crate) fn remove_task(&mut self, sub_id: &str) -> bool { - self.tasks.swap_remove(sub_id); - self.tasks.is_empty() + pub(crate) fn detach_task(&mut self, sub_id: &str) -> (Option, bool) { + let task = self.tasks.swap_remove(sub_id); + (task, self.tasks.is_empty()) } pub(crate) fn drain_tasks(&mut self) -> Vec { @@ -107,6 +109,8 @@ pub(crate) struct TurnState { mailbox_delivery_phase: MailboxDeliveryPhase, granted_permissions: Option, strict_auto_review_enabled: bool, + rawr_auto_compaction_signals: RawrAutoCompactionSignals, + rawr_completed_plan_steps: usize, pub(crate) tool_calls: u64, pub(crate) has_memory_citation: bool, pub(crate) token_usage_at_turn_start: TokenUsage, @@ -263,6 +267,22 @@ impl TurnState { pub(crate) fn strict_auto_review_enabled(&self) -> bool { self.strict_auto_review_enabled } + + pub(crate) fn rawr_auto_compaction_signals(&self) -> RawrAutoCompactionSignals { + self.rawr_auto_compaction_signals.clone() + } + + pub(crate) fn rawr_auto_compaction_signals_mut(&mut self) -> &mut RawrAutoCompactionSignals { + &mut self.rawr_auto_compaction_signals + } + + pub(crate) fn note_rawr_plan_update(&mut self, update: &UpdatePlanArgs) { + crate::rawr_auto_compaction::rawr_note_plan_update( + &mut self.rawr_auto_compaction_signals, + &mut self.rawr_completed_plan_steps, + update, + ); + } } impl ActiveTurn { diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index b0ec96cfedbb..2db0c11e60a9 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -10,11 +10,14 @@ use std::time::Duration; use std::time::Instant; use futures::future::BoxFuture; +use tokio::fs::OpenOptions; +use tokio::io::AsyncWriteExt; use tokio::select; use tokio::sync::Notify; use tokio_util::sync::CancellationToken; use tokio_util::task::AbortOnDropHandle; use tracing::Instrument; +use tracing::debug; use tracing::info_span; use tracing::trace; use tracing::warn; @@ -50,6 +53,10 @@ use codex_protocol::protocol::TurnCompleteEvent; use codex_protocol::protocol::WarningEvent; use codex_protocol::user_input::UserInput; +use codex_analytics::CompactionPhase; +use codex_analytics::CompactionReason; +use codex_config::types::RawrAutoCompactionMode; +use codex_config::types::RawrAutoCompactionPacketAuthor; use codex_features::Feature; use codex_protocol::models::ContentItem; pub(crate) use compact::CompactTask; @@ -378,8 +385,16 @@ impl Session { } if !task_cancellation_token.is_cancelled() { // Emit completion uniformly from spawn site so all tasks share the same lifecycle. - sess.on_task_finished(Arc::clone(&ctx_for_finish), last_agent_message) + let finished_task = sess + .on_task_finished( + Arc::clone(&ctx_for_finish), + last_agent_message, + task_kind, + ) .await; + done_clone.notify_waiters(); + drop(finished_task); + return; } done_clone.notify_waiters(); } @@ -495,27 +510,23 @@ impl Session { self: &Arc, turn_context: Arc, last_agent_message: Option, - ) { + task_kind: TaskKind, + ) -> Option { turn_context .turn_metadata_state .cancel_git_enrichment_task(); + let last_agent_message_for_rawr = last_agent_message.clone(); let mut pending_input = Vec::::new(); - let mut should_clear_active_turn = false; let mut token_usage_at_turn_start = None; let mut turn_had_memory_citation = false; let mut turn_tool_calls = 0_u64; let turn_state = { - let mut active = self.active_turn.lock().await; - if let Some(at) = active.as_mut() - && at.remove_task(&turn_context.sub_id) + let active = self.active_turn.lock().await; + if let Some(at) = active.as_ref() + && at.tasks.contains_key(&turn_context.sub_id) { - should_clear_active_turn = true; - let turn_state = Arc::clone(&at.turn_state); - if should_clear_active_turn { - *active = None; - } - Some(turn_state) + Some(Arc::clone(&at.turn_state)) } else { None } @@ -654,6 +665,42 @@ impl Session { .lock() .await .clear_turn(&turn_context.sub_id); + let rawr_signals = if task_kind == TaskKind::Regular { + Some( + self.rawr_auto_compaction_signals_for_turn(&turn_context.sub_id) + .await, + ) + } else { + None + }; + + let (finished_task, should_clear_active_turn) = { + let mut active = self.active_turn.lock().await; + if let Some(at) = active.as_mut() + && at.tasks.contains_key(&turn_context.sub_id) + { + let (finished_task, should_clear) = at.detach_task(&turn_context.sub_id); + if should_clear { + *active = None; + } + (finished_task, should_clear) + } else { + (None, false) + } + }; + + if task_kind == TaskKind::Regular { + debug!( + turn_id = %turn_context.sub_id, + "checking rawr post-turn auto-compaction" + ); + self.maybe_run_rawr_post_turn_auto_compaction( + Arc::clone(&turn_context), + last_agent_message_for_rawr, + rawr_signals.unwrap_or_default(), + ) + .await; + } if should_clear_active_turn { let session = Arc::clone(self); @@ -663,6 +710,345 @@ impl Session { }); }); } + + finished_task + } + + async fn maybe_run_rawr_post_turn_auto_compaction( + self: &Arc, + turn_context: Arc, + last_agent_message: Option, + signals: crate::rawr_auto_compaction::RawrAutoCompactionSignals, + ) { + if !turn_context.features.enabled(Feature::RawrAutoCompaction) { + debug!( + turn_id = %turn_context.sub_id, + "rawr auto-compaction skipped: feature disabled" + ); + return; + } + + let Some(model_context_window) = turn_context.model_context_window() else { + debug!( + turn_id = %turn_context.sub_id, + "rawr auto-compaction skipped: model context window missing" + ); + return; + }; + if model_context_window <= 0 { + debug!( + turn_id = %turn_context.sub_id, + model_context_window, + "rawr auto-compaction skipped: invalid model context window" + ); + return; + } + + let total_usage_tokens = self.get_total_token_usage().await; + let remaining = model_context_window.saturating_sub(total_usage_tokens); + let percent_remaining = + (remaining.saturating_mul(100) / model_context_window).clamp(0, 100); + if !crate::rawr_auto_compaction::rawr_should_compact_at_turn_complete( + turn_context.config.as_ref(), + percent_remaining, + &signals, + ) { + debug!( + turn_id = %turn_context.sub_id, + total_usage_tokens, + model_context_window, + percent_remaining, + "rawr auto-compaction skipped: policy not eligible" + ); + return; + } + debug!( + turn_id = %turn_context.sub_id, + total_usage_tokens, + model_context_window, + percent_remaining, + "rawr auto-compaction eligible" + ); + let Some(tier) = crate::rawr_auto_compaction::rawr_compaction_tier( + turn_context.config.as_ref(), + percent_remaining, + ) else { + debug!( + turn_id = %turn_context.sub_id, + percent_remaining, + "rawr auto-compaction skipped: eligible state had no tier" + ); + return; + }; + let is_emergency = matches!( + tier, + crate::rawr_auto_compaction::RawrAutoCompactionTier::Emergency + ); + let boundaries_present = + crate::rawr_auto_compaction::rawr_boundaries_present(&signals, true); + + match crate::rawr_auto_compaction::rawr_auto_compaction_mode(turn_context.config.as_ref()) { + RawrAutoCompactionMode::Tag | RawrAutoCompactionMode::Suggest => { + self.send_event( + turn_context.as_ref(), + EventMsg::Warning(WarningEvent { + message: format!( + "rawr auto-compaction is eligible at {percent_remaining}% context remaining." + ), + }), + ) + .await; + return; + } + RawrAutoCompactionMode::Auto => {} + } + + if !is_emergency + && let Some(decision_prompt_path) = + crate::rawr_auto_compaction::rawr_policy_decision_prompt_path( + turn_context.config.as_ref(), + tier, + ) + { + match crate::rawr_auto_compaction_model::request_rawr_auto_compaction_judgment( + self, + turn_context.as_ref(), + decision_prompt_path.as_str(), + rawr_tier_name(tier), + percent_remaining, + &boundaries_present, + last_agent_message.as_deref().unwrap_or(""), + ) + .await + { + Ok(judgment) if !judgment.should_compact => { + self.send_event( + turn_context.as_ref(), + EventMsg::Warning(WarningEvent { + message: format!( + "rawr auto-compaction skipped by judgment: {}", + judgment.reason + ), + }), + ) + .await; + return; + } + Ok(_) => {} + Err(err) => { + self.send_event( + turn_context.as_ref(), + EventMsg::Warning(WarningEvent { + message: format!( + "rawr auto-compaction judgment failed; continuing with static policy: {err}" + ), + }), + ) + .await; + } + } + } + + let do_scratch = crate::rawr_auto_compaction::rawr_should_schedule_scratch_write( + crate::rawr_auto_compaction::rawr_scratch_write_enabled(turn_context.config.as_ref()), + is_emergency, + &signals, + ); + let configured_scratch_file = do_scratch.then(|| { + crate::rawr_auto_compaction::rawr_scratch_file_rel_path( + turn_context.config.as_ref(), + &turn_context.session_source, + &self.conversation_id, + ) + }); + let packet_author = crate::rawr_auto_compaction::rawr_auto_compaction_packet_author( + turn_context.config.as_ref(), + ); + let agent_artifacts = if packet_author == RawrAutoCompactionPacketAuthor::Agent + || do_scratch + { + let packet_prompt = if packet_author == RawrAutoCompactionPacketAuthor::Agent { + let raw_packet_prompt = crate::rawr_auto_compaction::rawr_load_agent_packet_prompt( + turn_context.config.as_ref(), + ); + let raw_scratch_prompt = do_scratch.then(|| { + crate::rawr_auto_compaction::rawr_load_scratch_write_prompt( + turn_context.config.as_ref(), + ) + }); + Some( + crate::rawr_auto_compaction::rawr_build_agent_continuation_packet_prompt( + raw_packet_prompt.as_str(), + raw_scratch_prompt.as_deref().unwrap_or(""), + do_scratch, + configured_scratch_file.as_deref(), + Some(self.conversation_id), + ), + ) + } else { + None + }; + let scratch_prompt = if packet_author == RawrAutoCompactionPacketAuthor::Watcher { + configured_scratch_file.as_deref().map(|scratch_path| { + crate::rawr_auto_compaction::rawr_build_scratch_write_prompt( + crate::rawr_auto_compaction::rawr_load_scratch_write_prompt( + turn_context.config.as_ref(), + ) + .as_str(), + scratch_path, + Some(self.conversation_id), + ) + }) + } else { + None + }; + match crate::rawr_auto_compaction_model::request_rawr_auto_compaction_agent_artifacts( + self, + turn_context.as_ref(), + packet_prompt.as_deref(), + scratch_prompt.as_deref(), + rawr_tier_name(tier), + percent_remaining, + &boundaries_present, + last_agent_message.as_deref().unwrap_or(""), + configured_scratch_file.as_deref(), + ) + .await + { + Ok(artifacts) => Some(artifacts), + Err(err) => { + self.send_event( + turn_context.as_ref(), + EventMsg::Warning(WarningEvent { + message: format!( + "rawr pre-compact artifact generation failed; falling back: {err}" + ), + }), + ) + .await; + None + } + } + } else { + None + }; + + let scratch_file = if let (Some(rel_path), Some(contents)) = ( + configured_scratch_file.as_deref(), + agent_artifacts + .as_ref() + .and_then(|artifacts| artifacts.scratchpad_contents.as_deref()) + .map(str::trim) + .filter(|contents| !contents.is_empty()), + ) { + match write_rawr_scratchpad(turn_context.as_ref(), rel_path, contents).await { + Ok(()) => Some(rel_path.to_string()), + Err(err) => { + self.send_event( + turn_context.as_ref(), + EventMsg::Warning(WarningEvent { + message: format!("rawr scratch write failed; continuing without scratch reference: {err}"), + }), + ) + .await; + None + } + } + } else { + None + }; + + let packet = match packet_author { + RawrAutoCompactionPacketAuthor::Watcher => { + crate::rawr_auto_compaction::rawr_build_watcher_post_compact_packet( + crate::rawr_auto_compaction::rawr_load_watcher_packet_prompt( + turn_context.config.as_ref(), + ) + .as_str(), + percent_remaining, + &signals, + last_agent_message.as_deref(), + crate::rawr_auto_compaction::rawr_packet_max_tail_chars( + turn_context.config.as_ref(), + ), + ) + } + RawrAutoCompactionPacketAuthor::Agent => agent_artifacts + .as_ref() + .and_then(|artifacts| artifacts.continuation_packet.as_deref()) + .map(str::trim) + .filter(|packet| !packet.is_empty()) + .map(ToOwned::to_owned) + .unwrap_or_else(|| { + crate::rawr_auto_compaction::rawr_build_watcher_post_compact_packet( + crate::rawr_auto_compaction::rawr_load_watcher_packet_prompt( + turn_context.config.as_ref(), + ) + .as_str(), + percent_remaining, + &signals, + last_agent_message.as_deref(), + crate::rawr_auto_compaction::rawr_packet_max_tail_chars( + turn_context.config.as_ref(), + ), + ) + }), + }; + + let compaction_context = if let Some(model) = + crate::rawr_auto_compaction::rawr_compaction_model(turn_context.config.as_ref()) + { + Arc::new( + turn_context + .with_model(model, &self.services.models_manager) + .await, + ) + } else { + Arc::clone(&turn_context) + }; + + let compaction_result = + if crate::compact::should_use_remote_compact_task(compaction_context.provider.info()) { + crate::compact_remote::run_inline_remote_auto_compact_task( + Arc::clone(self), + Arc::clone(&compaction_context), + crate::compact::InitialContextInjection::BeforeLastUserMessage, + CompactionReason::ContextLimit, + CompactionPhase::StandaloneTurn, + ) + .await + } else { + crate::compact::run_inline_auto_compact_task( + Arc::clone(self), + Arc::clone(&compaction_context), + crate::compact::InitialContextInjection::BeforeLastUserMessage, + CompactionReason::ContextLimit, + CompactionPhase::StandaloneTurn, + ) + .await + }; + + if let Err(err) = compaction_result { + self.send_event( + turn_context.as_ref(), + EventMsg::Warning(WarningEvent { + message: format!("rawr auto-compaction failed: {err}"), + }), + ) + .await; + return; + } + let handoff = crate::rawr_auto_compaction::rawr_build_post_compact_handoff_message( + packet, + scratch_file.as_deref(), + ); + self.prepend_response_items_for_next_turn(vec![ResponseInputItem::from(vec![ + UserInput::Text { + text: handoff, + text_elements: Vec::new(), + }, + ])]) + .await; } async fn take_active_turn(&self) -> Option { @@ -753,6 +1139,41 @@ impl Session { } } +fn rawr_tier_name(tier: crate::rawr_auto_compaction::RawrAutoCompactionTier) -> &'static str { + match tier { + crate::rawr_auto_compaction::RawrAutoCompactionTier::Early => "early", + crate::rawr_auto_compaction::RawrAutoCompactionTier::Ready => "ready", + crate::rawr_auto_compaction::RawrAutoCompactionTier::Asap => "asap", + crate::rawr_auto_compaction::RawrAutoCompactionTier::Emergency => "emergency", + } +} + +async fn write_rawr_scratchpad( + turn_context: &TurnContext, + rel_path: &str, + contents: &str, +) -> std::io::Result<()> { + let path = turn_context.cwd.join(rel_path); + if let Some(parent) = path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + let existing_len = tokio::fs::metadata(&path) + .await + .map(|metadata| metadata.len()) + .unwrap_or(0); + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(&path) + .await?; + if existing_len > 0 { + file.write_all(b"\n\n").await?; + } + file.write_all(contents.trim().as_bytes()).await?; + file.write_all(b"\n").await?; + Ok(()) +} + #[cfg(test)] #[path = "mod_tests.rs"] mod tests; diff --git a/codex-rs/core/src/tools/js_repl/mod.rs b/codex-rs/core/src/tools/js_repl/mod.rs index 2f494adc35a2..a100207f4360 100644 --- a/codex-rs/core/src/tools/js_repl/mod.rs +++ b/codex-rs/core/src/tools/js_repl/mod.rs @@ -867,6 +867,13 @@ impl JsReplManager { let (stdin, pending_execs, exec_contexts, child, recent_stderr) = { let mut kernel = self.kernel.lock().await; + let stale_kernel = match kernel.as_ref() { + Some(state) => Self::kernel_child_has_exited(&state.child).await, + None => false, + }; + if stale_kernel { + kernel.take(); + } if kernel.is_none() { let dependency_env = session.dependency_env().await; let mut state = self @@ -1236,6 +1243,18 @@ impl JsReplManager { } } + async fn kernel_child_has_exited(child: &Arc>) -> bool { + let mut guard = child.lock().await; + match guard.try_wait() { + Ok(Some(_)) => true, + Ok(None) => false, + Err(err) => { + warn!(error = %err, "failed to inspect js_repl kernel before reuse"); + false + } + } + } + #[expect( clippy::await_holding_invalid_type, reason = "js_repl child shutdown must serialize process inspection and termination" diff --git a/codex-rs/core/src/tools/js_repl/mod_tests.rs b/codex-rs/core/src/tools/js_repl/mod_tests.rs index 38bd71e1a33e..a21dbdf4d444 100644 --- a/codex-rs/core/src/tools/js_repl/mod_tests.rs +++ b/codex-rs/core/src/tools/js_repl/mod_tests.rs @@ -730,13 +730,13 @@ async fn interrupt_active_exec_stops_aborted_kernel_before_later_exec() -> anyho r#" const {{ promises: fs }} = await import("fs"); -const paths = [{first_file_js}, {second_file_js}]; -for (let i = 0; i < paths.length; i++) {{ - await fs.writeFile(paths[i], `${{i + 1}}`); - if (i + 1 < paths.length) {{ - await new Promise((resolve) => setTimeout(resolve, 1000)); - }} -}} + const paths = [{first_file_js}, {second_file_js}]; + for (let i = 0; i < paths.length; i++) {{ + await fs.writeFile(paths[i], `${{i + 1}}`); + if (i + 1 < paths.length) {{ + await new Promise((resolve) => setTimeout(resolve, 10000)); + }} + }} "# ); @@ -844,22 +844,6 @@ async fn js_repl_forced_kernel_exit_recovers_on_next_exec() -> anyhow::Result<() Arc::clone(&state.child) }; JsReplManager::kill_kernel_child(&child, "test_crash").await; - tokio::time::timeout(Duration::from_secs(1), async { - loop { - let cleared = { - let guard = manager.kernel.lock().await; - guard - .as_ref() - .is_none_or(|state| !Arc::ptr_eq(&state.child, &child)) - }; - if cleared { - return; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .expect("host should clear dead kernel state promptly"); let result = manager .execute( diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index 527cef36f844..4d2dc6040937 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -215,6 +215,8 @@ pub enum Feature { ResponsesWebsocketsV2, /// Enable workspace dependency support. WorkspaceDependencies, + /// Enable RAWR automatic post-turn compaction policy. + RawrAutoCompaction, } impl Feature { @@ -1026,6 +1028,12 @@ pub const FEATURES: &[FeatureSpec] = &[ stage: Stage::Stable, default_enabled: true, }, + FeatureSpec { + id: Feature::RawrAutoCompaction, + key: "rawr_auto_compaction", + stage: Stage::UnderDevelopment, + default_enabled: false, + }, ]; pub fn unstable_features_warning_event( diff --git a/docs/agents.md b/docs/agents.md index 04414fb6d1df..c54c3e76c781 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -5,18 +5,21 @@ This repo is on macOS a lot, and macOS filesystems are commonly case-insensitive ## Non-Negotiable Rule: How Work Starts Here Any fix or work in this repo MUST: + - Use Graphite (`gt`) for branching/commits/submission. - Start new stacks from `codex/integration-upstream-main` using `gt create`. - Be based on the integration upstream main trunk (do not start new unrelated work from an existing PR branch). - Exception: only stack on an existing PR branch when the change is tightly coupled and intended to merge together. Preflight (do this before you write code): + ```bash git status --porcelain gt branch info --no-interactive ``` Canonical branch start: + ```bash gt checkout codex/integration-upstream-main gt create codex/ @@ -27,12 +30,14 @@ gt create codex/ This file intentionally does not duplicate runbooks. It provides the correct command entrypoints and routes you to canonical docs/scripts. ### 1) Run the rawr fork locally (side-by-side) + - Install/switch locally: - `bash rawr/install-local.sh` - Canonical instructions: - `rawr/INSTALL.md` ### 2) Publish locally (PATH switching) + - Recommended: - `bash rawr/publish-local.sh` - Canonical instructions: @@ -41,6 +46,7 @@ This file intentionally does not duplicate runbooks. It provides the correct com - `rawr/publish-local.sh` ### 3) Golden-path local release + - Release local: - `bash rawr/release-local.sh` - Canonical instructions: @@ -49,6 +55,7 @@ This file intentionally does not duplicate runbooks. It provides the correct com - `rawr/release-local.sh` ### 4) Upstream update / rebase checkpoint + - Preferred checkpoint automation: - `bash rawr/rebase-daily.sh` - Fallback/manual checkpoint flow: @@ -56,11 +63,13 @@ This file intentionally does not duplicate runbooks. It provides the correct com - `rawr/sync-upstream.sh codex/integration-upstream-main` Canonical runbooks (read these; don’t freestyle rebases): + - `rawr/UPDATING.md` - `docs/projects/rawr/rebase-runbook.md` - `docs/projects/rawr/rebase-gotchas.md` ### 5) Rust development (codex-rs) + - Install/dev setup: - `docs/install.md` - Repo tasks: @@ -71,6 +80,7 @@ Canonical runbooks (read these; don’t freestyle rebases): ## PR / Submission Workflow (Graphite) Use Graphite for commits and PR updates: + - Commit/update: - `gt modify -c -a -m ": "` - Submit: @@ -81,9 +91,10 @@ Avoid raw `git push --force` unless a specific runbook explicitly tells you to. ## Recovery: Started Work On The Wrong Branch High-level guardrail: + - If `gt branch info` shows you’re on a branch with an unrelated PR attached, stop and restart from `codex/integration-upstream-main` (or explicitly stack only if tightly coupled). If you already have edits: + - Keep the repo clean at the end of the fix (no dangling local changes). - Follow the relevant runbook/scripts for the workflow you were attempting rather than inventing a new one. - diff --git a/docs/projects/rawr/auto-compaction-system.md b/docs/projects/rawr/auto-compaction-system.md index 5f5ca5c03573..0c978cac2547 100644 --- a/docs/projects/rawr/auto-compaction-system.md +++ b/docs/projects/rawr/auto-compaction-system.md @@ -1,13 +1,15 @@ # RAWR Auto-Compaction — Canonical System + State Machine Spec -Status: living spec (fork-specific, current behavior + intent) +Status: living spec, but mixed history. The current durable branch uses core session/task lifecycle ownership for turn-complete RAWR plus internal non-transcript judgment/artifact calls. Older sections below that describe TUI watcher-owned orchestration or `Op::RawrAutoCompactionJudgment` are historical reference, not current implementation truth. -This document is the **canonical packet** describing how RAWR auto-compaction works end-to-end in this fork: +This document is the **canonical packet** describing how RAWR auto-compaction works end-to-end in this fork. +Current implementation truth is the core session/task lifecycle path; older watcher-oriented sections are retained +only as design history until this spec is fully collapsed: -- **Actors / systems** (TUI watcher, core sampling loop, compaction task, model provider) +- **Actors / systems** (core session/task lifecycle, compaction task, model provider) - **Config + prompts** (what exists, where it is loaded from, when it is used) -- **State machines** (TUI watcher and core mid-turn), and the **decision policy** (tiers/boundaries/judgment) -- **All major paths** (watcher auto, watcher suggest/tag, watcher preflight, core mid-turn, built-in auto-compact) +- **State machines** (turn-complete lifecycle and mid-turn), and the **decision policy** (tiers/boundaries/judgment) +- **All major paths** (turn-complete auto-compaction, mid-turn compaction, built-in auto-compact) - **Constraints + rationale** (what we optimized for, what we explicitly avoided) If the UX still feels bad (too eager, confusing, brittle), this doc is intended to make it obvious whether the root cause is: @@ -26,18 +28,19 @@ If the UX still feels bad (too eager, confusing, brittle), this doc is intended Fork-specific RAWR features (guarded behind `Feature::RawrAutoCompaction`): -- TUI watcher compaction orchestration (turn-complete + preflight) - - `codex-rs/tui/src/chatwidget.rs` +- Core session/task compaction orchestration after `EventMsg::TurnComplete` + - `codex-rs/core/src/session/mod.rs` + - `codex-rs/core/src/tasks/mod.rs` - Core mid-turn RAWR compaction orchestration (inside one user turn) - - `codex-rs/core/src/codex.rs` - `codex-rs/core/src/rawr_auto_compaction.rs` - Scratch-write pre-compact request (`.scratch/agent-.scratch.md`) - - core + TUI + - core internal non-transcript model calls - Config-driven tier policy matrix: `rawr_auto_compaction.policy.` - - core + TUI + - core - Optional **internal / non-transcript** judgment decision step - - protocol `Op::RawrAutoCompactionJudgment` + `EventMsg::RawrAutoCompactionJudgmentResult` - - core implementation in `codex-rs/core/src/rawr_auto_compaction_judgment.rs` + - `codex-rs/core/src/rawr_auto_compaction_model.rs` +- Prompt/config/path helpers + - `codex-rs/core/src/rawr_prompts.rs` ### What we do not alter (Codex-native / upstream behaviors) @@ -54,24 +57,23 @@ Fork-specific RAWR features (guarded behind `Feature::RawrAutoCompaction`): - **User**: human driving the CLI. - **In-session agent**: the assistant responding to the user (model sampling loop). -- **Watcher (TUI)**: deterministic orchestrator in the TUI that decides *when* to compact at turn completion. +- **Core session/task lifecycle**: deterministic orchestrator that evaluates turn-complete signals and decides *when* to compact. - **Core**: the engine that runs sampling loops, executes tasks, and mutates session state/history. - **Model provider / backend**: OpenAI (or other providers) handling model requests. - **Filesystem**: scratchpad persistence under `.scratch/`. ### Components (where to look in code) -- TUI watcher and injection: - - `codex-rs/tui/src/chatwidget.rs` +- Turn-complete lifecycle hook and injection: + - `codex-rs/core/src/session/mod.rs` + - `codex-rs/core/src/tasks/mod.rs` - Core mid-turn RAWR compaction: - - `codex-rs/core/src/codex.rs` (sampling loop) - `codex-rs/core/src/rawr_auto_compaction.rs` (tier policy + prompt building) - Compaction task (history rewrite): - `codex-rs/core/src/compact.rs` - `codex-rs/core/src/compaction_audit.rs` - Judgment (internal model call, non-transcript): - - `codex-rs/core/src/rawr_auto_compaction_judgment.rs` - - protocol definitions in `codex-rs/protocol/src/protocol.rs` + - `codex-rs/core/src/rawr_auto_compaction_model.rs` --- @@ -116,11 +118,21 @@ If this is `false`, RAWR watcher and mid-turn logic do not run (and built-in aut mode = "auto" # tag | suggest | auto packet_author = "agent" # watcher | agent scratch_write_enabled = true +packet_max_tail_chars = 1200 # Optional overrides used for watcher-triggered compaction tasks # compaction_model = "gpt-5.2" -# compaction_reasoning_effort = "high" -# compaction_verbosity = "high" +auto_compact_prompt_path = "auto-compact.md" +scratch_write_prompt_path = "scratch-write.md" +watcher_packet_prompt_path = "watcher-packet.md" +judgment_context_prompt_path = "judgment-context.md" +scratch_file_template = ".scratch/agent-{agentName}.scratch.md" + +[rawr_auto_compaction.semantic_signals] +agent_done_phrases = ["done", "completed", "finished", "shipped", "pushed"] +agent_done_negative_phrases = ["not done", "not completed", "not finished"] +topic_shift_phrases = ["moving on", "switching to", "next:", "next up"] +concluding_thought_phrases = ["in summary", "final thoughts", "next steps"] [rawr_auto_compaction.policy.early] percent_remaining_lt = 85 @@ -143,24 +155,20 @@ percent_remaining_lt = 15 # Emergency tier is a hard bypass; boundaries/judgment are ignored ``` -### Optional: structured state store + repo observation (fork-only, best-effort) - -When `Feature::RawrAutoCompaction` is enabled, core also writes a **side-channel** structured state store -under `codex_home/rawr/auto_compaction/threads//`: - -- `events.jsonl`: structured boundary events (turn start, plan update/checkpoint, commit, PR checkpoint, semantic boundaries, compaction completed), optionally including repo snapshots. -- `decisions.jsonl`: shadow arbiter decisions derived from policy + state (record-only; not enforced yet). To avoid noise, decisions are only persisted when the system is under token-pressure (tier eligible) or after compaction completes. -- `state.json`: latest snapshot (turn signals, last repo snapshot, last compaction, last decision). +### Runtime signal collection (current durable path) -Note: token-pressure (mid-turn) shadow decisions are further deduped in-memory and only emitted when the tier escalates (e.g. `early` → `ready` → `asap` → `emergency`). +The durable branch no longer treats a side-channel JSONL store as the runtime control plane. +Instead, turn-complete RAWR derives its boundary snapshot from current upstream-native session +events: -Repo observation is lightweight and best-effort: +- `PlanUpdate` for plan update / checkpoint semantics +- successful `ExecCommandEnd` for commit / PR checkpoint heuristics +- finalized assistant output for semantic boundary heuristics (`agent_done`, `topic_shift`, + `concluding_thought`) +- `TurnComplete` as the fallback natural boundary -```toml -[rawr_auto_compaction.repo_observation] -graphite_enabled = true -graphite_max_chars = 4096 -``` +This keeps RAWR’s fork delta concentrated in policy and prompt behavior rather than maintaining a +parallel event universe. #### Policy precedence rules (critical) @@ -174,17 +182,21 @@ Both TUI watcher and core mid-turn use the same precedence: This lets you control “fires too often” primarily by adjusting config rather than code. -### Important: `mode` only affects the TUI watcher +### Important: `mode` affects turn-complete watcher orchestration -`rawr_auto_compaction.mode` (`tag`/`suggest`/`auto`) is a **TUI watcher** behavior knob: +`rawr_auto_compaction.mode` (`tag`/`suggest`/`auto`) is a watcher-style behavior knob for +turn-complete orchestration: - `tag`: print “would compact” informational message - `suggest`: print “recommend compact” informational message - `auto`: actually run the watcher orchestration (packet/scratch/compact/handoff) -Core mid-turn RAWR compaction **does not consult `mode`**; it is gated by the feature flag and the -mid-turn decision policy (tiers/boundaries/judgment). This is intentional: mid-turn exists to prevent -context exhaustion while the agent still needs follow-up inside the sampling loop. +Core emergency/mid-turn RAWR compaction should not consult `mode`; it is gated by the feature flag +and the mid-turn decision policy (tiers/boundaries/judgment). This is intentional: mid-turn exists +to prevent context exhaustion while the agent still needs follow-up inside the sampling loop. + +The current session-lifecycle app-server parity hook runs after `TurnComplete`, so it follows the +turn-complete watcher semantics above. --- @@ -193,25 +205,43 @@ context exhaustion while the agent still needs follow-up inside the sampling loo ### Curated prompt files (codex_home/auto-compact) All RAWR auto-compaction prompts live under **codex_home/auto-compact/** at runtime. If the files -do not exist, Codex seeds them from the in-repo defaults under `rawr/prompts/` so you can edit -them without code changes. +do not exist, Codex seeds them from embedded defaults in +`codex-rs/core/src/rawr_prompts.rs` so you can edit them without code changes. | Prompt file | Used by | When used | How used | |---|---|---|---| -| `codex_home/auto-compact/auto-compact.md` | Core + TUI | Pre-compact request: continuation packet | YAML frontmatter drives thresholds + packet rules; body becomes the packet prompt (supports `{scratchFile}` / `{scratch_file}` and `{threadId}`). | -| `codex_home/auto-compact/scratch-write.md` | Core + TUI | Pre-compact request: scratch write | Template with `{scratchFile}` / `{scratch_file}` and `{threadId}`. Injected as a synthetic user turn (TUI watcher) or prepended before packet prompt (core mid-turn, and TUI when `packet_author="agent"`). | -| `codex_home/auto-compact/judgment.md` | Core (executed), TUI (requests) | Optional decision step before compaction | Base instructions for the schema-constrained internal judgment call. TUI triggers via `Op::RawrAutoCompactionJudgment`, core emits `EventMsg::RawrAutoCompactionJudgmentResult`. | +| `codex_home/auto-compact/auto-compact.md` | Core | Pre-compact request: continuation packet | Body becomes the agent-authored continuation packet instructions (supports `{scratchFile}` / `{scratch_file}` and `{threadId}`). | +| `codex_home/auto-compact/scratch-write.md` | Core | Pre-compact request: scratch write | Template with `{scratchFile}` / `{scratch_file}` and `{threadId}` used for internal scratch generation before compaction. | +| `codex_home/auto-compact/judgment.md` | Core | Optional decision step before compaction | Base instructions for the schema-constrained internal judgment call. | | `codex_home/auto-compact/judgment-context.md` | Core | Decision context template | Human-editable template expanded with runtime placeholders (see below). | +| `codex_home/auto-compact/watcher-packet.md` | Core | Watcher-authored fallback packet | Template used for code-authored handoff packets. Supports `{triggerPercentRemaining}`, `{trigger_percent_remaining}`, `{boundarySignals}`, `{boundary_signals}`, `{lastAgentMessage}`, and `{last_agent_message}`. | | `rawr/prompts/rawr-auto-compact-heads-up.md` | Currently unused (spec only) | Proposed UX: heads-up before packet/compact | Exists as an artifact for a future “heads-up banner” UX. Not referenced by current code paths. | +Each prompt path can be overridden in `config.toml` with: + +- `rawr_auto_compaction.auto_compact_prompt_path` +- `rawr_auto_compaction.scratch_write_prompt_path` +- `rawr_auto_compaction.watcher_packet_prompt_path` +- `rawr_auto_compaction.judgment_context_prompt_path` + +Relative prompt paths resolve under `codex_home/auto-compact/`; absolute paths are read as given. + +Other externally tunable behavior controls: + +- `rawr_auto_compaction.semantic_signals.*` overrides the phrase lists used for `agent_done`, + `topic_shift`, and `concluding_thought` signal derivation. +- `rawr_auto_compaction.scratch_file_template` controls the relative scratchpad path. It supports + `{agentName}`, `{agent_name}`, and `{threadId}`. Unsafe absolute or parent-traversing paths fall + back to `.scratch/agent-{agentName}.scratch.md`. + ### Inline fallbacks (only if prompt files cannot be read) If the prompt directory is missing or unreadable, defaults embedded in the binary are used (and a warning is logged): -- Defaults are embedded from `rawr/prompts/*` via `codex_core::rawr_prompts`. -- Watcher-authored packet (when `packet_author="watcher"`) is still built in code: - - `ChatWidget::rawr_build_post_compact_packet(...)` in `codex-rs/tui/src/chatwidget.rs` +- Defaults are embedded in `codex-rs/core/src/rawr_prompts.rs`. +- Watcher-authored packet (when `packet_author="watcher"`) is still built in code from the live + signal snapshot after compaction completes. ### Judgment context placeholders (judgment-context.md) diff --git a/rawr/INSTALL.md b/rawr/INSTALL.md index 48b5cd186030..b31f0660bd76 100644 --- a/rawr/INSTALL.md +++ b/rawr/INSTALL.md @@ -81,8 +81,8 @@ rawr_auto_compaction = true When enabled: - This fork owns compaction timing (Codex’s built-in auto-compaction is bypassed). -- The watcher can compact **mid-turn** (between sampling requests) at natural boundaries, and can also compact at turn completion. -- Core writes a best-effort, side-channel structured state store under `~/.codex-rawr/rawr/auto_compaction/threads//` for later inspectability (no transcript pollution). +- The fork can compact at natural boundaries and at turn completion using current core session/task ownership. +- Turn-complete RAWR runs after `TurnComplete`, evaluates live turn signals, optionally runs a non-transcript judgment gate, and then uses the normal local/remote compaction runners. Default behavior: suggest mode (prints a recommendation once context window drops below 75% remaining). @@ -93,28 +93,32 @@ mode = "auto" # tag | suggest | auto packet_author = "agent" # watcher | agent scratch_write_enabled = true packet_max_tail_chars = 1200 -# Defaults to "GPT-5.2 (high)" for watcher-triggered compactions: -# gpt-5.2 + ReasoningEffort::High. # compaction_model = "gpt-5.2" -# compaction_reasoning_effort = "high" -# compaction_verbosity = "high" - -[rawr_auto_compaction.repo_observation] -graphite_enabled = true -graphite_max_chars = 4096 +# auto_compact_prompt_path = "auto-compact.md" +# scratch_write_prompt_path = "scratch-write.md" +# watcher_packet_prompt_path = "watcher-packet.md" +# judgment_context_prompt_path = "judgment-context.md" +# scratch_file_template = ".scratch/agent-{agentName}.scratch.md" + +[rawr_auto_compaction.semantic_signals] +# Override these lists when local workflow language differs from the defaults. +# agent_done_phrases = ["done", "completed", "finished", "shipped", "pushed"] +# agent_done_negative_phrases = ["not done", "not completed", "not finished"] +# topic_shift_phrases = ["moving on", "switching to", "next:", "next up"] +# concluding_thought_phrases = ["in summary", "final thoughts", "next steps"] # Preferred: config-driven per-tier policy matrix (overrides thresholds + boundaries). [rawr_auto_compaction.policy.early] percent_remaining_lt = 85 requires_any_boundary = ["plan_checkpoint", "plan_update", "pr_checkpoint", "topic_shift"] plan_boundaries_require_semantic_break = true -# decision_prompt_path = "~/.codex-rawr/prompts/rawr-auto-compaction-judgment.md" +# decision_prompt_path = "judgment.md" [rawr_auto_compaction.policy.ready] percent_remaining_lt = 75 requires_any_boundary = ["commit", "plan_checkpoint", "plan_update", "pr_checkpoint", "topic_shift"] plan_boundaries_require_semantic_break = true -# decision_prompt_path = "~/.codex-rawr/prompts/rawr-auto-compaction-judgment.md" +# decision_prompt_path = "judgment.md" [rawr_auto_compaction.policy.asap] percent_remaining_lt = 65 @@ -126,12 +130,20 @@ percent_remaining_lt = 15 ``` ## Packet prompt + defaults (auditable/editable) -The prompt lives in-repo at `rawr/prompts/rawr-auto-compact.md` and is embedded into the binary at build time. +At runtime, editable prompt files live under `CODEX_HOME/auto-compact/`: + +- `auto-compact.md`: continuation packet prompt used for internal pre-compact artifact generation when `packet_author = "agent"`. +- `scratch-write.md`: scratch-write prompt used for internal scratch generation when `scratch_write_enabled = true`. +- `judgment.md`: optional judgment gate prompt referenced by `decision_prompt_path`. +- `judgment-context.md`: template expanded into the judgment call context. +- `watcher-packet.md`: watcher-authored fallback packet template used when `packet_author = "watcher"` or agent packet generation fails. + +If these files are missing, Codex creates them with built-in defaults. Config-driven thresholds and boundaries still live in `config.toml`; config overrides win. -- YAML frontmatter: default thresholds/boundaries (config overrides win). -- Markdown body: continuation packet prompt when `packet_author = "agent"`. - Compaction decision: code-driven tier policy + boundary gating; plan-based boundaries additionally require a semantic break (agent-done/topic-shift/concluding) in Early/Ready tiers so we don’t compact mid-thought just because the plan tool ran. -- Scratch write prompt: `rawr/prompts/rawr-scratch-write.md` (enabled by `scratch_write_enabled = true`). +- `packet_author = "watcher"` keeps the continuation packet watcher-authored; `packet_author = "agent"` uses an internal non-transcript model call to generate the continuation packet before compaction. +- Prompt path overrides are resolved relative to `CODEX_HOME/auto-compact/` unless absolute. +- `scratch_file_template` is a safe relative path template with `{agentName}`, `{agent_name}`, and `{threadId}` placeholders. ## Using with Happy Coder Happy Coder’s CLI supports `happy codex` (Codex mode). If your `PATH` resolves `codex` to this fork (e.g. via the symlink above), `happy codex` will launch the fork. @@ -185,6 +197,14 @@ The script verifies both versions and hashes after patching. If `/Applications/C sudo CODEX_RAWR_BIN="$HOME/.local/bin/codex-rawr-bin" bash rawr/patch-desktop-app.sh --apply ``` +For a post-update check that only patches when the Desktop helper no longer matches the rawr binary: + +```bash +bash rawr/patch-desktop-app.sh --ensure +``` + +This is the preferred command for a manual post-update hook or LaunchAgent. It compares hashes first, so repeated runs do not create extra backups when the app bundle is already patched. + ### Desktop rollback Use the latest backup created during patching: diff --git a/rawr/patch-desktop-app.sh b/rawr/patch-desktop-app.sh index 3be396a1b8eb..48e5944daa5c 100755 --- a/rawr/patch-desktop-app.sh +++ b/rawr/patch-desktop-app.sh @@ -18,6 +18,7 @@ Patch the local Desktop app bundle to run the current rawr CLI binary. Modes: --dry-run, --report Report current state without changing files (default). --apply Backup bundled binary, install ~/.local/bin/codex-rawr-bin, then verify. + --ensure Apply only when bundled binary does not match rawr binary. --rollback [BACKUP_PATH] Restore from a backup. Defaults to the latest rawr backup. Options: @@ -162,6 +163,23 @@ apply_patch_to_app() { echo "Warning: Codex app updates can overwrite this local patch. Re-run --apply after updates." } +ensure_patch() { + require_file "$BUNDLED_BIN" "bundled Codex binary" + require_file "$RAWR_BIN" "rawr Codex binary" + + local bundled_hash rawr_hash + bundled_hash="$(hash_file "$BUNDLED_BIN")" + rawr_hash="$(hash_file "$RAWR_BIN")" + + if [[ "$bundled_hash" == "$rawr_hash" ]]; then + log "Desktop app already uses rawr build: $(version_of "$BUNDLED_BIN")" + return 0 + fi + + log "Desktop app bundled binary differs from rawr build; patching" + apply_patch_to_app +} + rollback_patch() { local backup_path="$rollback_backup" if [[ -z "$backup_path" ]]; then @@ -202,6 +220,10 @@ while [[ $# -gt 0 ]]; do mode="apply" shift ;; + --ensure) + mode="ensure" + shift + ;; --rollback) mode="rollback" rollback_backup="${2:-}" @@ -231,6 +253,9 @@ case "$mode" in apply) apply_patch_to_app ;; + ensure) + ensure_patch + ;; rollback) rollback_patch ;; diff --git a/scripts/stage_npm_packages.py b/scripts/stage_npm_packages.py index 5bbee755e928..a41ea23748d8 100755 --- a/scripts/stage_npm_packages.py +++ b/scripts/stage_npm_packages.py @@ -47,6 +47,14 @@ def parse_args() -> argparse.Namespace: "--workflow-url", help="Optional workflow URL to reuse for native artifacts.", ) + parser.add_argument( + "--github-repo", + default=os.environ.get("CODEX_RELEASE_GITHUB_REPO", GITHUB_REPO), + help=( + "GitHub repo used to resolve rust-release workflow runs " + f"(default: {GITHUB_REPO}; env: CODEX_RELEASE_GITHUB_REPO)." + ), + ) parser.add_argument( "--output-dir", type=Path, @@ -78,12 +86,14 @@ def expand_packages(packages: list[str]) -> list[str]: return expanded -def resolve_release_workflow(version: str) -> dict: +def resolve_release_workflow(version: str, github_repo: str) -> dict: stdout = subprocess.check_output( [ "gh", "run", "list", + "--repo", + github_repo, "--branch", f"rust-v{version}", "--json", @@ -102,11 +112,13 @@ def resolve_release_workflow(version: str) -> dict: return workflow -def resolve_workflow_url(version: str, override: str | None) -> tuple[str, str | None]: +def resolve_workflow_url( + version: str, override: str | None, github_repo: str +) -> tuple[str, str | None]: if override: return override, None - workflow = resolve_release_workflow(version) + workflow = resolve_release_workflow(version, github_repo) return workflow["url"], workflow.get("headSha") @@ -157,7 +169,7 @@ def main() -> int: try: if native_components: workflow_url, resolved_head_sha = resolve_workflow_url( - args.release_version, args.workflow_url + args.release_version, args.workflow_url, args.github_repo ) vendor_temp_root = Path(tempfile.mkdtemp(prefix="npm-native-", dir=runner_temp)) install_native_components(workflow_url, native_components, vendor_temp_root)