diff --git a/plugins/modelopt/skills/evaluation/recipes/examples/example_eval_next.yaml b/plugins/modelopt/skills/evaluation/recipes/examples/example_eval_next.yaml index 9c58bb50e38..de7bf94e833 100644 --- a/plugins/modelopt/skills/evaluation/recipes/examples/example_eval_next.yaml +++ b/plugins/modelopt/skills/evaluation/recipes/examples/example_eval_next.yaml @@ -6,7 +6,7 @@ # "$SKILL_DIR/scripts/nel-next.sh" --setup-only # set -a && source .env && set +a # HF_TOKEN, AWS_*, NEL_NEXT_EVAL_IMAGE, HARBOR_*_ECR_REPOSITORY (from modelopttools:eval-config) # "$SKILL_DIR/scripts/nel-next.sh" eval run "$SKILL_DIR/recipes/examples/example_eval_next.yaml" --dry-run -# ... --submit -O benchmarks.0.max_problems=2 -O benchmarks.0.repeats=1 -O benchmarks.0.max_concurrent=2 # canary +# ... --submit -O benchmarks.0.max_problems=2 -O benchmarks.0.repeats=1 -O benchmarks.0.max_concurrent=2 -O benchmarks.0.sandbox.concurrency=2 # canary # ... --submit # full # Internal harbor infra (eval_image + ECR) comes from .env via ${VAR}; all blocks # validate against the pinned nel-next build (extra="forbid"). `???` = fill. @@ -68,7 +68,7 @@ services: benchmarks: - playbook: terminal_bench_2_1 # SWE-bench: swebench_verified (see recipe) repeats: 8 # AA count — keep for a scored run (canary: -O benchmarks.0.repeats=1) - max_concurrent: 50 # canonical TB2.1 (nano-class; larger models 15); keep == sandbox.concurrency + max_concurrent: 8 # conservative per shard; keep == sandbox.concurrency solver: service: model timeout_strategy: max # canonical TB2.1; "task" = leaderboard-comparable @@ -78,7 +78,7 @@ benchmarks: max_task_lifetime_sec: 21600 # ModelOpt 6h: agent budget plus setup/verification region: ${HARBOR_ECS_REGION:-us-east-1} # SWE-bench: us-east-2 + ${HARBOR_SWEBENCH_ECR_REPOSITORY} ecr_repository: ${HARBOR_ECR_REPOSITORY} # from modelopttools:eval-config - concurrency: 50 + concurrency: 8 log_stream_prefix: terminalbench-21-??? cluster: diff --git a/plugins/modelopt/skills/evaluation/recipes/tasks/aa_next/terminal_bench_2_1.md b/plugins/modelopt/skills/evaluation/recipes/tasks/aa_next/terminal_bench_2_1.md index 5d0289f54c9..be538b10fe0 100644 --- a/plugins/modelopt/skills/evaluation/recipes/tasks/aa_next/terminal_bench_2_1.md +++ b/plugins/modelopt/skills/evaluation/recipes/tasks/aa_next/terminal_bench_2_1.md @@ -21,7 +21,7 @@ pinned via a vendored registry override shipped in the `nemo-evaluator` package. | `sandbox.region` | `${HARBOR_ECS_REGION:-us-east-1}` | | `sandbox.ecr_repository` | `${HARBOR_ECR_REPOSITORY}` — set by `modelopttools:eval-config`; repo name tracks the region (`harbor-`) | | `cluster.container_env.AWS_DEFAULT_REGION` | match `sandbox.region` | -| `max_concurrent` / `sandbox.concurrency` | `50` (canonical, nano-class); **`15` for larger models** (upstream non-nano leaves). Keep both equal and fixed across baseline/candidate (see Sharding) | +| `max_concurrent` / `sandbox.concurrency` | `8` per shard (conservative starting point); keep both equal and fixed across comparisons | | timeout_strategy | `max` + `run_timeout: 14400` (ModelOpt default); use `task` for leaderboard-comparable | | `cluster.eval_image` | **`0.5.0.1-harbor`** (`${NEL_NEXT_EVAL_IMAGE}`, multi-arch) *(shared — see `references/nel-next.md`)* | | `proxy.request_timeout` | `3600` — set explicitly (TB2.1 has no benchmark key for it); must be **≥** `agent_kwargs.llm_kwargs.timeout` *(shared — see `references/nel-next.md`)* | @@ -32,7 +32,7 @@ pinned via a vendored registry override shipped in the `nemo-evaluator` package. | `output.export_config.mlflow.tags` | `task_name: terminal-bench-2.1` | | scope | 89 tasks × `repeats: 8` | -Except for the ModelOpt timeout/lifetime overrides below, these values follow +Except for concurrency and the ModelOpt timeout/lifetime overrides below, these values follow the canonical TB2.1 config — re-check it before a scored run: `configs/benchmarks/terminal-bench-2.1/bench.yaml` (+ `manifest.yaml`) in nvidia-eval-factory-benchmarking (`dl/JoC/competitive_evaluation/…`), with the image pin in @@ -44,7 +44,7 @@ nvidia-eval-factory-benchmarking (`dl/JoC/competitive_evaluation/…`), with the benchmarks: - playbook: terminal_bench_2_1 repeats: 8 - max_concurrent: 50 # nano-class; LARGE models use 15. Keep == sandbox.concurrency + max_concurrent: 8 # conservative per shard; keep == sandbox.concurrency solver: service: timeout_strategy: max # canonical bench.yaml; use "task" for leaderboard-comparable @@ -56,7 +56,7 @@ benchmarks: max_task_lifetime_sec: 21600 # 6h: allow agent execution plus setup/verification region: ${HARBOR_ECS_REGION:-us-east-1} # repo name below tracks it ecr_repository: ${HARBOR_ECR_REPOSITORY} # from eval-config (internal harbor account/region) - concurrency: 50 + concurrency: 8 log_stream_prefix: terminalbench-21-- ``` @@ -81,6 +81,13 @@ intended trial set. Sharding or concurrency changes can still affect scores when serving speed or queueing changes timeout rates. `shards: 4` suits 89 × r8 = 712 trials. Check `N × concurrency` against the Fargate quota and `N × gpus_per_node` against your allocation. +Use `8` as a conservative starting point, not a benchmark-mandated value. Model +endpoints are shared by trials within each shard. Reduce concurrency for capacity +only with latency/queueing evidence; match concurrency and effective settings +across comparisons. Keep benchmark-prescribed limits, temperature, top_p, and +output settings fixed; never raise timeouts merely to improve scores. This default +change has not been benchmarked and does not establish an improvement. + ## Score Extraction Before reporting `pass@1`, complete the evaluation skill's diff --git a/plugins/modelopt/skills/evaluation/references/nel-next.md b/plugins/modelopt/skills/evaluation/references/nel-next.md index 14f4420aa06..e3485e66d40 100644 --- a/plugins/modelopt/skills/evaluation/references/nel-next.md +++ b/plugins/modelopt/skills/evaluation/references/nel-next.md @@ -225,7 +225,7 @@ with its own `run_id`, copying the shared `services:` block. ```bash set -a && source .env && set +a; NEL="$SKILL_DIR/scripts/nel-next.sh" "$NEL" eval run .yaml --dry-run # validate/render (no SSH) -"$NEL" eval run .yaml --submit -O benchmarks.0.max_problems=2 -O benchmarks.0.repeats=1 -O benchmarks.0.max_concurrent=2 # canary +"$NEL" eval run .yaml --submit -O benchmarks.0.max_problems=2 -O benchmarks.0.repeats=1 -O benchmarks.0.max_concurrent=2 -O benchmarks.0.sandbox.concurrency=2 # canary "$NEL" eval run .yaml --submit # full "$NEL" eval {status|logs -f|report -f markdown|merge} -r # lifecycle "$NEL" mlflow-push -r -c .yaml # post-run: push merged bundle(s) to MLflow @@ -243,7 +243,7 @@ kwargs after `--` (e.g. `-- -o copy_artifacts=true`). ## Non-fatal noise - AWS `RegisterTaskDefinition: ThrottlingException` at high concurrency — SDK - retries (≤8); lower `max_concurrent` if frequent. + retries (≤8); lower matched `max_concurrent`/`sandbox.concurrency` only with capacity evidence. - `solver failed — grading 0.0` agent crashes are rare but real (e.g. terminus-2 tmux `send-keys`) — distinguish from genuine misses + task timeouts; the final report breaks them out.