diff --git a/Magpie/scripts/benchmark/atom_mi300x.sh b/Magpie/scripts/benchmark/atom_mi300x.sh index eb3eff5..9e22259 100644 --- a/Magpie/scripts/benchmark/atom_mi300x.sh +++ b/Magpie/scripts/benchmark/atom_mi300x.sh @@ -20,10 +20,11 @@ # instead of localhost:$PORT, and forces PHASE=client (no local server # launch, no server-side cleanup, no SERVER_PID monitoring). -source "$(dirname "$0")/benchmark_lib.sh" -source "$(dirname "$0")/server_cleanup.sh" +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" || exit $? +source "$SCRIPT_DIR/benchmark_lib.sh" +source "$SCRIPT_DIR/server_cleanup.sh" # shellcheck source=magpie_bench_remote_compat.sh -source "$(dirname "$0")/magpie_bench_remote_compat.sh" +source "$SCRIPT_DIR/magpie_bench_remote_compat.sh" PHASE="${MAGPIE_RUN_PHASE:-all}" case "$PHASE" in diff --git a/Magpie/scripts/benchmark/atom_mi355x.sh b/Magpie/scripts/benchmark/atom_mi355x.sh index bd8e2b1..c88143c 100644 --- a/Magpie/scripts/benchmark/atom_mi355x.sh +++ b/Magpie/scripts/benchmark/atom_mi355x.sh @@ -16,10 +16,11 @@ # with vLLM, so the bench client uses --backend vllm unchanged. See # atom_mi300x.sh for the full contract. -source "$(dirname "$0")/benchmark_lib.sh" -source "$(dirname "$0")/server_cleanup.sh" +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" || exit $? +source "$SCRIPT_DIR/benchmark_lib.sh" +source "$SCRIPT_DIR/server_cleanup.sh" # shellcheck source=magpie_bench_remote_compat.sh -source "$(dirname "$0")/magpie_bench_remote_compat.sh" +source "$SCRIPT_DIR/magpie_bench_remote_compat.sh" PHASE="${MAGPIE_RUN_PHASE:-all}" case "$PHASE" in diff --git a/Magpie/scripts/benchmark/sglang_mi300x.sh b/Magpie/scripts/benchmark/sglang_mi300x.sh index bde2280..74cdefa 100644 --- a/Magpie/scripts/benchmark/sglang_mi300x.sh +++ b/Magpie/scripts/benchmark/sglang_mi300x.sh @@ -15,10 +15,11 @@ # cluster, externally managed). Leave the env unset to keep the # default behaviour of launching a local server. -source "$(dirname "$0")/benchmark_lib.sh" -source "$(dirname "$0")/server_cleanup.sh" +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" || exit $? +source "$SCRIPT_DIR/benchmark_lib.sh" +source "$SCRIPT_DIR/server_cleanup.sh" # shellcheck source=magpie_bench_remote_compat.sh -source "$(dirname "$0")/magpie_bench_remote_compat.sh" +source "$SCRIPT_DIR/magpie_bench_remote_compat.sh" PHASE="${MAGPIE_RUN_PHASE:-all}" case "$PHASE" in diff --git a/Magpie/scripts/benchmark/sglang_mi355x.sh b/Magpie/scripts/benchmark/sglang_mi355x.sh index ea5aeaf..1c68444 100644 --- a/Magpie/scripts/benchmark/sglang_mi355x.sh +++ b/Magpie/scripts/benchmark/sglang_mi355x.sh @@ -16,10 +16,11 @@ # instead of localhost:$PORT, and forces PHASE=client (no local server # launch). See sglang_mi300x.sh for the full contract. -source "$(dirname "$0")/benchmark_lib.sh" -source "$(dirname "$0")/server_cleanup.sh" +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" || exit $? +source "$SCRIPT_DIR/benchmark_lib.sh" +source "$SCRIPT_DIR/server_cleanup.sh" # shellcheck source=magpie_bench_remote_compat.sh -source "$(dirname "$0")/magpie_bench_remote_compat.sh" +source "$SCRIPT_DIR/magpie_bench_remote_compat.sh" PHASE="${MAGPIE_RUN_PHASE:-all}" case "$PHASE" in diff --git a/Magpie/scripts/benchmark/sglang_radeon8060s.sh b/Magpie/scripts/benchmark/sglang_radeon8060s.sh index 12402ab..40d6f3d 100755 --- a/Magpie/scripts/benchmark/sglang_radeon8060s.sh +++ b/Magpie/scripts/benchmark/sglang_radeon8060s.sh @@ -16,10 +16,11 @@ # instead of localhost:$PORT, and forces PHASE=client (no local server # launch). See sglang_mi300x.sh for the full contract. -source "$(dirname "$0")/benchmark_lib.sh" -source "$(dirname "$0")/server_cleanup.sh" +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" || exit $? +source "$SCRIPT_DIR/benchmark_lib.sh" +source "$SCRIPT_DIR/server_cleanup.sh" # shellcheck source=magpie_bench_remote_compat.sh -source "$(dirname "$0")/magpie_bench_remote_compat.sh" +source "$SCRIPT_DIR/magpie_bench_remote_compat.sh" PHASE="${MAGPIE_RUN_PHASE:-all}" case "$PHASE" in diff --git a/Magpie/scripts/benchmark/vllm_mi300x.sh b/Magpie/scripts/benchmark/vllm_mi300x.sh index d852f6d..0bf5997 100644 --- a/Magpie/scripts/benchmark/vllm_mi300x.sh +++ b/Magpie/scripts/benchmark/vllm_mi300x.sh @@ -19,10 +19,11 @@ # cluster, externally managed). Leave the env unset to keep the # default behaviour of launching a local server. -source "$(dirname "$0")/benchmark_lib.sh" -source "$(dirname "$0")/server_cleanup.sh" +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" || exit $? +source "$SCRIPT_DIR/benchmark_lib.sh" +source "$SCRIPT_DIR/server_cleanup.sh" # shellcheck source=magpie_bench_remote_compat.sh -source "$(dirname "$0")/magpie_bench_remote_compat.sh" +source "$SCRIPT_DIR/magpie_bench_remote_compat.sh" PHASE="${MAGPIE_RUN_PHASE:-all}" case "$PHASE" in diff --git a/Magpie/scripts/benchmark/vllm_mi300x_mm.sh b/Magpie/scripts/benchmark/vllm_mi300x_mm.sh index b3ee431..aef3c1f 100644 --- a/Magpie/scripts/benchmark/vllm_mi300x_mm.sh +++ b/Magpie/scripts/benchmark/vllm_mi300x_mm.sh @@ -26,10 +26,11 @@ # Phases (via MAGPIE_RUN_PHASE): all | server | client (default all). # Server-only writes PID to MAGPIE_SERVER_PID_FILE then disowns and exits. -source "$(dirname "$0")/benchmark_lib.sh" -source "$(dirname "$0")/server_cleanup.sh" +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" || exit $? +source "$SCRIPT_DIR/benchmark_lib.sh" +source "$SCRIPT_DIR/server_cleanup.sh" # shellcheck source=magpie_bench_remote_compat.sh -[[ -f "$(dirname "$0")/magpie_bench_remote_compat.sh" ]] && source "$(dirname "$0")/magpie_bench_remote_compat.sh" +[[ -f "$SCRIPT_DIR/magpie_bench_remote_compat.sh" ]] && source "$SCRIPT_DIR/magpie_bench_remote_compat.sh" # Fallback: some server_cleanup.sh variants (e.g. InferenceX's) do not define # magpie_stop_benchmark_server_stack. Provide a safe inline version so the diff --git a/Magpie/scripts/benchmark/vllm_mi355x.sh b/Magpie/scripts/benchmark/vllm_mi355x.sh index 1b42e62..a3f8e3f 100644 --- a/Magpie/scripts/benchmark/vllm_mi355x.sh +++ b/Magpie/scripts/benchmark/vllm_mi355x.sh @@ -16,10 +16,11 @@ # instead of localhost:$PORT, and forces PHASE=client (no local server # launch). See vllm_mi300x.sh for the full contract. -source "$(dirname "$0")/benchmark_lib.sh" -source "$(dirname "$0")/server_cleanup.sh" +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" || exit $? +source "$SCRIPT_DIR/benchmark_lib.sh" +source "$SCRIPT_DIR/server_cleanup.sh" # shellcheck source=magpie_bench_remote_compat.sh -source "$(dirname "$0")/magpie_bench_remote_compat.sh" +source "$SCRIPT_DIR/magpie_bench_remote_compat.sh" PHASE="${MAGPIE_RUN_PHASE:-all}" case "$PHASE" in diff --git a/Magpie/scripts/benchmark/vllm_mi355x_mm.sh b/Magpie/scripts/benchmark/vllm_mi355x_mm.sh index fd025a3..d7c20ee 100644 --- a/Magpie/scripts/benchmark/vllm_mi355x_mm.sh +++ b/Magpie/scripts/benchmark/vllm_mi355x_mm.sh @@ -26,10 +26,11 @@ # Phases (via MAGPIE_RUN_PHASE): all | server | client (default all). # Server-only writes PID to MAGPIE_SERVER_PID_FILE then disowns and exits. -source "$(dirname "$0")/benchmark_lib.sh" -source "$(dirname "$0")/server_cleanup.sh" +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" || exit $? +source "$SCRIPT_DIR/benchmark_lib.sh" +source "$SCRIPT_DIR/server_cleanup.sh" # shellcheck source=magpie_bench_remote_compat.sh -[[ -f "$(dirname "$0")/magpie_bench_remote_compat.sh" ]] && source "$(dirname "$0")/magpie_bench_remote_compat.sh" +[[ -f "$SCRIPT_DIR/magpie_bench_remote_compat.sh" ]] && source "$SCRIPT_DIR/magpie_bench_remote_compat.sh" # Fallback: some server_cleanup.sh variants (e.g. InferenceX's) do not define # magpie_stop_benchmark_server_stack. Provide a safe inline version so the diff --git a/Magpie/scripts/benchmark/vllm_radeon8060s.sh b/Magpie/scripts/benchmark/vllm_radeon8060s.sh index a49aae7..ba30c11 100755 --- a/Magpie/scripts/benchmark/vllm_radeon8060s.sh +++ b/Magpie/scripts/benchmark/vllm_radeon8060s.sh @@ -16,10 +16,11 @@ # instead of localhost:$PORT, and forces PHASE=client (no local server # launch). See vllm_mi300x.sh for the full contract. -source "$(dirname "$0")/benchmark_lib.sh" -source "$(dirname "$0")/server_cleanup.sh" +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" || exit $? +source "$SCRIPT_DIR/benchmark_lib.sh" +source "$SCRIPT_DIR/server_cleanup.sh" # shellcheck source=magpie_bench_remote_compat.sh -source "$(dirname "$0")/magpie_bench_remote_compat.sh" +source "$SCRIPT_DIR/magpie_bench_remote_compat.sh" PHASE="${MAGPIE_RUN_PHASE:-all}" case "$PHASE" in diff --git a/tests/test_eval_source_paths.py b/tests/test_eval_source_paths.py new file mode 100644 index 0000000..4fd2854 --- /dev/null +++ b/tests/test_eval_source_paths.py @@ -0,0 +1,94 @@ +"""Generic scripts keep sourced eval resources valid after persisted eval changes cwd.""" + +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +SCRIPTS = Path(__file__).parents[1] / "Magpie" / "scripts" / "benchmark" +EVAL_SCRIPTS = [ + path.name + for path in sorted(SCRIPTS.glob("*.sh")) + if '/benchmark_lib.sh"' in path.read_text(encoding="utf-8") +] + +# InferenceX 3d5581562 resolves these paths inside functions, after Magpie's +# persisted eval wrapper changes cwd. Keep real Bash source-stack resolution; +# a stub that captures the root at source time would conceal the regression. +BENCHMARK_LIB = r''' +_eval_patches_dir() { + cd "$(dirname "${BASH_SOURCE[0]}")/../utils/evals/patches" && pwd +} + +run_eval() { + local _repo_root + _repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" || return 1 + [[ "$_repo_root" == "$EXPECTED_REPO_ROOT" ]] || return 2 + [[ "$PWD" == "$EVAL_RESULT_DIR" ]] || return 3 + cp "$(_eval_patches_dir)/lm_eval_sitecustomize.py" "$EVAL_RESULT_DIR/sitecustomize.py" || return 4 + cp "$_repo_root/utils/evals/gsm8k.yaml" "$EVAL_RESULT_DIR/task.yaml" || return 5 + printf '%s\n' '{"results":{"gsm8k":{"exact_match,strict-match":0.5}}}' > "$EVAL_RESULT_DIR/results_fixture.json" +} + +# Exercise the entrypoint's actual dependency loading and persisted eval +# wrapper, exiting before any server, GPU command, or throughput client runs. +check_env_vars() { + magpie_run_eval_persisted --framework lm-eval --port 8888 + local eval_rc=$? + [[ "$PWD" == "$EXPECTED_REPO_ROOT" ]] || exit 6 + exit "$eval_rc" +} +''' + + +@pytest.mark.parametrize("script_name", EVAL_SCRIPTS) +@pytest.mark.parametrize("absolute_entrypoint", [False, True]) +def test_persisted_eval_keeps_deferred_source_paths( + tmp_path: Path, script_name: str, absolute_entrypoint: bool +): + checkout = tmp_path / "InferenceX checkout" + scripts = checkout / "benchmarks" + shutil.copytree(SCRIPTS, scripts) + library = scripts / "benchmark_lib.sh" + library.write_text(BENCHMARK_LIB, encoding="utf-8") + patches = checkout / "utils" / "evals" / "patches" + patches.mkdir(parents=True) + (patches / "lm_eval_sitecustomize.py").write_text("# eval patch fixture\n") + (patches.parent / "gsm8k.yaml").write_text("task: gsm8k\n") + result_dir = tmp_path / "session results" + eval_dir = result_dir / "eval output" + env = { + "PATH": os.environ["PATH"], + "MAGPIE_RUN_PHASE": "client", + "MAGPIE_ACCURACY_REPORT_PYTHON": sys.executable, + "RESULT_DIR": str(result_dir), + "EVAL_RESULT_DIR": str(eval_dir), + "EXPECTED_REPO_ROOT": str(checkout), + "MODEL": "fixture-model", + "CONC": "1", + } + entrypoint = scripts / script_name + if not absolute_entrypoint: + entrypoint = entrypoint.relative_to(checkout) + completed = subprocess.run( + ["bash", str(entrypoint)], + cwd=checkout, + env=env, + capture_output=True, + text=True, + check=False, + timeout=15, + ) + assert completed.returncode == 0, completed.stdout + completed.stderr + assert (eval_dir / "sitecustomize.py").read_text() == "# eval patch fixture\n" + assert (eval_dir / "task.yaml").read_text() == "task: gsm8k\n" + summary = json.loads((result_dir / "accuracy_report.json").read_text()) + assert summary["status"] == "COMPLETED" + assert summary["score"] == 0.5 + assert (result_dir / summary["source_result"]).is_file() + assert library.read_text() == BENCHMARK_LIB + assert not (checkout / "results_fixture.json").exists()