From a3e60b86368e4c55378ca08b3c637571f69d6c7e Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Fri, 7 Aug 2026 10:38:25 -0300 Subject: [PATCH] fix(evaluator): load globbed dataset files in a deterministic order `discover_files` returned whatever order `rglob`/`glob` got back from the filesystem. Rows from those files are concatenated in that order and their positions become `row_index` on the resulting scores, so the same fileset scored twice could pair a row's score with a different input row. This is how `test_fileset_fragment_and_glob_datasets` was failing intermittently: globbing `part-*.json` over two files yielded either `[1.0, 0.0, 1.0]` (part-a first) or `[1.0, 1.0, 0.0]` (part-b first). The test had pinned the latter, so it passed or failed on readdir order. Sort both branches, matching what `agent_seeds.py`, `harbor_runtime.py`, and `fabric/skills.py` already do for their directory walks. The e2e expectation moves to sorted order, and two unit tests pin the guarantee directly -- the existing glob test wraps both sides in `sorted()`, so it could not have caught this. Signed-off-by: Sandy Chapman --- e2e/test_evaluator_plugin.py | 2 +- .../src/nemo_evaluator_sdk/datasets/loader.py | 6 +++--- .../tests/execution/test_metric_execution.py | 20 +++++++++++++++++++ .../beta/evaluator/datasets/loader.py | 6 +++--- 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/e2e/test_evaluator_plugin.py b/e2e/test_evaluator_plugin.py index 3c1de977dc..7abc1edfe2 100644 --- a/e2e/test_evaluator_plugin.py +++ b/e2e/test_evaluator_plugin.py @@ -476,7 +476,7 @@ def test_fileset_fragment_and_glob_datasets(evaluator_sdk: NeMoPlatform) -> None cases = { "specific file": (f"{workspace}/{fileset_name}#part-a.json", [1.0, 0.0]), - "glob": (f"{workspace}/{fileset_name}#part-*.json", [1.0, 1.0, 0.0]), + "glob": (f"{workspace}/{fileset_name}#part-*.json", [1.0, 0.0, 1.0]), } for label, (reference, expected_scores) in cases.items(): job = evaluator_sdk.evaluator.submit( diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/datasets/loader.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/datasets/loader.py index 78a799be26..eeca77445f 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/datasets/loader.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/datasets/loader.py @@ -163,7 +163,7 @@ def discover_files(base_path: Path, pattern: str | None) -> list[Path]: pattern: Optional explicit file name or glob pattern. Returns: - List of discovered files. + List of discovered files, sorted by path. Raises: DatasetLoadError: If files cannot be found or selected paths are invalid. @@ -172,7 +172,7 @@ def discover_files(base_path: Path, pattern: str | None) -> list[Path]: raise DatasetLoadError(f"Dataset directory not found: {base_path}") if pattern is None: - files = [f for f in base_path.rglob("*") if f.is_file()] + files = sorted(f for f in base_path.rglob("*") if f.is_file()) if not files: raise DatasetLoadError(f"No files found in {base_path}") return files @@ -184,7 +184,7 @@ def discover_files(base_path: Path, pattern: str | None) -> list[Path]: return [file_path] if is_glob_pattern(pattern): - files = list(base_path.glob(pattern)) + files = sorted(base_path.glob(pattern)) if not files: raise DatasetLoadError(f"No files found matching pattern '{pattern}' in {base_path}") return [f for f in files if f.is_file()] diff --git a/packages/nemo_evaluator_sdk/tests/execution/test_metric_execution.py b/packages/nemo_evaluator_sdk/tests/execution/test_metric_execution.py index fe606b291f..e3577f88ae 100644 --- a/packages/nemo_evaluator_sdk/tests/execution/test_metric_execution.py +++ b/packages/nemo_evaluator_sdk/tests/execution/test_metric_execution.py @@ -520,6 +520,26 @@ def test_glob_pattern_discovers_files(self, tmp_path: Path): assert sorted(discover_files(tmp_path / "splits", "**/*.jsonl")) == sorted([train_path, validation_path]) + def test_glob_pattern_returns_files_in_sorted_order(self, tmp_path: Path): + for name in ("part-c.json", "part-a.json", "part-b.json"): + (tmp_path / name).touch() + + assert discover_files(tmp_path, "part-*.json") == [ + tmp_path / "part-a.json", + tmp_path / "part-b.json", + tmp_path / "part-c.json", + ] + + def test_directory_without_a_pattern_returns_files_in_sorted_order(self, tmp_path: Path): + for name in ("c.jsonl", "a.jsonl", "b.jsonl"): + (tmp_path / name).touch() + + assert discover_files(tmp_path, None) == [ + tmp_path / "a.jsonl", + tmp_path / "b.jsonl", + tmp_path / "c.jsonl", + ] + class TestIsCompletionsEndpoint: @pytest.mark.parametrize( diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/datasets/loader.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/datasets/loader.py index 95b7419b66..6c5641e0f0 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/datasets/loader.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/datasets/loader.py @@ -163,7 +163,7 @@ def discover_files(base_path: Path, pattern: str | None) -> list[Path]: pattern: Optional explicit file name or glob pattern. Returns: - List of discovered files. + List of discovered files, sorted by path. Raises: DatasetLoadError: If files cannot be found or selected paths are invalid. @@ -172,7 +172,7 @@ def discover_files(base_path: Path, pattern: str | None) -> list[Path]: raise DatasetLoadError(f"Dataset directory not found: {base_path}") if pattern is None: - files = [f for f in base_path.rglob("*") if f.is_file()] + files = sorted(f for f in base_path.rglob("*") if f.is_file()) if not files: raise DatasetLoadError(f"No files found in {base_path}") return files @@ -184,7 +184,7 @@ def discover_files(base_path: Path, pattern: str | None) -> list[Path]: return [file_path] if is_glob_pattern(pattern): - files = list(base_path.glob(pattern)) + files = sorted(base_path.glob(pattern)) if not files: raise DatasetLoadError(f"No files found matching pattern '{pattern}' in {base_path}") return [f for f in files if f.is_file()]