sukesh-amd · Oct 30, 2026 · Oct 30, 2026 · Oct 30, 2026
@@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - Single-node 8-GPU guide configs `mi300x_rccl_single_node_config.json` and `mi350x_rccl_single_node_config.json`.
 - Platform `test_check_be_nic_link_speed` checks backend NIC link speed against `nic_link_speed` (default `400000` Mb/s). Existing configs fail on NICs running at another speed, or on a node with fewer backend NICs unless `nic_link_interfaces` is set.
 - Platform host config `fw_dict` includes `PLDM_BUNDLE`. Set it to the version reported by `amd-smi firmware` before the GPU firmware check; the check looks up every firmware ID `amd-smi` reports.
+- Primus Megatron configs accept `train_params.primus_cli_overrides` as either a raw CLI string or a JSON object rendered as `--key value` pairs. Primus 405B auto-splits decoder first/last pipeline layers from `train_params.pipeline_parallelism` when needed.
 
 ### Changed
 
@@ -53,6 +54,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Aorta script mode (the `aorta_single` default) now applies `training_overrides`; previously runs silently used the YAML values. The applied config is saved as `effective_config.yaml`.
 - Kernel-log health checks no longer echo each node's full dmesg or journal into the run log and HTML report. Each scan logs one summary line, and detected errors are still reported.
+- Megatron and Primus no longer write the Hugging Face token into training, tokenizer-download, or job-launch commands. Each node reads `paths.hf_token_file` inside the container when that command runs. If that file is missing, empty, or not mounted, Megatron fails the test and the remaining training stages are skipped. Primus exports an empty `HF_TOKEN` and the run continues.
 - ibperf:
   - Runs on nodes with fewer than 8 GPUs instead of failing with `KeyError: 'card4'`.
   - `verify_bw` now enforces `expected_results`; previously tests passed regardless of measured bandwidth. Configs with `verify_bw` set to `"True"` may now fail; re-baseline thresholds for your cluster.

@@ -50,7 +50,9 @@
     "micro_batch_size": "4",
     "global_batch_size": "128",
     "precision": "BF16",
-    "training_iterations": "<changeme>"
+    "training_iterations": "<changeme>",
+    "_primus_cli_overrides_comment": "Optional Primus-only extra CLI args. Set as a raw string (for example \"--use_backend triton\") or override per sweep combination with the same key.",
+    "primus_cli_overrides": {}
   },
   "_smoke_comment": "test_smoke loads the model and runs a small fixed cell without metric gating. enabled=false skips it.",
   "smoke": {

@@ -50,7 +50,9 @@
     "micro_batch_size": "3",
     "global_batch_size": "96",
     "precision": "BF16",
-    "training_iterations": "<changeme>"
+    "training_iterations": "<changeme>",
+    "_primus_cli_overrides_comment": "Optional Primus-only extra CLI args. Set as a raw string (for example \"--use_backend triton\") or override per sweep combination with the same key.",
+    "primus_cli_overrides": {}
   },
   "_smoke_comment": "test_smoke loads the model and runs a small fixed cell without metric gating. enabled=false skips it.",
   "smoke": {

@@ -57,7 +57,9 @@
     "micro_batch_size": "4",
     "global_batch_size": "128",
     "precision": "BF16",
-    "training_iterations": "<changeme>"
+    "training_iterations": "<changeme>",
+    "_primus_cli_overrides_comment": "Optional Primus-only extra CLI args. Set as a raw string (for example \"--use_backend triton\") or override per sweep combination with the same key.",
+    "primus_cli_overrides": {}
   },
   "verify_network_errors": "True",
   "_smoke_comment": "test_smoke loads the model and runs a small fixed cell without metric gating. enabled=false skips it.",

@@ -52,7 +52,9 @@
     "micro_batch_size": "4",
     "global_batch_size": "128",
     "precision": "BF16",
-    "training_iterations": "<changeme>"
+    "training_iterations": "<changeme>",
+    "_primus_cli_overrides_comment": "Optional Primus-only extra CLI args. Set as a raw string (for example \"--use_backend triton\") or override per sweep combination with the same key.",
+    "primus_cli_overrides": {}
   },
   "_smoke_comment": "test_smoke loads the model and runs a small fixed cell without metric gating. enabled=false skips it.",
   "smoke": {

@@ -57,7 +57,9 @@
     "micro_batch_size": "1",
     "global_batch_size": "64",
     "precision": "BF16",
-    "training_iterations": "<changeme>"
+    "training_iterations": "<changeme>",
+    "_primus_cli_overrides_comment": "Optional Primus-only extra CLI args. Set as a raw string (for example \"--use_backend triton\") or override per sweep combination with the same key.",
+    "primus_cli_overrides": {}
   },
   "verify_network_errors": "True",
   "_smoke_comment": "test_smoke loads the model and runs a small fixed cell without metric gating. enabled=false skips it.",

@@ -57,7 +57,9 @@
     "micro_batch_size": "4",
     "global_batch_size": "128",
     "precision": "BF16",
-    "training_iterations": "<changeme>"
+    "training_iterations": "<changeme>",
+    "_primus_cli_overrides_comment": "Optional Primus-only extra CLI args. Set as a raw string (for example \"--use_backend triton\") or override per sweep combination with the same key.",
+    "primus_cli_overrides": {}
   },
   "verify_network_errors": "True",
   "_smoke_comment": "test_smoke loads the model and runs a small fixed cell without metric gating. enabled=false skips it.",

@@ -52,7 +52,9 @@
     "micro_batch_size": "4",
     "global_batch_size": "128",
     "precision": "BF16",
-    "training_iterations": "<changeme>"
+    "training_iterations": "<changeme>",
+    "_primus_cli_overrides_comment": "Optional Primus-only extra CLI args. Set as a raw string (for example \"--use_backend triton\") or override per sweep combination with the same key.",
+    "primus_cli_overrides": {}
   },
   "_smoke_comment": "test_smoke loads the model and runs a small fixed cell without metric gating. enabled=false skips it.",
   "smoke": {

@@ -57,7 +57,9 @@
     "micro_batch_size": "1",
     "global_batch_size": "64",
     "precision": "BF16",
-    "training_iterations": "<changeme>"
+    "training_iterations": "<changeme>",
+    "_primus_cli_overrides_comment": "Optional Primus-only extra CLI args. Set as a raw string (for example \"--use_backend triton\") or override per sweep combination with the same key.",
+    "primus_cli_overrides": {}
   },
   "verify_network_errors": "True",
   "_smoke_comment": "test_smoke loads the model and runs a small fixed cell without metric gating. enabled=false skips it.",

@@ -52,7 +52,9 @@
     "micro_batch_size": "3",
     "global_batch_size": "96",
     "precision": "BF16",
-    "training_iterations": "<changeme>"
+    "training_iterations": "<changeme>",
+    "_primus_cli_overrides_comment": "Optional Primus-only extra CLI args. Set as a raw string (for example \"--use_backend triton\") or override per sweep combination with the same key.",
+    "primus_cli_overrides": {}
   },
   "_smoke_comment": "test_smoke loads the model and runs a small fixed cell without metric gating. enabled=false skips it.",
   "smoke": {

@@ -11,6 +11,7 @@
 import time
 
 from cvs.lib import globals
+from cvs.lib.env_lib import redact_secrets
 from cvs.lib.utils_lib import *
 from cvs.lib.verify_lib import *
 from cvs.lib import linux_utils
@@ -34,6 +35,60 @@ def _default_nic_type(gpu_name):
     return 'thor2'
 
 
+def _hf_token_export_line(token_file):
+    """Shell line that exports ``HF_TOKEN`` by reading ``token_file`` on the
+    node the command runs on (as SGLang/vLLM/jaxmaxtext do).
+
+    Embedding HF_TOKEN's value directly in a shell command would print it,
+    because the orchestrator logs the full command. A missing or empty file
+    fails the shell command instead of continuing with an empty token.
+    """
+    path = str(token_file or '').strip()
+    if not path:
+        return "echo 'paths.hf_token_file is not configured' >&2; exit 1; "
+    quoted = shlex.quote(path)
+    return (
+        f"if [ -s {quoted} ]; then export HF_TOKEN=\"$(tr -d '[:space:]' < {quoted})\"; "
+        f"else echo 'HF token file is missing or not mounted inside the container' >&2; exit 1; fi; "
+    )
+
+
+def _require_hf_token_file(orch, token_file):
+    """Fail the test when token_file is missing or empty inside the container.
+
+    The training command reads the file on the node, so a path that is not
+    mounted into the container must stop the test before the job is launched.
+    """
+    path = str(token_file or '').strip()
+    if not path:
+        msg = (
+            'paths.hf_token_file is not configured. Megatron reads the token inside the '
+            'container on each node and fails the test when the file is missing.'
+        )
+        fail_test(msg)
+        raise FileNotFoundError(msg)
+    quoted = shlex.quote(path)
+    result = orch.exec(f'test -s {quoted}', detailed=True, timeout=30) or {}
+    missing = []
+    seen = set()
+    for host, info in result.items():
+        seen.add(str(host))
+        code = info.get('exit_code') if isinstance(info, dict) else None
+        if code != 0:
+            missing.append(str(host))
+    for host in getattr(orch, 'hosts', None) or []:
+        if str(host) not in seen:
+            missing.append(str(host))
+    if not result or missing:
+        where = ', '.join(missing) if missing else 'no node returned a result'
+        msg = (
+            f'HF token file {path} is missing, empty, or not mounted inside the container on {where}. '
+            'Megatron fails the test here. Mount paths.hf_token_file into the container on each node.'
+        )
+        fail_test(msg)
+        raise FileNotFoundError(msg)
+
+
 # Ordered fallback chains for parsing Megatron-LM training output.
 # Each chain is tried in order; first non-empty match wins. Seeded with
 # [new, old] so newer Megatron output (e.g. `throughput per GPU
@@ -225,7 +280,6 @@ def __init__(
         self,
         orch,
         variant_config,
-        hf_token,
         micro_batch_size,
         global_batch_size,
         precision=None,
@@ -241,7 +295,6 @@ def __init__(
         Args:
           orch: Orchestrator handle for container and host command execution.
           variant_config: MegatronVariantConfig holding container, config, train_params, sweep.
-          hf_token: Hugging Face token passed to the job environment.
           batch_size: Global batch size for this sweep cell (overrides train_params).
           micro_batch_size: Micro batch size for this sweep cell (overrides train_params).
           precision: Optional precision override for this sweep cell. When None, falls
@@ -253,7 +306,6 @@ def __init__(
 
         self.orch = orch
         self.model_name = variant_config.train_params["model_name"]
-        self.hf_token = hf_token
         self.tune_model_params = tune_model_params
 
         self.job_cmd = ''
@@ -306,6 +358,8 @@ def __init__(
         self.megatron_root = tdict['megatron_root']
         self.training_scripts = tdict['training_scripts']
 
+        self.hf_token_file = tdict['hf_token_file']
+
         pdict = dict(variant_config.train_params)
         pdict.update(sweep_overrides or {})
         pdict['micro_batch_size'] = micro_batch_size
@@ -412,9 +466,9 @@ def download_tokenizer_model(self):
 
         local_dir = f'{self.data_cache_dir}/{self.model_name}'
         log.info('Downloading tokenizer.model for %s into %s', self.model_name, local_dir)
+        _require_hf_token_file(self.orch, self.hf_token_file)
         self.orch.exec(
-            f'export HF_TOKEN={shlex.quote(self.hf_token)}; '
-            f'huggingface-cli download {self.tokenizer_model} '
+            _hf_token_export_line(self.hf_token_file) + f'huggingface-cli download {self.tokenizer_model} '
             f'--include "tokenizer.model" '
             f'--local-dir {local_dir}'
         )
@@ -535,7 +589,7 @@ def build_training_job_cmd(
             cmd
             + f'cd {self.megatron_root}; export MOCK_DATA=1; '
             + f'export IMAGE={self.container_image}; '
-            + f'export HF_TOKEN="{self.hf_token}"; '
+            + _hf_token_export_line(self.hf_token_file)
             + f'export DATA_CACHE_PATH={self.data_cache_dir}; '
             + f'export TOKENIZER_MODEL={self.local_tokenizer_path if self.local_tokenizer_path else self.tokenizer_model}; '
             + f'export LD_LIBRARY_PATH=/usr/local/lib/:{self.rocm_path}/lib:$LD_LIBRARY_PATH; '
@@ -605,8 +659,9 @@ def start_training_job(self, timeout=500):
            by build_training_job_cmd() prior to invocation.
         """
         log.info('start training job')
-        log.info('%s', self.job_cmd_list)
-        log.info("%s", self.job_cmd)
+        _require_hf_token_file(self.orch, self.hf_token_file)
+        log.info('%s', [redact_secrets(c) for c in self.job_cmd_list])
+        log.info("%s", redact_secrets(self.job_cmd))
         n = len(self.orch.hosts)
 
         # Create per-node log dirs inside container on all hosts in parallel