From f3846f9b4695267e01a80f10bdf1b5686823e957 Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Tue, 20 Oct 2026 18:07:28 +0000 Subject: [PATCH] Give the Megatron worker pool 300 s to report ready DistributedWorkerPool waited 120 s for its workers to import the Megatron/TE/ modelopt stack and report ready. The start takes 75-77 s in 14 modules of one PR run and has a p90 of 94-98 s, so slow hosts time out and the whole module errors: 15 jobs (903 job-minutes) in two weeks. Wait 300 s. Testing: ruff check and format with the pinned ruff 0.15.20; py_compile. Not run on a GPU runner. Signed-off-by: Shengliang Xu --- tests/_test_utils/torch/distributed/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/_test_utils/torch/distributed/utils.py b/tests/_test_utils/torch/distributed/utils.py index d5bbebb5404..885db1f82c7 100644 --- a/tests/_test_utils/torch/distributed/utils.py +++ b/tests/_test_utils/torch/distributed/utils.py @@ -132,7 +132,7 @@ def _test_fn(rank, size): ... """ def __init__( - self, world_size, backend="nccl", teardown_fn=default_worker_teardown, ready_timeout=120 + self, world_size, backend="nccl", teardown_fn=default_worker_teardown, ready_timeout=300 ): assert world_size > 0, "World size must be greater than 0" self.world_size = world_size @@ -162,6 +162,8 @@ def __init__( p.start() self._processes.append(p) + # ready_timeout is 300 s: cold imports of the Megatron/TE/modelopt stack take 75-100 s + # under coverage in CI try: for _ in range(world_size): msg = self._get(ready_timeout)