Feature: add dispatch-correlated same-host multi-rank swimlane timeline

conftest.py

@@ -722,24 +722,22 @@ def sort_key(item):
 
     items.sort(key=sort_key)
 
-    # L3 perf collection is not supported yet: a single L3 case forks N chip-processes
-    # that all write chip_swimlane_records_<ts>.json to the same directory with
-    # second-precision timestamps, so they trample each other. Block the
-    # combination up front; waiting for a proper device-id-in-filename fix.
+    # The automatic rankN/dN layout is scoped to one same-host L3 Worker. Every
+    # level above NODE owns several L3 Workers, each of which numbers its local
+    # chips from zero, so accepting one would reintroduce directory collisions.
     if config.getoption("--enable-chip-swimlane", default=0) and config.getoption("--rounds", default=1) <= 1:
-        l3_items = [
-            i
-            for i in items
-            if _item_scene_level(i) == SceneTestLevel.NODE and not any(m.name == "skip" for m in i.iter_markers())
+        multi_node_items = [
+            item
+            for item in items
+            if (_item_scene_level(item) or SceneTestLevel.CHIP) > SceneTestLevel.NODE
+            and not any(marker.name == "skip" for marker in item.iter_markers())
         ]
-        if l3_items:
-            sample = ", ".join(sorted({i.nodeid for i in l3_items})[:3])
-            more = "" if len(l3_items) <= 3 else f" (+{len(l3_items) - 3} more)"
+        if multi_node_items:
+            sample = ", ".join(sorted({item.nodeid for item in multi_node_items})[:3])
+            more = "" if len(multi_node_items) <= 3 else f" (+{len(multi_node_items) - 3} more)"
             raise pytest.UsageError(
-                f"--enable-chip-swimlane is not supported for L3 tests yet — "
-                f"multi-chip-process filename collision unresolved. "
-                f"L3 items in this session: {sample}{more}. "
-                f"Either drop --enable-chip-swimlane or scope to L2 with --level 2."
+                "--enable-chip-swimlane supports automatic multi-Rank merging only for same-host L3 tests; "
+                f"NETWORK1/L4 needs a node namespace before it is safe. Items: {sample}{more}."
             )
 
 
@@ -872,7 +870,45 @@ def _strip_value_options(args, options):
     return stripped
 
 
-def _resource_child_command(spec, device_ids, platform, manual_mode):
+# Options that change what a case does rather than which case runs. The
+# resource child's argv is built from scratch so it can be narrowed to one
+# nodeid (see _build in the dispatcher), which means nothing reaches it that is
+# not listed here — a diagnostic the parent asked for is silently dropped
+# otherwise, and the run passes while producing no artifact at all.
+#
+# The options that select which case runs — --manual and --case — are forwarded
+# by _resource_child_command alongside the nodeid they refine, because a nodeid
+# names a whole SceneTestCase class: `--case` filters inside its single
+# `test_run` item at run time, so a child that does not receive it runs every
+# case of the class the parent narrowed to one.
+_RESOURCE_CHILD_VALUE_OPTIONS = (
+    ("--rounds", 1),
+    ("--enable-chip-swimlane", 0),
+    ("--dump-args", 0),
+    ("--enable-pmu", 0),
+)
+_RESOURCE_CHILD_FLAG_OPTIONS = (
+    "--skip-golden",
+    "--enable-dep-gen",
+    "--enable-scope-stats",
+    "--enable-swimlane-overhead",
+)
+
+
+def _resource_child_diagnostic_argv(cfg):
+    """Forward the parent's diagnostic and round selection to a resource child."""
+    argv = []
+    for option, unset in _RESOURCE_CHILD_VALUE_OPTIONS:
+        value = cfg.getoption(option, default=unset)
+        if value != unset:
+            argv.extend([option, str(value)])
+    for option in _RESOURCE_CHILD_FLAG_OPTIONS:
+        if cfg.getoption(option, default=False):
+            argv.append(option)
+    return argv
+
+
+def _resource_child_command(spec, device_ids, platform, manual_mode, cfg):
     command = [
         sys.executable,
         "-m",
@@ -888,6 +924,9 @@ def _resource_child_command(spec, device_ids, platform, manual_mode):
     if platform:
         command.extend(["--platform", platform])
     command.extend(["--manual", manual_mode])
+    for selector in cfg.getoption("--case", default=None) or []:
+        command.extend(["--case", str(selector)])
+    command.extend(_resource_child_diagnostic_argv(cfg))
     return command
 
 
@@ -1050,7 +1089,7 @@ def _build(ids, _spec=spec):
                 # this job in the same subprocess, which has only this job's
                 # allocated devices — e.g. TestL3Group (needs 2) would fail
                 # inside TestL3ChildMemory's 1-device subprocess.
-                return _resource_child_command(_spec, ids, platform, manual_mode)
+                return _resource_child_command(_spec, ids, platform, manual_mode, session.config)
 
             jobs.append(
                 _ps.Job(

docs/dfx/chip-swimlane-profiling.md

@@ -165,6 +165,43 @@ runs):
 Filenames are fixed (no per-file timestamp) — the directory is the
 per-task uniqueness boundary.
 
+For L3 runs, each forked ChipWorker writes below its own `rankN/dN` directory.
+The filenames above are fixed, so N children sharing one `output_prefix` would
+overwrite each other — the separation therefore covers **every** diagnostic that
+writes below `output_prefix`, not just the swimlane:
+
+```text
+<output_prefix>/
+├── rank0/d0/
+│   ├── chip_swimlane_records.json    # --enable-chip-swimlane
+│   ├── dispatch_identity.json        # always, whenever any diagnostic is on
+│   ├── deps.json                     # --enable-dep-gen
+│   └── scope_stats/                  # --enable-scope-stats
+├── rank1/d0/
+│   └── ...
+└── l3_swimlane.json                  # cross-Rank trace (added by converter)
+```
+
+Here `rankN` is the logical ChipWorker index and `dN` is that worker's local
+capture index. It is a storage-order suffix, not a globally comparable dispatch
+ID. `dispatch_identity.json` records the parent scheduler identity: `run_id`,
+`task_slot`, `group_index`, and `group_size`, plus the endpoint-local dispatch
+and pipeline diagnostics. All members submitted through one
+`submit_next_level_group` share `(run_id, task_slot)` and have distinct
+`group_index` values. Individually submitted tasks do not share that identity;
+the current postprocessor therefore retains local-capture-index pairing for
+them and requires symmetric `dN` sets.
+
+Automatic merging is limited to one same-host L3 Worker. NETWORK1/L4 is
+rejected until the layout also carries a node namespace. Every Rank must expose
+the same complete set of local capture indexes; the postprocessor refuses
+asymmetric sets instead of guessing pairings.
+
+Cross-Rank merging needs `--enable-chip-swimlane 4` on every Rank, because the
+Host/Device clock anchors that level 4 collects are what put the Ranks on a
+common timeline. A lower level still captures per Rank; the postprocessor then
+converts each `rankN/dN` capture on its own relative timeline and says so.
+
 `chip_swimlane_records.json` carries the raw records. **There are two
 layers to be aware of:**
 
@@ -338,11 +375,74 @@ python -m simpler_setup.tools.swimlane_converter \
 # Custom output path
 python -m simpler_setup.tools.swimlane_converter \
     outputs/<case>_<ts>/chip_swimlane_records.json -o my_trace.json
+
+# Same-host L3: merge rankN/d0 captures onto one CLOCK_MONOTONIC timeline
+python -m simpler_setup.tools.swimlane_converter \
+    build_output/<case>/dfx_outputs --dispatch d0
+
+# Prefer the parent group identity when Rank-local dN suffixes differ
+python -m simpler_setup.tools.swimlane_converter \
+    build_output/<case>/dfx_outputs --dispatch-id 17:5
 ```
 
-The output is `outputs/<case>_<ts>/merged_swimlane.json` (or your
-`-o` override). Open <https://ui.perfetto.dev/> and drag the file
-in. The trace contains:
+For directory input, the default output is `dfx_outputs/l3_swimlane.json`.
+Every Rank must be a level-4 capture under
+`rankN/<dispatch>/`, with successful clock anchors and the same
+`metadata.host_clock_domain_id`. The converter preserves real Rank start skew,
+adds Rank-specific PID/name/flow namespaces, and reports clock uncertainty and
+anchor-group observer overhead in trace metadata.
+
+For new group captures, `--dispatch-id RUN_ID:TASK_SLOT` selects the common
+parent DAG node and resolves each Rank's actual `dN` path through
+`dispatch_identity.json`. SceneTest does this automatically. `--dispatch dN`
+remains the compatibility selector for old captures and for independently
+submitted per-Rank tasks; it fails if available sidecars show that the selected
+paths belong to different parent groups.
+
+Host-orchestrated level-4 runs retain their existing clock anchors. For
+Device/AICPU orchestration, anchors are additionally enabled only when the
+ChipWorker marks the capture with `CallConfig.capture_clock_anchors`, which it
+does for an L3 chip-swimlane capture, at the common launch boundary before
+collectors and kernels start. Both modes sample again after AICPU/AICore
+execution completes. Existing single-card Device/AICPU level-4 captures
+therefore keep their prior relative timeline and do not pay the new anchor cost.
+
+`capture_clock_anchors` says only *what the runtime does* — sample the two
+clocks — never why. Rank, group and merge are concepts of the layer above: the
+platform runner that reads this flag has no notion of a Rank, and no runtime or
+platform code parses the `rankN/dN` path. The two are deliberately separate
+switches, because the directory is artifact separation that every diagnostic
+needs while the anchors are consumed only by the swimlane reader. An L3 run with
+`--enable-dep-gen` alone therefore gets its own `rankN/dN` directory and pays no
+anchor cost.
+
+**The opening anchor sits at a different point in each runtime**, because each
+takes it at the earliest point preceding every device timestamp it records:
+
+| Runtime | Opening anchor | Calibrated interval covers |
+| ------- | -------------- | -------------------------- |
+| `host_build_graph` | before Host orchestration (`host_phase_pool_arm`) | bind, H2D, and execution |
+| `tensormap_and_ringbuffer` | before kernel launch (`start_shared_collectors_for_run`) | execution only |
+
+Both close on `post_device_execution`. So the two runtimes' calibrated intervals
+are not comparable in length, and a `host_build_graph` interpolation spans work
+a `tensormap_and_ringbuffer` one does not. This does not affect
+`max_uncertainty_ns`, which depends only on each anchor group's own sampling
+RTT. The serialized position name `pre_host_orchestration` predates the
+Device/AICPU case — read it as "start of the calibrated interval", not as a
+claim about Host orchestration.
+
+The default output depends on which input form was used, and `-o` overrides
+either:
+
+| Input | Default output |
+| ----- | -------------- |
+| a records file | `outputs/<case>_<ts>/merged_swimlane.json` |
+| a `dfx_outputs` directory | `<dfx_outputs>/l3_swimlane.json` |
+
+Open <https://ui.perfetto.dev/> and drag the file in. Both forms produce the
+same lane structure — the directory form repeats it once per Rank under the
+`rankN / <view>` process names. The trace contains:
 
 - **Orchestrator** (pid=1) — per-submit `orch_submit` envelope
   blocks (level >= 4).

docs/dfx/hbg-bind-phases.md

@@ -347,8 +347,9 @@ silently:
    `bind phase=` lines alone, so Recipe A's environment collects no records.
 2. **A diagnostic flag must be on**, because that is what makes
    `CallConfig.output_prefix` non-empty. `--enable-scope-stats` is the cheapest
-   for an L3 case; `--enable-chip-swimlane` raises `NotImplementedError` for
-   `level=3` (per-chip-process filename collision).
+   choice when only Host phase records are needed. `--enable-chip-swimlane` is
+   also supported for same-host L3 and writes each ChipWorker capture below a
+   separate `rankN/dN` directory, but it collects substantially more data.
 3. **`--rounds` must be 1.** `rounds > 1` force-disables every diagnostic flag —
    this one does warn, `<flag> disabled: --rounds > 1` per flag
    ([`simpler_setup/scene_test.py`](../../simpler_setup/scene_test.py)), but the

docs/task-flow.md

@@ -220,6 +220,7 @@ struct CallConfig {
     int32_t enable_pmu = 0;           // 0 = disabled; >0 selects PMU event type
     int32_t enable_dep_gen = 0;
     int32_t enable_scope_stats = 0;
+    int32_t capture_clock_anchors = 0; // set by the ChipWorker child, not by callers
     char    output_prefix[1024] = {};
     // future fields here - same POD used at all levels
 };

docs/user/reference/python-api.md

@@ -140,6 +140,7 @@ takes a device pointer; `DataType` carries the element types.
 | `enable_pmu` | `0` | `0` off; `>0` selects the event type |
 | `enable_dep_gen` | `0` | Emit the dependency graph |
 | `enable_scope_stats` | `0` | Writes `<output_prefix>/scope_stats/scope_stats.jsonl` |
+| `capture_clock_anchors` | `False` | Anchors the Host and Device clocks so device timestamps land on an absolute Host timeline. Set by the ChipWorker child for an L3 chip-swimlane capture; not a caller knob |
 | `output_prefix` | `""` | **Required whenever any diagnostic is enabled** |
 | `runtime_env` | — | `ring_task_window`, `ring_heap`, `ring_dep_pool`; `tensormap_and_ringbuffer` only |
 

python/bindings/task_interface.cpp

@@ -2787,6 +2787,15 @@ NB_MODULE(_task_interface, m) {
                 c.enable_scope_stats = v ? 1 : 0;
             }
         )
+        .def_prop_rw(
+            "capture_clock_anchors",
+            [](const CallConfig &c) {
+                return static_cast<bool>(c.capture_clock_anchors);
+            },
+            [](CallConfig &c, bool v) {
+                c.capture_clock_anchors = v ? 1 : 0;
+            }
+        )
         .def_prop_rw(
             "output_prefix",
             [](const CallConfig &c) -> std::string {
@@ -2809,7 +2818,8 @@ NB_MODULE(_task_interface, m) {
                << ", enable_chip_swimlane=" << self.enable_chip_swimlane
                << ", enable_dump_args=" << self.enable_dump_args << ", enable_pmu=" << self.enable_pmu
                << ", enable_dep_gen=" << (self.enable_dep_gen ? "True" : "False")
-               << ", enable_scope_stats=" << (self.enable_scope_stats ? "True" : "False");
+               << ", enable_scope_stats=" << (self.enable_scope_stats ? "True" : "False")
+               << ", capture_clock_anchors=" << (self.capture_clock_anchors ? "True" : "False");
             if (self.runtime_env.any()) {
                 append_ring_values(os, "runtime_env.ring_task_window", true, self.runtime_env.ring_task_window);
                 append_ring_values(os, "runtime_env.ring_heap", true, self.runtime_env.ring_heap);