feat(ds4): add external ROCTX phase markers

server/CMakeLists.txt

@@ -390,6 +390,7 @@ add_library(dflash_common STATIC
     # DeepSeek V4 Flash target arch
     src/deepseek4/deepseek4_loader.cpp
     src/deepseek4/deepseek4_graph.cpp
+    src/deepseek4/deepseek4_roctx.cpp
     src/deepseek4/deepseek4_backend.cpp
     src/deepseek4/deepseek4_daemon.cpp
     src/deepseek4/deepseek4_layer_split_adapter.cpp
@@ -756,6 +757,8 @@ target_link_libraries(dflash_common
         ${DFLASH27B_GGML_BACKEND_TARGET}
         ggml-base
         nlohmann_json::nlohmann_json
+    PRIVATE
+        ${CMAKE_DL_LIBS}
 )
 # OpenMP for parallel MoE expert compute kernel (saturate memory bandwidth).
 find_package(OpenMP)
@@ -1006,6 +1009,20 @@ if(DFLASH27B_TESTS)
         endif()
         list(APPEND _raw_unit_test_targets test_deepseek4_unit)
     endif()
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_deepseek4_roctx.cpp")
+        add_executable(test_deepseek4_roctx
+            test/test_deepseek4_roctx.cpp
+            src/deepseek4/deepseek4_roctx.cpp
+            src/common/target_shard_ipc_daemon.cpp)
+        target_include_directories(test_deepseek4_roctx PRIVATE
+            ${CMAKE_CURRENT_SOURCE_DIR}/src)
+        if(DFLASH27B_GPU_BACKEND STREQUAL "hip")
+            target_compile_definitions(test_deepseek4_roctx PRIVATE
+                DFLASH27B_BACKEND_HIP=1)
+        endif()
+        target_link_libraries(test_deepseek4_roctx PRIVATE ${CMAKE_DL_LIBS})
+        list(APPEND _raw_unit_test_targets test_deepseek4_roctx)
+    endif()
     if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_ggml_rmsnorm_batch.cpp")
         add_executable(test_ggml_rmsnorm_batch test/test_ggml_rmsnorm_batch.cpp)
         target_include_directories(test_ggml_rmsnorm_batch PRIVATE ${DFLASH27B_SRC_INCLUDE_DIRS})

server/docs/DS4.md

@@ -231,6 +231,7 @@ The runtime logs the chosen split with a `[deepseek4-split] auto-split:` banner.
 |----------|---------|
 | `DFLASH_DS4_CUDA_LAYERS` | Override the auto-split heuristic and pin the first `N` DeepSeek4 layers to CUDA. The remaining `43 - N` layers run on the Halo shard. |
 | `DFLASH_DS4_TIMING` | Enable DS4 timing logs for the layer-split parent and target-shard daemon. Useful for profiling prefill/decode breakdowns; leave unset for normal runs. |
+| `DFLASH_DS4_ROCTX` | HIP-only, default-off semantic ROCTX ranges for an external rocprof trace. The library is loaded dynamically only when set to `1`, `true`, `yes`, or `on`. |
 | `DFLASH_DS4_SPEC` / `DFLASH_DS4_DRAFT` | Enable DSpark and select its GGUF. |
 | `DFLASH_DS4_DRAFT_BACKEND` / `DFLASH_DS4_DRAFT_GPU` | Backend and device for the in-process drafter. |
 | `DFLASH_DS4_MOE_TP` | Enable routed-expert partitioning. |
@@ -261,6 +262,21 @@ The runtime logs the chosen split with a `[deepseek4-split] auto-split:` banner.
 The old per-expert IPC worker is retired. The `DFLASH_DS4_MOE_TP*` variables
 above configure the in-process route-owner implementation.
 
+### External ROCm traces
+
+Set `DFLASH_DS4_ROCTX=1` when collecting an external rocprof marker trace.
+The runtime emits balanced `ds4.prefill`, `ds4.spec_decode`, and
+`ds4.layer_range` ranges with the applicable mode, token count, layer bounds,
+and device. The marker layer does not use HIP events, synchronize a stream, or
+calculate timings; rocprof remains the timing authority. Non-HIP builds emit
+no markers or ROCTX library calls, and an unset or false value does not load
+ROCTX.
+
+Use the marker-trace option supported by the installed rocprof version (for
+example, `rocprofv3 --marker-trace -- <lucebox command>`), then correlate these
+host scopes with the kernel and memory-copy tracks. A target-machine trace is
+still required to assess instrumentation perturbation.
+
 ## DSpark Speculative Decode
 
 DeepSeek4 uses the shared DFlash DSpark head implementation together with a

server/docs/ENVIRONMENT.md

@@ -42,6 +42,7 @@ consolidation of this list into CLI flags is tracked as follow-up work.
 | `GGML_SCHED_PROFILE` / `GGML_SCHED_PROFILE_MIN_SPLITS` | unset / 1 | DEBUG: report scheduler splits, copy volume, submission time, and source/destination synchronization time. |
 | `DFLASH_DS4_TP_FUSED_CACHE_SLOTS` | 2 | BURN-IN: number of heterogeneous verifier schedulers retained; higher values retain substantially more scratch on both GPUs. |
 | `DFLASH_DS4_VERIFY_FORCE_GRAPH_REPLAY` | unset | OPT-IN: bypass graph property scans only after warmup; scheduler-generation checks remain mandatory. |
+| `DFLASH_DS4_ROCTX` | unset | DEBUG: on HIP builds, dynamically load ROCTX and emit semantic DS4 prefill, speculative-decode, and layer-range markers for external rocprof traces. No events, timing, or device synchronization are added. |
 | `GGML_DS4_FA_SERIAL_INDEX_SCAN` | unset | DEBUG/A-B: restore the serial indexed-attention mask scan instead of the long-context HIP parallel scan. |
 | `DFLASH_MOE_PREFILL_PERSISTENT_OWNER_ALLOC` | 1 for qualified long heterogeneous prefill | KILL SWITCH: =0 restores per-layer route/owner scratch allocation. |
 | `DFLASH_MOE_TP_*` / `DFLASH_MOE_HYBRID_PREFILL_EAGER` | unset | BURN-IN: model-neutral names for common heterogeneous-MoE scheduling and kernel policy. Existing `DFLASH_DS4_*` names remain compatibility aliases. |
@@ -107,6 +108,7 @@ consolidation of this list into CLI flags is tracked as follow-up work.
 - `DFLASH_DS4_MOE_TP_INPROC` - deepseek4_backend.cpp
 - `DFLASH_DS4_MOE_TP_PEER_HOT` - deepseek4_backend.cpp
 - `DFLASH_DS4_ROUTING_STATS_OUT` - deepseek4_backend.cpp
+- `DFLASH_DS4_ROCTX` - deepseek4_roctx.cpp
 - `DFLASH_DS4_SEQ_VERIFY` - deepseek4_dspark_spec.cpp
 - `DFLASH_DS4_SPEC` - deepseek4_backend.cpp
 - `DFLASH_DS4_SPEC_REFERENCE_EXACT` - deepseek4_dspark_spec.cpp

server/src/common/inference_phase.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include <cstdint>
+
+namespace dflash::common {
+
+enum class InferencePhase : int32_t {
+    Unspecified = 0,
+    Exact = 1,
+    Dense = 2,
+    Sparse = 3,
+    Decode = 4,
+    Verify = 5,
+    ReferenceExact = 6,
+    Sequential = 7,
+    Batched = 8,
+};
+
+constexpr int32_t inference_phase_wire_value(InferencePhase phase) {
+    return static_cast<int32_t>(phase);
+}
+
+constexpr bool inference_phase_from_wire_value(int32_t value,
+                                                InferencePhase & phase) {
+    if (value < inference_phase_wire_value(InferencePhase::Unspecified) ||
+        value > inference_phase_wire_value(InferencePhase::Batched)) {
+        return false;
+    }
+    phase = static_cast<InferencePhase>(value);
+    return true;
+}
+
+} // namespace dflash::common

server/src/common/target_shard_ipc.cpp

@@ -231,6 +231,7 @@ bool TargetShardIpcSession::forward(
     const size_t bytes = boundary_activation.size() * sizeof(float);
     const int want_argmax = req.want_argmax ? 1 : 0;
     const int want_logits = req.want_logits ? 1 : 0;
+    const int32_t semantic_phase = inference_phase_wire_value(req.semantic_phase);
     const int forward_ubatch = ubatch > 0 ? ubatch : n_tokens;
     std::vector<int32_t> empty_ids;
     const std::vector<int32_t> * ids = token_ids ? token_ids : &empty_ids;
@@ -243,14 +244,16 @@ bool TargetShardIpcSession::forward(
                          bytes, process_.shared_payload_capacity());
             return false;
         }
-        std::fprintf(cmd, "forward_shared %d %d %d %d %zu %" PRIu64 " %d %d %d\n",
+        std::fprintf(cmd, "forward_shared %d %d %d %d %zu %" PRIu64 " %d %d %d %d\n",
                      base_pos, n_tokens, want_argmax, want_logits, bytes, seq,
-                     token_ids ? 1 : 0, forward_ubatch, (int)ids->size());
+                     token_ids ? 1 : 0, forward_ubatch, (int)ids->size(),
+                     semantic_phase);
         std::fflush(cmd);
     } else if (payload_fd >= 0) {
-        std::fprintf(cmd, "forward_pipe %d %d %d %d %zu %d %d %d\n",
+        std::fprintf(cmd, "forward_pipe %d %d %d %d %zu %d %d %d %d\n",
                      base_pos, n_tokens, want_argmax, want_logits, bytes,
-                     token_ids ? 1 : 0, forward_ubatch, (int)ids->size());
+                     token_ids ? 1 : 0, forward_ubatch, (int)ids->size(),
+                     semantic_phase);
         std::fflush(cmd);
         if (!write_exact_fd(payload_fd, boundary_activation.data(), bytes)) {
             std::fprintf(stderr, "target-shard payload write failed\n");

server/src/common/target_shard_ipc.h

@@ -3,6 +3,7 @@
 #pragma once
 
 #include "backend_ipc.h"
+#include "inference_phase.h"
 
 #include "ggml.h"
 #include "ggml-backend.h"
@@ -39,6 +40,7 @@ struct TargetShardForwardRequest {
     int ubatch = 0;
     bool want_argmax = false;
     bool want_logits = false;
+    InferencePhase semantic_phase = InferencePhase::Unspecified;
 };
 
 struct TargetShardForwardResponse {

server/src/common/target_shard_ipc_daemon.cpp

@@ -11,6 +11,7 @@
 #include <cstdio>
 #include <cstring>
 #include <iostream>
+#include <limits>
 #include <sstream>
 #include <string>
 #include <vector>
@@ -39,6 +40,21 @@ bool stream_daemon_status(const TargetShardDaemonCallbacks & callbacks,
     return true;
 }
 
+bool forward_payload_bytes(int hidden, int n_tokens, size_t & bytes) {
+    if (hidden <= 0 || n_tokens <= 0 ||
+        static_cast<size_t>(n_tokens) >
+            std::numeric_limits<size_t>::max() / static_cast<size_t>(hidden)) {
+        return false;
+    }
+    const size_t elements = static_cast<size_t>(n_tokens) *
+        static_cast<size_t>(hidden);
+    if (elements > std::numeric_limits<size_t>::max() / sizeof(float)) {
+        return false;
+    }
+    bytes = elements * sizeof(float);
+    return true;
+}
+
 }  // namespace
 
 int run_target_shard_ipc_daemon_loop(
@@ -117,34 +133,40 @@ int run_target_shard_ipc_daemon_loop(
         int has_token_ids = 0;
         int forward_ubatch = 0;
         int token_count = 0;
+        int32_t semantic_phase_value = 0;
         size_t bytes = 0;
         bool payload_ok = false;
+        bool framing_fields_present = false;
+        bool command_fields_ok = false;
+        bool pipe_forward = false;
+        bool shared_forward = false;
+        uint64_t shared_seq = 0;
 
         if (cmd == "forward_pipe") {
+            pipe_forward = true;
             iss >> base_pos >> n_tokens >> want_argmax >> want_logits >> bytes >>
                 has_token_ids >> forward_ubatch >> token_count;
-            const size_t expected_bytes =
-                (size_t)std::max(0, n_tokens) * (size_t)hidden * sizeof(float);
-            if (payload_fd >= 0 && n_tokens > 0 && bytes == expected_bytes) {
-                host_act.assign(bytes / sizeof(float), 0.0f);
-                payload_ok = read_exact_fd(payload_fd, host_act.data(), bytes);
+            framing_fields_present = static_cast<bool>(iss);
+            if (framing_fields_present) {
+                iss >> semantic_phase_value;
+                const bool phase_present = static_cast<bool>(iss);
+                if (phase_present) {
+                    iss >> std::ws;
+                    command_fields_ok = iss.eof();
+                }
             }
         } else if (cmd == "forward_shared") {
-            uint64_t seq = 0;
-            iss >> base_pos >> n_tokens >> want_argmax >> want_logits >> bytes >> seq >>
-                has_token_ids >> forward_ubatch >> token_count;
-            const size_t expected_bytes =
-                (size_t)std::max(0, n_tokens) * (size_t)hidden * sizeof(float);
-            const auto * header =
-                static_cast<const BackendIpcSharedPayloadHeader *>(shared_payload);
-            if (shared_payload && shared_payload != MAP_FAILED && shared_payload_data &&
-                seq != 0 && n_tokens > 0 && bytes == expected_bytes &&
-                backend_ipc_payload_in_bounds(0, bytes, shared_payload_capacity) &&
-                backend_ipc_shared_payload_header_matches(
-                    header, seq, static_cast<uint64_t>(bytes))) {
-                host_act.assign(bytes / sizeof(float), 0.0f);
-                std::memcpy(host_act.data(), shared_payload_data, bytes);
-                payload_ok = true;
+            shared_forward = true;
+            iss >> base_pos >> n_tokens >> want_argmax >> want_logits >> bytes >>
+                shared_seq >> has_token_ids >> forward_ubatch >> token_count;
+            framing_fields_present = static_cast<bool>(iss);
+            if (framing_fields_present) {
+                iss >> semantic_phase_value;
+                const bool phase_present = static_cast<bool>(iss);
+                if (phase_present) {
+                    iss >> std::ws;
+                    command_fields_ok = iss.eof();
+                }
             }
         } else {
             if (cmd == "reset_request_state") {
@@ -194,19 +216,46 @@ int run_target_shard_ipc_daemon_loop(
             continue;
         }
 
-        bool ok = payload_ok && base_pos >= 0 && n_tokens > 0;
-        if (ok && has_token_ids) {
-            ok = payload_fd >= 0 && token_count == n_tokens;
-            if (ok) {
-                token_ids.assign((size_t)n_tokens, 0);
-                ok = read_exact_fd(payload_fd, token_ids.data(),
-                                   sizeof(int32_t) * token_ids.size());
+        size_t expected_bytes = 0;
+        const bool framing_ok = framing_fields_present && base_pos >= 0 &&
+            forward_ubatch >= 0 && (want_argmax == 0 || want_argmax == 1) &&
+            (want_logits == 0 || want_logits == 1) &&
+            (has_token_ids == 0 || has_token_ids == 1) &&
+            forward_payload_bytes(hidden, n_tokens, expected_bytes) &&
+            bytes == expected_bytes &&
+            token_count == (has_token_ids ? n_tokens : 0);
+
+        if (framing_ok && pipe_forward && payload_fd >= 0) {
+            host_act.assign(bytes / sizeof(float), 0.0f);
+            payload_ok = read_exact_fd(payload_fd, host_act.data(), bytes);
+        } else if (framing_ok && shared_forward) {
+            const auto * header =
+                static_cast<const BackendIpcSharedPayloadHeader *>(shared_payload);
+            if (shared_payload && shared_payload != MAP_FAILED &&
+                shared_payload_data && shared_seq != 0 &&
+                backend_ipc_payload_in_bounds(0, bytes, shared_payload_capacity) &&
+                backend_ipc_shared_payload_header_matches(
+                    header, shared_seq, static_cast<uint64_t>(bytes))) {
+                host_act.assign(bytes / sizeof(float), 0.0f);
+                std::memcpy(host_act.data(), shared_payload_data, bytes);
+                payload_ok = true;
             }
-        } else {
-            token_ids.clear();
-            ok = ok && token_count == 0;
         }
 
+        const int token_fd = payload_fd >= 0 ? payload_fd : stream_fd;
+        bool token_ids_ok = !has_token_ids;
+        token_ids.clear();
+        if (framing_ok && has_token_ids) {
+            token_ids.assign(static_cast<size_t>(n_tokens), 0);
+            token_ids_ok = read_exact_fd(
+                token_fd, token_ids.data(), sizeof(int32_t) * token_ids.size());
+        }
+
+        InferencePhase semantic_phase = InferencePhase::Unspecified;
+        const bool semantic_phase_ok = command_fields_ok &&
+            inference_phase_from_wire_value(semantic_phase_value, semantic_phase);
+        bool ok = framing_ok && payload_ok && token_ids_ok && semantic_phase_ok;
+
         TargetShardDaemonForwardResponse resp;
         if (ok) {
             TargetShardDaemonForwardRequest req;
@@ -215,13 +264,15 @@ int run_target_shard_ipc_daemon_loop(
             req.ubatch = forward_ubatch > 0 ? forward_ubatch : n_tokens;
             req.want_argmax = want_argmax != 0;
             req.want_logits = want_logits != 0;
+            req.semantic_phase = semantic_phase;
             req.boundary_activation = &host_act;
             req.token_ids = has_token_ids ? &token_ids : nullptr;
             ok = callbacks.forward(req, resp);
         }
 
         const int32_t status = ok ? 0 : -1;
         if (!write_exact_fd(stream_fd, &status, sizeof(status))) break;
+        if (!framing_ok) break;
         if (!ok) continue;
 
         if (!write_exact_fd(stream_fd, &resp.last_tok, sizeof(resp.last_tok))) break;

server/src/common/target_shard_ipc_daemon.h

@@ -2,6 +2,8 @@
 
 #pragma once
 
+#include "inference_phase.h"
+
 #include <cstddef>
 #include <cstdint>
 #include <functional>
@@ -15,6 +17,7 @@ struct TargetShardDaemonForwardRequest {
     int ubatch = 0;
     bool want_argmax = false;
     bool want_logits = false;
+    InferencePhase semantic_phase = InferencePhase::Unspecified;
     const std::vector<float> * boundary_activation = nullptr;
     const std::vector<int32_t> * token_ids = nullptr;
 };

server/src/deepseek4/deepseek4_backend.cpp

@@ -1,4 +1,5 @@
 // DeepSeek4Backend implementation — AR-only decode, chunked prefill.
+#include "deepseek4_roctx.h"
 
 #include "deepseek4_backend.h"
 #include "deepseek4_internal.h"
@@ -1311,6 +1312,12 @@ int DeepSeek4Backend::do_prefill(const std::vector<int32_t> & tokens,
                                   int kv_offset,
                                   int snap_slot,
                                   int snap_pos) {
+    const InferencePhase phase = deepseek4_roctx_prefill_phase(
+        prefill_attention_mode_name(cfg_.prefill_mode));
+    const DeepSeek4RoctxPhaseScope roctx_phase(phase);
+    const DeepSeek4RoctxRange roctx_range(
+        "ds4.prefill",
+        {phase, static_cast<int>(tokens.size()), 0, w_.n_layer, cfg_.device.gpu});
     // The all-hot layer-range path supports causal chunked prefill. The
     // optimized graph snapshots the previous raw SWA window, attends over
     // that snapshot plus the current ubatch, and commits only the final SWA
@@ -1540,6 +1547,7 @@ bool DeepSeek4Backend::do_decode(int committed, int n_gen,
                                   const DaemonIO & io,
                                   const BudgetHook & budget_hook,
                                   bool * forced_close_out) {
+    const DeepSeek4RoctxPhaseScope roctx_phase(InferencePhase::Decode);
     if (forced_close_out) *forced_close_out = false;
     const bool timing = env_flag_enabled("DFLASH_DS4_TIMING");
     const auto phase_t0 = Clock::now();

server/src/deepseek4/deepseek4_dspark_spec.cpp

@@ -23,6 +23,7 @@
 
 #include "deepseek4_dspark.h"
 #include "deepseek4_internal.h"
+#include "deepseek4_roctx.h"
 #include "internal.h"
 #include "common/dspark_head.h"
 
@@ -585,6 +586,7 @@ bool deepseek4_dspark_verify_forward(ggml_backend_t backend,
                                      MoeHybridStorage * moe_hybrid,
                                      MoeExpertComputeRuntime * expert_runtime,
                                      MoeHybridRoutingStats * routing_stats) {
+    const DeepSeek4RoctxPhaseScope roctx_phase(InferencePhase::Verify);
     std::vector<float> hc_state;
     std::vector<float> all_logits;
     std::vector<float> last_logits;
@@ -664,6 +666,12 @@ bool run_deepseek4_dspark_spec_decode(
         spec_env_flag("DFLASH_DS4_FULL_SNAP");
     const bool seq_verify_mode = reference_exact ||
         spec_env_flag("DFLASH_DS4_SEQ_VERIFY");
+    const InferencePhase roctx_phase = reference_exact
+        ? InferencePhase::ReferenceExact
+        : (seq_verify_mode ? InferencePhase::Sequential : InferencePhase::Batched);
+    const DeepSeek4RoctxRange roctx_range(
+        "ds4.spec_decode",
+        {roctx_phase, n_gen, 0, target_w.n_layer, device});
     const bool async_rollback = spec_env_flag("DFLASH_DS4_ASYNC_ROLLBACK");
     const bool pinned_rollback = spec_env_flag("DFLASH_DS4_PINNED_ROLLBACK");
     const bool draft_overlap_probe =