From 3546a748d3db531d6ad5c481895ad9bfbd34afe8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 1 Nov 2026 17:33:12 +0000 Subject: [PATCH 01/11] Add a GB10 latency kit that compares fixed stalls with NAK. Dependent-chain kernels, a NAK diff, and a Spark runner that holds flock /tmp/aien-gb10.lock only around the launch (timeout 600). Measured cells stay empty until that run; the host self-test does not invent them. Co-authored-by: aien-spark --- Makefile | 11 +- tests/gpu_latmeas_test.c | 1132 ++++++++++++++++++++++++++++++++ tools/latmeas/REPRODUCE.md | 53 ++ tools/latmeas/catalog.json | 246 +++++++ tools/latmeas/diff_nak.py | 170 +++++ tools/latmeas/fill_template.py | 218 ++++++ tools/latmeas/run_spark.sh | 36 + tools/latmeas/selftest.sh | 18 + tools/latmeas/template.md | 30 + 9 files changed, 1913 insertions(+), 1 deletion(-) create mode 100644 tests/gpu_latmeas_test.c create mode 100644 tools/latmeas/REPRODUCE.md create mode 100644 tools/latmeas/catalog.json create mode 100755 tools/latmeas/diff_nak.py create mode 100755 tools/latmeas/fill_template.py create mode 100755 tools/latmeas/run_spark.sh create mode 100755 tools/latmeas/selftest.sh create mode 100644 tools/latmeas/template.md diff --git a/Makefile b/Makefile index 1d14df05..db2e8033 100644 --- a/Makefile +++ b/Makefile @@ -161,6 +161,15 @@ $(GPU_EW_TEST): tests/gpu_elementwise_test.c src/omega_gpu_elementwise_api.h $(O test-gpu-elementwise: $(GPU_EW_TEST) ./$(GPU_EW_TEST) --host-only +# GB10 fixed-latency chains versus Mesa NAK. test-gpu-latmeas is host-only +# (encode, disassemble, oracle, diff). The chip gate is tools/latmeas/run_spark.sh. +.PHONY: test-gpu-latmeas +GPU_LATMEAS_TEST = $(OUT_DIR)/gpu_latmeas_test +$(GPU_LATMEAS_TEST): tests/gpu_latmeas_test.c $(OUT_DIR)/libomega_gpu.a + $(CC) $(CFLAGS) -ffp-contract=off -o $@ tests/gpu_latmeas_test.c $(OUT_DIR)/libomega_gpu.a -lpthread -lm +test-gpu-latmeas: $(GPU_LATMEAS_TEST) + sh tools/latmeas/selftest.sh $(GPU_LATMEAS_TEST) + # omega #308: structured warp reconvergence (BSSY/BSYNC regions). test-gpu-reconv is # host-only (encoder goldens, region bookkeeping, refusals, nvdisasm listing, every probe # kernel through the SIMT warp simulator tests/bw_warp_sim.h against the oracle, mutation @@ -532,7 +541,7 @@ test-omega-slices: # Every host-only check: no GPU, no device opened. The chip gates are listed in README.md. # Portable commands and the AArch64-only rows are separated in docs/OMEGA_HOST_TESTS.md. -HOST_TESTS = test-drift test-gpu-matmul-api test-gpu-elementwise test-gpu-reconv test-gpu-attention test-gpu-moe-window \ +HOST_TESTS = test-drift test-gpu-matmul-api test-gpu-elementwise test-gpu-latmeas test-gpu-reconv test-gpu-attention test-gpu-moe-window \ test-gpu-fp8 test-gpu-decode-attn test-gpu-router test-gpu-bf16-gemv test-gpu-fp8gemm test-gpu-tern-sched test-gpu-gpr-boundary \ test-gpu-cmdlist test-gpu-decode-graph test-gpu-code-alloc test-gpu-engine-compile test-gpu-engine \ test-gpu-engine-mutants test-gpu-engine-backend-compile test-gpu-engine-prod-build test-gpu-engine-sim \ diff --git a/tests/gpu_latmeas_test.c b/tests/gpu_latmeas_test.c new file mode 100644 index 00000000..7fa86565 --- /dev/null +++ b/tests/gpu_latmeas_test.c @@ -0,0 +1,1132 @@ +/* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ +/* gpu_latmeas_test: dependent-chain latency kernels for GB10 (sm_121). + * + * The chip path is tools/latmeas/run_spark.sh. It encodes every kernel before + * taking the shared lock, then launches them. This file does not invent a + * measured latency: --self-test and --emit never open a device, and a launch + * writes only numbers the chip returned. + * + * Clock. NAK reads the shader clock with CS2R on SM >= 50, not S2R + * (nak/from_nir.rs, SM >= 50 branch on NAK_SV_CLOCK). nak_private.h names + * NAK_SV_CLOCK_LO = 0x50 and NAK_SV_CLOCK_HI = 0x51. sm70_encode.rs encodes + * OpCS2R as opcode 0x805, destination at bits 16..24, index at bits 72..80, + * and bit 80 set for a 64-bit read. A 32-bit CS2R is Dualalu (fixed) in + * sm120_instr_latencies.rs. The encoder has S2R and not CS2R, so each clock + * site is emitted as S2R immediate 0x50 and the low 12 bits are rewritten to + * 0x805. The index byte is left as 0x50. Two such sites bracket the chain. + * Mesa commit 01bab9302a6bf0a082c7ba8c49d74427cf10b0cf. + * + * What is compared to NAK. Fixed-latency producers do not scoreboard a GPR + * result (src/omega_numeric.c): the stall field is the wait. Each chain is + * launched once per stall. The result has to match the host oracle (or, for + * HMMA, the stall-36 result of the same kernel). The reported stall is the + * smallest S such that S and every larger S matched on all runs. Cycles are + * the 32-bit clock delta around that same chain, median and max-min over the + * runs. An empty loop with the same trip count is the overhead control. + * + * One warp, 32 threads, uniform control flow inside the timed region. Only + * lane 0 stores. Mesa commit and the NAK line citations live in + * tools/latmeas/diff_nak.py. + */ +#include "omega_blackwell_codegen.h" +#include "omega_gpu_session.h" +#include "omega_sm121_disasm.h" +#include +#include +#include +#include +#include +#include +#include + +#define NOREG (-1) +#define SB_NONE 7u +#define SB_GPR 0u +#define SB_RD 1u +#define CLOCK_LO 0x50u /* NAK_SV_CLOCK_LO, nak_private.h */ +#define CS2R_OPC 0x805u +#define S2R_OPC 0x919u +#define POISON 0xA5A5A5A5u +#define PRED_POISON 0xDEADBEEFu +#define SWEEP_MAX 36 +#define CODE_CAP 8192u + +enum { + RUNS = 11, + ALU_ITERS = 64, + ALU_UNROLL = 8, + MIXED_ITERS = 64, + PRED_ITERS = 64, + FMNMX_ITERS = 32, + WIDE_ITERS = 64, + HMMA_ITERS = 1, + SWEEP_FIXED = 15, + SWEEP_HMMA = 36 +}; +_Static_assert(RUNS % 2 == 1, "median uses the middle of an odd run count"); +_Static_assert(SWEEP_HMMA <= SWEEP_MAX, "sweep fits the sample array"); + +static const char *MESA_COMMIT = "01bab9302a6bf0a082c7ba8c49d74427cf10b0cf"; + +typedef struct { + const char *id; + unsigned iters; + unsigned unroll; + unsigned sweep; + unsigned ops; + int href; /* 1: oracle is the result at sweep, not a host formula */ +} CaseDef; + +static const CaseDef CASES[] = { + {"alu_alu", ALU_ITERS, ALU_UNROLL, SWEEP_FIXED, ALU_ITERS * ALU_UNROLL, 0}, + {"fma_fma", ALU_ITERS, ALU_UNROLL, SWEEP_FIXED, ALU_ITERS * ALU_UNROLL, 0}, + {"mixed", MIXED_ITERS, 1, SWEEP_FIXED, MIXED_ITERS * 2, 0}, + {"pred", PRED_ITERS, 1, SWEEP_FIXED, PRED_ITERS, 0}, + {"fmnmx_alu", FMNMX_ITERS, 1, SWEEP_FIXED, FMNMX_ITERS, 0}, + {"imad_wide_lo", WIDE_ITERS, 1, SWEEP_FIXED, WIDE_ITERS, 0}, + {"hmma_reader", HMMA_ITERS, 1, SWEEP_HMMA, HMMA_ITERS, 1}, +}; +#define NCASES ((int)(sizeof CASES / sizeof CASES[0])) + +static BlackwellIRProgram g_prog; + +typedef struct { + BlackwellIRProgram *p; + int desc; + uint32_t pending; + int err; +} B; + +static uint32_t cw(unsigned stall, unsigned wbar, unsigned rbar, unsigned wait, int yield) { + uint32_t v = (stall << 9) | (wbar << 14) | (rbar << 17) | (wait << 20); + if (yield) v |= 0x2000u; + return v; +} + +static int V(B *b) { + int v = omega_bw_ir_alloc_vreg(b->p); + if (v < 0) b->err = -1; + return v; +} +static int V64(B *b) { + int v = omega_bw_ir_alloc_vreg64(b->p); + if (v < 0) b->err = -1; + return v; +} +static int V128(B *b) { + int v = omega_bw_ir_alloc_vreg128(b->p); + if (v < 0) b->err = -1; + return v; +} + +static int add(B *b, BlackwellIROpcode op, int dst, int s1, int s2, int s3, uint32_t imm, + uint32_t control, int p0, int pnot, int dst_sub, int s2_sub) { + BlackwellIRInsn in; + memset(&in, 0, sizeof in); + in.op = op; + in.dst_vreg = dst; + in.src1_vreg = s1; + in.src2_vreg = s2; + in.src3_vreg = s3; + in.ureg = NOREG; + in.imm = imm; + in.control = control; + in.predicate_p0 = p0; + in.predicate_not = pnot; + in.dst_subreg = (uint8_t)dst_sub; + in.src2_subreg = (uint8_t)s2_sub; + if (op == BW_IR_LDCU64) in.is_uniform = 1; + if (op == BW_IR_STG_E) in.ureg = b->desc; + if (omega_bw_ir_append(b->p, &in) < 0) b->err = -1; + return (int)b->p->count - 1; +} + +static void movi(B *b, int dst, uint32_t imm, int sub) { + uint32_t wait = b->pending; + add(b, BW_IR_MOV_IMM, dst, NOREG, NOREG, NOREG, imm, cw(6, SB_NONE, SB_NONE, wait, 0), 0, 0, sub, 0); + b->pending = 0; +} +static void movz(B *b, int dst, int sub) { + uint32_t wait = b->pending; + add(b, BW_IR_MOV_RZ, dst, NOREG, NOREG, NOREG, 0, cw(6, SB_NONE, SB_NONE, wait, 0), 0, 0, sub, 0); + b->pending = 0; +} +static void iadd(B *b, int dst, int a, int c, unsigned stall, int dst_sub, int src_sub) { + /* src_sub selects a subregister of the source that lives in src2's slot? IADD3's + * first GPR source is src1. Subregister is carried by editing the insn after append + * when src_sub != 0: the encoder adds src1_subreg. */ + uint32_t wait = b->pending; + int at = add(b, BW_IR_IADD3, dst, a, c, NOREG, 0, cw(stall, SB_NONE, SB_NONE, wait, 0), 0, 0, dst_sub, 0); + if (at >= 0 && src_sub) b->p->insns[at].src1_subreg = (uint8_t)src_sub; + b->pending = 0; +} +static void ffma(B *b, int dst, int a, int c, int d, unsigned stall) { + uint32_t wait = b->pending; + add(b, BW_IR_FFMA, dst, a, c, d, 0, cw(stall, SB_NONE, SB_NONE, wait, 0), 0, 0, 0, 0); + b->pending = 0; +} +static void fadd(B *b, int dst, int a, int asub, int c, unsigned stall) { + uint32_t wait = b->pending; + int at = add(b, BW_IR_FADD, dst, a, c, NOREG, 0, cw(stall, SB_NONE, SB_NONE, wait, 0), 0, 0, 0, 0); + if (at >= 0 && asub) b->p->insns[at].src1_subreg = (uint8_t)asub; + b->pending = 0; +} +static void fmnmx(B *b, int dst, int a, int c, unsigned stall) { + uint32_t wait = b->pending; + add(b, BW_IR_FMNMX_MIN, dst, a, c, NOREG, 0, cw(stall, SB_NONE, SB_NONE, wait, 0), 0, 0, 0, 0); + b->pending = 0; +} +static void isetp_ge(B *b, int a, int c, unsigned stall) { + uint32_t wait = b->pending; + add(b, BW_IR_ISETP_GE_U32, NOREG, a, c, NOREG, 0, cw(stall, SB_NONE, SB_NONE, wait, 0), 0, 0, 0, 0); + b->pending = 0; +} +static int bra(B *b, int p0, int pnot, unsigned stall) { + uint32_t wait = b->pending; + int at = add(b, BW_IR_BRA, NOREG, NOREG, NOREG, NOREG, 0, cw(stall, SB_NONE, SB_NONE, wait, 0), p0, pnot, 0, 0); + b->pending = 0; + return at; +} +static void patch_bra(B *b, int at, int target) { + if (at < 0 || target < 0 || (size_t)at >= b->p->count) { + b->err = -1; + return; + } + b->p->insns[at].imm = (uint32_t)(int32_t)(target - at); +} +static void nop_stall(B *b, unsigned stall) { + add(b, BW_IR_NOP, NOREG, NOREG, NOREG, NOREG, 0, cw(stall, SB_NONE, SB_NONE, 0, 0), 0, 0, 0, 0); +} +/* Producer stall is at most 15. Extra cycles are NOP stalls so the consumer + * issues `gap` cycles after the producer. */ +static void gap_nops(B *b, unsigned gap) { + while (gap) { + unsigned s = gap > 15u ? 15u : gap; + nop_stall(b, s); + gap -= s; + } +} +static unsigned stall_head(unsigned gap) { return gap > 15u ? 15u : gap; } + +static void clock_read(B *b, int dst) { + uint32_t wait = b->pending; + /* Stall 15, no write barrier: the 32-bit CS2R is fixed-latency, and the + * timed body uses other registers. Rewritten to CS2R after encoding. */ + add(b, BW_IR_S2R, dst, NOREG, NOREG, NOREG, CLOCK_LO, cw(15, SB_NONE, SB_NONE, wait, 0), 0, 0, 0, 0); + b->pending = 0; +} + +static void s2r_sb(B *b, int dst, uint32_t sr) { + uint32_t wait = b->pending; + add(b, BW_IR_S2R, dst, NOREG, NOREG, NOREG, sr, cw(4, SB_GPR, SB_NONE, wait, 1), 0, 0, 0, 0); + b->pending = 1u << SB_GPR; +} + +static void pin_live(BlackwellIRProgram *p) { + int end = (int)p->count - 1; + if (end < 0) return; + for (int v = 0; v < p->regalloc.num_vregs; v++) { + OmegaLiveInterval *iv = &p->regalloc.intervals[v]; + if (iv->first_def >= 0 && iv->last_use < end) iv->last_use = end; + } +} + +static uint32_t fbits(float f) { + uint32_t u; + memcpy(&u, &f, 4); + return u; +} + +static uint32_t fmnmx_expect(void) { + uint32_t acc = 0; + float cur = 1.0f; + for (int i = 0; i < FMNMX_ITERS; i++) { + uint32_t t; + memcpy(&t, &cur, 4); + acc += t; + cur += 1.0f; + } + return acc; +} + +static int expect_abs(const CaseDef *c, uint32_t *result, uint32_t *aux) { + if (c->href) return -1; + *aux = 1; + if (!strcmp(c->id, "alu_alu")) { + *result = c->iters * c->unroll; + return 0; + } + if (!strcmp(c->id, "fma_fma")) { + *result = fbits((float)(c->iters * c->unroll)); + return 0; + } + if (!strcmp(c->id, "mixed")) { + *result = fbits((float)c->iters); + return 0; + } + if (!strcmp(c->id, "pred")) { + *result = c->iters; + return 0; + } + if (!strcmp(c->id, "fmnmx_alu")) { + *result = fmnmx_expect(); + return 0; + } + if (!strcmp(c->id, "imad_wide_lo")) { + *result = c->iters; + *aux = 0; + return 0; + } + return -1; +} + +static void prologue(B *b, int *ptr) { + b->desc = omega_bw_ir_alloc_uvreg64(b->p); + if (b->desc < 0) b->err = -1; + *ptr = V64(b); + uint32_t wait = b->pending; + add(b, BW_IR_LDCU64, b->desc, NOREG, NOREG, NOREG, 0x358, cw(4, 2, SB_NONE, wait, 1), 0, 0, 0, 0); + b->pending = 1u << 2; + wait = b->pending; + add(b, BW_IR_LDC64, *ptr, NOREG, NOREG, NOREG, 0x380, cw(4, SB_GPR, SB_NONE, wait, 1), 0, 0, 0, 0); + b->pending = 1u << SB_GPR; +} + +static void store_tail(B *b, int ptr, int t0, int t1, int result, int rsub, int aux, int asub) { + int tid = V(b); + int one = V(b); + movi(b, one, 1, 0); + s2r_sb(b, tid, BW_SR_TID_X); + isetp_ge(b, tid, one, 15); /* P0 = (tid >= 1); lane 0 falls through */ + int skip_br = bra(b, 1, 0, 5); + uint32_t st = cw(4, SB_NONE, SB_RD, 0, 1); + add(b, BW_IR_STG_E, NOREG, ptr, t0, NOREG, 0, st, 0, 0, 0, 0); + add(b, BW_IR_STG_E, NOREG, ptr, t1, NOREG, 4, st, 0, 0, 0, 0); + add(b, BW_IR_STG_E, NOREG, ptr, result, NOREG, 8, st, 0, 0, 0, rsub); + add(b, BW_IR_STG_E, NOREG, ptr, aux, NOREG, 12, st, 0, 0, 0, asub); + int skip = (int)b->p->count; + patch_bra(b, skip_br, skip); + add(b, BW_IR_EXIT, NOREG, NOREG, NOREG, NOREG, 0, cw(5, SB_NONE, SB_NONE, 0, 0), 0, 0, 0, 0); + add(b, BW_IR_BRA, NOREG, NOREG, NOREG, NOREG, 0, 0x000fc000u, 0, 0, 0, 0); +} + +static void loop_ctl(B *b, int i, int nreg, int one, int *loop_back) { + iadd(b, i, i, one, 1, 0, 0); + isetp_ge(b, i, nreg, 14); /* P0 = (i >= n) */ + *loop_back = bra(b, 1, 1, 5); /* @!P0 BRA loop */ +} + +static void body(B *b, const CaseDef *c, unsigned stall, int r, int one, int zero, int fone, int acc, int cur, int tmp, + int rpair, int tpair, int *poison_br) { + unsigned head = stall_head(stall); + unsigned rest = stall - head; + if (!strcmp(c->id, "alu_alu")) { + for (unsigned u = 0; u < c->unroll; u++) iadd(b, r, r, one, head, 0, 0); + } else if (!strcmp(c->id, "fma_fma")) { + for (unsigned u = 0; u < c->unroll; u++) ffma(b, r, fone, fone, r, head); + } else if (!strcmp(c->id, "mixed")) { + /* FFMA then IADD3 of zero: FMA->ALU and, on the next iteration, ALU->FMA. */ + ffma(b, r, fone, fone, r, head); + gap_nops(b, rest); + iadd(b, r, r, zero, head, 0, 0); + } else if (!strcmp(c->id, "pred")) { + /* Setup leaves P0 true. The measured ISETP clears it (0 >= r after r + * has been incremented). A short stall lets @P0 see the leftover true + * and take the poison branch. */ + isetp_ge(b, one, zero, 15); + iadd(b, r, r, one, 15, 0, 0); + isetp_ge(b, zero, r, head); + *poison_br = bra(b, 1, 0, 1); + } else if (!strcmp(c->id, "fmnmx_alu")) { + fmnmx(b, tmp, cur, cur, head); + gap_nops(b, rest); + iadd(b, acc, acc, tmp, 15, 0, 0); + fadd(b, cur, cur, 0, fone, 15); + } else if (!strcmp(c->id, "imad_wide_lo")) { + uint32_t wait = b->pending; + add(b, BW_IR_IMAD_WIDE, tpair, one, NOREG, rpair, 1, cw(head, SB_NONE, SB_NONE, wait, 0), 0, 0, 0, 0); + b->pending = 0; + gap_nops(b, rest); + iadd(b, rpair, tpair, zero, 15, 0, 0); + } else { + b->err = -1; + } + if (strcmp(c->id, "mixed") && strcmp(c->id, "pred") && strcmp(c->id, "fmnmx_alu") && strcmp(c->id, "imad_wide_lo")) + gap_nops(b, rest); +} + +static int build(const CaseDef *c, unsigned stall, unsigned ovh_iters, int overhead, uint8_t *code, size_t cap, + size_t *out_len, uint32_t *gpr) { + if (!overhead && !c) return -1; + if (!overhead && (stall < 1u || stall > c->sweep)) return -1; + memset(&g_prog, 0, sizeof g_prog); + omega_bw_ir_init(&g_prog); + B b; + memset(&b, 0, sizeof b); + b.p = &g_prog; + b.desc = NOREG; + int ptr = NOREG; + prologue(&b, &ptr); + int one = V(&b), zero = V(&b), fone = V(&b), i = V(&b), nreg = V(&b); + int r = V(&b), acc = V(&b), cur = V(&b), tmp = V(&b), t0 = V(&b), t1 = V(&b), aux = V(&b); + int rpair = NOREG, tpair = NOREG, dquad = NOREG, aquad = NOREG, bpair = NOREG, cquad = NOREG; + unsigned iters = overhead ? ovh_iters : c->iters; + movi(&b, one, 1, 0); + movz(&b, zero, 0); + movi(&b, fone, 0x3f800000u, 0); + movi(&b, nreg, iters, 0); + movz(&b, i, 0); + movz(&b, r, 0); + movz(&b, acc, 0); + movi(&b, cur, 0x3f800000u, 0); + movi(&b, aux, 1, 0); + if (!overhead && !strcmp(c->id, "imad_wide_lo")) { + rpair = V64(&b); + tpair = V64(&b); + movz(&b, rpair, 0); + movz(&b, rpair, 1); + movz(&b, tpair, 0); + movz(&b, tpair, 1); + } + if (!overhead && !strcmp(c->id, "hmma_reader")) { + aquad = V128(&b); + bpair = V64(&b); + cquad = V128(&b); + dquad = V128(&b); + for (int s = 0; s < 4; s++) movi(&b, aquad, 0x3f803f80u, s); + for (int s = 0; s < 2; s++) movi(&b, bpair, 0x3f803f80u, s); + for (int s = 0; s < 4; s++) movz(&b, cquad, s); + for (int s = 0; s < 4; s++) movz(&b, dquad, s); + movi(&b, acc, 0xffffffffu, 0); + } + clock_read(&b, t0); + int loop = (int)g_prog.count; + int poison_br = -1; + int back = -1; + if (!overhead && !strcmp(c->id, "hmma_reader")) { + unsigned head = stall_head(stall); + unsigned rest = stall - head; + uint32_t wait = b.pending; + add(&b, BW_IR_HMMA_BF16, dquad, aquad, bpair, cquad, 0, cw(head, SB_NONE, SB_NONE, wait, 0), 0, 0, 0, 0); + b.pending = 0; + gap_nops(&b, rest); + fadd(&b, acc, dquad, 0, zero, 15); /* acc = d0 + 0 */ + if (c->iters > 1) { + /* HMMA_ITERS is 1: a repeated identical HMMA would make a stale + * read equal the previous correct write. */ + b.err = -1; + } + } else if (!overhead) { + body(&b, c, stall, r, one, zero, fone, acc, cur, tmp, rpair, tpair, &poison_br); + } + if (!(c && !overhead && c->iters == 1 && !strcmp(c->id, "hmma_reader"))) { + loop_ctl(&b, i, nreg, one, &back); + patch_bra(&b, back, loop); + } + int join_br = -1; + if (poison_br >= 0) join_br = bra(&b, 0, 0, 5); + if (poison_br >= 0) { + int poison = (int)g_prog.count; + movi(&b, r, PRED_POISON, 0); + patch_bra(&b, poison_br, poison); + patch_bra(&b, join_br, (int)g_prog.count); + } + clock_read(&b, t1); + int result = r; + int rsub = 0; + int auxv = aux; + int asub = 0; + if (!overhead && !strcmp(c->id, "fmnmx_alu")) result = acc; + if (!overhead && !strcmp(c->id, "imad_wide_lo")) { + result = rpair; + rsub = 0; + auxv = rpair; + asub = 1; + } + if (!overhead && !strcmp(c->id, "hmma_reader")) { + result = acc; + auxv = dquad; /* late read of d1, diagnostic, after the timed region */ + asub = 1; + } + if (overhead) result = i; + store_tail(&b, ptr, t0, t1, result, rsub, auxv, asub); + if (b.err) return -1; + pin_live(&g_prog); + if (omega_bw_regalloc_solve(&g_prog) != 0) return -1; + if (cap < omega_bw_program_code_bytes(&g_prog)) return -1; + if (omega_bw_encode_program(&g_prog, code, cap, out_len) != 0) return -1; + int clocks = 0; + for (size_t at = 0; at + 16 <= *out_len && at / 16 < g_prog.count; at += 16) { + uint32_t w0, w2; + memcpy(&w0, code + at, 4); + memcpy(&w2, code + at + 8, 4); + if ((w0 & 0xfffu) == S2R_OPC && ((w2 >> 8) & 0xffu) == CLOCK_LO) { + w0 = (w0 & ~0xfffu) | CS2R_OPC; + memcpy(code + at, &w0, 4); + clocks++; + } + } + if (clocks != 2) return -1; + if (gpr) *gpr = omega_bw_gpr_declared(g_prog.regalloc.peak_gpr_usage); + return 0; +} + +static uint32_t rd32(const uint8_t *p) { + uint32_t w; + memcpy(&w, p, 4); + return w; +} +static unsigned stall_of(const uint8_t *insn) { return (rd32(insn + 12) >> 9) & 15u; } +static unsigned opc_of(const uint8_t *insn) { return rd32(insn) & 0xfffu; } + +static int count_opc_stall(const uint8_t *code, size_t ninsn, unsigned opc, unsigned stall) { + int n = 0; + for (size_t i = 0; i < ninsn; i++) { + const uint8_t *ins = code + i * 16; + if (opc_of(ins) == opc && stall_of(ins) == stall) n++; + } + return n; +} + +static int self_test(void) { + int failed = 0; + uint8_t code[CODE_CAP]; + size_t len = 0; + uint32_t gpr = 0; +#define CHECK(cond, ...) \ + do { \ + if (!(cond)) { \ + failed++; \ + printf("FAIL: "); \ + printf(__VA_ARGS__); \ + printf("\n"); \ + } \ + } while (0) + CHECK(build(&CASES[0], 0, 0, 0, code, sizeof code, &len, &gpr) != 0, "stall 0 must be refused"); + CHECK(build(&CASES[0], 99, 0, 0, code, sizeof code, &len, &gpr) != 0, "stall past the sweep must be refused"); + uint32_t er = 0, ea = 0; + CHECK(expect_abs(&CASES[0], &er, &ea) == 0 && er == 512 && ea == 1, "alu oracle"); + CHECK(expect_abs(&CASES[1], &er, &ea) == 0 && er == fbits(512.0f), "fma oracle"); + CHECK(expect_abs(&CASES[2], &er, &ea) == 0 && er == fbits(64.0f), "mixed oracle"); + CHECK(expect_abs(&CASES[3], &er, &ea) == 0 && er == 64, "pred oracle"); + CHECK(expect_abs(&CASES[4], &er, &ea) == 0 && er == fmnmx_expect() && er != 0, "fmnmx oracle"); + CHECK(expect_abs(&CASES[5], &er, &ea) == 0 && er == 64 && ea == 0, "wide oracle"); + CHECK(expect_abs(&CASES[6], &er, &ea) != 0, "hmma has no closed host formula"); + + for (int ci = 0; ci < NCASES; ci++) { + unsigned stall = CASES[ci].href ? 29u : 4u; + len = 0; + int rc = build(&CASES[ci], stall, 0, 0, code, sizeof code, &len, &gpr); + CHECK(rc == 0, "%s encode rc %d", CASES[ci].id, rc); + if (rc != 0) continue; + int clocks = 0, s2r_tid = 0; + size_t ninsn = g_prog.count; + for (size_t i = 0; i < ninsn; i++) { + unsigned opc = opc_of(code + i * 16); + unsigned idx = (rd32(code + i * 16 + 8) >> 8) & 0xffu; + if (opc == CS2R_OPC && idx == CLOCK_LO) clocks++; + if (opc == S2R_OPC && idx == BW_SR_TID_X) s2r_tid++; + } + CHECK(clocks == 2, "%s clock sites %d", CASES[ci].id, clocks); + CHECK(s2r_tid == 1, "%s lane filter %d", CASES[ci].id, s2r_tid); + char *text = NULL; + size_t tlen = 0; + FILE *mf = open_memstream(&text, &tlen); + int unk = mf ? omega_sm121_listing(code, len, mf) : -1; + if (mf) fclose(mf); + CHECK(unk == 2, "%s disasm unknowns %d (the two CS2R sites)", CASES[ci].id, unk); + if (!strcmp(CASES[ci].id, "alu_alu")) { + CHECK(text && strstr(text, "IADD3"), "alu listing"); + CHECK(count_opc_stall(code, ninsn, 0x210, 4) == ALU_UNROLL, "alu chain stall count"); + } else if (!strcmp(CASES[ci].id, "fma_fma")) { + CHECK(text && strstr(text, "FFMA"), "fma listing"); + CHECK(count_opc_stall(code, ninsn, 0x223, 4) == ALU_UNROLL, "fma chain stall count"); + } else if (!strcmp(CASES[ci].id, "mixed")) { + CHECK(count_opc_stall(code, ninsn, 0x223, 4) == 1, "mixed ffma"); + CHECK(count_opc_stall(code, ninsn, 0x210, 4) == 1, "mixed iadd"); + } else if (!strcmp(CASES[ci].id, "pred")) { + CHECK(text && strstr(text, "ISETP"), "pred listing"); + CHECK(text && strstr(text, "BRA"), "pred branch"); + CHECK(count_opc_stall(code, ninsn, 0x20c, 4) == 1, "measured isetp stall"); + } else if (!strcmp(CASES[ci].id, "fmnmx_alu")) { + CHECK(text && strstr(text, "FMNMX"), "fmnmx listing"); + CHECK(count_opc_stall(code, ninsn, 0x209, 4) == 1, "fmnmx stall"); + } else if (!strcmp(CASES[ci].id, "imad_wide_lo")) { + CHECK(text && strstr(text, "IMAD.WIDE"), "wide listing"); + CHECK(count_opc_stall(code, ninsn, 0x825, 4) == 1, "wide stall"); + int saw = 0; + for (size_t i = 0; i + 1 < ninsn; i++) { + if (opc_of(code + i * 16) == 0x825 && stall_of(code + i * 16) == 4) { + unsigned dst = (rd32(code + i * 16) >> 16) & 0xffu; + unsigned src = (rd32(code + (i + 1) * 16) >> 24) & 0xffu; + CHECK(opc_of(code + (i + 1) * 16) == 0x210, "wide consumer is IADD3"); + CHECK(src == dst, "IADD3 reads the wide low half"); + saw = 1; + } + } + CHECK(saw, "wide pair found"); + } else if (!strcmp(CASES[ci].id, "hmma_reader")) { + CHECK(text && strstr(text, "HMMA"), "hmma listing"); + CHECK(count_opc_stall(code, ninsn, 0x23c, 15) == 1, "hmma stall 15"); + CHECK(count_opc_stall(code, ninsn, 0x918, 14) == 1, "hmma padding nop stall 14 (15+14=29)"); + int saw = 0; + for (size_t i = 0; i + 2 < ninsn; i++) { + if (opc_of(code + i * 16) == 0x23c && stall_of(code + i * 16) == 15 && + opc_of(code + (i + 1) * 16) == 0x918 && stall_of(code + (i + 1) * 16) == 14 && + opc_of(code + (i + 2) * 16) == 0x221) { + unsigned dst = (rd32(code + i * 16) >> 16) & 0xffu; + unsigned src = (rd32(code + (i + 2) * 16) >> 24) & 0xffu; + CHECK(src == dst, "FADD reads HMMA d0"); + saw = 1; + } + } + CHECK(saw, "hmma, nop, fadd sequence"); + } + for (size_t i = 0; i < ninsn; i++) { + if (opc_of(code + i * 16) == CS2R_OPC) { + unsigned guard = (rd32(code + i * 16) >> 12) & 0xfu; + CHECK(guard == 7, "%s CS2R guard %u", CASES[ci].id, guard); + } + } + free(text); + } + len = 0; + CHECK(build(NULL, 0, 64, 1, code, sizeof code, &len, &gpr) == 0, "overhead kernel"); + + int pass[RUNS][SWEEP_MAX]; + memset(pass, 0, sizeof pass); + for (int r = 0; r < RUNS; r++) + for (int s = 5; s <= 15; s++) pass[r][s - 1] = 1; + /* declared below */ +#undef CHECK + printf("gpu_latmeas_test --self-test %s (%d checks failed)\n", failed ? "FAIL" : "PASS", failed); + return failed ? 1 : 0; +} + +/* The suffix helper is tested from main after self_test's static copy. */ +static int required_stall(const int pass[][SWEEP_MAX], int runs, int sweep) { + for (int s = 1; s <= sweep; s++) { + int ok = 1; + for (int t = s; t <= sweep && ok; t++) + for (int r = 0; r < runs && ok; r++) + if (!pass[r][t - 1]) ok = 0; + if (ok) return s; + } + return -1; +} + +static int self_test_suffix(void) { + int failed = 0; + int pass[RUNS][SWEEP_MAX]; + memset(pass, 0, sizeof pass); + for (int r = 0; r < RUNS; r++) + for (int s = 5; s <= 15; s++) pass[r][s - 1] = 1; + if (required_stall(pass, RUNS, 15) != 5) { + printf("FAIL: suffix threshold want 5\n"); + failed++; + } + pass[3][4] = 0; /* run 3 fails stall 5; 6..15 still pass */ + if (required_stall(pass, RUNS, 15) != 6) { + printf("FAIL: one-run hole must move the threshold to 6\n"); + failed++; + } + memset(pass, 0, sizeof pass); + if (required_stall(pass, RUNS, 15) != -1) { + printf("FAIL: empty sweep must not invent a stall\n"); + failed++; + } + return failed; +} + +static int plan_launches(void) { + int n = 0; + int seen64 = 0, seen32 = 0, seen1 = 0; + for (int i = 0; i < NCASES; i++) { + n += RUNS * (int)CASES[i].sweep; + if (CASES[i].iters == 64) seen64 = 1; + if (CASES[i].iters == 32) seen32 = 1; + if (CASES[i].iters == 1) seen1 = 1; + } + n += RUNS * (seen64 + seen32 + seen1); + return n; +} + +static void print_plan(void) { + int n = plan_launches(); + printf("latmeas plan: %d cases, %d runs, sweep 1..15 (HMMA 1..36), %d launches\n", NCASES, RUNS, n); + printf("estimated chip window: 90 s upper bound (%d launches; 50 ms each covers a cold shader invalidate). Usually shorter.\n", + n); + printf("The runner wraps only the launch with: flock /tmp/aien-gb10.lock timeout 600\n"); + printf("One warp, 32 threads. Clock: CS2R idx 0x50. Mesa %s\n", MESA_COMMIT); +} + +static int mkdir_p(const char *dir); + +static int emit_kernels(const char *dir) { + if (mkdir_p(dir) != 0) return -1; + char path[512], manp[512]; + snprintf(manp, sizeof manp, "%s/manifest.txt", dir); + FILE *man = fopen(manp, "w"); + if (!man) return -1; + uint8_t code[CODE_CAP]; + for (int ci = 0; ci < NCASES; ci++) { + uint32_t expv = 0, auxv = 1; + const char *mode = CASES[ci].href ? "ref" : "abs"; + if (!CASES[ci].href && expect_abs(&CASES[ci], &expv, &auxv) != 0) { + fclose(man); + return -1; + } + for (unsigned s = 1; s <= CASES[ci].sweep; s++) { + size_t len = 0; + uint32_t gpr = 0; + if (build(&CASES[ci], s, 0, 0, code, sizeof code, &len, &gpr) != 0) { + fprintf(stderr, "emit %s stall %u failed\n", CASES[ci].id, s); + fclose(man); + return -1; + } + snprintf(path, sizeof path, "%s/%s_%u.bin", dir, CASES[ci].id, s); + FILE *f = fopen(path, "wb"); + if (!f || fwrite(code, 1, len, f) != len) { + if (f) fclose(f); + fclose(man); + return -1; + } + fclose(f); + fprintf(man, "%s %u %s %u %u %u %u %u %zu\n", CASES[ci].id, s, mode, expv, auxv, CASES[ci].iters, + CASES[ci].ops, gpr, len); + } + } + unsigned ovh_iters[] = {64, 32, 1}; + for (int k = 0; k < 3; k++) { + size_t len = 0; + uint32_t gpr = 0; + if (build(NULL, 0, ovh_iters[k], 1, code, sizeof code, &len, &gpr) != 0) { + fclose(man); + return -1; + } + snprintf(path, sizeof path, "%s/ovh%u_0.bin", dir, ovh_iters[k]); + FILE *f = fopen(path, "wb"); + if (!f || fwrite(code, 1, len, f) != len) { + if (f) fclose(f); + fclose(man); + return -1; + } + fclose(f); + fprintf(man, "ovh%u 0 abs %u 1 %u 0 %u %zu\n", ovh_iters[k], ovh_iters[k], ovh_iters[k], gpr, len); + } + if (fclose(man) != 0) return -1; + printf("emitted kernels in %s (%d launches when run)\n", dir, plan_launches()); + return 0; +} + +static int mkdir_p(const char *dir) { + if (!dir || !dir[0]) return -1; + if (mkdir(dir, 0755) == 0 || errno == EEXIST) return 0; + return -1; +} + +static int cmp_u64(const void *a, const void *b) { + uint64_t x = *(const uint64_t *)a, y = *(const uint64_t *)b; + return (x > y) - (x < y); +} +static uint64_t median_spread(uint64_t *v, int n, uint64_t *spread) { + if (n <= 0) { + *spread = 0; + return 0; + } + qsort(v, (size_t)n, sizeof *v, cmp_u64); + *spread = v[n - 1] - v[0]; + return v[n / 2]; +} + +typedef struct { + char id[64]; + unsigned stall, iters, ops, gpr; + char mode[8]; + uint32_t expect, aux; + size_t nbytes; + uint8_t *code; +} Kern; + +static Kern *load_manifest(const char *dir, int *nout) { + char manp[512]; + snprintf(manp, sizeof manp, "%s/manifest.txt", dir); + FILE *man = fopen(manp, "r"); + if (!man) return NULL; + int cap = 256, n = 0; + Kern *k = calloc((size_t)cap, sizeof *k); + if (!k) { + fclose(man); + return NULL; + } + while (1) { + Kern t; + memset(&t, 0, sizeof t); + unsigned stall, expv, auxv, iters, ops, gpr; + size_t nbytes = 0; + int got = fscanf(man, "%63s %u %7s %u %u %u %u %u %zu\n", t.id, &stall, t.mode, &expv, &auxv, &iters, &ops, + &gpr, &nbytes); + if (got != 9) break; + if (n == cap) { + cap *= 2; + Kern *g = realloc(k, (size_t)cap * sizeof *k); + if (!g) { + free(k); + fclose(man); + return NULL; + } + k = g; + } + t.stall = stall; + t.expect = expv; + t.aux = auxv; + t.iters = iters; + t.ops = ops; + t.gpr = gpr; + t.nbytes = nbytes; + char path[512]; + snprintf(path, sizeof path, "%s/%s_%u.bin", dir, t.id, t.stall); + FILE *f = fopen(path, "rb"); + t.code = f ? malloc(nbytes ? nbytes : 1) : NULL; + if (!f || !t.code || fread(t.code, 1, nbytes, f) != nbytes) { + if (f) fclose(f); + free(t.code); + free(k); + fclose(man); + return NULL; + } + fclose(f); + k[n++] = t; + } + fclose(man); + *nout = n; + return k; +} + +static Kern *find_kern(Kern *k, int n, const char *id, unsigned stall) { + for (int i = 0; i < n; i++) + if (!strcmp(k[i].id, id) && k[i].stall == stall) return &k[i]; + return NULL; +} + +static void read_driver(char *dst, size_t n) { + dst[0] = 0; + FILE *f = fopen("/proc/driver/nvidia/version", "r"); + if (!f) return; + if (!fgets(dst, (int)n, f)) dst[0] = 0; + fclose(f); + size_t L = strlen(dst); + while (L && (dst[L - 1] == '\n' || dst[L - 1] == '\r')) dst[--L] = 0; +} + +static int launch_kern(const Kern *k, OmegaGpuMem *code, OmegaGpuMem *buf, uint32_t outw[4], uint64_t *cycles) { + if (k->nbytes > code->size) return -1; + memset(code->cpu, 0, code->size); + memcpy(code->cpu, k->code, k->nbytes); + uint32_t *w = buf->cpu; + for (int i = 0; i < 4; i++) w[i] = POISON; + if (omega_gpu_session_sync_to_gpu(code) || omega_gpu_session_sync_to_gpu(buf)) return -1; + uint32_t args[2] = {(uint32_t)buf->va, (uint32_t)(buf->va >> 32)}; + OmegaGpuLaunch L = {.code_va = code->va, + .gpr_count = k->gpr, + .threads_x = 32, + .threads_y = 1, + .grid_x = 1, + .grid_y = 1, + .num_elements = 4, + .args = args, + .n_args = 2, + .timeout_ms = 3000, + .spin_us = 200000, + .target_sm = 121}; + if (omega_gpu_session_launch(&L, NULL, NULL) != 0) return -1; + if (omega_gpu_session_sync_to_cpu(buf) != 0) return -1; + memcpy(outw, buf->cpu, 16); + uint32_t t0 = outw[0], t1 = outw[1]; + if (t0 == POISON && t1 == POISON && outw[2] == POISON && outw[3] == POISON) return -2; + *cycles = t1 >= t0 ? (uint64_t)t1 - t0 : (1ull << 32) + t1 - t0; + return 0; +} + +static void json_escape(const char *s, char *dst, size_t n) { + size_t j = 0; + if (!s) s = ""; + for (size_t i = 0; s[i] && j + 2 < n; i++) { + unsigned char c = (unsigned char)s[i]; + if (c == '"' || c == '\\') { + if (j + 3 >= n) break; + dst[j++] = '\\'; + } + if (c < 32) c = ' '; + dst[j++] = (char)c; + } + dst[j] = 0; +} + +static int run_launch(const char *kdir, const char *out_path) { + int nk = 0; + Kern *kerns = load_manifest(kdir, &nk); + if (!kerns || nk < 1) { + fprintf(stderr, "no kernels in %s (run --emit first, outside the lock)\n", kdir); + return 2; + } + print_plan(); + char driver[256]; + read_driver(driver, sizeof driver); + struct utsname ut; + if (uname(&ut) != 0) memset(&ut, 0, sizeof ut); + + int pass[NCASES][RUNS][SWEEP_MAX]; + uint64_t cyc[NCASES][RUNS][SWEEP_MAX]; + uint32_t href[NCASES][RUNS]; + int href_ok[NCASES][RUNS]; + uint64_t ovh_cyc[3][RUNS]; + memset(pass, 0, sizeof pass); + memset(cyc, 0, sizeof cyc); + memset(href, 0, sizeof href); + memset(href_ok, 0, sizeof href_ok); + memset(ovh_cyc, 0, sizeof ovh_cyc); + char notes[NCASES][384]; + memset(notes, 0, sizeof notes); + int device_error = 0; + const char *err = ""; + + omega_gpu_session_lock(); + if (!omega_gpu_session_open()) { + err = omega_gpu_session_last_error(); + fprintf(stderr, "GPU not available (%s)\n", err); + device_error = 1; + goto write_json; + } + if (omega_gpu_session_target_sm() != 121) { + err = "opened device is not sm_121"; + fprintf(stderr, "%s (%u)\n", err, omega_gpu_session_target_sm()); + device_error = 1; + goto write_json; + } + OmegaGpuMem code = {0}, buf = {0}; + if (omega_gpu_session_alloc_code(CODE_CAP, &code) != 0 || omega_gpu_session_alloc(4096, &buf) != 0) { + err = omega_gpu_session_last_error(); + device_error = 1; + goto write_json; + } + + unsigned ovh_list[] = {64, 32, 1}; + for (int oi = 0; oi < 3 && !device_error; oi++) { + char id[16]; + snprintf(id, sizeof id, "ovh%u", ovh_list[oi]); + Kern *k = find_kern(kerns, nk, id, 0); + if (!k) { + err = "missing overhead kernel"; + device_error = 1; + break; + } + for (int r = 0; r < RUNS; r++) { + uint32_t w[4]; + uint64_t cy = 0; + int rc = launch_kern(k, &code, &buf, w, &cy); + printf("overhead %s run %d/%d rc %d\n", id, r + 1, RUNS, rc); + fflush(stdout); + if (rc != 0) { + err = omega_gpu_session_last_error(); + device_error = 1; + break; + } + ovh_cyc[oi][r] = cy; + if (w[2] != ovh_list[oi] || w[3] != 1) err = "overhead loop count did not match"; + } + } + + for (int ci = 0; ci < NCASES && !device_error; ci++) { + const CaseDef *c = &CASES[ci]; + for (int r = 0; r < RUNS && !device_error; r++) { + printf("case %s run %d/%d\n", c->id, r + 1, RUNS); + fflush(stdout); + uint32_t refv = 0; + int have_ref = 0; + if (c->href) { + Kern *kr = find_kern(kerns, nk, c->id, c->sweep); + uint32_t w[4]; + uint64_t cy = 0; + if (!kr || launch_kern(kr, &code, &buf, w, &cy) != 0) { + err = omega_gpu_session_last_error(); + device_error = 1; + break; + } + refv = w[2]; + have_ref = (w[3] != POISON && refv != 0 && refv != POISON); + href[ci][r] = refv; + href_ok[ci][r] = have_ref; + cyc[ci][r][c->sweep - 1] = cy; + pass[ci][r][c->sweep - 1] = have_ref; + if (!have_ref) + snprintf(notes[ci], sizeof notes[ci], + "HMMA reference at stall %u is 0x%08x (aux 0x%08x); d0 did not move", c->sweep, w[2], w[3]); + } + for (unsigned s = 1; s <= c->sweep && !device_error; s++) { + if (c->href && s == c->sweep) continue; + Kern *k = find_kern(kerns, nk, c->id, s); + uint32_t w[4]; + uint64_t cy = 0; + if (!k || launch_kern(k, &code, &buf, w, &cy) != 0) { + err = k ? omega_gpu_session_last_error() : "missing kernel"; + device_error = 1; + break; + } + cyc[ci][r][s - 1] = cy; + int ok = 0; + if (c->href) { + ok = href_ok[ci][r] && w[2] == refv; + } else { + uint32_t expv = 0, auxv = 0; + if (expect_abs(c, &expv, &auxv) == 0) ok = (w[2] == expv && w[3] == auxv && w[0] != POISON); + } + pass[ci][r][s - 1] = ok; + } + } + } + + omega_gpu_session_free(&code); + omega_gpu_session_free(&buf); + +write_json:; + int locked = 1; + char tmp[512]; + snprintf(tmp, sizeof tmp, "%s.tmp", out_path); + FILE *jf = fopen(tmp, "w"); + if (!jf) { + if (locked) { + omega_gpu_session_unlock(); + omega_gpu_session_close(); + } + free(kerns); + return 1; + } + char err_esc[512], drv_esc[512], mach_esc[128]; + json_escape(err, err_esc, sizeof err_esc); + json_escape(driver, drv_esc, sizeof drv_esc); + json_escape(ut.machine, mach_esc, sizeof mach_esc); + fprintf(jf, + "{\n \"kind\": \"cudapp-latmeas\",\n \"synthetic\": false,\n \"measured\": %s,\n \"mesa_commit\": \"%s\",\n" + " \"driver\": \"%s\",\n \"machine\": \"%s\",\n \"runs\": %d,\n \"threads\": 32,\n \"warps\": 1,\n" + " \"clock\": \"CS2R idx 0x50 (NAK_SV_CLOCK_LO)\",\n \"launches\": %d,\n \"error\": \"%s\",\n \"cases\": {\n", + device_error ? "false" : "true", MESA_COMMIT, drv_esc, mach_esc, RUNS, plan_launches(), err_esc); + for (int ci = 0; ci < NCASES; ci++) { + const CaseDef *c = &CASES[ci]; + int req = device_error ? -1 : required_stall(pass[ci], RUNS, (int)c->sweep); + int spread_s = 0; + int row_req[RUNS]; + int all_rows = 1; + for (int r = 0; r < RUNS; r++) { + int one[1][SWEEP_MAX]; + memcpy(one[0], pass[ci][r], sizeof one[0]); + row_req[r] = required_stall(one, 1, (int)c->sweep); + if (row_req[r] < 0) all_rows = 0; + } + if (!all_rows) req = -1; + int smin = 1000, smax = -1; + if (all_rows && !device_error) { + for (int r = 0; r < RUNS; r++) { + if (row_req[r] < smin) smin = row_req[r]; + if (row_req[r] > smax) smax = row_req[r]; + } + spread_s = smax - smin; + } + uint64_t samples[RUNS]; + uint64_t cmed = 0, cspread = 0; + int have_cyc = 0; + if (req > 0) { + for (int r = 0; r < RUNS; r++) samples[r] = cyc[ci][r][req - 1]; + uint64_t copy[RUNS]; + memcpy(copy, samples, sizeof copy); + cmed = median_spread(copy, RUNS, &cspread); + have_cyc = 1; + for (int r = 0; r < RUNS; r++) + if (samples[r] == 0) have_cyc = 0; + } + unsigned ovh_iters = c->iters; + int oi = ovh_iters == 64 ? 0 : ovh_iters == 32 ? 1 : 2; + uint64_t oc[RUNS]; + memcpy(oc, ovh_cyc[oi], sizeof oc); + uint64_t ospread = 0; + uint64_t omed = median_spread(oc, RUNS, &ospread); + (void)ospread; + char note_esc[512]; + json_escape(notes[ci], note_esc, sizeof note_esc); + fprintf(jf, " \"%s\": {\n", c->id); + if (req > 0 && !device_error) + fprintf(jf, " \"stall_required\": %d,\n", req); + else + fprintf(jf, " \"stall_required\": null,\n"); + fprintf(jf, " \"stall_spread\": %d,\n", all_rows && req > 0 && !device_error ? spread_s : 0); + if (have_cyc && !device_error) { + fprintf(jf, " \"cycle_median\": %llu,\n \"cycle_spread\": %llu,\n", (unsigned long long)cmed, + (unsigned long long)cspread); + } else { + fprintf(jf, " \"cycle_median\": null,\n \"cycle_spread\": null,\n"); + } + if (device_error) + fprintf(jf, " \"overhead_cycle_median\": null,\n"); + else + fprintf(jf, " \"overhead_cycle_median\": %llu,\n", (unsigned long long)omed); + fprintf(jf, " \"iters\": %u,\n \"ops\": %u,\n", c->iters, c->ops); + fprintf(jf, " \"note\": \"%s\"\n }%s\n", note_esc, ci + 1 == NCASES ? "" : ","); + } + fprintf(jf, " }\n}\n"); + if (fclose(jf) != 0) { + omega_gpu_session_unlock(); + omega_gpu_session_close(); + free(kerns); + return 1; + } + if (rename(tmp, out_path) != 0) { + omega_gpu_session_unlock(); + omega_gpu_session_close(); + free(kerns); + return 1; + } + omega_gpu_session_unlock(); + omega_gpu_session_close(); + for (int i = 0; i < nk; i++) free(kerns[i].code); + free(kerns); + printf("wrote %s\n", out_path); + return device_error ? 1 : 0; +} + +static void usage(void) { + fprintf(stderr, + "usage:\n" + " gpu_latmeas_test --self-test\n" + " gpu_latmeas_test --print-plan\n" + " gpu_latmeas_test --emit DIR\n" + " gpu_latmeas_test --launch DIR --out raw.json\n"); +} + +int main(int argc, char **argv) { + const char *emit = NULL, *launch = NULL, *out = NULL; + int self = 0, plan = 0; + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--self-test")) self = 1; + else if (!strcmp(argv[i], "--print-plan")) plan = 1; + else if (!strcmp(argv[i], "--emit") && i + 1 < argc) emit = argv[++i]; + else if (!strcmp(argv[i], "--launch") && i + 1 < argc) launch = argv[++i]; + else if (!strcmp(argv[i], "--out") && i + 1 < argc) out = argv[++i]; + else { + usage(); + return 2; + } + } + if (self) { + int f = self_test(); + f += self_test_suffix(); + return f ? 1 : 0; + } + if (plan && !emit && !launch) { + print_plan(); + return 0; + } + if (emit && !launch) return emit_kernels(emit) == 0 ? 0 : 1; + if (launch && out) return run_launch(launch, out); + usage(); + return 2; +} diff --git a/tools/latmeas/REPRODUCE.md b/tools/latmeas/REPRODUCE.md new file mode 100644 index 00000000..b2b143ff --- /dev/null +++ b/tools/latmeas/REPRODUCE.md @@ -0,0 +1,53 @@ +# Reproduce the GB10 latency table + +This measures fixed instruction latencies on an NVIDIA GB10 (DGX Spark, sm_121) and compares them with Mesa NAK. Nouveau does not run on the Spark yet. The kernels are CUDA++ machine code. Nothing in the launch links `libcuda`, `libcudart`, or `libnvidia-ml`. + +## Command + +From a CUDA++ checkout, on the Spark, with `/dev/nvidia0` and `/dev/nvidia-uvm` present and the open kernel module loaded: + +```sh +sh tools/latmeas/run_spark.sh latmeas-out +``` + +The script builds and encodes first. It then runs `make doctor` and stops if the machine is not ready. Only the launch is inside `flock /tmp/aien-gb10.lock`, and that launch is wrapped in `timeout 600`. The binary prints an upper bound before it opens the device: about 1400 launches, 90 seconds if each one takes 50 ms, usually shorter. The hard stop is 600 seconds. + +Outputs, all under `latmeas-out/`: + +- `raw.json` — clock deltas, the stall that matched, spread. No number in this file was typed in by hand. +- `sm121_vs_NAK.md` — the table. A cell stays blank when that chain did not produce a stall. +- `diff.md` — every fixed-latency claim that already differs from NAK, with file and line. +- `kernels/` — the machine code that was launched. +- `head.txt` — `git rev-parse HEAD` of the tree that produced the binaries. + +Do not treat a blank cell as zero. + +## What a row means + +One warp, 32 threads, all of them executing the same chain. Lane 0 stores four words: the two clock reads, the chain result, and an auxiliary word. + +The clock is `CS2R` with index `0x50`. NAK's `nak_private.h` names that `NAK_SV_CLOCK_LO`. On SM 50 and later, `from_nir.rs` emits `CS2R` rather than `S2R` (`sm70_encode.rs`: opcode `0x805`, index at bits 72..80). CUDA++'s encoder has `S2R` only, so the test emits `S2R` immediate `0x50` and rewrites the opcode to `0x805`. + +NAK's table is the stall the next instruction must wait, not a scoreboard. These chips do not interlock a fixed-latency GPR write (`src/omega_numeric.c`). The runner therefore tries stall 1 through 15 (1 through 36 for HMMA, because the field is only four bits and the rest is `NOP`s). The reported stall is the smallest S such that S and every larger S matched the oracle on all 11 runs. The cycle median is the 32-bit clock delta at that S. An empty loop with the same trip count is the overhead control; its median is in `raw.json`. + +The oracle for ALU, FMA, mixed ALU/FMA, the predicate chain, FMNMX, and IMAD.WIDE is a host formula (exact integer or exact f32). HMMA's oracle is the same kernel at stall 36: lane 0's `d0` must be non-zero there, and shorter stalls must match it. A zero reference leaves the HMMA cell blank. + +## NAK pin + +Mesa commit `01bab9302a6bf0a082c7ba8c49d74427cf10b0cf`. + +- `src/nouveau/compiler/nak/sm120_instr_latencies.rs` — SM100 tables plus one cycle, and +9 / +7 for HMMA / IMMA. +- `src/nouveau/compiler/latencies/sm100/reg_raw.csv` — rows are readers, columns are writers. +- `src/nouveau/compiler/latencies/sm100/pred_raw.csv` — predicate write to a guard is row `redirected_fp64`, column `dualalu`, 13, plus the SM120 one. + +`python3 tools/latmeas/diff_nak.py --verify-mesa` re-downloads those files and checks the cells. It is not part of `make test`. + +## What this does not measure + +Loads, stores, and other scoreboarded ops. BSSY, BSYNC, BRA, EXIT, and BAR: NAK calls them decoupled, and the diff lists the encoder's fixed stall, but there is no GPR chain for them. QMMA: the encoder waits 30 and NAK's IMMA-to-ALU total is 26; the opcode is a patched placeholder, so this kit does not launch it. + +Priority 1 (ALU, FMA, mixed, predicate) matches NAK on paper. That match is circular until this table is filled on the chip. The attention emitter still programs predicate stall 13; the predicate kernel is the measurement for that row too. + +## Host-only check + +`make test-gpu-latmeas` encodes every chain, checks it with `sm121dis`, checks the oracles, and dry-runs the template on synthetic input that is deleted afterwards. It does not open a GPU and it does not write a result into the tree. diff --git a/tools/latmeas/catalog.json b/tools/latmeas/catalog.json new file mode 100644 index 00000000..5d2da80a --- /dev/null +++ b/tools/latmeas/catalog.json @@ -0,0 +1,246 @@ +{ + "mesa_commit": "01bab9302a6bf0a082c7ba8c49d74427cf10b0cf", + "mesa_clock": "nak_private.h:135 NAK_SV_CLOCK_LO = 0x50; from_nir.rs:3522 SM>=50 uses CS2R; sm70_encode.rs:4278 opcode 0x805, index bits 72..80", + "rows": [ + { + "id": "alu_alu", + "priority": 1, + "pair": "ALU -> ALU (GPR RAW)", + "encoder_stall": 5, + "encoder": "5 (4 + 1, tightened from the blanket 6)", + "encoder_file": "src/omega_gpu_elementwise_api.c", + "encoder_line": 800, + "encoder_substr": "(5u << 9)", + "nak_stall": 5, + "nak": "4 + 1 = 5", + "nak_where": "latencies/sm100/reg_raw.csv:2 row alu column alu = 4; sm120_instr_latencies.rs:457 GPR raw adds max(adjust, 1); IAdd3 is Alu at line 44", + "measure_id": "alu_alu", + "kernel": "tests/gpu_latmeas_test.c (alu_alu)", + "class_note": "" + }, + { + "id": "fma_fma", + "priority": 1, + "pair": "FMA -> FMA (GPR RAW)", + "encoder_stall": 5, + "encoder": "5 (4 + 1, tightened from the blanket 6)", + "encoder_file": "src/omega_gpu_elementwise_api.c", + "encoder_line": 787, + "encoder_substr": "BW_IR_FFMA", + "nak_stall": 5, + "nak": "4 + 1 = 5", + "nak_where": "latencies/sm100/reg_raw.csv:5 row fma column fma = 4; sm120_instr_latencies.rs:59 FFma is Fma; line 457 adds 1", + "measure_id": "fma_fma", + "kernel": "tests/gpu_latmeas_test.c (fma_fma)", + "class_note": "" + }, + { + "id": "mixed", + "priority": 1, + "pair": "Mixed ALU <-> FMA (GPR RAW)", + "encoder_stall": 6, + "encoder": "6 (5 + 1; different fixed_pair_class values are left at the blanket 6)", + "encoder_file": "src/omega_gpu_elementwise_api.c", + "encoder_line": 65, + "encoder_substr": "CW(6,", + "nak_stall": 6, + "nak": "5 + 1 = 6", + "nak_where": "latencies/sm100/reg_raw.csv:2 row alu column fma = 5 and line 5 row fma column alu = 5; sm120_instr_latencies.rs:457 adds 1", + "measure_id": "mixed", + "kernel": "tests/gpu_latmeas_test.c (mixed)", + "class_note": "" + }, + { + "id": "pred", + "priority": 1, + "pair": "Predicate write -> guard", + "encoder_stall": 14, + "encoder": "14 (elementwise, MoE, and the other emitters)", + "encoder_file": "src/omega_gpu_elementwise_api.c", + "encoder_line": 66, + "encoder_substr": "CW(14,", + "nak_stall": 14, + "nak": "13 + 1 = 14", + "nak_where": "latencies/sm100/pred_raw.csv:11 row redirected_fp64 column dualalu = 13; sm120_instr_latencies.rs:474 predicate raw(write, None) uses reader RedirectedFp64 and adds 1; ISetP is Dualalu at line 49", + "measure_id": "pred", + "kernel": "tests/gpu_latmeas_test.c (pred)", + "class_note": "" + }, + { + "id": "pred_attention", + "priority": 2, + "pair": "Predicate write -> guard, attention emitter", + "encoder_stall": 13, + "encoder": "13", + "encoder_file": "src/omega_gpu_attention_api.c", + "encoder_line": 74, + "encoder_substr": "CW(13,", + "nak_stall": 14, + "nak": "13 + 1 = 14", + "nak_where": "same predicate cell as the Priority 1 row; this emitter omits SM120's +1", + "measure_id": "pred", + "kernel": "tests/gpu_latmeas_test.c (pred)", + "class_note": "" + }, + { + "id": "hmma_reader", + "priority": 2, + "pair": "HMMA -> non-HMMA reader (FADD of d0)", + "encoder_stall": 29, + "encoder": "15 + NOP 14 = 29", + "encoder_file": "src/omega_blackwell_codegen_tc.c", + "encoder_line": 33, + "encoder_substr": "nop(e, 14)", + "nak_stall": 28, + "nak": "19 + 9 = 28", + "nak_where": "latencies/sm100/reg_raw.csv:2 row alu column hmma = 19; sm120_instr_latencies.rs:431 latency_adjust(Hmma) = 9, added at line 457. HMMA -> HMMA is row hmma column hmma = 20 + 9 = 29, which matches the encoder; the disagreement is the non-HMMA reader", + "measure_id": "hmma_reader", + "kernel": "tests/gpu_latmeas_test.c (hmma_reader)", + "class_note": "" + }, + { + "id": "imad_wide_lo", + "priority": 2, + "pair": "IMAD.WIDE low half -> IADD3", + "encoder_stall": 6, + "encoder": "6 (blanket K_FIXED; IMAD.WIDE is not in fixed_pair_class)", + "encoder_file": "src/omega_gpu_elementwise_api.c", + "encoder_line": 788, + "encoder_substr": "default: return 0", + "nak_stall": 4, + "nak": "3 + 1 = 4", + "nak_where": "latencies/sm100/reg_raw.csv:2 row alu column imad_wide_write_dl = 3; sm120_instr_latencies.rs:28 IMad64 write class and line 457 adds 1. The high half column imad_wide_write_dh is 5 + 1 = 6, which matches the blanket", + "measure_id": "imad_wide_lo", + "kernel": "tests/gpu_latmeas_test.c (imad_wide_lo)", + "class_note": "" + }, + { + "id": "fmnmx_alu", + "priority": 2, + "pair": "FMNMX (dualalu) -> IADD3 (alu)", + "encoder_stall": 6, + "encoder": "6 (blanket K_FIXED; FMNMX is not in fixed_pair_class)", + "encoder_file": "src/omega_gpu_elementwise_api.c", + "encoder_line": 788, + "encoder_substr": "default: return 0", + "nak_stall": 5, + "nak": "4 + 1 = 5", + "nak_where": "latencies/sm100/reg_raw.csv:2 row alu column dualalu = 4; sm120_instr_latencies.rs:62 FMnMx is Dualalu; line 457 adds 1. dualalu -> dualalu is the same 4 + 1", + "measure_id": "fmnmx_alu", + "kernel": "tests/gpu_latmeas_test.c (fmnmx_alu)", + "class_note": "" + }, + { + "id": "qmma_reader", + "priority": 2, + "pair": "QMMA (IMMA) -> next reader", + "encoder_stall": 30, + "encoder": "15 + NOP 15 = 30", + "encoder_file": "src/omega_blackwell_codegen_qmma.c", + "encoder_line": 42, + "encoder_substr": "nop(e, 15)", + "nak_stall": 26, + "nak": "19 + 7 = 26", + "nak_where": "latencies/sm100/reg_raw.csv:2 row alu column imma = 19; sm120_instr_latencies.rs:431 latency_adjust(Imma) = 7. IMMA -> IMMA is row imma (line 17) column imma = 20 + 7 = 27", + "measure_id": "", + "kernel": "not generated (patched QMMA placeholder; see REPRODUCE.md)", + "class_note": "" + }, + { + "id": "numeric_min", + "priority": 2, + "pair": "Fixed-producer minimum in the numeric checker", + "encoder_stall": 5, + "encoder": "5 (any fixed producer; mixed pairs need 6)", + "encoder_file": "src/omega_numeric.c", + "encoder_line": 1183, + "encoder_substr": "at least 5", + "nak_stall": 6, + "nak": "5 + 1 = 6 for mixed ALU/FMA", + "nak_where": "same mixed cell as Priority 1. The scheduler uses 6 there; this checker still accepts 5", + "measure_id": "mixed", + "kernel": "tests/gpu_latmeas_test.c (mixed)", + "class_note": "" + }, + { + "id": "bssy", + "priority": 2, + "pair": "BSSY stall", + "encoder_stall": 1, + "encoder": "1, no scoreboard", + "encoder_file": "src/omega_gpu_elementwise_api.c", + "encoder_line": 78, + "encoder_substr": "CW(1,", + "nak_stall": null, + "nak": "decoupled (1 & sb)", + "nak_where": "sm120_instr_latencies.rs:137 Op::BSSy is Decoupled, which needs a scoreboard (needs_scoreboards)", + "measure_id": "", + "kernel": "not generated (NAK class is decoupled, not a fixed GPR RAW)", + "class_note": "class" + }, + { + "id": "bsync", + "priority": 2, + "pair": "BSYNC stall", + "encoder_stall": 5, + "encoder": "5, no scoreboard", + "encoder_file": "src/omega_gpu_elementwise_api.c", + "encoder_line": 79, + "encoder_substr": "CW(5,", + "nak_stall": null, + "nak": "decoupled (1 & sb)", + "nak_where": "sm120_instr_latencies.rs:140 Op::BSync is Decoupled", + "measure_id": "", + "kernel": "not generated (NAK class is decoupled, not a fixed GPR RAW)", + "class_note": "class" + }, + { + "id": "bra", + "priority": 2, + "pair": "BRA stall", + "encoder_stall": 5, + "encoder": "5", + "encoder_file": "src/omega_gpu_elementwise_api.c", + "encoder_line": 73, + "encoder_substr": "case K_BRA:", + "nak_stall": null, + "nak": "decoupled (1 & sb)", + "nak_where": "sm120_instr_latencies.rs:136 Op::Bra is Decoupled", + "measure_id": "", + "kernel": "not generated (NAK class is decoupled, not a fixed GPR RAW)", + "class_note": "class" + }, + { + "id": "exit_op", + "priority": 2, + "pair": "EXIT stall", + "encoder_stall": 5, + "encoder": "5", + "encoder_file": "src/omega_gpu_elementwise_api.c", + "encoder_line": 72, + "encoder_substr": "case K_EXIT:", + "nak_stall": null, + "nak": "decoupled (1 & sb)", + "nak_where": "sm120_instr_latencies.rs:139 Op::Exit is Decoupled", + "measure_id": "", + "kernel": "not generated (NAK class is decoupled, not a fixed GPR RAW)", + "class_note": "class" + }, + { + "id": "bar", + "priority": 2, + "pair": "BAR.SYNC stall", + "encoder_stall": 6, + "encoder": "6", + "encoder_file": "src/omega_gpu_elementwise_api.c", + "encoder_line": 70, + "encoder_substr": "case K_SYNC:", + "nak_stall": null, + "nak": "decoupled AGU (1 & sb)", + "nak_where": "sm120_instr_latencies.rs:132 Op::Bar is DecoupledAgu", + "measure_id": "", + "kernel": "not generated (NAK class is decoupled, not a fixed GPR RAW)", + "class_note": "class" + } + ] +} diff --git a/tools/latmeas/diff_nak.py b/tools/latmeas/diff_nak.py new file mode 100755 index 00000000..b7b19ac9 --- /dev/null +++ b/tools/latmeas/diff_nak.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +"""Diff CUDA++ fixed-latency stalls against Mesa NAK at one pinned commit. + +The numbers in catalog.json are citations (file, line, cell), not a copy of +Mesa. --check reads the CUDA++ lines and checks the SM120 +1 arithmetic. +--verify-mesa re-downloads the pinned files and checks the cited cells. +It does not write a chip result. +""" +import argparse +import json +import os +import sys +import urllib.request + +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +CATALOG = os.path.join(os.path.dirname(__file__), "catalog.json") + + +def load(): + with open(CATALOG, encoding="utf-8") as f: + return json.load(f) + + +def sm120_gpr(base, adjust): + """sm120_instr_latencies.rs GPR raw: table value + max(adjust, 1).""" + return base + max(adjust, 1) + + +def check_sources(cat): + errors = [] + for row in cat["rows"]: + path = os.path.join(ROOT, row["encoder_file"]) + with open(path, encoding="utf-8") as f: + lines = f.read().splitlines() + n = row["encoder_line"] + if n < 1 or n > len(lines) or row["encoder_substr"] not in lines[n - 1]: + errors.append("%s: %s:%d does not contain %r" % (row["id"], row["encoder_file"], n, row["encoder_substr"])) + # The pair tightener's default class excludes IMAD.WIDE and FMNMX. + sched = os.path.join(ROOT, "src/omega_gpu_elementwise_api.c") + with open(sched, encoding="utf-8") as f: + text = f.read().splitlines() + window = "\n".join(text[780:789]) + if "IMAD_WIDE" in window or "FMNMX" in window: + errors.append("fixed_pair_class now names IMAD_WIDE or FMNMX; the catalog's blanket-6 claim changed") + if "case K_FIXED: in.control = CW(6," not in text[64]: + errors.append("K_FIXED blanket stall is not 6 at src/omega_gpu_elementwise_api.c:65") + return errors + + +def disagreements(cat): + out = [] + for row in cat["rows"]: + nak = row["nak_stall"] + enc = row["encoder_stall"] + if nak is None or enc != nak: + out.append(row) + return out + + +def render(cat, rows): + lines = [] + lines.append("# CUDA++ fixed latencies vs NAK") + lines.append("") + lines.append("Mesa commit `%s`." % cat["mesa_commit"]) + lines.append("Clock: %s." % cat["mesa_clock"]) + lines.append("Priority 1 pairs match NAK on paper. They are measured because that match is circular.") + lines.append("Rows below are the ones that differ. A blank NAK stall means NAK calls the op decoupled.") + lines.append("") + lines.append("| id | pair | CUDA++ | NAK | kernel |") + lines.append("|---|---|---|---|---|") + for row in rows: + nak = row["nak"] + lines.append("| %s | %s | %s | %s | %s |" % (row["id"], row["pair"], row["encoder"], nak, row["kernel"])) + lines.append("") + return "\n".join(lines) + "\n" + + +def self_check(cat): + """Negative control: an equal pair is not a disagreement; 1 vs 2 is.""" + fake = {"rows": [ + {"encoder_stall": 5, "nak_stall": 5, "id": "equal"}, + {"encoder_stall": 1, "nak_stall": 2, "id": "differ"}, + {"encoder_stall": 5, "nak_stall": None, "id": "class"}, + ]} + got = [r["id"] for r in disagreements(fake)] + if got != ["differ", "class"]: + return "negative control listed %s" % got + if sm120_gpr(4, 0) != 5 or sm120_gpr(19, 9) != 28 or sm120_gpr(3, 0) != 4: + return "SM120 +1 arithmetic drifted" + if sm120_gpr(20, 9) != 29 or sm120_gpr(19, 7) != 26 or (13 + 1) != 14: + return "cited NAK totals drifted" + # Priority 1 must not show up as a disagreement. + p1 = [r["id"] for r in disagreements(cat) if r["priority"] == 1] + if p1: + return "priority 1 rows differ from NAK: %s" % p1 + return None + + +def verify_mesa(commit): + base = "https://gitlab.freedesktop.org/mesa/mesa/-/raw/%s/" % commit + reg = urllib.request.urlopen(base + "src/nouveau/compiler/latencies/sm100/reg_raw.csv", timeout=60).read().decode() + pred = urllib.request.urlopen(base + "src/nouveau/compiler/latencies/sm100/pred_raw.csv", timeout=60).read().decode() + rs = urllib.request.urlopen(base + "src/nouveau/compiler/nak/sm120_instr_latencies.rs", timeout=60).read().decode() + lines = reg.splitlines() + hdr = lines[0].split(",") + alu = lines[1].split(",") + fma = lines[4].split(",") + + def cell(parts, name): + return parts[hdr.index(name)] + + checks = [ + (cell(alu, "alu"), "4"), + (cell(alu, "fma"), "5"), + (cell(alu, "dualalu"), "4"), + (cell(alu, "hmma"), "19"), + (cell(alu, "imma"), "19"), + (cell(alu, "imad_wide_write_dl"), "3"), + (cell(alu, "imad_wide_write_dh"), "5"), + (cell(fma, "fma"), "4"), + (cell(fma, "alu"), "5"), + ] + for got, want in checks: + if got != want: + raise SystemExit("reg_raw cell %s != %s" % (got, want)) + pred_lines = pred.splitlines() + pred_hdr = pred_lines[0].split(",") + redir = pred_lines[10].split(",") + if redir[0] != "redirected_fp64" or redir[pred_hdr.index("dualalu")] != "13": + raise SystemExit("pred_raw redirected_fp64/dualalu is not 13") + for needle in ("RegLatencySM100::Hmma => 9", "RegLatencySM100::Imma => 7", "NAK_SV_CLOCK" ): + # CLOCK is not in this file. Check the adjust arms and the +1 in raw(). + pass + if "RegLatencySM100::Hmma => 9" not in rs or "RegLatencySM100::Imma => 7" not in rs: + raise SystemExit("sm120 latency_adjust does not match the catalog") + if "+ 1" not in rs: + raise SystemExit("sm120 raw no longer adds padding") + print("mesa %s cells match the catalog" % commit) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--check", action="store_true") + p.add_argument("--md", help="write the disagreement table") + p.add_argument("--verify-mesa", action="store_true") + args = p.parse_args() + cat = load() + errors = check_sources(cat) + bad = self_check(cat) + if bad: + errors.append(bad) + if errors: + for e in errors: + print("FAIL:", e, file=sys.stderr) + return 1 + text = render(cat, disagreements(cat)) + sys.stdout.write(text) + if args.md: + with open(args.md, "w", encoding="utf-8") as f: + f.write(text) + if args.verify_mesa: + verify_mesa(cat["mesa_commit"]) + if not args.check and not args.md and not args.verify_mesa: + return 0 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/latmeas/fill_template.py b/tools/latmeas/fill_template.py new file mode 100755 index 00000000..2717e1af --- /dev/null +++ b/tools/latmeas/fill_template.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +"""Fill the GB10-vs-NAK markdown from a launch JSON file. + +A synthetic input is refused unless --allow-synthetic is set, and then every +measured cell is the word FAKE. Missing stall_required stays blank. This +script never invents a cycle count. +""" +import argparse +import json +import os +import sys +import tempfile + +CATALOG = os.path.join(os.path.dirname(__file__), "catalog.json") + + +def load_catalog(): + with open(CATALOG, encoding="utf-8") as f: + return json.load(f) + + +def cell(text): + return "" if text is None or text == "" else str(text) + + +def agree(measured, nak_stall, enc_stall): + if measured is None or nak_stall is None: + return "" + if measured == nak_stall and measured == enc_stall: + return "NAK and nvcc" + if measured == nak_stall: + return "NAK" + if measured == enc_stall: + return "nvcc" + return "neither" + + +def runs_text(case, runs): + if not case or case.get("stall_required") is None: + return "" + parts = ["%d runs" % runs, "stall spread %s" % case.get("stall_spread")] + if case.get("cycle_median") is not None: + parts.append("dep cycles median %s spread %s" % (case["cycle_median"], case.get("cycle_spread"))) + if case.get("overhead_cycle_median") is not None: + parts.append("loop overhead median %s" % case["overhead_cycle_median"]) + return "; ".join(parts) + + +def lookup(results, measure_id): + if not measure_id: + return None + cases = results.get("cases") or {} + return cases.get(measure_id) + + +def build_md(cat, results, fake): + results = results or {} + measured_flag = bool(results.get("measured")) and not fake + runs = results.get("runs") or 0 + driver = results.get("driver") or "" + commit = cat["mesa_commit"] + lines = [] + lines.append("# GB10 (sm_121) measured latencies vs NAK") + lines.append("") + lines.append("Chip: NVIDIA GB10 (DGX Spark), sm_121") + if driver: + lines.append("Driver: %s. No NVIDIA user-space libraries." % driver) + else: + lines.append("Driver: NVIDIA open kernel module , no NVIDIA user-space libraries") + lines.append("Method: dependent-chain timing kernel. One warp (32 threads), lane 0 stores.") + lines.append("Each fixed-latency producer is issued at a programmed stall. The GB10 column is the smallest stall S such that S and every larger S in the sweep matched the oracle on every run. Cycles are the CS2R clock delta (NAK_SV_CLOCK_LO, 0x50) around that chain, median and max-min.") + lines.append("Source: tests/gpu_latmeas_test.c in aien-dev/CUDApp") + lines.append("How to rerun: sh tools/latmeas/run_spark.sh OUTPUT_DIR") + lines.append("") + lines.append("NAK reference: `nak/sm120_instr_latencies.rs` plus `latencies/sm100/*.csv` at Mesa commit %s" % commit) + if fake: + lines.append("") + lines.append("FAKE: this table was rendered from synthetic input. It is not a measurement.") + elif not measured_flag: + lines.append("") + lines.append("NOT RUN: no chip measurement is recorded in this file. Empty cells are empty on purpose.") + lines.append("") + lines.append("## Priority 1: values CUDA++ currently borrows from NAK (needs independent measurement)") + lines.append("") + lines.append("| Writer -> reader | NAK value | NAK source line | GB10 measured | Delta | Runs / spread | Kernel |") + lines.append("|---|---|---|---|---|---|---|") + notes = [] + for row in cat["rows"]: + if row["priority"] != 1: + continue + case = lookup(results, row["measure_id"]) if measured_flag else None + stall = None if fake or not case else case.get("stall_required") + delta = "" + shown = "FAKE" if fake else cell(stall) + if not fake and stall is not None and row["nak_stall"] is not None: + delta = str(stall - row["nak_stall"]) + spread = "FAKE" if fake else runs_text(case, runs) + lines.append("| %s | %s | %s | %s | %s | %s | %s |" % ( + row["pair"], row["nak"], row["nak_where"], shown, delta, spread, row["kernel"])) + if case and case.get("note"): + notes.append("%s: %s" % (row["id"], case["note"])) + lines.append("") + lines.append("## Priority 2: instructions where CUDA++ (nvcc-derived) and NAK disagree") + lines.append("") + lines.append("| Instruction / pair | NAK value | nvcc sm_121 stall | GB10 measured | Agrees with | Kernel |") + lines.append("|---|---|---|---|---|---|") + for row in cat["rows"]: + if row["priority"] != 2: + continue + case = lookup(results, row["measure_id"]) if measured_flag else None + stall = None if fake or not case else case.get("stall_required") + shown = "FAKE" if fake else cell(stall) + who = "FAKE" if fake else agree(stall, row["nak_stall"], row["encoder_stall"]) + lines.append("| %s | %s | %s | %s | %s | %s |" % ( + row["pair"], row["nak"], row["encoder"], shown, who, row["kernel"])) + if case and case.get("note"): + notes.append("%s: %s" % (row["id"], case["note"])) + lines.append("") + lines.append("## Notes") + lines.append("- Anything not fixed-latency (LDG/LDS, scoreboard waits) is out of scope for this table.") + lines.append("- Decoupled ops (BSSY, BSYNC, BRA, EXIT, BAR) are listed because the encoder gives them a fixed stall and NAK does not. They have no dependent-chain kernel.") + lines.append("- HMMA -> HMMA in NAK is 20 + 9 = 29, the same total the encoder uses. The HMMA row is the non-HMMA reader (28).") + lines.append("- QMMA is not generated here. Its encoder stall is 30 and NAK's IMMA -> ALU total is 26 (IMMA -> IMMA is 27).") + err = (results or {}).get("error") or "" + if err: + notes.append("runner: %s" % err) + if notes: + for n in notes: + lines.append("- %s" % n) + else: + lines.append("- Confidence notes and anomalies go here.") + lines.append("") + return "\n".join(lines) + + +def write_or_refuse(results, dest, allow_synthetic): + fake = bool(results.get("synthetic")) + if fake and not allow_synthetic: + print("refusing synthetic input (pass --allow-synthetic to render a FAKE table)", file=sys.stderr) + return 2 + text = build_md(load_catalog(), results, fake) + if fake and "FAKE" not in text: + print("synthetic render did not stamp FAKE", file=sys.stderr) + return 1 + if dest: + with open(dest, "w", encoding="utf-8") as f: + f.write(text) + else: + sys.stdout.write(text) + return 0 + + +def self_test(): + cat = load_catalog() + synthetic = { + "synthetic": True, + "measured": True, + "runs": 11, + "driver": "synthetic", + "cases": {"alu_alu": {"stall_required": 99, "stall_spread": 0, "cycle_median": 99, "cycle_spread": 1, + "overhead_cycle_median": 1, "note": ""}}, + } + rc = write_or_refuse(synthetic, None, False) + if rc != 2: + print("FAIL: synthetic input was accepted", file=sys.stderr) + return 1 + fd, path = tempfile.mkstemp(prefix="latmeas-fake-", suffix=".md") + os.close(fd) + try: + rc = write_or_refuse(synthetic, path, True) + text = open(path, encoding="utf-8").read() + finally: + os.unlink(path) + if rc != 0 or "FAKE" not in text or "99" in text: + print("FAIL: synthetic table leaked a number or missed FAKE", file=sys.stderr) + return 1 + blank = {"synthetic": False, "measured": True, "runs": 11, "cases": {"alu_alu": {"stall_required": None}}} + text = build_md(cat, blank, False) + # The ALU row's measured cell is between the NAK citation and Delta. It must be empty. + alu = [ln for ln in text.splitlines() if ln.startswith("| ALU -> ALU")] + if len(alu) != 1 or "99" in text: + print("FAIL: blank measured row missing", file=sys.stderr) + return 1 + parts = [p.strip() for p in alu[0].strip("|").split("|")] + if parts[3] != "": + print("FAIL: null stall rendered as %r" % parts[3], file=sys.stderr) + return 1 + filled = {"synthetic": False, "measured": False, "runs": 11, + "cases": {"alu_alu": {"stall_required": 5, "stall_spread": 0}}} + text = build_md(cat, filled, False) + alu = [ln for ln in text.splitlines() if ln.startswith("| ALU -> ALU")][0] + parts = [p.strip() for p in alu.strip("|").split("|")] + if parts[3] != "": + print("FAIL: measured:false still filled a cell (%r)" % parts[3], file=sys.stderr) + return 1 + print("fill_template self-test PASS") + return 0 + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--results") + p.add_argument("--md") + p.add_argument("--allow-synthetic", action="store_true") + p.add_argument("--self-test", action="store_true") + args = p.parse_args() + if args.self_test: + return self_test() + if not args.results: + p.error("--results is required") + with open(args.results, encoding="utf-8") as f: + results = json.load(f) + return write_or_refuse(results, args.md, args.allow_synthetic) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/latmeas/run_spark.sh b/tools/latmeas/run_spark.sh new file mode 100755 index 00000000..f5212bec --- /dev/null +++ b/tools/latmeas/run_spark.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# One-shot GB10 latency run. Builds and encodes before taking the shared lock. +# Usage: sh tools/latmeas/run_spark.sh OUTPUT_DIR +set -eu +if [ $# -ne 1 ] || [ -z "$1" ]; then + echo "usage: sh tools/latmeas/run_spark.sh OUTPUT_DIR" >&2 + exit 2 +fi +root=$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd) +out=$1 +case "$out" in + /*) ;; + *) out=$root/$out ;; +esac +cd "$root" +mkdir -p "$out" +echo "latmeas: building (outside the chip lock)" +make -j"$(nproc)" build/gpu_latmeas_test build/cudapp-doctor +echo "latmeas: doctor (outside the chip lock; NOT READY aborts before flock)" +./build/cudapp-doctor +git rev-parse HEAD > "$out/head.txt" +./build/gpu_latmeas_test --print-plan +./build/gpu_latmeas_test --emit "$out/kernels" +python3 tools/latmeas/diff_nak.py --check --md "$out/diff.md" +echo "latmeas: taking flock /tmp/aien-gb10.lock for the launch only; timeout 600" +echo "latmeas: estimated chip window 90 s upper bound (printed again by the binary)" +# flock waits here without holding the lock. timeout covers only the launch. +set +e +flock /tmp/aien-gb10.lock timeout 600 ./build/gpu_latmeas_test --launch "$out/kernels" --out "$out/raw.json" +rc=$? +set -e +if [ -f "$out/raw.json" ]; then + python3 tools/latmeas/fill_template.py --results "$out/raw.json" --md "$out/sm121_vs_NAK.md" +fi +exit "$rc" diff --git a/tools/latmeas/selftest.sh b/tools/latmeas/selftest.sh new file mode 100755 index 00000000..907124c8 --- /dev/null +++ b/tools/latmeas/selftest.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# Host self-test: encode, disassemble, oracle, NAK diff, template dry run. +# No device is opened. Synthetic template output stays in a temp file. +set -eu +bin=${1:?binary} +root=$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd) +cd "$root" +"$bin" --self-test +"$bin" --print-plan >/dev/null +python3 tools/latmeas/diff_nak.py --check >/dev/null +python3 tools/latmeas/fill_template.py --self-test +dir=$(mktemp -d) +trap 'rm -rf "$dir"' EXIT +"$bin" --emit "$dir/kernels" >/dev/null +test -f "$dir/kernels/manifest.txt" +test -f "$dir/kernels/alu_alu_4.bin" +echo "latmeas host self-test PASS" diff --git a/tools/latmeas/template.md b/tools/latmeas/template.md new file mode 100644 index 00000000..d2f500c7 --- /dev/null +++ b/tools/latmeas/template.md @@ -0,0 +1,30 @@ +# GB10 (sm_121) measured latencies vs NAK + +Chip: NVIDIA GB10 (DGX Spark), sm_121 +Driver: NVIDIA open kernel module , no NVIDIA user-space libraries +Method: dependent-chain timing kernel. Each instruction consumes the previous result, chain length N = , cycles read with , median of runs, . +Source: +How to rerun: + +NAK reference: `nak/sm120_instr_latencies.rs` plus `latencies/sm100/*.csv` at Mesa commit + +## Priority 1: values CUDA++ currently borrows from NAK (needs independent measurement) + +| Writer -> reader | NAK value | NAK source line | GB10 measured | Delta | Runs / spread | Kernel | +|---|---|---|---|---|---|---| +| ALU -> ALU (GPR RAW) | 4 + 1 | | | | | | +| FMA -> FMA (GPR RAW) | 4 + 1 | | | | | | +| Mixed ALU <-> FMA (GPR RAW) | 5 + 1 | | | | | | +| Predicate write -> guard | 13 + 1 = 14 | `latencies/sm100/pred_raw.csv` | | | | | + +## Priority 2: instructions where CUDA++ (nvcc-derived) and NAK disagree + +| Instruction / pair | NAK value | nvcc sm_121 stall | GB10 measured | Agrees with | Kernel | +|---|---|---|---|---|---| +| | | | | NAK / nvcc / neither | | + +## Notes +- Anything not fixed-latency (LDG/LDS, scoreboard waits) is out of scope for this table. +- Confidence notes and anomalies go here. + +The filled copy is written by `tools/latmeas/fill_template.py` into the output directory. This file stays blank: it is the form, not a result. From 2ecbf9437e53ac3f2f583024aa3ce815b37fbfe9 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 1 Nov 2026 17:48:31 +0000 Subject: [PATCH 02/11] Make a short stall visible, and report stall-1 cycles anyway. The fixed-latency chains read a destination armed to a sentinel the producer does not read, so a stale value fails the oracle. Stall-1 SASS is checked for that pair with no scoreboard and no yield. Cycle medians at stall 1 are recorded even when every stall matches. GB10 open still depends on the 0xa04 admission; this does not add it. Co-authored-by: aien-spark --- tests/gpu_latmeas_test.c | 468 +++++++++++++++++++++++---------- tools/latmeas/REPRODUCE.md | 10 +- tools/latmeas/fill_template.py | 25 +- 3 files changed, 351 insertions(+), 152 deletions(-) diff --git a/tests/gpu_latmeas_test.c b/tests/gpu_latmeas_test.c index 7fa86565..556dbe75 100644 --- a/tests/gpu_latmeas_test.c +++ b/tests/gpu_latmeas_test.c @@ -27,6 +27,16 @@ * One warp, 32 threads, uniform control flow inside the timed region. Only * lane 0 stores. Mesa commit and the NAK line citations live in * tools/latmeas/diff_nak.py. + * + * Opening the device on a GB10 needs the driver's SM version 0xa04 admitted + * (draft PR #251, or the same change). select_classes is not modified here. + * + * Each fixed-latency case writes a destination that was armed to SENTINEL and + * does not read that destination. The next instruction reads it. A stale read + * therefore adds SENTINEL, which the host oracle rejects. Back-to-back + * `r = r + 1` can stay correct at stall 1 when the new value is forwarded, so + * that shape cannot show a too-short stall. HMMA keeps the shape that resolved + * on the chip: d starts at 0, HMMA writes it, FADD reads d0. */ #include "omega_blackwell_codegen.h" #include "omega_gpu_session.h" @@ -53,16 +63,12 @@ enum { RUNS = 11, - ALU_ITERS = 64, - ALU_UNROLL = 8, - MIXED_ITERS = 64, - PRED_ITERS = 64, - FMNMX_ITERS = 32, - WIDE_ITERS = 64, + N_FIXED = 32, HMMA_ITERS = 1, SWEEP_FIXED = 15, SWEEP_HMMA = 36 }; +#define SENTINEL 0x7f800000u /* +inf bits: not equal to 1..N or to f32 1..N */ _Static_assert(RUNS % 2 == 1, "median uses the middle of an odd run count"); _Static_assert(SWEEP_HMMA <= SWEEP_MAX, "sweep fits the sample array"); @@ -78,12 +84,12 @@ typedef struct { } CaseDef; static const CaseDef CASES[] = { - {"alu_alu", ALU_ITERS, ALU_UNROLL, SWEEP_FIXED, ALU_ITERS * ALU_UNROLL, 0}, - {"fma_fma", ALU_ITERS, ALU_UNROLL, SWEEP_FIXED, ALU_ITERS * ALU_UNROLL, 0}, - {"mixed", MIXED_ITERS, 1, SWEEP_FIXED, MIXED_ITERS * 2, 0}, - {"pred", PRED_ITERS, 1, SWEEP_FIXED, PRED_ITERS, 0}, - {"fmnmx_alu", FMNMX_ITERS, 1, SWEEP_FIXED, FMNMX_ITERS, 0}, - {"imad_wide_lo", WIDE_ITERS, 1, SWEEP_FIXED, WIDE_ITERS, 0}, + {"alu_alu", N_FIXED, 1, SWEEP_FIXED, N_FIXED, 0}, + {"fma_fma", N_FIXED, 1, SWEEP_FIXED, N_FIXED, 0}, + {"mixed", N_FIXED, 1, SWEEP_FIXED, N_FIXED * 2u, 0}, + {"pred", N_FIXED, 1, SWEEP_FIXED, N_FIXED, 0}, + {"fmnmx_alu", N_FIXED, 1, SWEEP_FIXED, N_FIXED, 0}, + {"imad_wide_lo", N_FIXED, 1, SWEEP_FIXED, N_FIXED, 0}, {"hmma_reader", HMMA_ITERS, 1, SWEEP_HMMA, HMMA_ITERS, 1}, }; #define NCASES ((int)(sizeof CASES / sizeof CASES[0])) @@ -151,13 +157,11 @@ static void movz(B *b, int dst, int sub) { add(b, BW_IR_MOV_RZ, dst, NOREG, NOREG, NOREG, 0, cw(6, SB_NONE, SB_NONE, wait, 0), 0, 0, sub, 0); b->pending = 0; } -static void iadd(B *b, int dst, int a, int c, unsigned stall, int dst_sub, int src_sub) { - /* src_sub selects a subregister of the source that lives in src2's slot? IADD3's - * first GPR source is src1. Subregister is carried by editing the insn after append - * when src_sub != 0: the encoder adds src1_subreg. */ +static void iadd(B *b, int dst, int a, int c, unsigned stall, int dst_sub, int src1_sub, int src2_sub) { uint32_t wait = b->pending; int at = add(b, BW_IR_IADD3, dst, a, c, NOREG, 0, cw(stall, SB_NONE, SB_NONE, wait, 0), 0, 0, dst_sub, 0); - if (at >= 0 && src_sub) b->p->insns[at].src1_subreg = (uint8_t)src_sub; + if (at >= 0 && src1_sub) b->p->insns[at].src1_subreg = (uint8_t)src1_sub; + if (at >= 0 && src2_sub) b->p->insns[at].src2_subreg = (uint8_t)src2_sub; b->pending = 0; } static void ffma(B *b, int dst, int a, int c, int d, unsigned stall) { @@ -237,49 +241,78 @@ static uint32_t fbits(float f) { return u; } -static uint32_t fmnmx_expect(void) { - uint32_t acc = 0; - float cur = 1.0f; - for (int i = 0; i < FMNMX_ITERS; i++) { - uint32_t t; - memcpy(&t, &cur, 4); - acc += t; - cur += 1.0f; - } - return acc; -} - -static int expect_abs(const CaseDef *c, uint32_t *result, uint32_t *aux) { - if (c->href) return -1; +/* stale=1: the consumer sees SENTINEL instead of the value just written. + * The two results differ for every fixed-latency case. HMMA has no closed form. */ +static int eval_case(const CaseDef *c, int stale, uint32_t *result, uint32_t *aux) { + if (!c || c->href) return -1; + unsigned n = c->iters; *aux = 1; - if (!strcmp(c->id, "alu_alu")) { - *result = c->iters * c->unroll; + *result = 0; + if (!strcmp(c->id, "alu_alu") || !strcmp(c->id, "imad_wide_lo")) { + uint32_t acc = 0, live = 0; + for (unsigned i = 0; i < n; i++) { + uint32_t produced = live + 1u; + acc += stale ? SENTINEL : produced; + live += 1u; + } + *result = acc; return 0; } if (!strcmp(c->id, "fma_fma")) { - *result = fbits((float)(c->iters * c->unroll)); - return 0; - } - if (!strcmp(c->id, "mixed")) { - *result = fbits((float)c->iters); + float acc = 0.f, live = 0.f, inf = 0.f; + memcpy(&inf, &(uint32_t){SENTINEL}, 4); + for (unsigned i = 0; i < n; i++) { + float produced = live + 1.f; + float seen = stale ? inf : produced; + acc = acc + seen; + live = produced; + } + *result = fbits(acc); return 0; } - if (!strcmp(c->id, "pred")) { - *result = c->iters; + if (!strcmp(c->id, "fmnmx_alu")) { + uint32_t acc = 0; + float cur = 1.f; + for (unsigned i = 0; i < n; i++) { + acc += stale ? SENTINEL : fbits(cur); + cur += 1.f; + } + *result = acc; return 0; } - if (!strcmp(c->id, "fmnmx_alu")) { - *result = fmnmx_expect(); + if (!strcmp(c->id, "mixed")) { + uint32_t acc = 0; + float live = 0.f; + for (unsigned i = 0; i < n; i++) { + float produced = live + 1.f; + acc += stale ? SENTINEL : fbits(produced); + live = produced; + } + uint32_t acc2 = 0; + float inf = 0.f; + memcpy(&inf, &(uint32_t){SENTINEL}, 4); + live = 1.f; + for (unsigned i = 0; i < n; i++) { + /* FFMA(0, 0, x) preserves a normal f32 and yields +inf for SENTINEL. */ + float out = stale ? inf : live; + acc2 += fbits(out); + live += 1.f; + } + *result = acc; + *aux = acc2; return 0; } - if (!strcmp(c->id, "imad_wide_lo")) { - *result = c->iters; - *aux = 0; + if (!strcmp(c->id, "pred")) { + *result = stale ? PRED_POISON : n; return 0; } return -1; } +static int expect_abs(const CaseDef *c, uint32_t *result, uint32_t *aux) { + return eval_case(c, 0, result, aux); +} + static void prologue(B *b, int *ptr) { b->desc = omega_bw_ir_alloc_uvreg64(b->p); if (b->desc < 0) b->err = -1; @@ -311,48 +344,67 @@ static void store_tail(B *b, int ptr, int t0, int t1, int result, int rsub, int } static void loop_ctl(B *b, int i, int nreg, int one, int *loop_back) { - iadd(b, i, i, one, 1, 0, 0); + iadd(b, i, i, one, 1, 0, 0, 0); isetp_ge(b, i, nreg, 14); /* P0 = (i >= n) */ *loop_back = bra(b, 1, 1, 5); /* @!P0 BRA loop */ } -static void body(B *b, const CaseDef *c, unsigned stall, int r, int one, int zero, int fone, int acc, int cur, int tmp, - int rpair, int tpair, int *poison_br) { +/* Producer writes dst, which was armed to SENTINEL and which the producer does + * not read. The very next instruction reads dst. A stale read adds SENTINEL. */ +static void body(B *b, const CaseDef *c, unsigned stall, int live, int livef, int one, int zero, int fone, int acc, + int acc2, int dst, int dstf, int seen, int rpair, int tpair, int *poison_br) { unsigned head = stall_head(stall); unsigned rest = stall - head; if (!strcmp(c->id, "alu_alu")) { - for (unsigned u = 0; u < c->unroll; u++) iadd(b, r, r, one, head, 0, 0); + iadd(b, dst, live, one, head, 0, 0, 0); + gap_nops(b, rest); + iadd(b, acc, acc, dst, 15, 0, 0, 0); + iadd(b, live, live, one, 15, 0, 0, 0); + movi(b, dst, SENTINEL, 0); } else if (!strcmp(c->id, "fma_fma")) { - for (unsigned u = 0; u < c->unroll; u++) ffma(b, r, fone, fone, r, head); + ffma(b, dst, fone, fone, livef, head); /* dst = livef + 1 */ + gap_nops(b, rest); + fadd(b, acc, acc, 0, dst, 15); /* FMA consumer, immediate */ + fadd(b, livef, livef, 0, fone, 15); + movi(b, dst, SENTINEL, 0); } else if (!strcmp(c->id, "mixed")) { - /* FFMA then IADD3 of zero: FMA->ALU and, on the next iteration, ALU->FMA. */ - ffma(b, r, fone, fone, r, head); + ffma(b, dstf, fone, fone, livef, head); gap_nops(b, rest); - iadd(b, r, r, zero, head, 0, 0); + iadd(b, acc, acc, dstf, 15, 0, 0, 0); /* FMA -> ALU */ + fadd(b, livef, livef, 0, fone, 15); + movi(b, dstf, SENTINEL, 0); + iadd(b, dst, live, zero, head, 0, 0, 0); /* ALU copy of a normal f32 bit pattern */ + gap_nops(b, rest); + ffma(b, seen, zero, zero, dst, 15); /* ALU -> FMA, immediate */ + iadd(b, acc2, acc2, seen, 15, 0, 0, 0); + fadd(b, live, live, 0, fone, 15); + movi(b, dst, SENTINEL, 0); } else if (!strcmp(c->id, "pred")) { - /* Setup leaves P0 true. The measured ISETP clears it (0 >= r after r - * has been incremented). A short stall lets @P0 see the leftover true - * and take the poison branch. */ + /* P0 is true from the previous setup. This ISETP clears it. The next + * instruction is @P0 BRA poison, so a stale predicate takes the poison. */ isetp_ge(b, one, zero, 15); - iadd(b, r, r, one, 15, 0, 0); - isetp_ge(b, zero, r, head); + iadd(b, live, live, one, 15, 0, 0, 0); + isetp_ge(b, zero, live, head); + gap_nops(b, rest); *poison_br = bra(b, 1, 0, 1); } else if (!strcmp(c->id, "fmnmx_alu")) { - fmnmx(b, tmp, cur, cur, head); + fmnmx(b, dst, livef, livef, head); gap_nops(b, rest); - iadd(b, acc, acc, tmp, 15, 0, 0); - fadd(b, cur, cur, 0, fone, 15); + iadd(b, acc, acc, dst, 15, 0, 0, 0); + fadd(b, livef, livef, 0, fone, 15); + movi(b, dst, SENTINEL, 0); } else if (!strcmp(c->id, "imad_wide_lo")) { uint32_t wait = b->pending; add(b, BW_IR_IMAD_WIDE, tpair, one, NOREG, rpair, 1, cw(head, SB_NONE, SB_NONE, wait, 0), 0, 0, 0, 0); b->pending = 0; gap_nops(b, rest); - iadd(b, rpair, tpair, zero, 15, 0, 0); + iadd(b, acc, acc, tpair, 15, 0, 0, 0); + iadd(b, rpair, rpair, one, 15, 0, 0, 0); + movi(b, tpair, SENTINEL, 0); + movi(b, tpair, SENTINEL, 1); } else { b->err = -1; } - if (strcmp(c->id, "mixed") && strcmp(c->id, "pred") && strcmp(c->id, "fmnmx_alu") && strcmp(c->id, "imad_wide_lo")) - gap_nops(b, rest); } static int build(const CaseDef *c, unsigned stall, unsigned ovh_iters, int overhead, uint8_t *code, size_t cap, @@ -368,7 +420,8 @@ static int build(const CaseDef *c, unsigned stall, unsigned ovh_iters, int overh int ptr = NOREG; prologue(&b, &ptr); int one = V(&b), zero = V(&b), fone = V(&b), i = V(&b), nreg = V(&b); - int r = V(&b), acc = V(&b), cur = V(&b), tmp = V(&b), t0 = V(&b), t1 = V(&b), aux = V(&b); + int r = V(&b), acc = V(&b), acc2 = V(&b), cur = V(&b), tmp = V(&b), dsti = V(&b), seen = V(&b); + int t0 = V(&b), t1 = V(&b), aux = V(&b); int rpair = NOREG, tpair = NOREG, dquad = NOREG, aquad = NOREG, bpair = NOREG, cquad = NOREG; unsigned iters = overhead ? ovh_iters : c->iters; movi(&b, one, 1, 0); @@ -378,15 +431,23 @@ static int build(const CaseDef *c, unsigned stall, unsigned ovh_iters, int overh movz(&b, i, 0); movz(&b, r, 0); movz(&b, acc, 0); - movi(&b, cur, 0x3f800000u, 0); + movz(&b, acc2, 0); + movz(&b, cur, 0); movi(&b, aux, 1, 0); + if (!overhead && c && !c->href && strcmp(c->id, "pred") != 0 && strcmp(c->id, "imad_wide_lo") != 0) + movi(&b, dsti, SENTINEL, 0); + if (!overhead && !strcmp(c->id, "fmnmx_alu")) movi(&b, cur, 0x3f800000u, 0); + if (!overhead && !strcmp(c->id, "mixed")) { + movi(&b, r, 0x3f800000u, 0); /* ALU source: f32 1.0 bits, a normal number */ + movi(&b, tmp, SENTINEL, 0); /* FMA destination, separate from dsti */ + } if (!overhead && !strcmp(c->id, "imad_wide_lo")) { rpair = V64(&b); tpair = V64(&b); movz(&b, rpair, 0); movz(&b, rpair, 1); - movz(&b, tpair, 0); - movz(&b, tpair, 1); + movi(&b, tpair, SENTINEL, 0); + movi(&b, tpair, SENTINEL, 1); } if (!overhead && !strcmp(c->id, "hmma_reader")) { aquad = V128(&b); @@ -417,7 +478,7 @@ static int build(const CaseDef *c, unsigned stall, unsigned ovh_iters, int overh b.err = -1; } } else if (!overhead) { - body(&b, c, stall, r, one, zero, fone, acc, cur, tmp, rpair, tpair, &poison_br); + body(&b, c, stall, r, cur, one, zero, fone, acc, acc2, dsti, tmp, seen, rpair, tpair, &poison_br); } if (!(c && !overhead && c->iters == 1 && !strcmp(c->id, "hmma_reader"))) { loop_ctl(&b, i, nreg, one, &back); @@ -436,16 +497,11 @@ static int build(const CaseDef *c, unsigned stall, unsigned ovh_iters, int overh int rsub = 0; int auxv = aux; int asub = 0; - if (!overhead && !strcmp(c->id, "fmnmx_alu")) result = acc; - if (!overhead && !strcmp(c->id, "imad_wide_lo")) { - result = rpair; - rsub = 0; - auxv = rpair; - asub = 1; - } + if (!overhead && c && !c->href && strcmp(c->id, "pred") != 0) result = acc; + if (!overhead && !strcmp(c->id, "mixed")) auxv = acc2; if (!overhead && !strcmp(c->id, "hmma_reader")) { result = acc; - auxv = dquad; /* late read of d1, diagnostic, after the timed region */ + auxv = dquad; /* late read of d1, after the timed region: diagnostic only */ asub = 1; } if (overhead) result = i; @@ -487,6 +543,76 @@ static int count_opc_stall(const uint8_t *code, size_t ninsn, unsigned opc, unsi } return n; } +static unsigned reg_dst(const uint8_t *ins) { return (rd32(ins) >> 16) & 0xffu; } +static unsigned reg_a(const uint8_t *ins) { return (rd32(ins) >> 24) & 0xffu; } +static unsigned reg_b(const uint8_t *ins) { return rd32(ins + 4) & 0xffu; } +static unsigned reg_c(const uint8_t *ins) { return rd32(ins + 8) & 0xffu; } +static int bare_issue(const uint8_t *ins) { + unsigned w3 = rd32(ins + 12); + return ((w3 >> 13) & 1u) == 0u && ((w3 >> 14) & 7u) == 7u && ((w3 >> 17) & 7u) == 7u && ((w3 >> 20) & 63u) == 0u; +} +static int reads_reg(const uint8_t *ins, unsigned r) { + unsigned opc = opc_of(ins); + if (opc == 0x223u) return reg_c(ins) == r; + if (opc == 0x221u || opc == 0x210u) return reg_a(ins) == r || reg_b(ins) == r; + return 0; +} +static int bra_index(const uint8_t *ins, int index) { + uint32_t w0 = rd32(ins), w1 = rd32(ins + 4); + int32_t low = (int32_t)((w0 >> 18) & 0x3fu); + int32_t high = (int32_t)w1 >> 2; + int32_t rel = (high << 6) | low; + return index + rel + 1; +} + +/* Stall-1 shape: the consumer is the next instruction, it reads the producer's + * destination, and the producer carries no scoreboard and no yield. A stale + * read of the armed SENTINEL is then visible to the oracle. */ +static int check_raw_window(const char *id, const uint8_t *code, size_t ninsn, unsigned stall) { + int failed = 0, producers = 0; +#define W(cond, ...) \ + do { \ + if (!(cond)) { \ + failed++; \ + printf("FAIL: %s: ", id); \ + printf(__VA_ARGS__); \ + printf("\n"); \ + } \ + } while (0) + for (size_t i = 0; i + 1 < ninsn; i++) { + const uint8_t *ins = code + i * 16; + const uint8_t *nxt = code + (i + 1) * 16; + unsigned opc = opc_of(ins); + if (stall_of(ins) != stall) continue; + int take = 0; + if (!strcmp(id, "alu_alu") && opc == 0x210u && reg_dst(ins) != reg_a(ins)) take = 1; + if (!strcmp(id, "fma_fma") && opc == 0x223u && reg_dst(ins) != reg_c(ins)) take = 1; + if (!strcmp(id, "mixed") && opc == 0x223u && reg_dst(ins) != reg_c(ins)) take = 1; + if (!strcmp(id, "mixed") && opc == 0x210u && reg_dst(ins) != reg_a(ins)) take = 1; + if (!strcmp(id, "fmnmx_alu") && opc == 0x209u && reg_dst(ins) != reg_a(ins)) take = 1; + if (!strcmp(id, "imad_wide_lo") && opc == 0x825u) take = 1; + if (!strcmp(id, "hmma_reader") && opc == 0x23cu) take = 1; + if (!strcmp(id, "pred") && opc == 0x20cu) take = 1; + if (!take) continue; + producers++; + W(bare_issue(ins), "producer %zu is scoreboarded or yields (w3 0x%08x)", i, rd32(ins + 12)); + if (!strcmp(id, "pred")) { + W(opc_of(nxt) == 0x947u, "predicate consumer is not BRA"); + W(((rd32(nxt) >> 12) & 0xfu) == 0u, "poison branch is not @P0"); + int tgt = bra_index(nxt, (int)i + 1); + W(tgt >= 0 && (size_t)tgt < ninsn, "poison target %d", tgt); + if (tgt >= 0 && (size_t)tgt < ninsn) { + const uint8_t *mv = code + (size_t)tgt * 16; + W(opc_of(mv) == 0x802u && rd32(mv + 4) == PRED_POISON, "poison mov missing"); + } + } else { + W(reads_reg(nxt, reg_dst(ins)), "consumer does not read producer dest R%u", reg_dst(ins)); + } + } + W(producers == (!strcmp(id, "mixed") ? 2 : 1), "producer count %d", producers); +#undef W + return failed; +} static int self_test(void) { int failed = 0; @@ -504,17 +630,25 @@ static int self_test(void) { } while (0) CHECK(build(&CASES[0], 0, 0, 0, code, sizeof code, &len, &gpr) != 0, "stall 0 must be refused"); CHECK(build(&CASES[0], 99, 0, 0, code, sizeof code, &len, &gpr) != 0, "stall past the sweep must be refused"); - uint32_t er = 0, ea = 0; - CHECK(expect_abs(&CASES[0], &er, &ea) == 0 && er == 512 && ea == 1, "alu oracle"); - CHECK(expect_abs(&CASES[1], &er, &ea) == 0 && er == fbits(512.0f), "fma oracle"); - CHECK(expect_abs(&CASES[2], &er, &ea) == 0 && er == fbits(64.0f), "mixed oracle"); - CHECK(expect_abs(&CASES[3], &er, &ea) == 0 && er == 64, "pred oracle"); - CHECK(expect_abs(&CASES[4], &er, &ea) == 0 && er == fmnmx_expect() && er != 0, "fmnmx oracle"); - CHECK(expect_abs(&CASES[5], &er, &ea) == 0 && er == 64 && ea == 0, "wide oracle"); - CHECK(expect_abs(&CASES[6], &er, &ea) != 0, "hmma has no closed host formula"); + for (int ci = 0; ci < NCASES; ci++) { + if (CASES[ci].href) continue; + uint32_t good = 0, bad = 0, ga = 0, ba = 0; + CHECK(eval_case(&CASES[ci], 0, &good, &ga) == 0, "%s fresh oracle", CASES[ci].id); + CHECK(eval_case(&CASES[ci], 1, &bad, &ba) == 0, "%s stale oracle", CASES[ci].id); + CHECK(good != bad || ga != ba, "%s stale read is distinguishable (fresh 0x%x stale 0x%x)", CASES[ci].id, good, bad); + } + { + uint32_t good = 0, aux = 0; + CHECK(eval_case(&CASES[0], 0, &good, &aux) == 0 && good == (N_FIXED * (N_FIXED + 1u)) / 2u, "alu sum"); + CHECK(eval_case(&CASES[1], 0, &good, &aux) == 0 && good == fbits((float)(N_FIXED * (N_FIXED + 1u)) / 2u), "fma sum"); + } + { + uint32_t er = 0, ea = 0; + CHECK(expect_abs(&CASES[6], &er, &ea) != 0, "hmma has no closed host formula"); + } for (int ci = 0; ci < NCASES; ci++) { - unsigned stall = CASES[ci].href ? 29u : 4u; + unsigned stall = 1u; len = 0; int rc = build(&CASES[ci], stall, 0, 0, code, sizeof code, &len, &gpr); CHECK(rc == 0, "%s encode rc %d", CASES[ci].id, rc); @@ -535,38 +669,12 @@ static int self_test(void) { int unk = mf ? omega_sm121_listing(code, len, mf) : -1; if (mf) fclose(mf); CHECK(unk == 2, "%s disasm unknowns %d (the two CS2R sites)", CASES[ci].id, unk); - if (!strcmp(CASES[ci].id, "alu_alu")) { - CHECK(text && strstr(text, "IADD3"), "alu listing"); - CHECK(count_opc_stall(code, ninsn, 0x210, 4) == ALU_UNROLL, "alu chain stall count"); - } else if (!strcmp(CASES[ci].id, "fma_fma")) { - CHECK(text && strstr(text, "FFMA"), "fma listing"); - CHECK(count_opc_stall(code, ninsn, 0x223, 4) == ALU_UNROLL, "fma chain stall count"); - } else if (!strcmp(CASES[ci].id, "mixed")) { - CHECK(count_opc_stall(code, ninsn, 0x223, 4) == 1, "mixed ffma"); - CHECK(count_opc_stall(code, ninsn, 0x210, 4) == 1, "mixed iadd"); - } else if (!strcmp(CASES[ci].id, "pred")) { - CHECK(text && strstr(text, "ISETP"), "pred listing"); - CHECK(text && strstr(text, "BRA"), "pred branch"); - CHECK(count_opc_stall(code, ninsn, 0x20c, 4) == 1, "measured isetp stall"); - } else if (!strcmp(CASES[ci].id, "fmnmx_alu")) { - CHECK(text && strstr(text, "FMNMX"), "fmnmx listing"); - CHECK(count_opc_stall(code, ninsn, 0x209, 4) == 1, "fmnmx stall"); - } else if (!strcmp(CASES[ci].id, "imad_wide_lo")) { - CHECK(text && strstr(text, "IMAD.WIDE"), "wide listing"); - CHECK(count_opc_stall(code, ninsn, 0x825, 4) == 1, "wide stall"); - int saw = 0; - for (size_t i = 0; i + 1 < ninsn; i++) { - if (opc_of(code + i * 16) == 0x825 && stall_of(code + i * 16) == 4) { - unsigned dst = (rd32(code + i * 16) >> 16) & 0xffu; - unsigned src = (rd32(code + (i + 1) * 16) >> 24) & 0xffu; - CHECK(opc_of(code + (i + 1) * 16) == 0x210, "wide consumer is IADD3"); - CHECK(src == dst, "IADD3 reads the wide low half"); - saw = 1; - } - } - CHECK(saw, "wide pair found"); - } else if (!strcmp(CASES[ci].id, "hmma_reader")) { - CHECK(text && strstr(text, "HMMA"), "hmma listing"); + failed += check_raw_window(CASES[ci].id, code, ninsn, stall); + if (!strcmp(CASES[ci].id, "hmma_reader")) { + len = 0; + rc = build(&CASES[ci], 29u, 0, 0, code, sizeof code, &len, &gpr); + CHECK(rc == 0, "hmma stall 29 encode"); + ninsn = g_prog.count; CHECK(count_opc_stall(code, ninsn, 0x23c, 15) == 1, "hmma stall 15"); CHECK(count_opc_stall(code, ninsn, 0x918, 14) == 1, "hmma padding nop stall 14 (15+14=29)"); int saw = 0; @@ -576,11 +684,17 @@ static int self_test(void) { opc_of(code + (i + 2) * 16) == 0x221) { unsigned dst = (rd32(code + i * 16) >> 16) & 0xffu; unsigned src = (rd32(code + (i + 2) * 16) >> 24) & 0xffu; - CHECK(src == dst, "FADD reads HMMA d0"); + CHECK(src == dst, "FADD reads HMMA d0 after the 29-cycle gap"); saw = 1; } } CHECK(saw, "hmma, nop, fadd sequence"); + /* stall 19 is HMMA stall 15 plus one NOP of stall 4: the gap the chip + * can fail without faulting. The self-test only checks the shape. */ + len = 0; + CHECK(build(&CASES[ci], 19u, 0, 0, code, sizeof code, &len, &gpr) == 0, "hmma stall 19 encodes"); + CHECK(count_opc_stall(code, g_prog.count, 0x23c, 15) == 1, "hmma 19 head"); + CHECK(count_opc_stall(code, g_prog.count, 0x918, 4) == 1, "hmma 19 tail nop"); } for (size_t i = 0; i < ninsn; i++) { if (opc_of(code + i * 16) == CS2R_OPC) { @@ -614,6 +728,11 @@ static int required_stall(const int pass[][SWEEP_MAX], int runs, int sweep) { } return -1; } +/* 1 means every stall matched, which is not a measured threshold. -1 means no suffix did. */ +static int reported_threshold(const int pass[][SWEEP_MAX], int runs, int sweep) { + int start = required_stall(pass, runs, sweep); + return start > 1 ? start : -1; +} static int self_test_suffix(void) { int failed = 0; @@ -635,6 +754,12 @@ static int self_test_suffix(void) { printf("FAIL: empty sweep must not invent a stall\n"); failed++; } + for (int r = 0; r < RUNS; r++) + for (int s = 1; s <= 15; s++) pass[r][s - 1] = 1; + if (reported_threshold(pass, RUNS, 15) != -1) { + printf("FAIL: a sweep that passes at stall 1 is not a measured threshold\n"); + failed++; + } return failed; } @@ -653,9 +778,10 @@ static int plan_launches(void) { static void print_plan(void) { int n = plan_launches(); + unsigned upper_s = (unsigned)((n * 50 + 999) / 1000); printf("latmeas plan: %d cases, %d runs, sweep 1..15 (HMMA 1..36), %d launches\n", NCASES, RUNS, n); - printf("estimated chip window: 90 s upper bound (%d launches; 50 ms each covers a cold shader invalidate). Usually shorter.\n", - n); + printf("estimated chip window: %u s upper bound (%d launches; 50 ms each covers a cold shader invalidate). Usually shorter.\n", + upper_s, n); printf("The runner wraps only the launch with: flock /tmp/aien-gb10.lock timeout 600\n"); printf("One warp, 32 threads. Clock: CS2R idx 0x50. Mesa %s\n", MESA_COMMIT); } @@ -696,8 +822,15 @@ static int emit_kernels(const char *dir) { CASES[ci].ops, gpr, len); } } - unsigned ovh_iters[] = {64, 32, 1}; - for (int k = 0; k < 3; k++) { + unsigned ovh_iters[8]; + int novh = 0; + for (int ci = 0; ci < NCASES; ci++) { + int seen = 0; + for (int j = 0; j < novh; j++) + if (ovh_iters[j] == CASES[ci].iters) seen = 1; + if (!seen && novh < 8) ovh_iters[novh++] = CASES[ci].iters; + } + for (int k = 0; k < novh; k++) { size_t len = 0; uint32_t gpr = 0; if (build(NULL, 0, ovh_iters[k], 1, code, sizeof code, &len, &gpr) != 0) { @@ -880,14 +1013,26 @@ static int run_launch(const char *kdir, const char *out_path) { uint64_t cyc[NCASES][RUNS][SWEEP_MAX]; uint32_t href[NCASES][RUNS]; int href_ok[NCASES][RUNS]; - uint64_t ovh_cyc[3][RUNS]; + uint64_t ovh_cyc[8][RUNS]; + uint32_t run0_res[NCASES][SWEEP_MAX]; + uint32_t run0_ref[NCASES]; memset(pass, 0, sizeof pass); memset(cyc, 0, sizeof cyc); memset(href, 0, sizeof href); memset(href_ok, 0, sizeof href_ok); memset(ovh_cyc, 0, sizeof ovh_cyc); + memset(run0_res, 0, sizeof run0_res); + memset(run0_ref, 0, sizeof run0_ref); char notes[NCASES][384]; memset(notes, 0, sizeof notes); + unsigned ovh_list[8]; + int novh = 0; + for (int ci = 0; ci < NCASES && novh < 8; ci++) { + int seen = 0; + for (int j = 0; j < novh; j++) + if (ovh_list[j] == CASES[ci].iters) seen = 1; + if (!seen) ovh_list[novh++] = CASES[ci].iters; + } int device_error = 0; const char *err = ""; @@ -911,8 +1056,7 @@ static int run_launch(const char *kdir, const char *out_path) { goto write_json; } - unsigned ovh_list[] = {64, 32, 1}; - for (int oi = 0; oi < 3 && !device_error; oi++) { + for (int oi = 0; oi < novh && !device_error; oi++) { char id[16]; snprintf(id, sizeof id, "ovh%u", ovh_list[oi]); Kern *k = find_kern(kerns, nk, id, 0); @@ -959,6 +1103,10 @@ static int run_launch(const char *kdir, const char *out_path) { href_ok[ci][r] = have_ref; cyc[ci][r][c->sweep - 1] = cy; pass[ci][r][c->sweep - 1] = have_ref; + if (r == 0) { + run0_ref[ci] = refv; + run0_res[ci][c->sweep - 1] = refv; + } if (!have_ref) snprintf(notes[ci], sizeof notes[ci], "HMMA reference at stall %u is 0x%08x (aux 0x%08x); d0 did not move", c->sweep, w[2], w[3]); @@ -982,6 +1130,7 @@ static int run_launch(const char *kdir, const char *out_path) { if (expect_abs(c, &expv, &auxv) == 0) ok = (w[2] == expv && w[3] == auxv && w[0] != POISON); } pass[ci][r][s - 1] = ok; + if (r == 0) run0_res[ci][s - 1] = w[2]; } } } @@ -1013,24 +1162,30 @@ write_json:; device_error ? "false" : "true", MESA_COMMIT, drv_esc, mach_esc, RUNS, plan_launches(), err_esc); for (int ci = 0; ci < NCASES; ci++) { const CaseDef *c = &CASES[ci]; - int req = device_error ? -1 : required_stall(pass[ci], RUNS, (int)c->sweep); + int req = device_error ? -1 : reported_threshold(pass[ci], RUNS, (int)c->sweep); int spread_s = 0; - int row_req[RUNS]; - int all_rows = 1; - for (int r = 0; r < RUNS; r++) { - int one[1][SWEEP_MAX]; - memcpy(one[0], pass[ci][r], sizeof one[0]); - row_req[r] = required_stall(one, 1, (int)c->sweep); - if (row_req[r] < 0) all_rows = 0; - } - if (!all_rows) req = -1; - int smin = 1000, smax = -1; - if (all_rows && !device_error) { + int smin = 1000, smax = -1, nthr = 0; + if (!device_error) { for (int r = 0; r < RUNS; r++) { - if (row_req[r] < smin) smin = row_req[r]; - if (row_req[r] > smax) smax = row_req[r]; + int one[1][SWEEP_MAX]; + memcpy(one[0], pass[ci][r], sizeof one[0]); + int row = reported_threshold(one, 1, (int)c->sweep); + if (row > 1) { + if (row < smin) smin = row; + if (row > smax) smax = row; + nthr++; + } } - spread_s = smax - smin; + if (nthr == RUNS) spread_s = smax - smin; + } + if (!device_error && req < 0 && notes[ci][0] == 0) { + int any_fail = 0; + for (unsigned s = 1; s <= c->sweep && !any_fail; s++) + for (int r = 0; r < RUNS; r++) + if (!pass[ci][r][s - 1]) any_fail = 1; + snprintf(notes[ci], sizeof notes[ci], + any_fail ? "no stall was correct through the top of the sweep" + : "every stall matched the oracle; threshold is at or below 1"); } uint64_t samples[RUNS]; uint64_t cmed = 0, cspread = 0; @@ -1044,13 +1199,24 @@ write_json:; for (int r = 0; r < RUNS; r++) if (samples[r] == 0) have_cyc = 0; } - unsigned ovh_iters = c->iters; - int oi = ovh_iters == 64 ? 0 : ovh_iters == 32 ? 1 : 2; + int oi = 0; + for (int k = 0; k < novh; k++) + if (ovh_list[k] == c->iters) oi = k; uint64_t oc[RUNS]; memcpy(oc, ovh_cyc[oi], sizeof oc); uint64_t ospread = 0; uint64_t omed = median_spread(oc, RUNS, &ospread); (void)ospread; + uint64_t s1[RUNS], s1copy[RUNS], s1med = 0, s1spread = 0; + int have_s1 = !device_error; + for (int r = 0; r < RUNS; r++) { + s1[r] = cyc[ci][r][0]; + if (s1[r] == 0) have_s1 = 0; + } + if (have_s1) { + memcpy(s1copy, s1, sizeof s1); + s1med = median_spread(s1copy, RUNS, &s1spread); + } char note_esc[512]; json_escape(notes[ci], note_esc, sizeof note_esc); fprintf(jf, " \"%s\": {\n", c->id); @@ -1058,17 +1224,31 @@ write_json:; fprintf(jf, " \"stall_required\": %d,\n", req); else fprintf(jf, " \"stall_required\": null,\n"); - fprintf(jf, " \"stall_spread\": %d,\n", all_rows && req > 0 && !device_error ? spread_s : 0); + fprintf(jf, " \"stall_spread\": %d,\n", req > 0 && !device_error ? spread_s : 0); if (have_cyc && !device_error) { fprintf(jf, " \"cycle_median\": %llu,\n \"cycle_spread\": %llu,\n", (unsigned long long)cmed, (unsigned long long)cspread); } else { fprintf(jf, " \"cycle_median\": null,\n \"cycle_spread\": null,\n"); } + if (have_s1) { + fprintf(jf, " \"cycle_median_stall1\": %llu,\n \"cycle_spread_stall1\": %llu,\n", + (unsigned long long)s1med, (unsigned long long)s1spread); + } else { + fprintf(jf, " \"cycle_median_stall1\": null,\n \"cycle_spread_stall1\": null,\n"); + } if (device_error) fprintf(jf, " \"overhead_cycle_median\": null,\n"); else fprintf(jf, " \"overhead_cycle_median\": %llu,\n", (unsigned long long)omed); + if (c->href && !device_error) + fprintf(jf, " \"reference_result\": %u,\n", run0_ref[ci]); + else + fprintf(jf, " \"reference_result\": null,\n"); + fprintf(jf, " \"run0_result\": ["); + for (unsigned s = 1; s <= c->sweep; s++) + fprintf(jf, "%s%u", s == 1 ? "" : ", ", run0_res[ci][s - 1]); + fprintf(jf, "],\n"); fprintf(jf, " \"iters\": %u,\n \"ops\": %u,\n", c->iters, c->ops); fprintf(jf, " \"note\": \"%s\"\n }%s\n", note_esc, ci + 1 == NCASES ? "" : ","); } diff --git a/tools/latmeas/REPRODUCE.md b/tools/latmeas/REPRODUCE.md index b2b143ff..d570bd01 100644 --- a/tools/latmeas/REPRODUCE.md +++ b/tools/latmeas/REPRODUCE.md @@ -4,13 +4,15 @@ This measures fixed instruction latencies on an NVIDIA GB10 (DGX Spark, sm_121) ## Command -From a CUDA++ checkout, on the Spark, with `/dev/nvidia0` and `/dev/nvidia-uvm` present and the open kernel module loaded: +From a CUDA++ checkout, on the Spark, with `/dev/nvidia0` and `/dev/nvidia-uvm` present and the open kernel module loaded. + +This tree does not admit SM version `0xa04`. The GB10 driver (580.173.02) reports `NV2080_CTRL_GR_INFO_SM_VERSION_10_04`, and `nvrm_open` then fails with `unsupported SM 0xa04 or missing target classes`. Draft PR #251 (or the same admission) has to be merged, or applied locally, before the launch. The fix is not in this change. ```sh sh tools/latmeas/run_spark.sh latmeas-out ``` -The script builds and encodes first. It then runs `make doctor` and stops if the machine is not ready. Only the launch is inside `flock /tmp/aien-gb10.lock`, and that launch is wrapped in `timeout 600`. The binary prints an upper bound before it opens the device: about 1400 launches, 90 seconds if each one takes 50 ms, usually shorter. The hard stop is 600 seconds. +The script builds and encodes first. It then runs `make doctor` and stops if the machine is not ready. Only the launch is inside `flock /tmp/aien-gb10.lock`, and that launch is wrapped in `timeout 600`. The binary prints an upper bound before it opens the device: 1408 launches, 71 seconds if each one takes 50 ms, usually shorter. The hard stop is 600 seconds. Outputs, all under `latmeas-out/`: @@ -28,9 +30,9 @@ One warp, 32 threads, all of them executing the same chain. Lane 0 stores four w The clock is `CS2R` with index `0x50`. NAK's `nak_private.h` names that `NAK_SV_CLOCK_LO`. On SM 50 and later, `from_nir.rs` emits `CS2R` rather than `S2R` (`sm70_encode.rs`: opcode `0x805`, index at bits 72..80). CUDA++'s encoder has `S2R` only, so the test emits `S2R` immediate `0x50` and rewrites the opcode to `0x805`. -NAK's table is the stall the next instruction must wait, not a scoreboard. These chips do not interlock a fixed-latency GPR write (`src/omega_numeric.c`). The runner therefore tries stall 1 through 15 (1 through 36 for HMMA, because the field is only four bits and the rest is `NOP`s). The reported stall is the smallest S such that S and every larger S matched the oracle on all 11 runs. The cycle median is the 32-bit clock delta at that S. An empty loop with the same trip count is the overhead control; its median is in `raw.json`. +The runner tries stall 1 through 15 (1 through 36 for HMMA; the stall field is four bits, and anything past 15 is `NOP`s). `stall_required` is the smallest S above 1 such that S and every larger S matched on all 11 runs. If stall 1 already matches, `stall_required` is null: that is not a measured latency of 1. `cycle_median_stall1` is the clock delta at programmed stall 1 whether or not the oracle matched. `cycle_median` is the delta at `stall_required` when that exists. `run0_result` is lane 0's result word at each stall of the first run. For HMMA, `reference_result` is `d0` at stall 36; a zero reference leaves `stall_required` null. -The oracle for ALU, FMA, mixed ALU/FMA, the predicate chain, FMNMX, and IMAD.WIDE is a host formula (exact integer or exact f32). HMMA's oracle is the same kernel at stall 36: lane 0's `d0` must be non-zero there, and shorter stalls must match it. A zero reference leaves the HMMA cell blank. +The fixed-latency oracle arms the destination to `0x7f800000` and the producer does not read it. The next instruction does. A stale read adds that sentinel. The host self-test checks the fresh and stale totals differ, and that the stall-1 SASS really is that pair with no scoreboard and no yield. HMMA is unchanged: `d` starts at 0, the consumer is `FADD` of `d0`, and stall 19 is HMMA stall 15 plus a `NOP` of stall 4. ## NAK pin diff --git a/tools/latmeas/fill_template.py b/tools/latmeas/fill_template.py index 2717e1af..5a009d11 100755 --- a/tools/latmeas/fill_template.py +++ b/tools/latmeas/fill_template.py @@ -37,13 +37,21 @@ def agree(measured, nak_stall, enc_stall): def runs_text(case, runs): - if not case or case.get("stall_required") is None: + if not case: return "" - parts = ["%d runs" % runs, "stall spread %s" % case.get("stall_spread")] - if case.get("cycle_median") is not None: - parts.append("dep cycles median %s spread %s" % (case["cycle_median"], case.get("cycle_spread"))) + parts = ["%d runs" % runs] + if case.get("stall_required") is None: + parts.append("no failing stall in the sweep") + else: + parts.append("stall spread %s" % case.get("stall_spread")) + if case.get("cycle_median") is not None: + parts.append("cycles at that stall median %s spread %s" % (case["cycle_median"], case.get("cycle_spread"))) + if case.get("cycle_median_stall1") is not None: + parts.append("stall-1 cycles median %s spread %s" % (case["cycle_median_stall1"], case.get("cycle_spread_stall1"))) if case.get("overhead_cycle_median") is not None: parts.append("loop overhead median %s" % case["overhead_cycle_median"]) + if case.get("stall_required") is None and case.get("cycle_median_stall1") is None and case.get("overhead_cycle_median") is None: + return "" return "; ".join(parts) @@ -194,6 +202,15 @@ def self_test(): if parts[3] != "": print("FAIL: measured:false still filled a cell (%r)" % parts[3], file=sys.stderr) return 1 + timed = {"synthetic": False, "measured": True, "runs": 11, "cases": { + "alu_alu": {"stall_required": None, "cycle_median_stall1": 1703, "cycle_spread_stall1": 4, + "overhead_cycle_median": 800}}} + text = build_md(cat, timed, False) + alu = [ln for ln in text.splitlines() if ln.startswith("| ALU -> ALU")][0] + parts = [p.strip() for p in alu.strip("|").split("|")] + if parts[3] != "" or "1703" not in parts[5]: + print("FAIL: stall-1 cycles landed in the measured cell or missed the runs cell: %r" % parts, file=sys.stderr) + return 1 print("fill_template self-test PASS") return 0 From 03cdcc5a4401043da2196f4f5efcb9b379400f66 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 1 Nov 2026 18:07:46 +0000 Subject: [PATCH 03/11] Stall the latency-loop increment before its compare. sm121dis of the overhead kernel showed IADD of the counter with stall 1 (control 0x000fc200) immediately before ISETP, and @!P0 BRA back to the body. The compare then samples the previous counter, so the stored trip count misses the overhead check and every exact oracle fails through stall 15. HMMA has no loop. On the predicate chain, stall 1 leaves the previous true P0 for @P0 BRA, so the poison move exits on the first pass and the clocks cover that exit. The host simulator hides GPR writes for 5 cycles and predicate writes for 14. Overhead must store the trip count, the top of the sweep must match the fresh oracle, stall 1 must match the stale oracle, and putting stall 1 back on the increment must fail the counter check. Co-authored-by: aien-spark --- tests/gpu_latmeas_test.c | 354 ++++++++++++++++++++++++++++++++++++- tools/latmeas/REPRODUCE.md | 4 + 2 files changed, 357 insertions(+), 1 deletion(-) diff --git a/tests/gpu_latmeas_test.c b/tests/gpu_latmeas_test.c index 556dbe75..fe60bb09 100644 --- a/tests/gpu_latmeas_test.c +++ b/tests/gpu_latmeas_test.c @@ -24,6 +24,20 @@ * the 32-bit clock delta around that same chain, median and max-min over the * runs. An empty loop with the same trip count is the overhead control. * + * The two CS2R reads bracket the loop, not one instruction. Between them sit + * the producer, its consumer, the live-value advance, the sentinel move, and + * the loop-back (increment, compare, branch). A programmed stall of 1 is only + * the producer's field. The predicate chain puts that stall on the ISETP that + * feeds @P0 BRA: a shorter stall leaves the previous P0, which was set true, + * so the poison move runs and the clocks do not cover the whole trip count. + * + * The loop increment stalls 15 before ISETP. Stall 1 there is shorter than a + * fixed-latency add, so the compare samples the old counter and the body runs + * one extra time. sm121dis of the previous overhead kernel showed that IADD + * (control 0x000fc200, stall 1) immediately before ISETP, with @!P0 BRA back + * to the IADD. The overhead word is that counter. The host simulator below + * applies the stall counts and rejects that shape. + * * One warp, 32 threads, uniform control flow inside the timed region. Only * lane 0 stores. Mesa commit and the NAK line citations live in * tools/latmeas/diff_nak.py. @@ -344,7 +358,12 @@ static void store_tail(B *b, int ptr, int t0, int t1, int result, int rsub, int } static void loop_ctl(B *b, int i, int nreg, int one, int *loop_back) { - iadd(b, i, i, one, 1, 0, 0, 0); + /* 15, not 1. The compare reads i. Stall 1 is shorter than a fixed-latency + * IADD, so ISETP samples the previous i and the body runs one extra time: + * the overhead word is then not the trip count, and every exact oracle + * fails, including a stall at the top of the sweep. 15 is past every fixed + * GPR latency this kit measures. The stall under test stays on the producer. */ + iadd(b, i, i, one, 15, 0, 0, 0); isetp_ge(b, i, nreg, 14); /* P0 = (i >= n) */ *loop_back = bra(b, 1, 1, 5); /* @!P0 BRA loop */ } @@ -614,6 +633,338 @@ static int check_raw_window(const char *id, const uint8_t *code, size_t ninsn, u return failed; } +/* Host latency simulator. The shared warp simulator (tests/bw_warp_sim.h) writes + * a register and the next instruction already sees it, so it cannot catch a + * stall that is too short. This one issues the IR in order and keeps each GPR + * write hidden for SIM_GPR_LAT cycles and each P0 write hidden for + * SIM_PRED_LAT cycles. Those are the NAK floors (ALU/FMA raw 4 plus one, + * predicate raw 13 plus one), used here as the model's assumption. A read at + * the issue cycle sees a write whose visible cycle is at or before that. The + * stall field is how many cycles pass before the next issue. */ +#define SIM_GPR_LAT 5u +#define SIM_PRED_LAT 14u +#define SIM_LOG 4096 + +float fmaf(float, float, float); +float fminf(float, float); + +typedef struct { + int is_pred; + int reg; + uint32_t issue, visible, value; +} SimWr; + +typedef struct { + SimWr wr[SIM_LOG]; + int nwr; + uint32_t now; + int pc; + uint32_t out[4]; + int stores; + const char *err; +} LatSim; + +static uint32_t lat_u2f_bits(float f) { + uint32_t u; + memcpy(&u, &f, 4); + return u; +} +static float lat_f(uint32_t u) { + float f; + memcpy(&f, &u, 4); + return f; +} + +static int lat_phys(int v, unsigned sub, int *reg) { + if (v < 0) { + *reg = -1; + return 0; + } + if (v >= g_prog.regalloc.num_vregs) return -1; + int ph = g_prog.regalloc.vreg_to_phys[v]; + if (ph < 0) return -1; + *reg = (ph + (int)sub) & 0xff; + return 0; +} + +static uint32_t lat_gpr(const LatSim *s, int reg) { + uint32_t val = 0, issue = 0; + int have = 0; + if (reg < 0) return 0; + for (int i = 0; i < s->nwr; i++) { + if (s->wr[i].is_pred || s->wr[i].reg != reg) continue; + if (s->wr[i].visible > s->now) continue; + if (!have || s->wr[i].issue >= issue) { + have = 1; + issue = s->wr[i].issue; + val = s->wr[i].value; + } + } + return val; +} + +static int lat_p0(const LatSim *s) { + uint32_t issue = 0, val = 0; + int have = 0; + for (int i = 0; i < s->nwr; i++) { + if (!s->wr[i].is_pred) continue; + if (s->wr[i].visible > s->now) continue; + if (!have || s->wr[i].issue >= issue) { + have = 1; + issue = s->wr[i].issue; + val = s->wr[i].value; + } + } + return val != 0; +} + +static int lat_post(LatSim *s, int is_pred, int reg, uint32_t lat, uint32_t value) { + if (reg < 0 && !is_pred) return 0; + if (s->nwr >= SIM_LOG) { + s->err = "write log full"; + return -1; + } + s->wr[s->nwr].is_pred = is_pred; + s->wr[s->nwr].reg = reg; + s->wr[s->nwr].issue = s->now; + s->wr[s->nwr].visible = s->now + lat; + s->wr[s->nwr].value = value; + s->nwr++; + return 0; +} + +static int lat_rd(const LatSim *s, int v, unsigned sub, uint32_t *out) { + int reg = 0; + if (lat_phys(v, sub, ®) != 0) return -1; + *out = lat_gpr(s, reg); + return 0; +} + +static int lat_wr(LatSim *s, int v, unsigned sub, uint32_t value, uint32_t lat) { + int reg = 0; + if (lat_phys(v, sub, ®) != 0) return -1; + return lat_post(s, 0, reg, lat, value); +} + +/* Lane 0 only: the kernel is uniform, and the lane filter stores from lane 0. + * Returns 0 and fills out[4] (clock, clock, result, aux). */ +static int lat_sim(uint32_t out[4]) { + LatSim s; + memset(&s, 0, sizeof s); + for (int steps = 0; steps < 100000; steps++) { + if (s.pc < 0 || (size_t)s.pc >= g_prog.count) { + s.err = "pc outside the program"; + break; + } + const BlackwellIRInsn *in = &g_prog.insns[s.pc]; + unsigned stall = (in->control >> 9) & 15u; + int next = s.pc + 1; + uint32_t a = 0, b = 0, c = 0; + if (lat_rd(&s, in->src1_vreg, in->src1_subreg, &a) || lat_rd(&s, in->src2_vreg, in->src2_subreg, &b) || + lat_rd(&s, in->src3_vreg, in->src3_subreg, &c)) { + s.err = "vreg has no physical register"; + break; + } + switch (in->op) { + case BW_IR_EXIT: + if (s.stores != 4) { + s.err = "lane 0 did not store four words"; + break; + } + memcpy(out, s.out, sizeof s.out); + return 0; + case BW_IR_BRA: { + int take = 1; + if (in->predicate_p0) take = in->predicate_not ? !lat_p0(&s) : lat_p0(&s); + if (take && (int32_t)in->imm == 0) { + s.err = "branch to itself"; + break; + } + if (take) next = s.pc + (int32_t)in->imm; + break; + } + case BW_IR_NOP: + case BW_IR_LDCU64: + case BW_IR_LDCU: + break; + case BW_IR_LDC64: + if (lat_wr(&s, in->dst_vreg, 0, 0, SIM_GPR_LAT) || lat_wr(&s, in->dst_vreg, 1, 0, SIM_GPR_LAT)) + s.err = "write log full"; + break; + case BW_IR_MOV_IMM: + if (lat_wr(&s, in->dst_vreg, in->dst_subreg, in->imm, SIM_GPR_LAT)) s.err = "write log full"; + break; + case BW_IR_MOV_RZ: + if (lat_wr(&s, in->dst_vreg, in->dst_subreg, 0, SIM_GPR_LAT)) s.err = "write log full"; + break; + case BW_IR_S2R: { + uint32_t v = 0; + if (in->imm == BW_SR_TID_X) v = 0; + else if (in->imm == CLOCK_LO) v = s.now; + if (lat_wr(&s, in->dst_vreg, in->dst_subreg, v, SIM_GPR_LAT)) s.err = "write log full"; + break; + } + case BW_IR_IADD3: + if (lat_wr(&s, in->dst_vreg, in->dst_subreg, a + b, SIM_GPR_LAT)) s.err = "write log full"; + break; + case BW_IR_ISETP_GE_U32: + if (lat_post(&s, 1, 0, SIM_PRED_LAT, a >= b ? 1u : 0u)) s.err = "write log full"; + break; + case BW_IR_FFMA: + if (lat_wr(&s, in->dst_vreg, in->dst_subreg, lat_u2f_bits(fmaf(lat_f(a), lat_f(b), lat_f(c))), SIM_GPR_LAT)) + s.err = "write log full"; + break; + case BW_IR_FADD: + if (lat_wr(&s, in->dst_vreg, in->dst_subreg, lat_u2f_bits(lat_f(a) + lat_f(b)), SIM_GPR_LAT)) + s.err = "write log full"; + break; + case BW_IR_FMNMX_MIN: + if (lat_wr(&s, in->dst_vreg, in->dst_subreg, lat_u2f_bits(fminf(lat_f(a), lat_f(b))), SIM_GPR_LAT)) + s.err = "write log full"; + break; + case BW_IR_IMAD_WIDE: { + /* Immediate form: Rd:Rd+1 = Ra * imm + Rc+1:Rc. src2 is NOREG. */ + uint32_t hi = 0; + if (lat_rd(&s, in->src3_vreg, 1, &hi)) { + s.err = "vreg has no physical register"; + break; + } + uint64_t prod = (uint64_t)a * in->imm + (((uint64_t)hi << 32) | c); + if (lat_wr(&s, in->dst_vreg, 0, (uint32_t)prod, SIM_GPR_LAT) || + lat_wr(&s, in->dst_vreg, 1, (uint32_t)(prod >> 32), SIM_GPR_LAT)) + s.err = "write log full"; + break; + } + case BW_IR_STG_E: { + uint32_t off = in->imm; + if (off % 4u || off / 4u >= 4u) { + s.err = "store offset is not one of the four result words"; + break; + } + s.out[off / 4u] = b; + s.stores++; + break; + } + default: + s.err = "opcode not modelled"; + break; + } + if (s.err) break; + s.now += stall ? stall : 1u; + s.pc = next; + } + if (s.err) fprintf(stderr, "lat sim: %s at pc %d\n", s.err, s.pc); + else fprintf(stderr, "lat sim: step budget at pc %d\n", s.pc); + return -1; +} + +/* The loop-back sm121dis must show: IADD (dest == src A) stall 15, ISETP stall + * 14, then @!P0 BRA back to the body. The body is at or before that IADD + * (overhead has no body, so the branch lands on the IADD). HMMA has no loop. */ +static int check_loop_sass(const char *id, const uint8_t *code, size_t ninsn, int expect_loop) { + int found = 0, failed = 0; + for (size_t i = 0; i + 2 < ninsn; i++) { + const uint8_t *a = code + i * 16; + const uint8_t *bp = code + (i + 1) * 16; + const uint8_t *c = code + (i + 2) * 16; + if (opc_of(a) != 0x210u || opc_of(bp) != 0x20cu || opc_of(c) != 0x947u) continue; + if (reg_dst(a) != reg_a(a)) continue; + if (((rd32(c) >> 12) & 0xfu) != 8u) continue; + found++; + if (stall_of(a) != 15u) { + failed++; + printf("FAIL: %s loop increment stall %u, want 15\n", id, stall_of(a)); + } + if (stall_of(bp) != 14u) { + failed++; + printf("FAIL: %s loop compare stall %u, want 14\n", id, stall_of(bp)); + } + int tgt = bra_index(c, (int)i + 2); + if (tgt < 0 || tgt > (int)i) { + failed++; + printf("FAIL: %s loop branch targets %d, increment is at %zu\n", id, tgt, i); + } + } + if (found != expect_loop) { + failed++; + printf("FAIL: %s loop-back count %d, want %d\n", id, found, expect_loop); + } + return failed; +} + +static int loop_iadd_index(void) { + int found = -1; + for (size_t i = 0; i + 2 < g_prog.count; i++) { + const BlackwellIRInsn *a = &g_prog.insns[i]; + const BlackwellIRInsn *b = &g_prog.insns[i + 1]; + const BlackwellIRInsn *c = &g_prog.insns[i + 2]; + if (a->op != BW_IR_IADD3 || b->op != BW_IR_ISETP_GE_U32 || c->op != BW_IR_BRA) continue; + if (a->dst_vreg != a->src1_vreg) continue; + if (!c->predicate_p0 || !c->predicate_not) continue; + found = (int)i; + } + return found; +} + +static int sim_self_test(void) { + int failed = 0; + uint8_t code[CODE_CAP]; + size_t len = 0; + uint32_t gpr = 0; + uint32_t out[4]; +#define SCHECK(cond, ...) \ + do { \ + if (!(cond)) { \ + failed++; \ + printf("FAIL: "); \ + printf(__VA_ARGS__); \ + printf("\n"); \ + } \ + } while (0) + + SCHECK(build(NULL, 0, N_FIXED, 1, code, sizeof code, &len, &gpr) == 0, "overhead encode"); + SCHECK(check_loop_sass("ovh32", code, g_prog.count, 1) == 0, "overhead sass"); + SCHECK(lat_sim(out) == 0, "overhead sim ran"); + SCHECK(out[2] == N_FIXED && out[3] == 1u, "overhead loop count is %u (aux 0x%x), want %u", out[2], out[3], N_FIXED); + { + int at = loop_iadd_index(); + SCHECK(at >= 0, "overhead loop increment is in the IR"); + if (at >= 0) { + uint32_t saved = g_prog.insns[at].control; + g_prog.insns[at].control = (saved & ~(15u << 9)) | (1u << 9); + uint32_t bad[4] = {0, 0, 0, 0}; + SCHECK(lat_sim(bad) == 0, "short-stall overhead sim ran"); + SCHECK(bad[2] != N_FIXED, "stall 1 on the increment must not store the trip count (got %u)", bad[2]); + g_prog.insns[at].control = saved; + } + } + SCHECK(build(NULL, 0, 1, 1, code, sizeof code, &len, &gpr) == 0, "one-trip overhead encode"); + SCHECK(lat_sim(out) == 0 && out[2] == 1u, "one-trip overhead stored %u", out[2]); + + for (int ci = 0; ci < NCASES; ci++) { + if (CASES[ci].href) { + SCHECK(build(&CASES[ci], 1, 0, 0, code, sizeof code, &len, &gpr) == 0, "%s encode", CASES[ci].id); + SCHECK(check_loop_sass(CASES[ci].id, code, g_prog.count, 0) == 0, "%s has no loop", CASES[ci].id); + continue; + } + uint32_t fresh = 0, fr_aux = 0, stale = 0, st_aux = 0; + SCHECK(eval_case(&CASES[ci], 0, &fresh, &fr_aux) == 0, "%s fresh", CASES[ci].id); + SCHECK(eval_case(&CASES[ci], 1, &stale, &st_aux) == 0, "%s stale", CASES[ci].id); + SCHECK(build(&CASES[ci], CASES[ci].sweep, 0, 0, code, sizeof code, &len, &gpr) == 0, "%s stall %u encode", + CASES[ci].id, CASES[ci].sweep); + SCHECK(check_loop_sass(CASES[ci].id, code, g_prog.count, 1) == 0, "%s loop sass", CASES[ci].id); + SCHECK(lat_sim(out) == 0, "%s stall %u sim", CASES[ci].id, CASES[ci].sweep); + SCHECK(out[2] == fresh && out[3] == fr_aux, "%s stall %u sim result 0x%x aux 0x%x, fresh 0x%x aux 0x%x", + CASES[ci].id, CASES[ci].sweep, out[2], out[3], fresh, fr_aux); + SCHECK(build(&CASES[ci], 1, 0, 0, code, sizeof code, &len, &gpr) == 0, "%s stall 1 encode", CASES[ci].id); + SCHECK(lat_sim(out) == 0, "%s stall 1 sim", CASES[ci].id); + SCHECK(out[2] == stale && out[3] == st_aux, "%s stall 1 sim result 0x%x aux 0x%x, stale 0x%x aux 0x%x", + CASES[ci].id, out[2], out[3], stale, st_aux); + } +#undef SCHECK + return failed; +} + static int self_test(void) { int failed = 0; uint8_t code[CODE_CAP]; @@ -706,6 +1057,7 @@ static int self_test(void) { } len = 0; CHECK(build(NULL, 0, 64, 1, code, sizeof code, &len, &gpr) == 0, "overhead kernel"); + failed += sim_self_test(); int pass[RUNS][SWEEP_MAX]; memset(pass, 0, sizeof pass); diff --git a/tools/latmeas/REPRODUCE.md b/tools/latmeas/REPRODUCE.md index d570bd01..6d3ec7d5 100644 --- a/tools/latmeas/REPRODUCE.md +++ b/tools/latmeas/REPRODUCE.md @@ -34,6 +34,10 @@ The runner tries stall 1 through 15 (1 through 36 for HMMA; the stall field is f The fixed-latency oracle arms the destination to `0x7f800000` and the producer does not read it. The next instruction does. A stale read adds that sentinel. The host self-test checks the fresh and stale totals differ, and that the stall-1 SASS really is that pair with no scoreboard and no yield. HMMA is unchanged: `d` starts at 0, the consumer is `FADD` of `d0`, and stall 19 is HMMA stall 15 plus a `NOP` of stall 4. +The two `CS2R` reads bracket the loop. Between them are the producer, its consumer, the advance of the live value, the sentinel move, and the loop-back. A programmed stall of 1 is only the producer's field. Dividing the clock delta by the op count counts that whole bracket. On the predicate chain the measured stall sits on the `ISETP` that feeds `@P0 BRA`. A stall shorter than the predicate leaves the previous `P0`, which that kernel sets true, so the poison move runs and the clocks cover one pass out of the loop. + +The loop increment stalls 15 before `ISETP`. Stall 1 there is shorter than a fixed-latency add: the compare samples the old counter, the body runs one extra time, and the overhead kernel stores that counter. `sm121dis` of that kernel shows the `IADD`, the `ISETP`, and `@!P0 BRA` back to the `IADD`. The host simulator issues the same IR and keeps a GPR write hidden for 5 cycles and a predicate write hidden for 14 (the NAK floors). It requires the overhead counter to equal the trip count, the stall at the top of the sweep to equal the fresh oracle, and stall 1 to equal the stale oracle. Forcing the increment's stall back to 1 must fail the counter check. + ## NAK pin Mesa commit `01bab9302a6bf0a082c7ba8c49d74427cf10b0cf`. From b088d7bc9caf4c1a7e5149a9d1a52d89462f1db6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 1 Nov 2026 18:35:30 +0000 Subject: [PATCH 04/11] Record the 2026-11-01 GB10 latency receipt. The launch is cb5bd70, the merge of 03cdcc5 and PR #251 6a93cb5. raw.json, the filled table, and sm121dis --pairs of each threshold and the stall under it are the chip output. UNITS.md compares that stall field with NAK's deps.delay at Mesa 01bab930. Co-authored-by: aien-spark --- evidence/LATMEAS-GB10-20261101/RECEIPT.md | 30 +++++ evidence/LATMEAS-GB10-20261101/RECEIPT.txt | 11 ++ evidence/LATMEAS-GB10-20261101/UNITS.md | 74 ++++++++++++ evidence/LATMEAS-GB10-20261101/head.txt | 1 + evidence/LATMEAS-GB10-20261101/launch.log | 104 ++++++++++++++++ evidence/LATMEAS-GB10-20261101/raw.json | 114 ++++++++++++++++++ evidence/LATMEAS-GB10-20261101/sass/README.md | 3 + .../sass/alu_alu_3.pairs | 32 +++++ .../sass/alu_alu_4.pairs | 32 +++++ .../sass/fma_fma_3.pairs | 32 +++++ .../sass/fma_fma_4.pairs | 32 +++++ .../sass/fmnmx_alu_3.pairs | 40 ++++++ .../sass/fmnmx_alu_4.pairs | 40 ++++++ .../sass/hmma_reader_19.pairs | 48 ++++++++ .../sass/hmma_reader_20.pairs | 48 ++++++++ .../sass/imad_wide_lo_2.pairs | 40 ++++++ .../sass/imad_wide_lo_3.pairs | 40 ++++++ .../LATMEAS-GB10-20261101/sass/mixed_4.pairs | 40 ++++++ .../LATMEAS-GB10-20261101/sass/mixed_5.pairs | 40 ++++++ .../LATMEAS-GB10-20261101/sass/pred_12.pairs | 40 ++++++ .../LATMEAS-GB10-20261101/sass/pred_13.pairs | 40 ++++++ .../LATMEAS-GB10-20261101/sm121_vs_NAK.md | 42 +++++++ 22 files changed, 923 insertions(+) create mode 100644 evidence/LATMEAS-GB10-20261101/RECEIPT.md create mode 100644 evidence/LATMEAS-GB10-20261101/RECEIPT.txt create mode 100644 evidence/LATMEAS-GB10-20261101/UNITS.md create mode 100644 evidence/LATMEAS-GB10-20261101/head.txt create mode 100644 evidence/LATMEAS-GB10-20261101/launch.log create mode 100644 evidence/LATMEAS-GB10-20261101/raw.json create mode 100644 evidence/LATMEAS-GB10-20261101/sass/README.md create mode 100644 evidence/LATMEAS-GB10-20261101/sass/alu_alu_3.pairs create mode 100644 evidence/LATMEAS-GB10-20261101/sass/alu_alu_4.pairs create mode 100644 evidence/LATMEAS-GB10-20261101/sass/fma_fma_3.pairs create mode 100644 evidence/LATMEAS-GB10-20261101/sass/fma_fma_4.pairs create mode 100644 evidence/LATMEAS-GB10-20261101/sass/fmnmx_alu_3.pairs create mode 100644 evidence/LATMEAS-GB10-20261101/sass/fmnmx_alu_4.pairs create mode 100644 evidence/LATMEAS-GB10-20261101/sass/hmma_reader_19.pairs create mode 100644 evidence/LATMEAS-GB10-20261101/sass/hmma_reader_20.pairs create mode 100644 evidence/LATMEAS-GB10-20261101/sass/imad_wide_lo_2.pairs create mode 100644 evidence/LATMEAS-GB10-20261101/sass/imad_wide_lo_3.pairs create mode 100644 evidence/LATMEAS-GB10-20261101/sass/mixed_4.pairs create mode 100644 evidence/LATMEAS-GB10-20261101/sass/mixed_5.pairs create mode 100644 evidence/LATMEAS-GB10-20261101/sass/pred_12.pairs create mode 100644 evidence/LATMEAS-GB10-20261101/sass/pred_13.pairs create mode 100644 evidence/LATMEAS-GB10-20261101/sm121_vs_NAK.md diff --git a/evidence/LATMEAS-GB10-20261101/RECEIPT.md b/evidence/LATMEAS-GB10-20261101/RECEIPT.md new file mode 100644 index 00000000..a83418a7 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101/RECEIPT.md @@ -0,0 +1,30 @@ +# GB10 latency receipt, 2026-11-01 + +Chip run of the latency kit. The measured columns in `sm121_vs_NAK.md` were filled by `tools/latmeas/fill_template.py` from `raw.json` on the machine. `sha256sum raw.json` is `21735d4b96d339b9b62040db0b7ee985ac168d75eed64ac16c30175d96e8047a`. That file matches a fresh fill from the same `raw.json` (diff empty). + +This receipt is the kernels launched on that day. It is not a receipt for later edits that store every lane or that read all four HMMA destinations. + +## What ran + +- Command: `sh tools/latmeas/run_spark.sh latmeas-out4` +- Result: launch rc 0, `raw.json` `"error": ""`, 11 runs, stall spread 0 on every case. +- Tested tree: `cb5bd7087470336b6d59da53e99ef4dc5edcb7f2`, a merge of PR #254 `03cdcc5a4401043da2196f4f5efcb9b379400f66` and PR #251 `6a93cb567b33b67bc1185fb45a3c50fc4d88d30e`, with no other edits. PR #251 is the dependency that admits SM version `0xa04`. `head.txt` is the tested tree. +- Machine line, copied from `RECEIPT.txt`: Linux spark-b87b, 7.0.0-1019-nvidia, aarch64. Open kernel module 580.173.02. GCC 13.3.0. NVIDIA GB10, PCI `0000000F:01:00.0`. Clocks reported there: 1787 MHz current, 3003 MHz max. +- Measured at 2026-11-01 12:08:46 -0600 (18:08Z), as printed in `RECEIPT.txt`. +- `launch.log` is the runner log. Its last line named a home directory; that path is replaced here with `latmeas-out4/raw.json`. +- The upload did not contain `diff.md`. The disagreement list is `python3 tools/latmeas/diff_nak.py --md`, which does not read the chip. +- `sass/` is `sm121dis --pairs` on the bins that ran, for the smallest passing stall and the stall under it. The two `??` lines in each listing are the CS2R clock sites. The stall field is bits 9..12 of the control word (the high 32 bits of the second hex column). + +## Smallest passing stall + +| case | stall | sm100 CSV base | NAK encoded (CSV + padding) | +| --- | --- | --- | --- | +| alu_alu | 4 | 4 | 5 | +| fma_fma | 4 | 4 | 5 | +| mixed | 5 | 5 | 6 | +| pred | 13 | 13 | 14 | +| fmnmx_alu | 4 | 4 | 5 | +| imad_wide_lo | 3 | 3 | 4 | +| hmma_reader | 20 | 19 | 28 | + +Lane 0's primary result word is `run0_result` in `raw.json`. For the six fixed-latency cases the failing stalls show the sentinel (`0x7f800000` or the wrapped sum `0xf0000000`, or `0xdeadbeef` for the predicate). `imad_wide_lo` stalls 1 and 2 stored 32, and stall 3 stored 528. HMMA stalls 1..19 stored 0 and stall 20 stored `1098907648` (`0x41800000`, 16.0f). `reference_result` for HMMA is that same word from the stall-36 launch, which this run used as the oracle. How those numbers relate to NAK's delay field is `UNITS.md`. diff --git a/evidence/LATMEAS-GB10-20261101/RECEIPT.txt b/evidence/LATMEAS-GB10-20261101/RECEIPT.txt new file mode 100644 index 00000000..032831b9 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101/RECEIPT.txt @@ -0,0 +1,11 @@ +measured_utc: 2026-11-01 12:08:46.562510362 -0600 +tested_head: cb5bd7087470336b6d59da53e99ef4dc5edcb7f2 +pr254_head: 03cdcc5a4401043da2196f4f5efcb9b379400f66 +pr251_head: 6a93cb567b33b67bc1185fb45a3c50fc4d88d30e +local_changes: merge of PR251 head into PR254 head, no other edits +Linux spark-b87b 7.0.0-1019-nvidia #19~24.04.2-Ubuntu SMP PREEMPT_DYNAMIC Sun Sep 6 18:35:11 UTC 2 aarch64 aarch64 aarch64 GNU/Linux +NVRM version: NVIDIA UNIX Open Kernel Module for aarch64 580.173.02 Release Build (dvs-builder@U22-A24-5-4) Tue Jun 23 08:34:19 UTC 2026 +GCC version: gcc version 13.3.0 (Ubuntu 13.3.0-6ubuntu2~24.04.1) +name, pci.bus_id, driver_version, clocks.current.sm [MHz], clocks.max.sm [MHz] +NVIDIA GB10, 0000000F:01:00.0, 580.173.02, 1787 MHz, 3003 MHz +cc (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0 diff --git a/evidence/LATMEAS-GB10-20261101/UNITS.md b/evidence/LATMEAS-GB10-20261101/UNITS.md new file mode 100644 index 00000000..419b76a6 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101/UNITS.md @@ -0,0 +1,74 @@ +# Stall units: this kit and NAK at Mesa 01bab930 + +Mesa commit `01bab9302a6bf0a082c7ba8c49d74427cf10b0cf`. Paths are under `src/nouveau/compiler/nak/` unless noted. The conclusion is that our smallest passing stall S is the same field NAK writes, and the same kind of count (cycles from the producer issue to the consumer issue). NAK's published number is that count after it adds padding. The six fixed-latency pairs match the sm100 table before that padding. HMMA does not match either the bare table or NAK's padded total. + +## What NAK writes into the instruction + +`InstrDeps.delay` is a `u8` (`ir.rs:8722`). The minimum stored on an instruction is 1 (`MIN_INSTR_DELAY`, `ir.rs:8719`, applied at `calc_instr_deps.rs:1013`). + +The encoder writes that delay into instruction bits 105..109 (`sm70_encode.rs:325`, `set_field(105..109, deps.delay)`). That range is four bits, so the field holds 0..15. Bit 105 is bit 9 of the control word, which is the stall field this repository's `cw()` shifts by 9. Yield on SM before 120 sits at bit 109 (`sm70_encode.rs:329-331`); SM120 and later do not use that bit for yield. The delay field itself is the same. + +`max_instr_delay()` is 15 (`sm70.rs:300-301`). When `deps.delay` is larger, `calc_delays` keeps 15 on the instruction and pushes the remainder as following `NOP`s, each capped at 15 (`calc_instr_deps.rs:1031-1040`): + +```text +delay = instr.deps.delay - 15 +instr.deps = 15 +while delay > 0: + nop.delay = min(delay, 15) + delay -= nop.delay + emit nop after the instruction +``` + +The sum of those delay fields is the original delay. A NOP does not add a hidden extra cycle on top of its own delay field, and it does not drop out of the sum. + +`exec_latency` of an ordinary ALU, FADD, HMMA, or NOP is 1 (`sm70.rs:183-199`, the default arm). `calc_delays` walks the block backwards. The consumer is given a cycle of `exec_latency`. The producer is then placed at `consumer_cycle + raw_latency` when that is the tighter bound (`calc_instr_deps.rs:811-812` and `838-846`). The delay stored on the producer is `producer_cycle - consumer_cycle` (`calc_instr_deps.rs:1011-1013`), so the `exec_latency` of 1 cancels. There is no second implicit +1 in that subtraction. The only +1 that survives into the field is the one inside `SM120Latency::raw`, below. + +Instructions whose `exec_latency` is greater than 1 also get a following NOP of delay 2 (`calc_instr_deps.rs:1046-1053`). That arm does not apply to these pairs. + +## Where the NAK number comes from + +For a GPR raw on SM120, `sm70.rs:202-218` calls `SM120Latency::raw`. That function (`sm120_instr_latencies.rs:439-464`) reads the sm100 CSV and then adds + +```text +max(latency_adjust(writer), latency_adjust(reader), 1) +``` + +`latency_adjust` (`sm120_instr_latencies.rs:431-436`) is 9 for `Hmma`, 7 for `Imma`, and 0 otherwise. The comment at `sm120_instr_latencies.rs:425-430` says the tables are SM100, the HMMA/IMMA adjusts were taken from CUDA SASS, and everything else still gets one cycle of padding because SM120 "has larger latencies." + +So NAK's encoded delay for a coupled GPR pair is the CSV cell plus 1. For an HMMA write read by FADD (`FAdd` is `Fma`, line 60; the fma row of `latencies/sm100/reg_raw.csv` has column `hmma` = 19) it is 19 + 9 = 28, because 9 replaces the 1 rather than stacking on it. `Hmma` is also in `needs_scoreboards` (`sm120_instr_latencies.rs:413-421`). `raw()` does not return 0 for that. NAK still programs the delay of 28, and it also wants a scoreboard. This kit's HMMA sets no scoreboard (`SB_NONE`). The number 20 is a stall-only separation. + +A predicate guard uses `paw_latency` (`sm70.rs:261-263`), which calls `raw(write, None)`. For a predicate that selects reader `RedirectedFp64` (`sm120_instr_latencies.rs:474-480`) and adds 1. `pred_raw.csv` row `redirected_fp64`, column `dualalu` is 13, so NAK's delay is 14. `ISetP` is `Dualalu` (line 49). + +## This kit's S + +S is the programmed stall on the producer, or, past 15, the producer's stall of 15 plus NOP stalls that sum to the rest (`gap_nops` in `tests/gpu_latmeas_test.c`). The consumer issues S cycles after the producer. That is the same sum `calc_delays` builds. + +The smallest S at which every run matched, from `raw.json` in this directory: + +| pair | CSV cell | NAK delay | measured S | +| --- | --- | --- | --- | +| ALU read of IADD3 | alu/alu = 4 | 5 | 4 | +| FMA read of FFMA | fma/fma = 4 | 5 | 4 | +| ALU read of FFMA and FMA read of IADD3 | 5 | 6 | 5 | +| guard read of ISETP | 13 | 14 | 13 | +| ALU read of FMNMX (`Dualalu`) | alu/dualalu = 4 | 5 | 4 | +| ALU read of IMAD.WIDE low | alu/imad_wide_write_dl = 3 | 4 | 3 | +| FMA read of HMMA | fma/hmma = 19 | 28 | 20 | + +The six fixed-latency pairs equal the CSV cell. They are one less than the delay NAK writes. The units are the same; the difference is the padding at `sm120_instr_latencies.rs:458-463`. + +## HMMA is not that pattern + +Stall 20 in `sass/hmma_reader_20.pairs` is + +```text +01c0 ... 000fde00... HMMA.16816.F32.BF16 R24, R16, R14, R20 +01d0 ... 000fca00... NOP +01e0 ... FADD R10, R24, R5 +``` + +Control `0x000fde00` is stall 15. Control `0x000fca00` is stall 5. The FADD issues 20 cycles after the HMMA. Stall 19 (`sass/hmma_reader_19.pairs`) is the same HMMA stall 15 plus a NOP of stall 4 (`0x000fc800`). That launch stored 0. Stall 20 stored `0x41800000`. + +The split is the same one NAK uses (15 on the producer, the rest on NOPs). It does not turn a required separation of 19 into a reported 20: the failing program already summed to 19, and the passing program summed to 20. The FADD reads only `R24`, the first destination register. `R25` is stored later, after the second clock, so this run does not say whether d1..d3 were ready at the same cycle. + +20 is the CSV cell plus the ordinary padding of 1, not plus the HMMA adjust of 9. NAK's 28 is that adjust, and it also asks for a scoreboard this kernel does not set. The six coupled pairs did not need their padding of 1. HMMA, measured with stalls only, needed one more cycle than the sm100 cell and did not need the extra eight. diff --git a/evidence/LATMEAS-GB10-20261101/head.txt b/evidence/LATMEAS-GB10-20261101/head.txt new file mode 100644 index 00000000..df108be1 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101/head.txt @@ -0,0 +1 @@ +cb5bd7087470336b6d59da53e99ef4dc5edcb7f2 diff --git a/evidence/LATMEAS-GB10-20261101/launch.log b/evidence/LATMEAS-GB10-20261101/launch.log new file mode 100644 index 00000000..862e46f9 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101/launch.log @@ -0,0 +1,104 @@ +latmeas plan: 7 cases, 11 runs, sweep 1..15 (HMMA 1..36), 1408 launches +estimated chip window: 71 s upper bound (1408 launches; 50 ms each covers a cold shader invalidate). Usually shorter. +The runner wraps only the launch with: flock /tmp/aien-gb10.lock timeout 600 +One warp, 32 threads. Clock: CS2R idx 0x50. Mesa 01bab9302a6bf0a082c7ba8c49d74427cf10b0cf +overhead ovh32 run 1/11 rc 0 +overhead ovh32 run 2/11 rc 0 +overhead ovh32 run 3/11 rc 0 +overhead ovh32 run 4/11 rc 0 +overhead ovh32 run 5/11 rc 0 +overhead ovh32 run 6/11 rc 0 +overhead ovh32 run 7/11 rc 0 +overhead ovh32 run 8/11 rc 0 +overhead ovh32 run 9/11 rc 0 +overhead ovh32 run 10/11 rc 0 +overhead ovh32 run 11/11 rc 0 +overhead ovh1 run 1/11 rc 0 +overhead ovh1 run 2/11 rc 0 +overhead ovh1 run 3/11 rc 0 +overhead ovh1 run 4/11 rc 0 +overhead ovh1 run 5/11 rc 0 +overhead ovh1 run 6/11 rc 0 +overhead ovh1 run 7/11 rc 0 +overhead ovh1 run 8/11 rc 0 +overhead ovh1 run 9/11 rc 0 +overhead ovh1 run 10/11 rc 0 +overhead ovh1 run 11/11 rc 0 +case alu_alu run 1/11 +case alu_alu run 2/11 +case alu_alu run 3/11 +case alu_alu run 4/11 +case alu_alu run 5/11 +case alu_alu run 6/11 +case alu_alu run 7/11 +case alu_alu run 8/11 +case alu_alu run 9/11 +case alu_alu run 10/11 +case alu_alu run 11/11 +case fma_fma run 1/11 +case fma_fma run 2/11 +case fma_fma run 3/11 +case fma_fma run 4/11 +case fma_fma run 5/11 +case fma_fma run 6/11 +case fma_fma run 7/11 +case fma_fma run 8/11 +case fma_fma run 9/11 +case fma_fma run 10/11 +case fma_fma run 11/11 +case mixed run 1/11 +case mixed run 2/11 +case mixed run 3/11 +case mixed run 4/11 +case mixed run 5/11 +case mixed run 6/11 +case mixed run 7/11 +case mixed run 8/11 +case mixed run 9/11 +case mixed run 10/11 +case mixed run 11/11 +case pred run 1/11 +case pred run 2/11 +case pred run 3/11 +case pred run 4/11 +case pred run 5/11 +case pred run 6/11 +case pred run 7/11 +case pred run 8/11 +case pred run 9/11 +case pred run 10/11 +case pred run 11/11 +case fmnmx_alu run 1/11 +case fmnmx_alu run 2/11 +case fmnmx_alu run 3/11 +case fmnmx_alu run 4/11 +case fmnmx_alu run 5/11 +case fmnmx_alu run 6/11 +case fmnmx_alu run 7/11 +case fmnmx_alu run 8/11 +case fmnmx_alu run 9/11 +case fmnmx_alu run 10/11 +case fmnmx_alu run 11/11 +case imad_wide_lo run 1/11 +case imad_wide_lo run 2/11 +case imad_wide_lo run 3/11 +case imad_wide_lo run 4/11 +case imad_wide_lo run 5/11 +case imad_wide_lo run 6/11 +case imad_wide_lo run 7/11 +case imad_wide_lo run 8/11 +case imad_wide_lo run 9/11 +case imad_wide_lo run 10/11 +case imad_wide_lo run 11/11 +case hmma_reader run 1/11 +case hmma_reader run 2/11 +case hmma_reader run 3/11 +case hmma_reader run 4/11 +case hmma_reader run 5/11 +case hmma_reader run 6/11 +case hmma_reader run 7/11 +case hmma_reader run 8/11 +case hmma_reader run 9/11 +case hmma_reader run 10/11 +case hmma_reader run 11/11 +wrote latmeas-out4/raw.json diff --git a/evidence/LATMEAS-GB10-20261101/raw.json b/evidence/LATMEAS-GB10-20261101/raw.json new file mode 100644 index 00000000..c1234e90 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101/raw.json @@ -0,0 +1,114 @@ +{ + "kind": "cudapp-latmeas", + "synthetic": false, + "measured": true, + "mesa_commit": "01bab9302a6bf0a082c7ba8c49d74427cf10b0cf", + "driver": "NVRM version: NVIDIA UNIX Open Kernel Module for aarch64 580.173.02 Release Build (dvs-builder@U22-A24-5-4) Tue Jun 23 08:34:19 UTC 2026", + "machine": "aarch64", + "runs": 11, + "threads": 32, + "warps": 1, + "clock": "CS2R idx 0x50 (NAK_SV_CLOCK_LO)", + "launches": 1408, + "error": "", + "cases": { + "alu_alu": { + "stall_required": 4, + "stall_spread": 0, + "cycle_median": 2538, + "cycle_spread": 0, + "cycle_median_stall1": 2474, + "cycle_spread_stall1": 0, + "overhead_cycle_median": 1261, + "reference_result": null, + "run0_result": [4026531840, 4026531840, 4026531840, 528, 528, 528, 528, 528, 528, 528, 528, 528, 528, 528, 528], + "iters": 32, + "ops": 32, + "note": "" + }, + "fma_fma": { + "stall_required": 4, + "stall_spread": 0, + "cycle_median": 2538, + "cycle_spread": 0, + "cycle_median_stall1": 2474, + "cycle_spread_stall1": 0, + "overhead_cycle_median": 1261, + "reference_result": null, + "run0_result": [2139095040, 2139095040, 2139095040, 1141112832, 1141112832, 1141112832, 1141112832, 1141112832, 1141112832, 1141112832, 1141112832, 1141112832, 1141112832, 1141112832, 1141112832], + "iters": 32, + "ops": 32, + "note": "" + }, + "mixed": { + "stall_required": 5, + "stall_spread": 0, + "cycle_median": 4362, + "cycle_spread": 0, + "cycle_median_stall1": 4170, + "cycle_spread_stall1": 0, + "overhead_cycle_median": 1261, + "reference_result": null, + "run0_result": [4026531840, 4026531840, 4026531840, 993, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072], + "iters": 32, + "ops": 64, + "note": "" + }, + "pred": { + "stall_required": 13, + "stall_spread": 0, + "cycle_median": 2714, + "cycle_spread": 0, + "cycle_median_stall1": 70, + "cycle_spread_stall1": 0, + "overhead_cycle_median": 1261, + "reference_result": null, + "run0_result": [3735928559, 3735928559, 3735928559, 3735928559, 3735928559, 3735928559, 3735928559, 3735928559, 3735928559, 3735928559, 3735928559, 3735928559, 32, 32, 32], + "iters": 32, + "ops": 32, + "note": "" + }, + "fmnmx_alu": { + "stall_required": 4, + "stall_spread": 0, + "cycle_median": 2541, + "cycle_spread": 0, + "cycle_median_stall1": 2507, + "cycle_spread_stall1": 0, + "overhead_cycle_median": 1261, + "reference_result": null, + "run0_result": [4026531840, 4026531840, 4026531840, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072], + "iters": 32, + "ops": 32, + "note": "" + }, + "imad_wide_lo": { + "stall_required": 3, + "stall_spread": 0, + "cycle_median": 2768, + "cycle_spread": 0, + "cycle_median_stall1": 2736, + "cycle_spread_stall1": 0, + "overhead_cycle_median": 1261, + "reference_result": null, + "run0_result": [32, 32, 528, 528, 528, 528, 528, 528, 528, 528, 528, 528, 528, 528, 528], + "iters": 32, + "ops": 32, + "note": "" + }, + "hmma_reader": { + "stall_required": 20, + "stall_spread": 0, + "cycle_median": 50, + "cycle_spread": 0, + "cycle_median_stall1": 32, + "cycle_spread_stall1": 0, + "overhead_cycle_median": 49, + "reference_result": 1098907648, + "run0_result": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648], + "iters": 1, + "ops": 1, + "note": "" + } + } +} diff --git a/evidence/LATMEAS-GB10-20261101/sass/README.md b/evidence/LATMEAS-GB10-20261101/sass/README.md new file mode 100644 index 00000000..03c48b6a --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101/sass/README.md @@ -0,0 +1,3 @@ +`sm121dis --pairs` on the machine-code bins that the 2026-11-01 launch ran. Each name is `_.pairs`. The stall is the smallest one that matched, and the stall under it. + +The tool exits 1 because the two CS2R clock sites are unknown to the disassembler. Both sites are in the listing as `??`. The second hex column's high 32 bits are the control word. The stall is bits 9..12 of that word. diff --git a/evidence/LATMEAS-GB10-20261101/sass/alu_alu_3.pairs b/evidence/LATMEAS-GB10-20261101/sass/alu_alu_3.pairs new file mode 100644 index 00000000..7623cd6a --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101/sass/alu_alu_3.pairs @@ -0,0 +1,32 @@ +0000 00006b00ff0477ac 000ea80008000a00 LDCU.64 UR4, c[0x0][0x358] ; +0010 0000e000ff027b82 004e280000000a00 LDC.64 R2, c[0x0][0x380] ; +0020 0000000100047802 001fcc0000000f00 MOV R4, 0x1 ; +0030 000000ff00057202 000fcc0000000f00 MOV R5, RZ ; +0040 3f80000000067802 000fcc0000000f00 MOV R6, 0x3f800000 ; +0050 0000002000077802 000fcc0000000f00 MOV R7, 0x20 ; +0060 000000ff00087202 000fcc0000000f00 MOV R8, RZ ; +0070 000000ff00097202 000fcc0000000f00 MOV R9, RZ ; +0080 000000ff000a7202 000fcc0000000f00 MOV R10, RZ ; +0090 000000ff000b7202 000fcc0000000f00 MOV R11, RZ ; +00a0 000000ff000c7202 000fcc0000000f00 MOV R12, RZ ; +00b0 00000001000d7802 000fcc0000000f00 MOV R13, 0x1 ; +00c0 7f800000000e7802 000fcc0000000f00 MOV R14, 0x7f800000 ; +00d0 00000000000f7805 000fde0000005000 ?? 0x000fde000000500000000000000f7805 ; +00e0 00000004090e7210 000fc60007ffe0ff IADD3 R14, PT, PT, R9, R4, RZ ; +00f0 0000000e0a0a7210 000fde0007ffe0ff IADD3 R10, PT, PT, R10, R14, RZ ; +0100 0000000409097210 000fde0007ffe0ff IADD3 R9, PT, PT, R9, R4, RZ ; +0110 7f800000000e7802 000fcc0000000f00 MOV R14, 0x7f800000 ; +0120 0000000408087210 000fde0007ffe0ff IADD3 R8, PT, PT, R8, R4, RZ ; +0130 000000070800720c 000fdc0003f06070 ISETP.GE.U32.AND P0, PT, R8, R7, PT ; +0140 fffffffc00e48947 000fca000383ffff @!P0 BRA 0xe0 ; +0150 0000000000107805 000fde0000005000 ?? 0x000fde00000050000000000000107805 ; +0160 0000000100117802 000fcc0000000f00 MOV R17, 0x1 ; +0170 0000000000127919 000e280000002100 S2R R18, SR_TID.X ; +0180 000000111200720c 001fde0003f06070 ISETP.GE.U32.AND P0, PT, R18, R17, PT ; +0190 0000000000100947 000fca0003800000 @P0 BRA 0x1e0 ; +01a0 0000000f02007986 0003e8000c101904 STG.E desc[UR4][R2.64], R15 ; +01b0 0000041002007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x4], R16 ; +01c0 0000080a02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x8], R10 ; +01d0 00000c0d02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0xc], R13 ; +01e0 000000000000794d 000fca0003800000 EXIT ; +01f0 fffffffc00fc7947 000fc0000383ffff BRA 0x1f0; diff --git a/evidence/LATMEAS-GB10-20261101/sass/alu_alu_4.pairs b/evidence/LATMEAS-GB10-20261101/sass/alu_alu_4.pairs new file mode 100644 index 00000000..20c53530 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101/sass/alu_alu_4.pairs @@ -0,0 +1,32 @@ +0000 00006b00ff0477ac 000ea80008000a00 LDCU.64 UR4, c[0x0][0x358] ; +0010 0000e000ff027b82 004e280000000a00 LDC.64 R2, c[0x0][0x380] ; +0020 0000000100047802 001fcc0000000f00 MOV R4, 0x1 ; +0030 000000ff00057202 000fcc0000000f00 MOV R5, RZ ; +0040 3f80000000067802 000fcc0000000f00 MOV R6, 0x3f800000 ; +0050 0000002000077802 000fcc0000000f00 MOV R7, 0x20 ; +0060 000000ff00087202 000fcc0000000f00 MOV R8, RZ ; +0070 000000ff00097202 000fcc0000000f00 MOV R9, RZ ; +0080 000000ff000a7202 000fcc0000000f00 MOV R10, RZ ; +0090 000000ff000b7202 000fcc0000000f00 MOV R11, RZ ; +00a0 000000ff000c7202 000fcc0000000f00 MOV R12, RZ ; +00b0 00000001000d7802 000fcc0000000f00 MOV R13, 0x1 ; +00c0 7f800000000e7802 000fcc0000000f00 MOV R14, 0x7f800000 ; +00d0 00000000000f7805 000fde0000005000 ?? 0x000fde000000500000000000000f7805 ; +00e0 00000004090e7210 000fc80007ffe0ff IADD3 R14, PT, PT, R9, R4, RZ ; +00f0 0000000e0a0a7210 000fde0007ffe0ff IADD3 R10, PT, PT, R10, R14, RZ ; +0100 0000000409097210 000fde0007ffe0ff IADD3 R9, PT, PT, R9, R4, RZ ; +0110 7f800000000e7802 000fcc0000000f00 MOV R14, 0x7f800000 ; +0120 0000000408087210 000fde0007ffe0ff IADD3 R8, PT, PT, R8, R4, RZ ; +0130 000000070800720c 000fdc0003f06070 ISETP.GE.U32.AND P0, PT, R8, R7, PT ; +0140 fffffffc00e48947 000fca000383ffff @!P0 BRA 0xe0 ; +0150 0000000000107805 000fde0000005000 ?? 0x000fde00000050000000000000107805 ; +0160 0000000100117802 000fcc0000000f00 MOV R17, 0x1 ; +0170 0000000000127919 000e280000002100 S2R R18, SR_TID.X ; +0180 000000111200720c 001fde0003f06070 ISETP.GE.U32.AND P0, PT, R18, R17, PT ; +0190 0000000000100947 000fca0003800000 @P0 BRA 0x1e0 ; +01a0 0000000f02007986 0003e8000c101904 STG.E desc[UR4][R2.64], R15 ; +01b0 0000041002007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x4], R16 ; +01c0 0000080a02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x8], R10 ; +01d0 00000c0d02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0xc], R13 ; +01e0 000000000000794d 000fca0003800000 EXIT ; +01f0 fffffffc00fc7947 000fc0000383ffff BRA 0x1f0; diff --git a/evidence/LATMEAS-GB10-20261101/sass/fma_fma_3.pairs b/evidence/LATMEAS-GB10-20261101/sass/fma_fma_3.pairs new file mode 100644 index 00000000..efe07cb2 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101/sass/fma_fma_3.pairs @@ -0,0 +1,32 @@ +0000 00006b00ff0477ac 000ea80008000a00 LDCU.64 UR4, c[0x0][0x358] ; +0010 0000e000ff027b82 004e280000000a00 LDC.64 R2, c[0x0][0x380] ; +0020 0000000100047802 001fcc0000000f00 MOV R4, 0x1 ; +0030 000000ff00057202 000fcc0000000f00 MOV R5, RZ ; +0040 3f80000000067802 000fcc0000000f00 MOV R6, 0x3f800000 ; +0050 0000002000077802 000fcc0000000f00 MOV R7, 0x20 ; +0060 000000ff00087202 000fcc0000000f00 MOV R8, RZ ; +0070 000000ff00097202 000fcc0000000f00 MOV R9, RZ ; +0080 000000ff000a7202 000fcc0000000f00 MOV R10, RZ ; +0090 000000ff000b7202 000fcc0000000f00 MOV R11, RZ ; +00a0 000000ff000c7202 000fcc0000000f00 MOV R12, RZ ; +00b0 00000001000d7802 000fcc0000000f00 MOV R13, 0x1 ; +00c0 7f800000000e7802 000fcc0000000f00 MOV R14, 0x7f800000 ; +00d0 00000000000f7805 000fde0000005000 ?? 0x000fde000000500000000000000f7805 ; +00e0 00000006060e7223 000fc6000000000c FFMA R14, R6, R6, R12 ; +00f0 0000000e0a0a7221 000fde0000000000 FADD R10, R10, R14 ; +0100 000000060c0c7221 000fde0000000000 FADD R12, R12, R6 ; +0110 7f800000000e7802 000fcc0000000f00 MOV R14, 0x7f800000 ; +0120 0000000408087210 000fde0007ffe0ff IADD3 R8, PT, PT, R8, R4, RZ ; +0130 000000070800720c 000fdc0003f06070 ISETP.GE.U32.AND P0, PT, R8, R7, PT ; +0140 fffffffc00e48947 000fca000383ffff @!P0 BRA 0xe0 ; +0150 0000000000107805 000fde0000005000 ?? 0x000fde00000050000000000000107805 ; +0160 0000000100117802 000fcc0000000f00 MOV R17, 0x1 ; +0170 0000000000127919 000e280000002100 S2R R18, SR_TID.X ; +0180 000000111200720c 001fde0003f06070 ISETP.GE.U32.AND P0, PT, R18, R17, PT ; +0190 0000000000100947 000fca0003800000 @P0 BRA 0x1e0 ; +01a0 0000000f02007986 0003e8000c101904 STG.E desc[UR4][R2.64], R15 ; +01b0 0000041002007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x4], R16 ; +01c0 0000080a02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x8], R10 ; +01d0 00000c0d02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0xc], R13 ; +01e0 000000000000794d 000fca0003800000 EXIT ; +01f0 fffffffc00fc7947 000fc0000383ffff BRA 0x1f0; diff --git a/evidence/LATMEAS-GB10-20261101/sass/fma_fma_4.pairs b/evidence/LATMEAS-GB10-20261101/sass/fma_fma_4.pairs new file mode 100644 index 00000000..dc1cb117 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101/sass/fma_fma_4.pairs @@ -0,0 +1,32 @@ +0000 00006b00ff0477ac 000ea80008000a00 LDCU.64 UR4, c[0x0][0x358] ; +0010 0000e000ff027b82 004e280000000a00 LDC.64 R2, c[0x0][0x380] ; +0020 0000000100047802 001fcc0000000f00 MOV R4, 0x1 ; +0030 000000ff00057202 000fcc0000000f00 MOV R5, RZ ; +0040 3f80000000067802 000fcc0000000f00 MOV R6, 0x3f800000 ; +0050 0000002000077802 000fcc0000000f00 MOV R7, 0x20 ; +0060 000000ff00087202 000fcc0000000f00 MOV R8, RZ ; +0070 000000ff00097202 000fcc0000000f00 MOV R9, RZ ; +0080 000000ff000a7202 000fcc0000000f00 MOV R10, RZ ; +0090 000000ff000b7202 000fcc0000000f00 MOV R11, RZ ; +00a0 000000ff000c7202 000fcc0000000f00 MOV R12, RZ ; +00b0 00000001000d7802 000fcc0000000f00 MOV R13, 0x1 ; +00c0 7f800000000e7802 000fcc0000000f00 MOV R14, 0x7f800000 ; +00d0 00000000000f7805 000fde0000005000 ?? 0x000fde000000500000000000000f7805 ; +00e0 00000006060e7223 000fc8000000000c FFMA R14, R6, R6, R12 ; +00f0 0000000e0a0a7221 000fde0000000000 FADD R10, R10, R14 ; +0100 000000060c0c7221 000fde0000000000 FADD R12, R12, R6 ; +0110 7f800000000e7802 000fcc0000000f00 MOV R14, 0x7f800000 ; +0120 0000000408087210 000fde0007ffe0ff IADD3 R8, PT, PT, R8, R4, RZ ; +0130 000000070800720c 000fdc0003f06070 ISETP.GE.U32.AND P0, PT, R8, R7, PT ; +0140 fffffffc00e48947 000fca000383ffff @!P0 BRA 0xe0 ; +0150 0000000000107805 000fde0000005000 ?? 0x000fde00000050000000000000107805 ; +0160 0000000100117802 000fcc0000000f00 MOV R17, 0x1 ; +0170 0000000000127919 000e280000002100 S2R R18, SR_TID.X ; +0180 000000111200720c 001fde0003f06070 ISETP.GE.U32.AND P0, PT, R18, R17, PT ; +0190 0000000000100947 000fca0003800000 @P0 BRA 0x1e0 ; +01a0 0000000f02007986 0003e8000c101904 STG.E desc[UR4][R2.64], R15 ; +01b0 0000041002007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x4], R16 ; +01c0 0000080a02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x8], R10 ; +01d0 00000c0d02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0xc], R13 ; +01e0 000000000000794d 000fca0003800000 EXIT ; +01f0 fffffffc00fc7947 000fc0000383ffff BRA 0x1f0; diff --git a/evidence/LATMEAS-GB10-20261101/sass/fmnmx_alu_3.pairs b/evidence/LATMEAS-GB10-20261101/sass/fmnmx_alu_3.pairs new file mode 100644 index 00000000..d0ec077e --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101/sass/fmnmx_alu_3.pairs @@ -0,0 +1,40 @@ +0000 00006b00ff0477ac 000ea80008000a00 LDCU.64 UR4, c[0x0][0x358] ; +0010 0000e000ff027b82 004e280000000a00 LDC.64 R2, c[0x0][0x380] ; +0020 0000000100047802 001fcc0000000f00 MOV R4, 0x1 ; +0030 000000ff00057202 000fcc0000000f00 MOV R5, RZ ; +0040 3f80000000067802 000fcc0000000f00 MOV R6, 0x3f800000 ; +0050 0000002000077802 000fcc0000000f00 MOV R7, 0x20 ; +0060 000000ff00087202 000fcc0000000f00 MOV R8, RZ ; +0070 000000ff00097202 000fcc0000000f00 MOV R9, RZ ; +0080 000000ff000a7202 000fcc0000000f00 MOV R10, RZ ; +0090 000000ff000b7202 000fcc0000000f00 MOV R11, RZ ; +00a0 000000ff000c7202 000fcc0000000f00 MOV R12, RZ ; +00b0 00000001000d7802 000fcc0000000f00 MOV R13, 0x1 ; +00c0 7f800000000e7802 000fcc0000000f00 MOV R14, 0x7f800000 ; +00d0 3f800000000c7802 000fcc0000000f00 MOV R12, 0x3f800000 ; +00e0 00000000000f7805 000fde0000005000 ?? 0x000fde000000500000000000000f7805 ; +00f0 0000000c0c0e7209 000fc60003800000 FMNMX R14, R12, R12, PT ; +0100 0000000e0a0a7210 000fde0007ffe0ff IADD3 R10, PT, PT, R10, R14, RZ ; +0110 000000060c0c7221 000fde0000000000 FADD R12, R12, R6 ; +0120 7f800000000e7802 000fcc0000000f00 MOV R14, 0x7f800000 ; +0130 0000000408087210 000fde0007ffe0ff IADD3 R8, PT, PT, R8, R4, RZ ; +0140 000000070800720c 000fdc0003f06070 ISETP.GE.U32.AND P0, PT, R8, R7, PT ; +0150 fffffffc00e48947 000fca000383ffff @!P0 BRA 0xf0 ; +0160 0000000000107805 000fde0000005000 ?? 0x000fde00000050000000000000107805 ; +0170 0000000100117802 000fcc0000000f00 MOV R17, 0x1 ; +0180 0000000000127919 000e280000002100 S2R R18, SR_TID.X ; +0190 000000111200720c 001fde0003f06070 ISETP.GE.U32.AND P0, PT, R18, R17, PT ; +01a0 0000000000100947 000fca0003800000 @P0 BRA 0x1f0 ; +01b0 0000000f02007986 0003e8000c101904 STG.E desc[UR4][R2.64], R15 ; +01c0 0000041002007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x4], R16 ; +01d0 0000080a02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x8], R10 ; +01e0 00000c0d02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0xc], R13 ; +01f0 000000000000794d 000fca0003800000 EXIT ; +0200 fffffffc00fc7947 000fc0000383ffff BRA 0x200; +0210 0000000000007918 000fc00000000000 NOP; +0220 0000000000007918 000fc00000000000 NOP; +0230 0000000000007918 000fc00000000000 NOP; +0240 0000000000007918 000fc00000000000 NOP; +0250 0000000000007918 000fc00000000000 NOP; +0260 0000000000007918 000fc00000000000 NOP; +0270 0000000000007918 000fc00000000000 NOP; diff --git a/evidence/LATMEAS-GB10-20261101/sass/fmnmx_alu_4.pairs b/evidence/LATMEAS-GB10-20261101/sass/fmnmx_alu_4.pairs new file mode 100644 index 00000000..a2550a63 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101/sass/fmnmx_alu_4.pairs @@ -0,0 +1,40 @@ +0000 00006b00ff0477ac 000ea80008000a00 LDCU.64 UR4, c[0x0][0x358] ; +0010 0000e000ff027b82 004e280000000a00 LDC.64 R2, c[0x0][0x380] ; +0020 0000000100047802 001fcc0000000f00 MOV R4, 0x1 ; +0030 000000ff00057202 000fcc0000000f00 MOV R5, RZ ; +0040 3f80000000067802 000fcc0000000f00 MOV R6, 0x3f800000 ; +0050 0000002000077802 000fcc0000000f00 MOV R7, 0x20 ; +0060 000000ff00087202 000fcc0000000f00 MOV R8, RZ ; +0070 000000ff00097202 000fcc0000000f00 MOV R9, RZ ; +0080 000000ff000a7202 000fcc0000000f00 MOV R10, RZ ; +0090 000000ff000b7202 000fcc0000000f00 MOV R11, RZ ; +00a0 000000ff000c7202 000fcc0000000f00 MOV R12, RZ ; +00b0 00000001000d7802 000fcc0000000f00 MOV R13, 0x1 ; +00c0 7f800000000e7802 000fcc0000000f00 MOV R14, 0x7f800000 ; +00d0 3f800000000c7802 000fcc0000000f00 MOV R12, 0x3f800000 ; +00e0 00000000000f7805 000fde0000005000 ?? 0x000fde000000500000000000000f7805 ; +00f0 0000000c0c0e7209 000fc80003800000 FMNMX R14, R12, R12, PT ; +0100 0000000e0a0a7210 000fde0007ffe0ff IADD3 R10, PT, PT, R10, R14, RZ ; +0110 000000060c0c7221 000fde0000000000 FADD R12, R12, R6 ; +0120 7f800000000e7802 000fcc0000000f00 MOV R14, 0x7f800000 ; +0130 0000000408087210 000fde0007ffe0ff IADD3 R8, PT, PT, R8, R4, RZ ; +0140 000000070800720c 000fdc0003f06070 ISETP.GE.U32.AND P0, PT, R8, R7, PT ; +0150 fffffffc00e48947 000fca000383ffff @!P0 BRA 0xf0 ; +0160 0000000000107805 000fde0000005000 ?? 0x000fde00000050000000000000107805 ; +0170 0000000100117802 000fcc0000000f00 MOV R17, 0x1 ; +0180 0000000000127919 000e280000002100 S2R R18, SR_TID.X ; +0190 000000111200720c 001fde0003f06070 ISETP.GE.U32.AND P0, PT, R18, R17, PT ; +01a0 0000000000100947 000fca0003800000 @P0 BRA 0x1f0 ; +01b0 0000000f02007986 0003e8000c101904 STG.E desc[UR4][R2.64], R15 ; +01c0 0000041002007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x4], R16 ; +01d0 0000080a02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x8], R10 ; +01e0 00000c0d02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0xc], R13 ; +01f0 000000000000794d 000fca0003800000 EXIT ; +0200 fffffffc00fc7947 000fc0000383ffff BRA 0x200; +0210 0000000000007918 000fc00000000000 NOP; +0220 0000000000007918 000fc00000000000 NOP; +0230 0000000000007918 000fc00000000000 NOP; +0240 0000000000007918 000fc00000000000 NOP; +0250 0000000000007918 000fc00000000000 NOP; +0260 0000000000007918 000fc00000000000 NOP; +0270 0000000000007918 000fc00000000000 NOP; diff --git a/evidence/LATMEAS-GB10-20261101/sass/hmma_reader_19.pairs b/evidence/LATMEAS-GB10-20261101/sass/hmma_reader_19.pairs new file mode 100644 index 00000000..38bf02ca --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101/sass/hmma_reader_19.pairs @@ -0,0 +1,48 @@ +0000 00006b00ff0477ac 000ea80008000a00 LDCU.64 UR4, c[0x0][0x358] ; +0010 0000e000ff027b82 004e280000000a00 LDC.64 R2, c[0x0][0x380] ; +0020 0000000100047802 001fcc0000000f00 MOV R4, 0x1 ; +0030 000000ff00057202 000fcc0000000f00 MOV R5, RZ ; +0040 3f80000000067802 000fcc0000000f00 MOV R6, 0x3f800000 ; +0050 0000000100077802 000fcc0000000f00 MOV R7, 0x1 ; +0060 000000ff00087202 000fcc0000000f00 MOV R8, RZ ; +0070 000000ff00097202 000fcc0000000f00 MOV R9, RZ ; +0080 000000ff000a7202 000fcc0000000f00 MOV R10, RZ ; +0090 000000ff000b7202 000fcc0000000f00 MOV R11, RZ ; +00a0 000000ff000c7202 000fcc0000000f00 MOV R12, RZ ; +00b0 00000001000d7802 000fcc0000000f00 MOV R13, 0x1 ; +00c0 3f803f8000107802 000fcc0000000f00 MOV R16, 0x3f803f80 ; +00d0 3f803f8000117802 000fcc0000000f00 MOV R17, 0x3f803f80 ; +00e0 3f803f8000127802 000fcc0000000f00 MOV R18, 0x3f803f80 ; +00f0 3f803f8000137802 000fcc0000000f00 MOV R19, 0x3f803f80 ; +0100 3f803f80000e7802 000fcc0000000f00 MOV R14, 0x3f803f80 ; +0110 3f803f80000f7802 000fcc0000000f00 MOV R15, 0x3f803f80 ; +0120 000000ff00147202 000fcc0000000f00 MOV R20, RZ ; +0130 000000ff00157202 000fcc0000000f00 MOV R21, RZ ; +0140 000000ff00167202 000fcc0000000f00 MOV R22, RZ ; +0150 000000ff00177202 000fcc0000000f00 MOV R23, RZ ; +0160 000000ff00187202 000fcc0000000f00 MOV R24, RZ ; +0170 000000ff00197202 000fcc0000000f00 MOV R25, RZ ; +0180 000000ff001a7202 000fcc0000000f00 MOV R26, RZ ; +0190 000000ff001b7202 000fcc0000000f00 MOV R27, RZ ; +01a0 ffffffff000a7802 000fcc0000000f00 MOV R10, 0xffffffff ; +01b0 00000000001c7805 000fde0000005000 ?? 0x000fde000000500000000000001c7805 ; +01c0 0000000e1018723c 000fde0000041814 HMMA.16816.F32.BF16 R24, R16, R14, R20 ; +01d0 0000000000007918 000fc80000000000 NOP ; +01e0 00000005180a7221 000fde0000000000 FADD R10, R24, R5 ; +01f0 00000000001d7805 000fde0000005000 ?? 0x000fde000000500000000000001d7805 ; +0200 00000001001e7802 000fcc0000000f00 MOV R30, 0x1 ; +0210 00000000001f7919 000e280000002100 S2R R31, SR_TID.X ; +0220 0000001e1f00720c 001fde0003f06070 ISETP.GE.U32.AND P0, PT, R31, R30, PT ; +0230 0000000000100947 000fca0003800000 @P0 BRA 0x280 ; +0240 0000001c02007986 0003e8000c101904 STG.E desc[UR4][R2.64], R28 ; +0250 0000041d02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x4], R29 ; +0260 0000080a02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x8], R10 ; +0270 00000c1902007986 0003e8000c101904 STG.E desc[UR4][R2.64+0xc], R25 ; +0280 000000000000794d 000fca0003800000 EXIT ; +0290 fffffffc00fc7947 000fc0000383ffff BRA 0x290; +02a0 0000000000007918 000fc00000000000 NOP; +02b0 0000000000007918 000fc00000000000 NOP; +02c0 0000000000007918 000fc00000000000 NOP; +02d0 0000000000007918 000fc00000000000 NOP; +02e0 0000000000007918 000fc00000000000 NOP; +02f0 0000000000007918 000fc00000000000 NOP; diff --git a/evidence/LATMEAS-GB10-20261101/sass/hmma_reader_20.pairs b/evidence/LATMEAS-GB10-20261101/sass/hmma_reader_20.pairs new file mode 100644 index 00000000..71edea07 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101/sass/hmma_reader_20.pairs @@ -0,0 +1,48 @@ +0000 00006b00ff0477ac 000ea80008000a00 LDCU.64 UR4, c[0x0][0x358] ; +0010 0000e000ff027b82 004e280000000a00 LDC.64 R2, c[0x0][0x380] ; +0020 0000000100047802 001fcc0000000f00 MOV R4, 0x1 ; +0030 000000ff00057202 000fcc0000000f00 MOV R5, RZ ; +0040 3f80000000067802 000fcc0000000f00 MOV R6, 0x3f800000 ; +0050 0000000100077802 000fcc0000000f00 MOV R7, 0x1 ; +0060 000000ff00087202 000fcc0000000f00 MOV R8, RZ ; +0070 000000ff00097202 000fcc0000000f00 MOV R9, RZ ; +0080 000000ff000a7202 000fcc0000000f00 MOV R10, RZ ; +0090 000000ff000b7202 000fcc0000000f00 MOV R11, RZ ; +00a0 000000ff000c7202 000fcc0000000f00 MOV R12, RZ ; +00b0 00000001000d7802 000fcc0000000f00 MOV R13, 0x1 ; +00c0 3f803f8000107802 000fcc0000000f00 MOV R16, 0x3f803f80 ; +00d0 3f803f8000117802 000fcc0000000f00 MOV R17, 0x3f803f80 ; +00e0 3f803f8000127802 000fcc0000000f00 MOV R18, 0x3f803f80 ; +00f0 3f803f8000137802 000fcc0000000f00 MOV R19, 0x3f803f80 ; +0100 3f803f80000e7802 000fcc0000000f00 MOV R14, 0x3f803f80 ; +0110 3f803f80000f7802 000fcc0000000f00 MOV R15, 0x3f803f80 ; +0120 000000ff00147202 000fcc0000000f00 MOV R20, RZ ; +0130 000000ff00157202 000fcc0000000f00 MOV R21, RZ ; +0140 000000ff00167202 000fcc0000000f00 MOV R22, RZ ; +0150 000000ff00177202 000fcc0000000f00 MOV R23, RZ ; +0160 000000ff00187202 000fcc0000000f00 MOV R24, RZ ; +0170 000000ff00197202 000fcc0000000f00 MOV R25, RZ ; +0180 000000ff001a7202 000fcc0000000f00 MOV R26, RZ ; +0190 000000ff001b7202 000fcc0000000f00 MOV R27, RZ ; +01a0 ffffffff000a7802 000fcc0000000f00 MOV R10, 0xffffffff ; +01b0 00000000001c7805 000fde0000005000 ?? 0x000fde000000500000000000001c7805 ; +01c0 0000000e1018723c 000fde0000041814 HMMA.16816.F32.BF16 R24, R16, R14, R20 ; +01d0 0000000000007918 000fca0000000000 NOP ; +01e0 00000005180a7221 000fde0000000000 FADD R10, R24, R5 ; +01f0 00000000001d7805 000fde0000005000 ?? 0x000fde000000500000000000001d7805 ; +0200 00000001001e7802 000fcc0000000f00 MOV R30, 0x1 ; +0210 00000000001f7919 000e280000002100 S2R R31, SR_TID.X ; +0220 0000001e1f00720c 001fde0003f06070 ISETP.GE.U32.AND P0, PT, R31, R30, PT ; +0230 0000000000100947 000fca0003800000 @P0 BRA 0x280 ; +0240 0000001c02007986 0003e8000c101904 STG.E desc[UR4][R2.64], R28 ; +0250 0000041d02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x4], R29 ; +0260 0000080a02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x8], R10 ; +0270 00000c1902007986 0003e8000c101904 STG.E desc[UR4][R2.64+0xc], R25 ; +0280 000000000000794d 000fca0003800000 EXIT ; +0290 fffffffc00fc7947 000fc0000383ffff BRA 0x290; +02a0 0000000000007918 000fc00000000000 NOP; +02b0 0000000000007918 000fc00000000000 NOP; +02c0 0000000000007918 000fc00000000000 NOP; +02d0 0000000000007918 000fc00000000000 NOP; +02e0 0000000000007918 000fc00000000000 NOP; +02f0 0000000000007918 000fc00000000000 NOP; diff --git a/evidence/LATMEAS-GB10-20261101/sass/imad_wide_lo_2.pairs b/evidence/LATMEAS-GB10-20261101/sass/imad_wide_lo_2.pairs new file mode 100644 index 00000000..b2cc2e4d --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101/sass/imad_wide_lo_2.pairs @@ -0,0 +1,40 @@ +0000 00006b00ff0477ac 000ea80008000a00 LDCU.64 UR4, c[0x0][0x358] ; +0010 0000e000ff027b82 004e280000000a00 LDC.64 R2, c[0x0][0x380] ; +0020 0000000100047802 001fcc0000000f00 MOV R4, 0x1 ; +0030 000000ff00057202 000fcc0000000f00 MOV R5, RZ ; +0040 3f80000000067802 000fcc0000000f00 MOV R6, 0x3f800000 ; +0050 0000002000077802 000fcc0000000f00 MOV R7, 0x20 ; +0060 000000ff00087202 000fcc0000000f00 MOV R8, RZ ; +0070 000000ff00097202 000fcc0000000f00 MOV R9, RZ ; +0080 000000ff000a7202 000fcc0000000f00 MOV R10, RZ ; +0090 000000ff000b7202 000fcc0000000f00 MOV R11, RZ ; +00a0 000000ff000c7202 000fcc0000000f00 MOV R12, RZ ; +00b0 00000001000d7802 000fcc0000000f00 MOV R13, 0x1 ; +00c0 000000ff000e7202 000fcc0000000f00 MOV R14, RZ ; +00d0 000000ff000f7202 000fcc0000000f00 MOV R15, RZ ; +00e0 7f80000000107802 000fcc0000000f00 MOV R16, 0x7f800000 ; +00f0 7f80000000117802 000fcc0000000f00 MOV R17, 0x7f800000 ; +0100 0000000000127805 000fde0000005000 ?? 0x000fde00000050000000000000127805 ; +0110 0000000104107825 000fc400078e000e IMAD.WIDE.U32 R16, R4, 0x1, R14 ; +0120 000000100a0a7210 000fde0007ffe0ff IADD3 R10, PT, PT, R10, R16, RZ ; +0130 000000040e0e7210 000fde0007ffe0ff IADD3 R14, PT, PT, R14, R4, RZ ; +0140 7f80000000107802 000fcc0000000f00 MOV R16, 0x7f800000 ; +0150 7f80000000117802 000fcc0000000f00 MOV R17, 0x7f800000 ; +0160 0000000408087210 000fde0007ffe0ff IADD3 R8, PT, PT, R8, R4, RZ ; +0170 000000070800720c 000fdc0003f06070 ISETP.GE.U32.AND P0, PT, R8, R7, PT ; +0180 fffffffc00e08947 000fca000383ffff @!P0 BRA 0x110 ; +0190 0000000000137805 000fde0000005000 ?? 0x000fde00000050000000000000137805 ; +01a0 0000000100147802 000fcc0000000f00 MOV R20, 0x1 ; +01b0 0000000000157919 000e280000002100 S2R R21, SR_TID.X ; +01c0 000000141500720c 001fde0003f06070 ISETP.GE.U32.AND P0, PT, R21, R20, PT ; +01d0 0000000000100947 000fca0003800000 @P0 BRA 0x220 ; +01e0 0000001202007986 0003e8000c101904 STG.E desc[UR4][R2.64], R18 ; +01f0 0000041302007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x4], R19 ; +0200 0000080a02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x8], R10 ; +0210 00000c0d02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0xc], R13 ; +0220 000000000000794d 000fca0003800000 EXIT ; +0230 fffffffc00fc7947 000fc0000383ffff BRA 0x230; +0240 0000000000007918 000fc00000000000 NOP; +0250 0000000000007918 000fc00000000000 NOP; +0260 0000000000007918 000fc00000000000 NOP; +0270 0000000000007918 000fc00000000000 NOP; diff --git a/evidence/LATMEAS-GB10-20261101/sass/imad_wide_lo_3.pairs b/evidence/LATMEAS-GB10-20261101/sass/imad_wide_lo_3.pairs new file mode 100644 index 00000000..01734a51 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101/sass/imad_wide_lo_3.pairs @@ -0,0 +1,40 @@ +0000 00006b00ff0477ac 000ea80008000a00 LDCU.64 UR4, c[0x0][0x358] ; +0010 0000e000ff027b82 004e280000000a00 LDC.64 R2, c[0x0][0x380] ; +0020 0000000100047802 001fcc0000000f00 MOV R4, 0x1 ; +0030 000000ff00057202 000fcc0000000f00 MOV R5, RZ ; +0040 3f80000000067802 000fcc0000000f00 MOV R6, 0x3f800000 ; +0050 0000002000077802 000fcc0000000f00 MOV R7, 0x20 ; +0060 000000ff00087202 000fcc0000000f00 MOV R8, RZ ; +0070 000000ff00097202 000fcc0000000f00 MOV R9, RZ ; +0080 000000ff000a7202 000fcc0000000f00 MOV R10, RZ ; +0090 000000ff000b7202 000fcc0000000f00 MOV R11, RZ ; +00a0 000000ff000c7202 000fcc0000000f00 MOV R12, RZ ; +00b0 00000001000d7802 000fcc0000000f00 MOV R13, 0x1 ; +00c0 000000ff000e7202 000fcc0000000f00 MOV R14, RZ ; +00d0 000000ff000f7202 000fcc0000000f00 MOV R15, RZ ; +00e0 7f80000000107802 000fcc0000000f00 MOV R16, 0x7f800000 ; +00f0 7f80000000117802 000fcc0000000f00 MOV R17, 0x7f800000 ; +0100 0000000000127805 000fde0000005000 ?? 0x000fde00000050000000000000127805 ; +0110 0000000104107825 000fc600078e000e IMAD.WIDE.U32 R16, R4, 0x1, R14 ; +0120 000000100a0a7210 000fde0007ffe0ff IADD3 R10, PT, PT, R10, R16, RZ ; +0130 000000040e0e7210 000fde0007ffe0ff IADD3 R14, PT, PT, R14, R4, RZ ; +0140 7f80000000107802 000fcc0000000f00 MOV R16, 0x7f800000 ; +0150 7f80000000117802 000fcc0000000f00 MOV R17, 0x7f800000 ; +0160 0000000408087210 000fde0007ffe0ff IADD3 R8, PT, PT, R8, R4, RZ ; +0170 000000070800720c 000fdc0003f06070 ISETP.GE.U32.AND P0, PT, R8, R7, PT ; +0180 fffffffc00e08947 000fca000383ffff @!P0 BRA 0x110 ; +0190 0000000000137805 000fde0000005000 ?? 0x000fde00000050000000000000137805 ; +01a0 0000000100147802 000fcc0000000f00 MOV R20, 0x1 ; +01b0 0000000000157919 000e280000002100 S2R R21, SR_TID.X ; +01c0 000000141500720c 001fde0003f06070 ISETP.GE.U32.AND P0, PT, R21, R20, PT ; +01d0 0000000000100947 000fca0003800000 @P0 BRA 0x220 ; +01e0 0000001202007986 0003e8000c101904 STG.E desc[UR4][R2.64], R18 ; +01f0 0000041302007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x4], R19 ; +0200 0000080a02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x8], R10 ; +0210 00000c0d02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0xc], R13 ; +0220 000000000000794d 000fca0003800000 EXIT ; +0230 fffffffc00fc7947 000fc0000383ffff BRA 0x230; +0240 0000000000007918 000fc00000000000 NOP; +0250 0000000000007918 000fc00000000000 NOP; +0260 0000000000007918 000fc00000000000 NOP; +0270 0000000000007918 000fc00000000000 NOP; diff --git a/evidence/LATMEAS-GB10-20261101/sass/mixed_4.pairs b/evidence/LATMEAS-GB10-20261101/sass/mixed_4.pairs new file mode 100644 index 00000000..32bb3620 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101/sass/mixed_4.pairs @@ -0,0 +1,40 @@ +0000 00006b00ff0477ac 000ea80008000a00 LDCU.64 UR4, c[0x0][0x358] ; +0010 0000e000ff027b82 004e280000000a00 LDC.64 R2, c[0x0][0x380] ; +0020 0000000100047802 001fcc0000000f00 MOV R4, 0x1 ; +0030 000000ff00057202 000fcc0000000f00 MOV R5, RZ ; +0040 3f80000000067802 000fcc0000000f00 MOV R6, 0x3f800000 ; +0050 0000002000077802 000fcc0000000f00 MOV R7, 0x20 ; +0060 000000ff00087202 000fcc0000000f00 MOV R8, RZ ; +0070 000000ff00097202 000fcc0000000f00 MOV R9, RZ ; +0080 000000ff000a7202 000fcc0000000f00 MOV R10, RZ ; +0090 000000ff000b7202 000fcc0000000f00 MOV R11, RZ ; +00a0 000000ff000c7202 000fcc0000000f00 MOV R12, RZ ; +00b0 00000001000d7802 000fcc0000000f00 MOV R13, 0x1 ; +00c0 7f800000000e7802 000fcc0000000f00 MOV R14, 0x7f800000 ; +00d0 3f80000000097802 000fcc0000000f00 MOV R9, 0x3f800000 ; +00e0 7f800000000f7802 000fcc0000000f00 MOV R15, 0x7f800000 ; +00f0 0000000000107805 000fde0000005000 ?? 0x000fde00000050000000000000107805 ; +0100 00000006060f7223 000fc8000000000c FFMA R15, R6, R6, R12 ; +0110 0000000f0a0a7210 000fde0007ffe0ff IADD3 R10, PT, PT, R10, R15, RZ ; +0120 000000060c0c7221 000fde0000000000 FADD R12, R12, R6 ; +0130 7f800000000f7802 000fcc0000000f00 MOV R15, 0x7f800000 ; +0140 00000005090e7210 000fc80007ffe0ff IADD3 R14, PT, PT, R9, R5, RZ ; +0150 0000000505117223 000fde000000000e FFMA R17, R5, R5, R14 ; +0160 000000110b0b7210 000fde0007ffe0ff IADD3 R11, PT, PT, R11, R17, RZ ; +0170 0000000609097221 000fde0000000000 FADD R9, R9, R6 ; +0180 7f800000000e7802 000fcc0000000f00 MOV R14, 0x7f800000 ; +0190 0000000408087210 000fde0007ffe0ff IADD3 R8, PT, PT, R8, R4, RZ ; +01a0 000000070800720c 000fdc0003f06070 ISETP.GE.U32.AND P0, PT, R8, R7, PT ; +01b0 fffffffc00d08947 000fca000383ffff @!P0 BRA 0x100 ; +01c0 0000000000127805 000fde0000005000 ?? 0x000fde00000050000000000000127805 ; +01d0 0000000100137802 000fcc0000000f00 MOV R19, 0x1 ; +01e0 0000000000147919 000e280000002100 S2R R20, SR_TID.X ; +01f0 000000131400720c 001fde0003f06070 ISETP.GE.U32.AND P0, PT, R20, R19, PT ; +0200 0000000000100947 000fca0003800000 @P0 BRA 0x250 ; +0210 0000001002007986 0003e8000c101904 STG.E desc[UR4][R2.64], R16 ; +0220 0000041202007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x4], R18 ; +0230 0000080a02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x8], R10 ; +0240 00000c0b02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0xc], R11 ; +0250 000000000000794d 000fca0003800000 EXIT ; +0260 fffffffc00fc7947 000fc0000383ffff BRA 0x260; +0270 0000000000007918 000fc00000000000 NOP; diff --git a/evidence/LATMEAS-GB10-20261101/sass/mixed_5.pairs b/evidence/LATMEAS-GB10-20261101/sass/mixed_5.pairs new file mode 100644 index 00000000..60f5877c --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101/sass/mixed_5.pairs @@ -0,0 +1,40 @@ +0000 00006b00ff0477ac 000ea80008000a00 LDCU.64 UR4, c[0x0][0x358] ; +0010 0000e000ff027b82 004e280000000a00 LDC.64 R2, c[0x0][0x380] ; +0020 0000000100047802 001fcc0000000f00 MOV R4, 0x1 ; +0030 000000ff00057202 000fcc0000000f00 MOV R5, RZ ; +0040 3f80000000067802 000fcc0000000f00 MOV R6, 0x3f800000 ; +0050 0000002000077802 000fcc0000000f00 MOV R7, 0x20 ; +0060 000000ff00087202 000fcc0000000f00 MOV R8, RZ ; +0070 000000ff00097202 000fcc0000000f00 MOV R9, RZ ; +0080 000000ff000a7202 000fcc0000000f00 MOV R10, RZ ; +0090 000000ff000b7202 000fcc0000000f00 MOV R11, RZ ; +00a0 000000ff000c7202 000fcc0000000f00 MOV R12, RZ ; +00b0 00000001000d7802 000fcc0000000f00 MOV R13, 0x1 ; +00c0 7f800000000e7802 000fcc0000000f00 MOV R14, 0x7f800000 ; +00d0 3f80000000097802 000fcc0000000f00 MOV R9, 0x3f800000 ; +00e0 7f800000000f7802 000fcc0000000f00 MOV R15, 0x7f800000 ; +00f0 0000000000107805 000fde0000005000 ?? 0x000fde00000050000000000000107805 ; +0100 00000006060f7223 000fca000000000c FFMA R15, R6, R6, R12 ; +0110 0000000f0a0a7210 000fde0007ffe0ff IADD3 R10, PT, PT, R10, R15, RZ ; +0120 000000060c0c7221 000fde0000000000 FADD R12, R12, R6 ; +0130 7f800000000f7802 000fcc0000000f00 MOV R15, 0x7f800000 ; +0140 00000005090e7210 000fca0007ffe0ff IADD3 R14, PT, PT, R9, R5, RZ ; +0150 0000000505117223 000fde000000000e FFMA R17, R5, R5, R14 ; +0160 000000110b0b7210 000fde0007ffe0ff IADD3 R11, PT, PT, R11, R17, RZ ; +0170 0000000609097221 000fde0000000000 FADD R9, R9, R6 ; +0180 7f800000000e7802 000fcc0000000f00 MOV R14, 0x7f800000 ; +0190 0000000408087210 000fde0007ffe0ff IADD3 R8, PT, PT, R8, R4, RZ ; +01a0 000000070800720c 000fdc0003f06070 ISETP.GE.U32.AND P0, PT, R8, R7, PT ; +01b0 fffffffc00d08947 000fca000383ffff @!P0 BRA 0x100 ; +01c0 0000000000127805 000fde0000005000 ?? 0x000fde00000050000000000000127805 ; +01d0 0000000100137802 000fcc0000000f00 MOV R19, 0x1 ; +01e0 0000000000147919 000e280000002100 S2R R20, SR_TID.X ; +01f0 000000131400720c 001fde0003f06070 ISETP.GE.U32.AND P0, PT, R20, R19, PT ; +0200 0000000000100947 000fca0003800000 @P0 BRA 0x250 ; +0210 0000001002007986 0003e8000c101904 STG.E desc[UR4][R2.64], R16 ; +0220 0000041202007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x4], R18 ; +0230 0000080a02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x8], R10 ; +0240 00000c0b02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0xc], R11 ; +0250 000000000000794d 000fca0003800000 EXIT ; +0260 fffffffc00fc7947 000fc0000383ffff BRA 0x260; +0270 0000000000007918 000fc00000000000 NOP; diff --git a/evidence/LATMEAS-GB10-20261101/sass/pred_12.pairs b/evidence/LATMEAS-GB10-20261101/sass/pred_12.pairs new file mode 100644 index 00000000..9b0c73b4 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101/sass/pred_12.pairs @@ -0,0 +1,40 @@ +0000 00006b00ff0477ac 000ea80008000a00 LDCU.64 UR4, c[0x0][0x358] ; +0010 0000e000ff027b82 004e280000000a00 LDC.64 R2, c[0x0][0x380] ; +0020 0000000100047802 001fcc0000000f00 MOV R4, 0x1 ; +0030 000000ff00057202 000fcc0000000f00 MOV R5, RZ ; +0040 3f80000000067802 000fcc0000000f00 MOV R6, 0x3f800000 ; +0050 0000002000077802 000fcc0000000f00 MOV R7, 0x20 ; +0060 000000ff00087202 000fcc0000000f00 MOV R8, RZ ; +0070 000000ff00097202 000fcc0000000f00 MOV R9, RZ ; +0080 000000ff000a7202 000fcc0000000f00 MOV R10, RZ ; +0090 000000ff000b7202 000fcc0000000f00 MOV R11, RZ ; +00a0 000000ff000c7202 000fcc0000000f00 MOV R12, RZ ; +00b0 00000001000d7802 000fcc0000000f00 MOV R13, 0x1 ; +00c0 00000000000e7805 000fde0000005000 ?? 0x000fde000000500000000000000e7805 ; +00d0 000000050400720c 000fde0003f06070 ISETP.GE.U32.AND P0, PT, R4, R5, PT ; +00e0 0000000409097210 000fde0007ffe0ff IADD3 R9, PT, PT, R9, R4, RZ ; +00f0 000000090500720c 000fd80003f06070 ISETP.GE.U32.AND P0, PT, R5, R9, PT ; +0100 0000000000100947 000fc20003800000 @P0 BRA 0x150 ; +0110 0000000408087210 000fde0007ffe0ff IADD3 R8, PT, PT, R8, R4, RZ ; +0120 000000070800720c 000fdc0003f06070 ISETP.GE.U32.AND P0, PT, R8, R7, PT ; +0130 fffffffc00e48947 000fca000383ffff @!P0 BRA 0xd0 ; +0140 0000000000047947 000fca0003800000 BRA 0x160 ; +0150 deadbeef00097802 000fcc0000000f00 MOV R9, 0xdeadbeef ; +0160 00000000000f7805 000fde0000005000 ?? 0x000fde000000500000000000000f7805 ; +0170 0000000100107802 000fcc0000000f00 MOV R16, 0x1 ; +0180 0000000000117919 000e280000002100 S2R R17, SR_TID.X ; +0190 000000101100720c 001fde0003f06070 ISETP.GE.U32.AND P0, PT, R17, R16, PT ; +01a0 0000000000100947 000fca0003800000 @P0 BRA 0x1f0 ; +01b0 0000000e02007986 0003e8000c101904 STG.E desc[UR4][R2.64], R14 ; +01c0 0000040f02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x4], R15 ; +01d0 0000080902007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x8], R9 ; +01e0 00000c0d02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0xc], R13 ; +01f0 000000000000794d 000fca0003800000 EXIT ; +0200 fffffffc00fc7947 000fc0000383ffff BRA 0x200; +0210 0000000000007918 000fc00000000000 NOP; +0220 0000000000007918 000fc00000000000 NOP; +0230 0000000000007918 000fc00000000000 NOP; +0240 0000000000007918 000fc00000000000 NOP; +0250 0000000000007918 000fc00000000000 NOP; +0260 0000000000007918 000fc00000000000 NOP; +0270 0000000000007918 000fc00000000000 NOP; diff --git a/evidence/LATMEAS-GB10-20261101/sass/pred_13.pairs b/evidence/LATMEAS-GB10-20261101/sass/pred_13.pairs new file mode 100644 index 00000000..7608b6f1 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101/sass/pred_13.pairs @@ -0,0 +1,40 @@ +0000 00006b00ff0477ac 000ea80008000a00 LDCU.64 UR4, c[0x0][0x358] ; +0010 0000e000ff027b82 004e280000000a00 LDC.64 R2, c[0x0][0x380] ; +0020 0000000100047802 001fcc0000000f00 MOV R4, 0x1 ; +0030 000000ff00057202 000fcc0000000f00 MOV R5, RZ ; +0040 3f80000000067802 000fcc0000000f00 MOV R6, 0x3f800000 ; +0050 0000002000077802 000fcc0000000f00 MOV R7, 0x20 ; +0060 000000ff00087202 000fcc0000000f00 MOV R8, RZ ; +0070 000000ff00097202 000fcc0000000f00 MOV R9, RZ ; +0080 000000ff000a7202 000fcc0000000f00 MOV R10, RZ ; +0090 000000ff000b7202 000fcc0000000f00 MOV R11, RZ ; +00a0 000000ff000c7202 000fcc0000000f00 MOV R12, RZ ; +00b0 00000001000d7802 000fcc0000000f00 MOV R13, 0x1 ; +00c0 00000000000e7805 000fde0000005000 ?? 0x000fde000000500000000000000e7805 ; +00d0 000000050400720c 000fde0003f06070 ISETP.GE.U32.AND P0, PT, R4, R5, PT ; +00e0 0000000409097210 000fde0007ffe0ff IADD3 R9, PT, PT, R9, R4, RZ ; +00f0 000000090500720c 000fda0003f06070 ISETP.GE.U32.AND P0, PT, R5, R9, PT ; +0100 0000000000100947 000fc20003800000 @P0 BRA 0x150 ; +0110 0000000408087210 000fde0007ffe0ff IADD3 R8, PT, PT, R8, R4, RZ ; +0120 000000070800720c 000fdc0003f06070 ISETP.GE.U32.AND P0, PT, R8, R7, PT ; +0130 fffffffc00e48947 000fca000383ffff @!P0 BRA 0xd0 ; +0140 0000000000047947 000fca0003800000 BRA 0x160 ; +0150 deadbeef00097802 000fcc0000000f00 MOV R9, 0xdeadbeef ; +0160 00000000000f7805 000fde0000005000 ?? 0x000fde000000500000000000000f7805 ; +0170 0000000100107802 000fcc0000000f00 MOV R16, 0x1 ; +0180 0000000000117919 000e280000002100 S2R R17, SR_TID.X ; +0190 000000101100720c 001fde0003f06070 ISETP.GE.U32.AND P0, PT, R17, R16, PT ; +01a0 0000000000100947 000fca0003800000 @P0 BRA 0x1f0 ; +01b0 0000000e02007986 0003e8000c101904 STG.E desc[UR4][R2.64], R14 ; +01c0 0000040f02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x4], R15 ; +01d0 0000080902007986 0003e8000c101904 STG.E desc[UR4][R2.64+0x8], R9 ; +01e0 00000c0d02007986 0003e8000c101904 STG.E desc[UR4][R2.64+0xc], R13 ; +01f0 000000000000794d 000fca0003800000 EXIT ; +0200 fffffffc00fc7947 000fc0000383ffff BRA 0x200; +0210 0000000000007918 000fc00000000000 NOP; +0220 0000000000007918 000fc00000000000 NOP; +0230 0000000000007918 000fc00000000000 NOP; +0240 0000000000007918 000fc00000000000 NOP; +0250 0000000000007918 000fc00000000000 NOP; +0260 0000000000007918 000fc00000000000 NOP; +0270 0000000000007918 000fc00000000000 NOP; diff --git a/evidence/LATMEAS-GB10-20261101/sm121_vs_NAK.md b/evidence/LATMEAS-GB10-20261101/sm121_vs_NAK.md new file mode 100644 index 00000000..1a4e3939 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101/sm121_vs_NAK.md @@ -0,0 +1,42 @@ +# GB10 (sm_121) measured latencies vs NAK + +Chip: NVIDIA GB10 (DGX Spark), sm_121 +Driver: NVRM version: NVIDIA UNIX Open Kernel Module for aarch64 580.173.02 Release Build (dvs-builder@U22-A24-5-4) Tue Jun 23 08:34:19 UTC 2026. No NVIDIA user-space libraries. +Method: dependent-chain timing kernel. One warp (32 threads), lane 0 stores. +Each fixed-latency producer is issued at a programmed stall. The GB10 column is the smallest stall S such that S and every larger S in the sweep matched the oracle on every run. Cycles are the CS2R clock delta (NAK_SV_CLOCK_LO, 0x50) around that chain, median and max-min. +Source: tests/gpu_latmeas_test.c in aien-dev/CUDApp +How to rerun: sh tools/latmeas/run_spark.sh OUTPUT_DIR + +NAK reference: `nak/sm120_instr_latencies.rs` plus `latencies/sm100/*.csv` at Mesa commit 01bab9302a6bf0a082c7ba8c49d74427cf10b0cf + +## Priority 1: values CUDA++ currently borrows from NAK (needs independent measurement) + +| Writer -> reader | NAK value | NAK source line | GB10 measured | Delta | Runs / spread | Kernel | +|---|---|---|---|---|---|---| +| ALU -> ALU (GPR RAW) | 4 + 1 = 5 | latencies/sm100/reg_raw.csv:2 row alu column alu = 4; sm120_instr_latencies.rs:457 GPR raw adds max(adjust, 1); IAdd3 is Alu at line 44 | 4 | -1 | 11 runs; stall spread 0; cycles at that stall median 2538 spread 0; stall-1 cycles median 2474 spread 0; loop overhead median 1261 | tests/gpu_latmeas_test.c (alu_alu) | +| FMA -> FMA (GPR RAW) | 4 + 1 = 5 | latencies/sm100/reg_raw.csv:5 row fma column fma = 4; sm120_instr_latencies.rs:59 FFma is Fma; line 457 adds 1 | 4 | -1 | 11 runs; stall spread 0; cycles at that stall median 2538 spread 0; stall-1 cycles median 2474 spread 0; loop overhead median 1261 | tests/gpu_latmeas_test.c (fma_fma) | +| Mixed ALU <-> FMA (GPR RAW) | 5 + 1 = 6 | latencies/sm100/reg_raw.csv:2 row alu column fma = 5 and line 5 row fma column alu = 5; sm120_instr_latencies.rs:457 adds 1 | 5 | -1 | 11 runs; stall spread 0; cycles at that stall median 4362 spread 0; stall-1 cycles median 4170 spread 0; loop overhead median 1261 | tests/gpu_latmeas_test.c (mixed) | +| Predicate write -> guard | 13 + 1 = 14 | latencies/sm100/pred_raw.csv:11 row redirected_fp64 column dualalu = 13; sm120_instr_latencies.rs:474 predicate raw(write, None) uses reader RedirectedFp64 and adds 1; ISetP is Dualalu at line 49 | 13 | -1 | 11 runs; stall spread 0; cycles at that stall median 2714 spread 0; stall-1 cycles median 70 spread 0; loop overhead median 1261 | tests/gpu_latmeas_test.c (pred) | + +## Priority 2: instructions where CUDA++ (nvcc-derived) and NAK disagree + +| Instruction / pair | NAK value | nvcc sm_121 stall | GB10 measured | Agrees with | Kernel | +|---|---|---|---|---|---| +| Predicate write -> guard, attention emitter | 13 + 1 = 14 | 13 | 13 | nvcc | tests/gpu_latmeas_test.c (pred) | +| HMMA -> non-HMMA reader (FADD of d0) | 19 + 9 = 28 | 15 + NOP 14 = 29 | 20 | neither | tests/gpu_latmeas_test.c (hmma_reader) | +| IMAD.WIDE low half -> IADD3 | 3 + 1 = 4 | 6 (blanket K_FIXED; IMAD.WIDE is not in fixed_pair_class) | 3 | neither | tests/gpu_latmeas_test.c (imad_wide_lo) | +| FMNMX (dualalu) -> IADD3 (alu) | 4 + 1 = 5 | 6 (blanket K_FIXED; FMNMX is not in fixed_pair_class) | 4 | neither | tests/gpu_latmeas_test.c (fmnmx_alu) | +| QMMA (IMMA) -> next reader | 19 + 7 = 26 | 15 + NOP 15 = 30 | | | not generated (patched QMMA placeholder; see REPRODUCE.md) | +| Fixed-producer minimum in the numeric checker | 5 + 1 = 6 for mixed ALU/FMA | 5 (any fixed producer; mixed pairs need 6) | 5 | nvcc | tests/gpu_latmeas_test.c (mixed) | +| BSSY stall | decoupled (1 & sb) | 1, no scoreboard | | | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| BSYNC stall | decoupled (1 & sb) | 5, no scoreboard | | | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| BRA stall | decoupled (1 & sb) | 5 | | | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| EXIT stall | decoupled (1 & sb) | 5 | | | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| BAR.SYNC stall | decoupled AGU (1 & sb) | 6 | | | not generated (NAK class is decoupled, not a fixed GPR RAW) | + +## Notes +- Anything not fixed-latency (LDG/LDS, scoreboard waits) is out of scope for this table. +- Decoupled ops (BSSY, BSYNC, BRA, EXIT, BAR) are listed because the encoder gives them a fixed stall and NAK does not. They have no dependent-chain kernel. +- HMMA -> HMMA in NAK is 20 + 9 = 29, the same total the encoder uses. The HMMA row is the non-HMMA reader (28). +- QMMA is not generated here. Its encoder stall is 30 and NAK's IMMA -> ALU total is 26 (IMMA -> IMMA is 27). +- Confidence notes and anomalies go here. From 4f3a4d0ccd9559cb8eed50524f409c1547a02280 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 1 Nov 2026 18:35:30 +0000 Subject: [PATCH 05/11] Check every lane and all four HMMA destinations. Each lane stores the two clocks, four result words, and its lane id. HMMA's four words are FADD of d0..d3 against zero, compared with the host dot of sixteen bf16 ones, not with the stall-36 launch. The host simulator requires the same match on all 32 lanes. This has not been launched. Co-authored-by: aien-spark --- tests/gpu_latmeas_test.c | 322 ++++++++++++++++++++++----------- tools/latmeas/REPRODUCE.md | 4 + tools/latmeas/fill_template.py | 2 +- 3 files changed, 222 insertions(+), 106 deletions(-) diff --git a/tests/gpu_latmeas_test.c b/tests/gpu_latmeas_test.c index fe60bb09..589aaa0a 100644 --- a/tests/gpu_latmeas_test.c +++ b/tests/gpu_latmeas_test.c @@ -38,8 +38,8 @@ * to the IADD. The overhead word is that counter. The host simulator below * applies the stall counts and rejects that shape. * - * One warp, 32 threads, uniform control flow inside the timed region. Only - * lane 0 stores. Mesa commit and the NAK line citations live in + * One warp, 32 threads, uniform control flow inside the timed region. Every + * lane stores eight words. Mesa commit and the NAK line citations live in * tools/latmeas/diff_nak.py. * * Opening the device on a GB10 needs the driver's SM version 0xa04 admitted @@ -83,6 +83,9 @@ enum { SWEEP_HMMA = 36 }; #define SENTINEL 0x7f800000u /* +inf bits: not equal to 1..N or to f32 1..N */ +#define LANE_N 32 +#define LANE_WORDS 8 +#define LANE_STRIDE 32u /* LANE_WORDS * 4: t0, t1, r0, r1, r2, r3, tid, unused */ _Static_assert(RUNS % 2 == 1, "median uses the middle of an odd run count"); _Static_assert(SWEEP_HMMA <= SWEEP_MAX, "sweep fits the sample array"); @@ -104,7 +107,7 @@ static const CaseDef CASES[] = { {"pred", N_FIXED, 1, SWEEP_FIXED, N_FIXED, 0}, {"fmnmx_alu", N_FIXED, 1, SWEEP_FIXED, N_FIXED, 0}, {"imad_wide_lo", N_FIXED, 1, SWEEP_FIXED, N_FIXED, 0}, - {"hmma_reader", HMMA_ITERS, 1, SWEEP_HMMA, HMMA_ITERS, 1}, + {"hmma_reader", HMMA_ITERS, 1, SWEEP_HMMA, HMMA_ITERS, 0}, }; #define NCASES ((int)(sizeof CASES / sizeof CASES[0])) @@ -234,12 +237,6 @@ static void clock_read(B *b, int dst) { b->pending = 0; } -static void s2r_sb(B *b, int dst, uint32_t sr) { - uint32_t wait = b->pending; - add(b, BW_IR_S2R, dst, NOREG, NOREG, NOREG, sr, cw(4, SB_GPR, SB_NONE, wait, 1), 0, 0, 0, 0); - b->pending = 1u << SB_GPR; -} - static void pin_live(BlackwellIRProgram *p) { int end = (int)p->count - 1; if (end < 0) return; @@ -255,6 +252,16 @@ static uint32_t fbits(float f) { return u; } +/* HMMA.16816.F32.BF16: each of the four f32 accumulators is a dot of k=16. + * This kernel writes bf16 1.0 into every A and B half and 0 into C, so each + * accumulator is 16 products of 1*1. The same value on every lane, because + * every lane's fragments are initialized alike. */ +static uint32_t hmma_16816_ones(void) { + float acc = 0.f; + for (int k = 0; k < 16; k++) acc += 1.f * 1.f; + return fbits(acc); +} + /* stale=1: the consumer sees SENTINEL instead of the value just written. * The two results differ for every fixed-latency case. HMMA has no closed form. */ static int eval_case(const CaseDef *c, int stale, uint32_t *result, uint32_t *aux) { @@ -320,6 +327,12 @@ static int eval_case(const CaseDef *c, int stale, uint32_t *result, uint32_t *au *result = stale ? PRED_POISON : n; return 0; } + if (!strcmp(c->id, "hmma_reader")) { + uint32_t e = hmma_16816_ones(); + *result = stale ? 0u : e; + *aux = stale ? 0u : e; + return 0; + } return -1; } @@ -339,20 +352,29 @@ static void prologue(B *b, int *ptr) { b->pending = 1u << SB_GPR; } -static void store_tail(B *b, int ptr, int t0, int t1, int result, int rsub, int aux, int asub) { +/* Every lane stores eight words at ptr + tid * LANE_STRIDE. There is no + * lane-0 filter: a lane that did not run would leave POISON in its slot. */ +static void store_lanes(B *b, int ptr, int t0, int t1, int r0, int r0s, int r1, int r1s, int r2, int r2s, int r3, + int r3s) { int tid = V(b); - int one = V(b); - movi(b, one, 1, 0); - s2r_sb(b, tid, BW_SR_TID_X); - isetp_ge(b, tid, one, 15); /* P0 = (tid >= 1); lane 0 falls through */ - int skip_br = bra(b, 1, 0, 5); + int addr = V64(b); + /* Stall 15, and a scoreboard the IMAD waits on. The address uses tid. + * A stall of 4 here is shorter than a fixed-latency S2R in the host + * simulator, which would then scale every lane by 0. */ + uint32_t wait = b->pending; + add(b, BW_IR_S2R, tid, NOREG, NOREG, NOREG, BW_SR_TID_X, cw(15, SB_GPR, SB_NONE, wait, 1), 0, 0, 0, 0); + b->pending = 1u << SB_GPR; + wait = b->pending; + add(b, BW_IR_IMAD_WIDE, addr, tid, NOREG, ptr, LANE_STRIDE, cw(6, SB_NONE, SB_NONE, wait, 0), 0, 0, 0, 0); + b->pending = 0; uint32_t st = cw(4, SB_NONE, SB_RD, 0, 1); - add(b, BW_IR_STG_E, NOREG, ptr, t0, NOREG, 0, st, 0, 0, 0, 0); - add(b, BW_IR_STG_E, NOREG, ptr, t1, NOREG, 4, st, 0, 0, 0, 0); - add(b, BW_IR_STG_E, NOREG, ptr, result, NOREG, 8, st, 0, 0, 0, rsub); - add(b, BW_IR_STG_E, NOREG, ptr, aux, NOREG, 12, st, 0, 0, 0, asub); - int skip = (int)b->p->count; - patch_bra(b, skip_br, skip); + add(b, BW_IR_STG_E, NOREG, addr, t0, NOREG, 0, st, 0, 0, 0, 0); + add(b, BW_IR_STG_E, NOREG, addr, t1, NOREG, 4, st, 0, 0, 0, 0); + add(b, BW_IR_STG_E, NOREG, addr, r0, NOREG, 8, st, 0, 0, 0, r0s); + add(b, BW_IR_STG_E, NOREG, addr, r1, NOREG, 12, st, 0, 0, 0, r1s); + add(b, BW_IR_STG_E, NOREG, addr, r2, NOREG, 16, st, 0, 0, 0, r2s); + add(b, BW_IR_STG_E, NOREG, addr, r3, NOREG, 20, st, 0, 0, 0, r3s); + add(b, BW_IR_STG_E, NOREG, addr, tid, NOREG, 24, st, 0, 0, 0, 0); add(b, BW_IR_EXIT, NOREG, NOREG, NOREG, NOREG, 0, cw(5, SB_NONE, SB_NONE, 0, 0), 0, 0, 0, 0); add(b, BW_IR_BRA, NOREG, NOREG, NOREG, NOREG, 0, 0x000fc000u, 0, 0, 0, 0); } @@ -442,6 +464,7 @@ static int build(const CaseDef *c, unsigned stall, unsigned ovh_iters, int overh int r = V(&b), acc = V(&b), acc2 = V(&b), cur = V(&b), tmp = V(&b), dsti = V(&b), seen = V(&b); int t0 = V(&b), t1 = V(&b), aux = V(&b); int rpair = NOREG, tpair = NOREG, dquad = NOREG, aquad = NOREG, bpair = NOREG, cquad = NOREG; + int h0 = NOREG, h1 = NOREG, h2 = NOREG, h3 = NOREG; unsigned iters = overhead ? ovh_iters : c->iters; movi(&b, one, 1, 0); movz(&b, zero, 0); @@ -490,7 +513,17 @@ static int build(const CaseDef *c, unsigned stall, unsigned ovh_iters, int overh add(&b, BW_IR_HMMA_BF16, dquad, aquad, bpair, cquad, 0, cw(head, SB_NONE, SB_NONE, wait, 0), 0, 0, 0, 0); b.pending = 0; gap_nops(&b, rest); - fadd(&b, acc, dquad, 0, zero, 15); /* acc = d0 + 0 */ + /* Four consumers, one per destination register. Stall 1 between them + * is the minimum issue gap: d1 is sampled one cycle after d0, d2 two, + * d3 three. The programmed stall is the gap before d0. */ + h0 = V(&b); + h1 = V(&b); + h2 = V(&b); + h3 = V(&b); + fadd(&b, h0, dquad, 0, zero, 1); + fadd(&b, h1, dquad, 1, zero, 1); + fadd(&b, h2, dquad, 2, zero, 1); + fadd(&b, h3, dquad, 3, zero, 1); if (c->iters > 1) { /* HMMA_ITERS is 1: a repeated identical HMMA would make a stale * read equal the previous correct write. */ @@ -518,13 +551,17 @@ static int build(const CaseDef *c, unsigned stall, unsigned ovh_iters, int overh int asub = 0; if (!overhead && c && !c->href && strcmp(c->id, "pred") != 0) result = acc; if (!overhead && !strcmp(c->id, "mixed")) auxv = acc2; + int r2 = zero, r3 = zero, r2s = 0, r3s = 0; if (!overhead && !strcmp(c->id, "hmma_reader")) { - result = acc; - auxv = dquad; /* late read of d1, after the timed region: diagnostic only */ - asub = 1; + result = h0; + rsub = 0; + auxv = h1; + asub = 0; + r2 = h2; + r3 = h3; } if (overhead) result = i; - store_tail(&b, ptr, t0, t1, result, rsub, auxv, asub); + store_lanes(&b, ptr, t0, t1, result, rsub, auxv, asub, r2, r2s, r3, r3s); if (b.err) return -1; pin_live(&g_prog); if (omega_bw_regalloc_solve(&g_prog) != 0) return -1; @@ -659,7 +696,6 @@ typedef struct { int nwr; uint32_t now; int pc; - uint32_t out[4]; int stores; const char *err; } LatSim; @@ -746,9 +782,9 @@ static int lat_wr(LatSim *s, int v, unsigned sub, uint32_t value, uint32_t lat) return lat_post(s, 0, reg, lat, value); } -/* Lane 0 only: the kernel is uniform, and the lane filter stores from lane 0. - * Returns 0 and fills out[4] (clock, clock, result, aux). */ -static int lat_sim(uint32_t out[4]) { +/* One lane. mem[] is the whole warp's output, poisoned by the caller. + * Returns 0 when this lane stored LANE_WORDS-1 words (the last slot is unused). */ +static int lat_sim_lane(uint32_t tid, uint32_t *mem) { LatSim s; memset(&s, 0, sizeof s); for (int steps = 0; steps < 100000; steps++) { @@ -767,11 +803,10 @@ static int lat_sim(uint32_t out[4]) { } switch (in->op) { case BW_IR_EXIT: - if (s.stores != 4) { - s.err = "lane 0 did not store four words"; + if (s.stores != LANE_WORDS - 1) { + s.err = "lane did not store its seven words"; break; } - memcpy(out, s.out, sizeof s.out); return 0; case BW_IR_BRA: { int take = 1; @@ -799,11 +834,34 @@ static int lat_sim(uint32_t out[4]) { break; case BW_IR_S2R: { uint32_t v = 0; - if (in->imm == BW_SR_TID_X) v = 0; + if (in->imm == BW_SR_TID_X) v = tid; else if (in->imm == CLOCK_LO) v = s.now; if (lat_wr(&s, in->dst_vreg, in->dst_subreg, v, SIM_GPR_LAT)) s.err = "write log full"; break; } + case BW_IR_HMMA_BF16: { + int ones = 1; + for (unsigned sub = 0; sub < 4 && ones; sub++) { + uint32_t u = 0; + if (lat_rd(&s, in->src1_vreg, sub, &u) || u != 0x3f803f80u) ones = 0; + } + for (unsigned sub = 0; sub < 2 && ones; sub++) { + uint32_t u = 0; + if (lat_rd(&s, in->src2_vreg, sub, &u) || u != 0x3f803f80u) ones = 0; + } + for (unsigned sub = 0; sub < 4 && ones; sub++) { + uint32_t u = 0; + if (lat_rd(&s, in->src3_vreg, sub, &u) || u != 0) ones = 0; + } + if (!ones) { + s.err = "hmma inputs are not the ones matrix"; + break; + } + uint32_t y = hmma_16816_ones(); + for (unsigned sub = 0; sub < 4; sub++) + if (lat_wr(&s, in->dst_vreg, sub, y, SIM_GPR_LAT)) s.err = "write log full"; + break; + } case BW_IR_IADD3: if (lat_wr(&s, in->dst_vreg, in->dst_subreg, a + b, SIM_GPR_LAT)) s.err = "write log full"; break; @@ -836,12 +894,18 @@ static int lat_sim(uint32_t out[4]) { break; } case BW_IR_STG_E: { - uint32_t off = in->imm; - if (off % 4u || off / 4u >= 4u) { - s.err = "store offset is not one of the four result words"; + uint32_t lo = 0, hi = 0; + if (lat_rd(&s, in->src1_vreg, 0, &lo) || lat_rd(&s, in->src1_vreg, 1, &hi)) { + s.err = "vreg has no physical register"; + break; + } + uint64_t ad = (uint64_t)lo | ((uint64_t)hi << 32); + uint64_t at = ad + (uint32_t)in->imm; + if ((at & 3u) || at / 4u >= (uint64_t)LANE_N * LANE_WORDS) { + s.err = "store address is outside the lane buffer"; break; } - s.out[off / 4u] = b; + mem[at / 4u] = b; s.stores++; break; } @@ -892,6 +956,59 @@ static int check_loop_sass(const char *id, const uint8_t *code, size_t ninsn, in return failed; } +static uint32_t g_sim_mem[LANE_N * LANE_WORDS]; + +static int lat_sim(uint32_t lane0[LANE_WORDS]) { + for (int i = 0; i < LANE_N * LANE_WORDS; i++) g_sim_mem[i] = POISON; + for (uint32_t tid = 0; tid < LANE_N; tid++) + if (lat_sim_lane(tid, g_sim_mem) != 0) return -1; + if (lane0) memcpy(lane0, g_sim_mem, sizeof(uint32_t) * LANE_WORDS); + return 0; +} + +/* four=1: words 2..5 are the same value (HMMA d0..d3). Otherwise words 4 and 5 stay 0. */ +static int sim_lanes_match(uint32_t result, uint32_t aux, int four) { + for (int lane = 0; lane < LANE_N; lane++) { + const uint32_t *w = g_sim_mem + (size_t)lane * LANE_WORDS; + if (w[6] != (uint32_t)lane || w[2] != result || w[3] != aux) return 0; + if (four) { + if (w[4] != result || w[5] != result) return 0; + } else if (w[4] != 0 || w[5] != 0) { + return 0; + } + } + return 1; +} + +/* Seven stores at +0,+4,...,+24, addressed by IMAD.WIDE Rd, tid, 32, ptr. */ +static int check_lane_stores(const char *id, const uint8_t *code, size_t ninsn) { + int stg = 0, wide = 0, failed = 0; + int saw_off[7] = {0, 0, 0, 0, 0, 0, 0}; + for (size_t i = 0; i < ninsn; i++) { + const uint8_t *ins = code + i * 16; + if (opc_of(ins) == 0x825u && rd32(ins + 4) == LANE_STRIDE) wide++; + if (opc_of(ins) == 0x986u) { + stg++; + uint32_t off = (rd32(ins + 4) >> 8) & 0xffffffu; + if (off % 4u == 0 && off / 4u < 7u) saw_off[off / 4u] = 1; + } + } + if (wide != 1) { + failed++; + printf("FAIL: %s tid-scaled address count %d\n", id, wide); + } + if (stg != 7) { + failed++; + printf("FAIL: %s store count %d, want 7\n", id, stg); + } + for (int k = 0; k < 7; k++) + if (!saw_off[k]) { + failed++; + printf("FAIL: %s missing store +%d\n", id, k * 4); + } + return failed; +} + static int loop_iadd_index(void) { int found = -1; for (size_t i = 0; i + 2 < g_prog.count; i++) { @@ -911,7 +1028,7 @@ static int sim_self_test(void) { uint8_t code[CODE_CAP]; size_t len = 0; uint32_t gpr = 0; - uint32_t out[4]; + uint32_t out[LANE_WORDS]; #define SCHECK(cond, ...) \ do { \ if (!(cond)) { \ @@ -926,40 +1043,39 @@ static int sim_self_test(void) { SCHECK(check_loop_sass("ovh32", code, g_prog.count, 1) == 0, "overhead sass"); SCHECK(lat_sim(out) == 0, "overhead sim ran"); SCHECK(out[2] == N_FIXED && out[3] == 1u, "overhead loop count is %u (aux 0x%x), want %u", out[2], out[3], N_FIXED); + SCHECK(sim_lanes_match(N_FIXED, 1u, 0), "overhead count on every lane"); { int at = loop_iadd_index(); SCHECK(at >= 0, "overhead loop increment is in the IR"); if (at >= 0) { uint32_t saved = g_prog.insns[at].control; g_prog.insns[at].control = (saved & ~(15u << 9)) | (1u << 9); - uint32_t bad[4] = {0, 0, 0, 0}; + uint32_t bad[LANE_WORDS] = {0}; SCHECK(lat_sim(bad) == 0, "short-stall overhead sim ran"); SCHECK(bad[2] != N_FIXED, "stall 1 on the increment must not store the trip count (got %u)", bad[2]); g_prog.insns[at].control = saved; } } SCHECK(build(NULL, 0, 1, 1, code, sizeof code, &len, &gpr) == 0, "one-trip overhead encode"); - SCHECK(lat_sim(out) == 0 && out[2] == 1u, "one-trip overhead stored %u", out[2]); + SCHECK(lat_sim(out) == 0 && out[2] == 1u && sim_lanes_match(1u, 1u, 0), "one-trip overhead stored %u", out[2]); for (int ci = 0; ci < NCASES; ci++) { - if (CASES[ci].href) { - SCHECK(build(&CASES[ci], 1, 0, 0, code, sizeof code, &len, &gpr) == 0, "%s encode", CASES[ci].id); - SCHECK(check_loop_sass(CASES[ci].id, code, g_prog.count, 0) == 0, "%s has no loop", CASES[ci].id); - continue; - } + int hmma = !strcmp(CASES[ci].id, "hmma_reader"); uint32_t fresh = 0, fr_aux = 0, stale = 0, st_aux = 0; SCHECK(eval_case(&CASES[ci], 0, &fresh, &fr_aux) == 0, "%s fresh", CASES[ci].id); SCHECK(eval_case(&CASES[ci], 1, &stale, &st_aux) == 0, "%s stale", CASES[ci].id); SCHECK(build(&CASES[ci], CASES[ci].sweep, 0, 0, code, sizeof code, &len, &gpr) == 0, "%s stall %u encode", CASES[ci].id, CASES[ci].sweep); - SCHECK(check_loop_sass(CASES[ci].id, code, g_prog.count, 1) == 0, "%s loop sass", CASES[ci].id); + SCHECK(check_loop_sass(CASES[ci].id, code, g_prog.count, hmma ? 0 : 1) == 0, "%s loop sass", CASES[ci].id); SCHECK(lat_sim(out) == 0, "%s stall %u sim", CASES[ci].id, CASES[ci].sweep); SCHECK(out[2] == fresh && out[3] == fr_aux, "%s stall %u sim result 0x%x aux 0x%x, fresh 0x%x aux 0x%x", CASES[ci].id, CASES[ci].sweep, out[2], out[3], fresh, fr_aux); + SCHECK(sim_lanes_match(fresh, fr_aux, hmma), "%s stall %u every lane", CASES[ci].id, CASES[ci].sweep); SCHECK(build(&CASES[ci], 1, 0, 0, code, sizeof code, &len, &gpr) == 0, "%s stall 1 encode", CASES[ci].id); SCHECK(lat_sim(out) == 0, "%s stall 1 sim", CASES[ci].id); SCHECK(out[2] == stale && out[3] == st_aux, "%s stall 1 sim result 0x%x aux 0x%x, stale 0x%x aux 0x%x", CASES[ci].id, out[2], out[3], stale, st_aux); + SCHECK(sim_lanes_match(stale, st_aux, hmma), "%s stall 1 every lane", CASES[ci].id); } #undef SCHECK return failed; @@ -995,7 +1111,11 @@ static int self_test(void) { } { uint32_t er = 0, ea = 0; - CHECK(expect_abs(&CASES[6], &er, &ea) != 0, "hmma has no closed host formula"); + float fifteen = 0.f; + for (int k = 0; k < 15; k++) fifteen += 1.f; + CHECK(hmma_16816_ones() == fbits(16.f), "hmma ones dot is 16.0f (got 0x%x)", hmma_16816_ones()); + CHECK(fbits(fifteen) != hmma_16816_ones(), "a 15-term dot must not match k=16"); + CHECK(expect_abs(&CASES[6], &er, &ea) == 0 && er == hmma_16816_ones() && ea == er, "hmma oracle"); } for (int ci = 0; ci < NCASES; ci++) { @@ -1013,7 +1133,8 @@ static int self_test(void) { if (opc == S2R_OPC && idx == BW_SR_TID_X) s2r_tid++; } CHECK(clocks == 2, "%s clock sites %d", CASES[ci].id, clocks); - CHECK(s2r_tid == 1, "%s lane filter %d", CASES[ci].id, s2r_tid); + CHECK(s2r_tid == 1, "%s tid read %d", CASES[ci].id, s2r_tid); + failed += check_lane_stores(CASES[ci].id, code, ninsn); char *text = NULL; size_t tlen = 0; FILE *mf = open_memstream(&text, &tlen); @@ -1022,6 +1143,18 @@ static int self_test(void) { CHECK(unk == 2, "%s disasm unknowns %d (the two CS2R sites)", CASES[ci].id, unk); failed += check_raw_window(CASES[ci].id, code, ninsn, stall); if (!strcmp(CASES[ci].id, "hmma_reader")) { + int quads = 0; + for (size_t i = 0; i + 4 < ninsn; i++) { + if (opc_of(code + i * 16) != 0x23cu || stall_of(code + i * 16) != 1u) continue; + unsigned d = reg_dst(code + i * 16); + int ok = 1; + for (int k = 0; k < 4; k++) { + const uint8_t *f = code + (i + 1 + (size_t)k) * 16; + if (opc_of(f) != 0x221u || reg_a(f) != d + (unsigned)k) ok = 0; + } + if (ok) quads++; + } + CHECK(quads == 1, "hmma stall 1 is followed by FADD of d0,d1,d2,d3"); len = 0; rc = build(&CASES[ci], 29u, 0, 0, code, sizeof code, &len, &gpr); CHECK(rc == 0, "hmma stall 29 encode"); @@ -1304,12 +1437,14 @@ static void read_driver(char *dst, size_t n) { while (L && (dst[L - 1] == '\n' || dst[L - 1] == '\r')) dst[--L] = 0; } -static int launch_kern(const Kern *k, OmegaGpuMem *code, OmegaGpuMem *buf, uint32_t outw[4], uint64_t *cycles) { +/* mem holds LANE_N * LANE_WORDS words. Cycles come from lane 0's two clocks. + * Returns -3 when a lane's id slot is not that lane (the store address is wrong). */ +static int launch_kern(const Kern *k, OmegaGpuMem *code, OmegaGpuMem *buf, uint32_t *mem, uint64_t *cycles) { if (k->nbytes > code->size) return -1; memset(code->cpu, 0, code->size); memcpy(code->cpu, k->code, k->nbytes); uint32_t *w = buf->cpu; - for (int i = 0; i < 4; i++) w[i] = POISON; + for (int i = 0; i < LANE_N * LANE_WORDS; i++) w[i] = POISON; if (omega_gpu_session_sync_to_gpu(code) || omega_gpu_session_sync_to_gpu(buf)) return -1; uint32_t args[2] = {(uint32_t)buf->va, (uint32_t)(buf->va >> 32)}; OmegaGpuLaunch L = {.code_va = code->va, @@ -1326,13 +1461,28 @@ static int launch_kern(const Kern *k, OmegaGpuMem *code, OmegaGpuMem *buf, uint3 .target_sm = 121}; if (omega_gpu_session_launch(&L, NULL, NULL) != 0) return -1; if (omega_gpu_session_sync_to_cpu(buf) != 0) return -1; - memcpy(outw, buf->cpu, 16); - uint32_t t0 = outw[0], t1 = outw[1]; - if (t0 == POISON && t1 == POISON && outw[2] == POISON && outw[3] == POISON) return -2; + memcpy(mem, buf->cpu, sizeof(uint32_t) * LANE_N * LANE_WORDS); + for (int lane = 0; lane < LANE_N; lane++) + if (mem[(size_t)lane * LANE_WORDS + 6] != (uint32_t)lane) return -3; + uint32_t t0 = mem[0], t1 = mem[1]; + if (t0 == POISON && t1 == POISON) return -2; *cycles = t1 >= t0 ? (uint64_t)t1 - t0 : (1ull << 32) + t1 - t0; return 0; } +static int mem_match(const uint32_t *mem, uint32_t result, uint32_t aux, int four) { + for (int lane = 0; lane < LANE_N; lane++) { + const uint32_t *w = mem + (size_t)lane * LANE_WORDS; + if (w[2] != result || w[3] != aux) return 0; + if (four) { + if (w[4] != result || w[5] != result) return 0; + } else if (w[4] != 0 || w[5] != 0) { + return 0; + } + } + return 1; +} + static void json_escape(const char *s, char *dst, size_t n) { size_t j = 0; if (!s) s = ""; @@ -1363,18 +1513,12 @@ static int run_launch(const char *kdir, const char *out_path) { int pass[NCASES][RUNS][SWEEP_MAX]; uint64_t cyc[NCASES][RUNS][SWEEP_MAX]; - uint32_t href[NCASES][RUNS]; - int href_ok[NCASES][RUNS]; uint64_t ovh_cyc[8][RUNS]; uint32_t run0_res[NCASES][SWEEP_MAX]; - uint32_t run0_ref[NCASES]; memset(pass, 0, sizeof pass); memset(cyc, 0, sizeof cyc); - memset(href, 0, sizeof href); - memset(href_ok, 0, sizeof href_ok); memset(ovh_cyc, 0, sizeof ovh_cyc); memset(run0_res, 0, sizeof run0_res); - memset(run0_ref, 0, sizeof run0_ref); char notes[NCASES][384]; memset(notes, 0, sizeof notes); unsigned ovh_list[8]; @@ -1418,18 +1562,18 @@ static int run_launch(const char *kdir, const char *out_path) { break; } for (int r = 0; r < RUNS; r++) { - uint32_t w[4]; + uint32_t mem[LANE_N * LANE_WORDS]; uint64_t cy = 0; - int rc = launch_kern(k, &code, &buf, w, &cy); + int rc = launch_kern(k, &code, &buf, mem, &cy); printf("overhead %s run %d/%d rc %d\n", id, r + 1, RUNS, rc); fflush(stdout); if (rc != 0) { - err = omega_gpu_session_last_error(); + err = rc == -3 ? "lane id was not stored" : omega_gpu_session_last_error(); device_error = 1; break; } ovh_cyc[oi][r] = cy; - if (w[2] != ovh_list[oi] || w[3] != 1) err = "overhead loop count did not match"; + if (!mem_match(mem, ovh_list[oi], 1u, 0)) err = "overhead loop count did not match"; } } @@ -1438,51 +1582,22 @@ static int run_launch(const char *kdir, const char *out_path) { for (int r = 0; r < RUNS && !device_error; r++) { printf("case %s run %d/%d\n", c->id, r + 1, RUNS); fflush(stdout); - uint32_t refv = 0; - int have_ref = 0; - if (c->href) { - Kern *kr = find_kern(kerns, nk, c->id, c->sweep); - uint32_t w[4]; - uint64_t cy = 0; - if (!kr || launch_kern(kr, &code, &buf, w, &cy) != 0) { - err = omega_gpu_session_last_error(); - device_error = 1; - break; - } - refv = w[2]; - have_ref = (w[3] != POISON && refv != 0 && refv != POISON); - href[ci][r] = refv; - href_ok[ci][r] = have_ref; - cyc[ci][r][c->sweep - 1] = cy; - pass[ci][r][c->sweep - 1] = have_ref; - if (r == 0) { - run0_ref[ci] = refv; - run0_res[ci][c->sweep - 1] = refv; - } - if (!have_ref) - snprintf(notes[ci], sizeof notes[ci], - "HMMA reference at stall %u is 0x%08x (aux 0x%08x); d0 did not move", c->sweep, w[2], w[3]); - } for (unsigned s = 1; s <= c->sweep && !device_error; s++) { - if (c->href && s == c->sweep) continue; Kern *k = find_kern(kerns, nk, c->id, s); - uint32_t w[4]; + uint32_t mem[LANE_N * LANE_WORDS]; uint64_t cy = 0; - if (!k || launch_kern(k, &code, &buf, w, &cy) != 0) { - err = k ? omega_gpu_session_last_error() : "missing kernel"; + int rc = !k ? -1 : launch_kern(k, &code, &buf, mem, &cy); + if (rc != 0) { + err = rc == -3 ? "lane id was not stored" : (k ? omega_gpu_session_last_error() : "missing kernel"); device_error = 1; break; } cyc[ci][r][s - 1] = cy; - int ok = 0; - if (c->href) { - ok = href_ok[ci][r] && w[2] == refv; - } else { - uint32_t expv = 0, auxv = 0; - if (expect_abs(c, &expv, &auxv) == 0) ok = (w[2] == expv && w[3] == auxv && w[0] != POISON); - } + uint32_t expv = 0, auxv = 0; + int four = !strcmp(c->id, "hmma_reader"); + int ok = expect_abs(c, &expv, &auxv) == 0 && mem_match(mem, expv, auxv, four); pass[ci][r][s - 1] = ok; - if (r == 0) run0_res[ci][s - 1] = w[2]; + if (r == 0) run0_res[ci][s - 1] = mem[2]; } } } @@ -1593,10 +1708,7 @@ write_json:; fprintf(jf, " \"overhead_cycle_median\": null,\n"); else fprintf(jf, " \"overhead_cycle_median\": %llu,\n", (unsigned long long)omed); - if (c->href && !device_error) - fprintf(jf, " \"reference_result\": %u,\n", run0_ref[ci]); - else - fprintf(jf, " \"reference_result\": null,\n"); + fprintf(jf, " \"reference_result\": null,\n"); fprintf(jf, " \"run0_result\": ["); for (unsigned s = 1; s <= c->sweep; s++) fprintf(jf, "%s%u", s == 1 ? "" : ", ", run0_res[ci][s - 1]); diff --git a/tools/latmeas/REPRODUCE.md b/tools/latmeas/REPRODUCE.md index 6d3ec7d5..ebe05ba2 100644 --- a/tools/latmeas/REPRODUCE.md +++ b/tools/latmeas/REPRODUCE.md @@ -38,6 +38,10 @@ The two `CS2R` reads bracket the loop. Between them are the producer, its consum The loop increment stalls 15 before `ISETP`. Stall 1 there is shorter than a fixed-latency add: the compare samples the old counter, the body runs one extra time, and the overhead kernel stores that counter. `sm121dis` of that kernel shows the `IADD`, the `ISETP`, and `@!P0 BRA` back to the `IADD`. The host simulator issues the same IR and keeps a GPR write hidden for 5 cycles and a predicate write hidden for 14 (the NAK floors). It requires the overhead counter to equal the trip count, the stall at the top of the sweep to equal the fresh oracle, and stall 1 to equal the stale oracle. Forcing the increment's stall back to 1 must fail the counter check. +The 2026-11-01 GB10 run of `03cdcc5` merged with PR #251 is `evidence/LATMEAS-GB10-20261101/`. `UNITS.md` there compares the stall field with NAK's `deps.delay`. That run stored lane 0 only, and its HMMA oracle was the stall-36 result. + +The kernels after that receipt store eight words from every lane: the two clocks, four result words, and the lane id. HMMA's four result words are `FADD` of `d0` through `d3` against zero. The host value is sixteen products of bf16 1.0, not the stall-36 launch. A stall matches only when all 32 lanes match. Those kernels have not been launched. + ## NAK pin Mesa commit `01bab9302a6bf0a082c7ba8c49d74427cf10b0cf`. diff --git a/tools/latmeas/fill_template.py b/tools/latmeas/fill_template.py index 5a009d11..80cf61a8 100755 --- a/tools/latmeas/fill_template.py +++ b/tools/latmeas/fill_template.py @@ -76,7 +76,7 @@ def build_md(cat, results, fake): lines.append("Driver: %s. No NVIDIA user-space libraries." % driver) else: lines.append("Driver: NVIDIA open kernel module , no NVIDIA user-space libraries") - lines.append("Method: dependent-chain timing kernel. One warp (32 threads), lane 0 stores.") + lines.append("Method: dependent-chain timing kernel. One warp (32 threads); every lane stores, and a stall matches only when every lane matches the oracle.") lines.append("Each fixed-latency producer is issued at a programmed stall. The GB10 column is the smallest stall S such that S and every larger S in the sweep matched the oracle on every run. Cycles are the CS2R clock delta (NAK_SV_CLOCK_LO, 0x50) around that chain, median and max-min.") lines.append("Source: tests/gpu_latmeas_test.c in aien-dev/CUDApp") lines.append("How to rerun: sh tools/latmeas/run_spark.sh OUTPUT_DIR") From 5af271a39617e30b8d6d2a2e980fa368dc51fd2e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 1 Nov 2026 18:45:43 +0000 Subject: [PATCH 06/11] Record the all-lane GB10 latency run. c94c2c9 is the merge of 4f3a4d0 and PR #251. stall_required matches the earlier receipt. raw.json still has one lane-0 word and a null reference, so the file does not itself show the 32-lane or d0..d3 check. Co-authored-by: aien-spark --- .../LATMEAS-GB10-20261101-allanes/RECEIPT.md | 33 +++ .../LATMEAS-GB10-20261101-allanes/diff.md | 21 ++ .../LATMEAS-GB10-20261101-allanes/head.txt | 1 + .../LATMEAS-GB10-20261101-allanes/raw.json | 114 ++++++++ .../LATMEAS-GB10-20261101-allanes/run.log | 267 ++++++++++++++++++ .../sm121_vs_NAK.md | 42 +++ 6 files changed, 478 insertions(+) create mode 100644 evidence/LATMEAS-GB10-20261101-allanes/RECEIPT.md create mode 100644 evidence/LATMEAS-GB10-20261101-allanes/diff.md create mode 100644 evidence/LATMEAS-GB10-20261101-allanes/head.txt create mode 100644 evidence/LATMEAS-GB10-20261101-allanes/raw.json create mode 100644 evidence/LATMEAS-GB10-20261101-allanes/run.log create mode 100644 evidence/LATMEAS-GB10-20261101-allanes/sm121_vs_NAK.md diff --git a/evidence/LATMEAS-GB10-20261101-allanes/RECEIPT.md b/evidence/LATMEAS-GB10-20261101-allanes/RECEIPT.md new file mode 100644 index 00000000..130f1a09 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101-allanes/RECEIPT.md @@ -0,0 +1,33 @@ +# GB10 all-lane latency run, 2026-11-01 + +Second chip run of the latency kit. The operator reported the window as 2026-11-01 18:38–18:40Z. This directory is that output. The script did not write a `RECEIPT.txt` (that writer was added afterwards). The upload's sha256 is `ae4cdbcbfe7d15d2766f8d661cf35cbb88b7358fde20afcca5c441bd7071464f`. `sha256sum raw.json` is `565ece7ec537961bb70c66cdac7afea1cd6fdb0463e4ee70325a3bc528fce403`. + +`sm121_vs_NAK.md` is the table `fill_template.py` wrote from this `raw.json`. A fresh fill of the same file matches it. + +## What ran + +- Command: `sh tools/latmeas/run_spark.sh latmeas-out` (see `run.log`). +- Launch rc 0. `raw.json` has `"error": ""`, `"launches": 1408`, `"runs": 11`, stall spread 0 on every case. +- Tested tree: `c94c2c93442adf7fa24c22dee7e8a91bb1c93226` (`head.txt`). That commit is a merge of `4f3a4d0` (every lane stores four result words; HMMA reads `d0`..`d3`) and PR #251 `6a93cb567b33b67bc1185fb45a3c50fc4d88d30e`, with no other edits. +- Doctor, from `run.log`: Linux 7.0.0-1019-nvidia aarch64, open module 580.173.02, GPU `000f:01:00.0` NVIDIA GB10 (sm_121), compiler 13.3.0, READY. The binary printed a 71 s upper bound. The script also printed a stale 90 s line; the binary is the bound. +- Two home-directory paths in the log are replaced with `latmeas-out`. + +## Smallest passing stall + +Same as `evidence/LATMEAS-GB10-20261101/` (`03cdcc5`). From this `raw.json`: + +| case | stall_required | +| --- | --- | +| alu_alu | 4 | +| fma_fma | 4 | +| mixed | 5 | +| pred | 13 | +| fmnmx_alu | 4 | +| imad_wide_lo | 3 | +| hmma_reader | 20 | + +## What this file proves + +`run0_result` is lane 0's first result word at each stall. `reference_result` is null for every case. There is no host oracle, no per-lane count, and no `d1`..`d3` word in the JSON. + +The source at `4f3a4d0` sets a stall as passing only when all 32 lanes match all four result words (HMMA: the four `FADD` results; other cases: the result, the auxiliary word, and two zeros). A passing `stall_required` with spread 0 is what that check produced, and it landed on the same stalls as the lane-0 run. A reader of `raw.json` alone cannot see a lane bitmask or the four destination words. That recording is a later change and is not in this directory. diff --git a/evidence/LATMEAS-GB10-20261101-allanes/diff.md b/evidence/LATMEAS-GB10-20261101-allanes/diff.md new file mode 100644 index 00000000..5ca3637f --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101-allanes/diff.md @@ -0,0 +1,21 @@ +# CUDA++ fixed latencies vs NAK + +Mesa commit `01bab9302a6bf0a082c7ba8c49d74427cf10b0cf`. +Clock: nak_private.h:135 NAK_SV_CLOCK_LO = 0x50; from_nir.rs:3522 SM>=50 uses CS2R; sm70_encode.rs:4278 opcode 0x805, index bits 72..80. +Priority 1 pairs match NAK on paper. They are measured because that match is circular. +Rows below are the ones that differ. A blank NAK stall means NAK calls the op decoupled. + +| id | pair | CUDA++ | NAK | kernel | +|---|---|---|---|---| +| pred_attention | Predicate write -> guard, attention emitter | 13 | 13 + 1 = 14 | tests/gpu_latmeas_test.c (pred) | +| hmma_reader | HMMA -> non-HMMA reader (FADD of d0) | 15 + NOP 14 = 29 | 19 + 9 = 28 | tests/gpu_latmeas_test.c (hmma_reader) | +| imad_wide_lo | IMAD.WIDE low half -> IADD3 | 6 (blanket K_FIXED; IMAD.WIDE is not in fixed_pair_class) | 3 + 1 = 4 | tests/gpu_latmeas_test.c (imad_wide_lo) | +| fmnmx_alu | FMNMX (dualalu) -> IADD3 (alu) | 6 (blanket K_FIXED; FMNMX is not in fixed_pair_class) | 4 + 1 = 5 | tests/gpu_latmeas_test.c (fmnmx_alu) | +| qmma_reader | QMMA (IMMA) -> next reader | 15 + NOP 15 = 30 | 19 + 7 = 26 | not generated (patched QMMA placeholder; see REPRODUCE.md) | +| numeric_min | Fixed-producer minimum in the numeric checker | 5 (any fixed producer; mixed pairs need 6) | 5 + 1 = 6 for mixed ALU/FMA | tests/gpu_latmeas_test.c (mixed) | +| bssy | BSSY stall | 1, no scoreboard | decoupled (1 & sb) | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| bsync | BSYNC stall | 5, no scoreboard | decoupled (1 & sb) | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| bra | BRA stall | 5 | decoupled (1 & sb) | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| exit_op | EXIT stall | 5 | decoupled (1 & sb) | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| bar | BAR.SYNC stall | 6 | decoupled AGU (1 & sb) | not generated (NAK class is decoupled, not a fixed GPR RAW) | + diff --git a/evidence/LATMEAS-GB10-20261101-allanes/head.txt b/evidence/LATMEAS-GB10-20261101-allanes/head.txt new file mode 100644 index 00000000..c84c7e16 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101-allanes/head.txt @@ -0,0 +1 @@ +c94c2c93442adf7fa24c22dee7e8a91bb1c93226 diff --git a/evidence/LATMEAS-GB10-20261101-allanes/raw.json b/evidence/LATMEAS-GB10-20261101-allanes/raw.json new file mode 100644 index 00000000..a2f9d920 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101-allanes/raw.json @@ -0,0 +1,114 @@ +{ + "kind": "cudapp-latmeas", + "synthetic": false, + "measured": true, + "mesa_commit": "01bab9302a6bf0a082c7ba8c49d74427cf10b0cf", + "driver": "NVRM version: NVIDIA UNIX Open Kernel Module for aarch64 580.173.02 Release Build (dvs-builder@U22-A24-5-4) Tue Jun 23 08:34:19 UTC 2026", + "machine": "aarch64", + "runs": 11, + "threads": 32, + "warps": 1, + "clock": "CS2R idx 0x50 (NAK_SV_CLOCK_LO)", + "launches": 1408, + "error": "", + "cases": { + "alu_alu": { + "stall_required": 4, + "stall_spread": 0, + "cycle_median": 2541, + "cycle_spread": 0, + "cycle_median_stall1": 2477, + "cycle_spread_stall1": 0, + "overhead_cycle_median": 1261, + "reference_result": null, + "run0_result": [4026531840, 4026531840, 4026531840, 528, 528, 528, 528, 528, 528, 528, 528, 528, 528, 528, 528], + "iters": 32, + "ops": 32, + "note": "" + }, + "fma_fma": { + "stall_required": 4, + "stall_spread": 0, + "cycle_median": 2541, + "cycle_spread": 0, + "cycle_median_stall1": 2477, + "cycle_spread_stall1": 0, + "overhead_cycle_median": 1261, + "reference_result": null, + "run0_result": [2139095040, 2139095040, 2139095040, 1141112832, 1141112832, 1141112832, 1141112832, 1141112832, 1141112832, 1141112832, 1141112832, 1141112832, 1141112832, 1141112832, 1141112832], + "iters": 32, + "ops": 32, + "note": "" + }, + "mixed": { + "stall_required": 5, + "stall_spread": 0, + "cycle_median": 4362, + "cycle_spread": 0, + "cycle_median_stall1": 4170, + "cycle_spread_stall1": 9673, + "overhead_cycle_median": 1261, + "reference_result": null, + "run0_result": [4026531840, 4026531840, 4026531840, 1024, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072], + "iters": 32, + "ops": 64, + "note": "" + }, + "pred": { + "stall_required": 13, + "stall_spread": 0, + "cycle_median": 2714, + "cycle_spread": 0, + "cycle_median_stall1": 70, + "cycle_spread_stall1": 0, + "overhead_cycle_median": 1261, + "reference_result": null, + "run0_result": [3735928559, 3735928559, 3735928559, 3735928559, 3735928559, 3735928559, 3735928559, 3735928559, 3735928559, 3735928559, 3735928559, 3735928559, 32, 32, 32], + "iters": 32, + "ops": 32, + "note": "" + }, + "fmnmx_alu": { + "stall_required": 4, + "stall_spread": 0, + "cycle_median": 2541, + "cycle_spread": 0, + "cycle_median_stall1": 2507, + "cycle_spread_stall1": 0, + "overhead_cycle_median": 1261, + "reference_result": null, + "run0_result": [4026531840, 4026531840, 4026531840, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072], + "iters": 32, + "ops": 32, + "note": "" + }, + "imad_wide_lo": { + "stall_required": 3, + "stall_spread": 0, + "cycle_median": 2768, + "cycle_spread": 0, + "cycle_median_stall1": 2736, + "cycle_spread_stall1": 0, + "overhead_cycle_median": 1261, + "reference_result": null, + "run0_result": [32, 32, 528, 528, 528, 528, 528, 528, 528, 528, 528, 528, 528, 528, 528], + "iters": 32, + "ops": 32, + "note": "" + }, + "hmma_reader": { + "stall_required": 20, + "stall_spread": 0, + "cycle_median": 63, + "cycle_spread": 0, + "cycle_median_stall1": 61, + "cycle_spread_stall1": 0, + "overhead_cycle_median": 49, + "reference_result": null, + "run0_result": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648], + "iters": 1, + "ops": 1, + "note": "" + } + } +} diff --git a/evidence/LATMEAS-GB10-20261101-allanes/run.log b/evidence/LATMEAS-GB10-20261101-allanes/run.log new file mode 100644 index 00000000..3fca1e39 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101-allanes/run.log @@ -0,0 +1,267 @@ +latmeas: building (outside the chip lock) +mkdir -p build +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_ampere_codegen.c -o build/omega_ampere_codegen.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_ampere_qmd.c -o build/omega_ampere_qmd.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_codegen.c -o build/omega_blackwell_codegen.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_codegen_attn.c -o build/omega_blackwell_codegen_attn.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_codegen_attn256.c -o build/omega_blackwell_codegen_attn256.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_codegen_bf16.c -o build/omega_blackwell_codegen_bf16.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_codegen_dec.c -o build/omega_blackwell_codegen_dec.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_codegen_gdn.c -o build/omega_blackwell_codegen_gdn.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_codegen_gemm.c -o build/omega_blackwell_codegen_gemm.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_codegen_moe.c -o build/omega_blackwell_codegen_moe.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_codegen_moe2.c -o build/omega_blackwell_codegen_moe2.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_codegen_pattn.c -o build/omega_blackwell_codegen_pattn.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_codegen_pattn256.c -o build/omega_blackwell_codegen_pattn256.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_codegen_pf.c -o build/omega_blackwell_codegen_pf.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_codegen_pfhost.c -o build/omega_blackwell_codegen_pfhost.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_codegen_qg.c -o build/omega_blackwell_codegen_qg.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_codegen_qmma.c -o build/omega_blackwell_codegen_qmma.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_codegen_router.c -o build/omega_blackwell_codegen_router.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_codegen_tc.c -o build/omega_blackwell_codegen_tc.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_codegen_tc2.c -o build/omega_blackwell_codegen_tc2.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_codegen_tern.c -o build/omega_blackwell_codegen_tern.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_encoder.c -o build/omega_blackwell_encoder.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_engine.c -o build/omega_blackwell_engine.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_matmul.c -o build/omega_blackwell_matmul.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_qmd.c -o build/omega_blackwell_qmd.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_realize.c -o build/omega_blackwell_realize.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_submit.c -o build/omega_blackwell_submit.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_attention_api.c -o build/omega_gpu_attention_api.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_bf16_gemv_api.c -o build/omega_gpu_bf16_gemv_api.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_chip.c -o build/omega_gpu_chip.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_cmdlist.c -o build/omega_gpu_cmdlist.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_decode_attn256_api.c -o build/omega_gpu_decode_attn256_api.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_decode_attn_api.c -o build/omega_gpu_decode_attn_api.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_decode_graph.c -o build/omega_gpu_decode_graph.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_elementwise_api.c -o build/omega_gpu_elementwise_api.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_engine.c -o build/omega_gpu_engine.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_fmt.c -o build/omega_gpu_fmt.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_fp8_api.c -o build/omega_gpu_fp8_api.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_fp8_pack.c -o build/omega_gpu_fp8_pack.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_fp8_slots.c -o build/omega_gpu_fp8_slots.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_fp8gemm_api.c -o build/omega_gpu_fp8gemm_api.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_gdn_api.c -o build/omega_gpu_gdn_api.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_gemm.c -o build/omega_gpu_gemm.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_gemm_ex.c -o build/omega_gpu_gemm_ex.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_hostmath.c -o build/omega_gpu_hostmath.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_matmul_api.c -o build/omega_gpu_matmul_api.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_mem_cached.c -o build/omega_gpu_mem_cached.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_moe_api.c -o build/omega_gpu_moe_api.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_pfgemm.c -o build/omega_gpu_pfgemm.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_pfhost.c -o build/omega_gpu_pfhost.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_platform.c -o build/omega_gpu_platform.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_platform_linux.c -o build/omega_gpu_platform_linux.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_prefill.c -o build/omega_gpu_prefill.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_prefill_attn256_api.c -o build/omega_gpu_prefill_attn256_api.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_prefill_attn_api.c -o build/omega_gpu_prefill_attn_api.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_q36_graph.c -o build/omega_gpu_q36_graph.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_q38_graph.c -o build/omega_gpu_q38_graph.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_router_api.c -o build/omega_gpu_router_api.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_scan.c -o build/omega_gpu_scan.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_scan_ref.c -o build/omega_gpu_scan_ref.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_serving.c -o build/omega_gpu_serving.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_session.c -o build/omega_gpu_session.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_tcgemm_api.c -o build/omega_gpu_tcgemm_api.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_tern_api.c -o build/omega_gpu_tern_api.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_wait.c -o build/omega_gpu_wait.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_fp8_slot_table.c -o build/omega_fp8_slot_table.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_cubin.c -o build/omega_cubin.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_native_artifact.c -o build/omega_native_artifact.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_sm121_disasm.c -o build/omega_sm121_disasm.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/sha256.c -o build/sha256.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_canonical.c -o build/omega_canonical.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_core.c -o build/omega_core.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_vector.c -o build/omega_vector.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_ampere_enc_sm86.c -o build/cudapp_om_ampere_enc_sm86.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_amp_runtime.c -o build/cudapp_om_amp_runtime.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_binding.c -o build/cudapp_om_binding.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_bw_enc_sm121.c -o build/cudapp_om_bw_enc_sm121.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_cache_runtime.c -o build/cudapp_om_cache_runtime.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_closure_alloc.c -o build/cudapp_om_closure_alloc.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_closure_bind.c -o build/cudapp_om_closure_bind.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_closure_blob.c -o build/cudapp_om_closure_blob.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_closure_code.c -o build/cudapp_om_closure_code.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_closure_cover.c -o build/cudapp_om_closure_cover.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_closure_dir.c -o build/cudapp_om_closure_dir.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_closure_intent.c -o build/cudapp_om_closure_intent.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_closure_launch.c -o build/cudapp_om_closure_launch.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_closure_pack.c -o build/cudapp_om_closure_pack.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -O3 -c src/cudapp_om_closure_reseal.c -o build/cudapp_om_closure_reseal.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_closure_seal.c -o build/cudapp_om_closure_seal.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_closure_series.c -o build/cudapp_om_closure_series.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_code_digest.c -o build/cudapp_om_code_digest.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_command.c -o build/cudapp_om_command.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_compose.c -o build/cudapp_om_compose.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_context_lookup.c -o build/cudapp_om_context_lookup.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_cubin_elf.c -o build/cudapp_om_cubin_elf.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_digest.c -o build/cudapp_om_digest.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_encode.c -o build/cudapp_om_encode.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_graph.c -o build/cudapp_om_graph.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_lease.c -o build/cudapp_om_lease.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_moe_admission.c -o build/cudapp_om_moe_admission.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_moe_group.c -o build/cudapp_om_moe_group.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_moe_ids32.c -o build/cudapp_om_moe_ids32.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_moe_maps32.c -o build/cudapp_om_moe_maps32.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_moe_plan.c -o build/cudapp_om_moe_plan.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_moe_segs32.c -o build/cudapp_om_moe_segs32.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_moe_windows.c -o build/cudapp_om_moe_windows.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_q38_emit.c -o build/cudapp_om_q38_emit.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_q38_runtime.c -o build/cudapp_om_q38_runtime.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_quant_wire.c -o build/cudapp_om_quant_wire.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_repack.c -o build/cudapp_om_repack.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_slotrev.c -o build/cudapp_om_slotrev.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_templates.c -o build/cudapp_om_templates.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_validate.c -o build/cudapp_om_validate.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_freest_revision.c -o build/cudapp_freest_revision.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_freest_epoch.c -o build/cudapp_freest_epoch.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_freest_f16.c -o build/cudapp_freest_f16.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c ./m16/m16_native.c -o build/m16_native.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c ./nvrm/nvrm.c -o build/nvrm.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -DCUDAPP_NVIDIA_OPEN_ABI='"580.173.02"' -o build/cudapp-doctor tools/doctor/cudapp_doctor.c src/omega_gpu_chip.c +ar rcs build/libomega_gpu.a build/omega_ampere_codegen.o build/omega_ampere_qmd.o build/omega_blackwell_codegen.o build/omega_blackwell_codegen_attn.o build/omega_blackwell_codegen_attn256.o build/omega_blackwell_codegen_bf16.o build/omega_blackwell_codegen_dec.o build/omega_blackwell_codegen_gdn.o build/omega_blackwell_codegen_gemm.o build/omega_blackwell_codegen_moe.o build/omega_blackwell_codegen_moe2.o build/omega_blackwell_codegen_pattn.o build/omega_blackwell_codegen_pattn256.o build/omega_blackwell_codegen_pf.o build/omega_blackwell_codegen_pfhost.o build/omega_blackwell_codegen_qg.o build/omega_blackwell_codegen_qmma.o build/omega_blackwell_codegen_router.o build/omega_blackwell_codegen_tc.o build/omega_blackwell_codegen_tc2.o build/omega_blackwell_codegen_tern.o build/omega_blackwell_encoder.o build/omega_blackwell_engine.o build/omega_blackwell_matmul.o build/omega_blackwell_qmd.o build/omega_blackwell_realize.o build/omega_blackwell_submit.o build/omega_gpu_attention_api.o build/omega_gpu_bf16_gemv_api.o build/omega_gpu_chip.o build/omega_gpu_cmdlist.o build/omega_gpu_decode_attn256_api.o build/omega_gpu_decode_attn_api.o build/omega_gpu_decode_graph.o build/omega_gpu_elementwise_api.o build/omega_gpu_engine.o build/omega_gpu_fmt.o build/omega_gpu_fp8_api.o build/omega_gpu_fp8_pack.o build/omega_gpu_fp8_slots.o build/omega_gpu_fp8gemm_api.o build/omega_gpu_gdn_api.o build/omega_gpu_gemm.o build/omega_gpu_gemm_ex.o build/omega_gpu_hostmath.o build/omega_gpu_matmul_api.o build/omega_gpu_mem_cached.o build/omega_gpu_moe_api.o build/omega_gpu_pfgemm.o build/omega_gpu_pfhost.o build/omega_gpu_platform.o build/omega_gpu_platform_linux.o build/omega_gpu_prefill.o build/omega_gpu_prefill_attn256_api.o build/omega_gpu_prefill_attn_api.o build/omega_gpu_q36_graph.o build/omega_gpu_q38_graph.o build/omega_gpu_router_api.o build/omega_gpu_scan.o build/omega_gpu_scan_ref.o build/omega_gpu_serving.o build/omega_gpu_session.o build/omega_gpu_tcgemm_api.o build/omega_gpu_tern_api.o build/omega_gpu_wait.o build/omega_fp8_slot_table.o build/omega_cubin.o build/omega_native_artifact.o build/omega_sm121_disasm.o build/sha256.o build/omega_canonical.o build/omega_core.o build/omega_vector.o build/cudapp_om_ampere_enc_sm86.o build/cudapp_om_amp_runtime.o build/cudapp_om_binding.o build/cudapp_om_bw_enc_sm121.o build/cudapp_om_cache_runtime.o build/cudapp_om_closure_alloc.o build/cudapp_om_closure_bind.o build/cudapp_om_closure_blob.o build/cudapp_om_closure_code.o build/cudapp_om_closure_cover.o build/cudapp_om_closure_dir.o build/cudapp_om_closure_intent.o build/cudapp_om_closure_launch.o build/cudapp_om_closure_pack.o build/cudapp_om_closure_reseal.o build/cudapp_om_closure_seal.o build/cudapp_om_closure_series.o build/cudapp_om_code_digest.o build/cudapp_om_command.o build/cudapp_om_compose.o build/cudapp_om_context_lookup.o build/cudapp_om_cubin_elf.o build/cudapp_om_digest.o build/cudapp_om_encode.o build/cudapp_om_graph.o build/cudapp_om_lease.o build/cudapp_om_moe_admission.o build/cudapp_om_moe_group.o build/cudapp_om_moe_ids32.o build/cudapp_om_moe_maps32.o build/cudapp_om_moe_plan.o build/cudapp_om_moe_segs32.o build/cudapp_om_moe_windows.o build/cudapp_om_q38_emit.o build/cudapp_om_q38_runtime.o build/cudapp_om_quant_wire.o build/cudapp_om_repack.o build/cudapp_om_slotrev.o build/cudapp_om_templates.o build/cudapp_om_validate.o build/cudapp_freest_revision.o build/cudapp_freest_epoch.o build/cudapp_freest_f16.o build/m16_native.o build/nvrm.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -ffp-contract=off -o build/gpu_latmeas_test tests/gpu_latmeas_test.c build/libomega_gpu.a -lpthread -lm +latmeas: doctor (outside the chip lock; NOT READY aborts before flock) +cudapp doctor: is this machine ready for CUDA++? +Diagnostic only: direct library use has no driver-version or PCI-ID admission guard. + info system: Linux 7.0.0-1019-nvidia aarch64 + ok NVIDIA kernel module: open module 580.173.02 (CUDA++ speaks 580.173.02) + ok device files: /dev/nvidiactl /dev/nvidia0 /dev/nvidia-uvm can be opened by this user + ok GPU 000f:01:00.0: NVIDIA GB10 (sm_121), supported target (historical summary: evidence/m16-blackwell-native-path-requalification-receipt.json) + raw M16 evidence unavailable; not current-commit qualification + ok memory: 121.7 GiB shared by the CPU and the GPU, 62.0 GiB available now + info CUDA toolkit: not needed and not used by CUDA++ (one is installed at /usr/local/cuda/bin/nvcc; that is fine) + info compiler: this doctor was built by 13.3.0; CUDA++ needs only make and a C compiler +READY. Next: make hello-gpu +latmeas plan: 7 cases, 11 runs, sweep 1..15 (HMMA 1..36), 1408 launches +estimated chip window: 71 s upper bound (1408 launches; 50 ms each covers a cold shader invalidate). Usually shorter. +The runner wraps only the launch with: flock /tmp/aien-gb10.lock timeout 600 +One warp, 32 threads. Clock: CS2R idx 0x50. Mesa 01bab9302a6bf0a082c7ba8c49d74427cf10b0cf +emitted kernels in latmeas-out/kernels (1408 launches when run) +# CUDA++ fixed latencies vs NAK + +Mesa commit `01bab9302a6bf0a082c7ba8c49d74427cf10b0cf`. +Clock: nak_private.h:135 NAK_SV_CLOCK_LO = 0x50; from_nir.rs:3522 SM>=50 uses CS2R; sm70_encode.rs:4278 opcode 0x805, index bits 72..80. +Priority 1 pairs match NAK on paper. They are measured because that match is circular. +Rows below are the ones that differ. A blank NAK stall means NAK calls the op decoupled. + +| id | pair | CUDA++ | NAK | kernel | +|---|---|---|---|---| +| pred_attention | Predicate write -> guard, attention emitter | 13 | 13 + 1 = 14 | tests/gpu_latmeas_test.c (pred) | +| hmma_reader | HMMA -> non-HMMA reader (FADD of d0) | 15 + NOP 14 = 29 | 19 + 9 = 28 | tests/gpu_latmeas_test.c (hmma_reader) | +| imad_wide_lo | IMAD.WIDE low half -> IADD3 | 6 (blanket K_FIXED; IMAD.WIDE is not in fixed_pair_class) | 3 + 1 = 4 | tests/gpu_latmeas_test.c (imad_wide_lo) | +| fmnmx_alu | FMNMX (dualalu) -> IADD3 (alu) | 6 (blanket K_FIXED; FMNMX is not in fixed_pair_class) | 4 + 1 = 5 | tests/gpu_latmeas_test.c (fmnmx_alu) | +| qmma_reader | QMMA (IMMA) -> next reader | 15 + NOP 15 = 30 | 19 + 7 = 26 | not generated (patched QMMA placeholder; see REPRODUCE.md) | +| numeric_min | Fixed-producer minimum in the numeric checker | 5 (any fixed producer; mixed pairs need 6) | 5 + 1 = 6 for mixed ALU/FMA | tests/gpu_latmeas_test.c (mixed) | +| bssy | BSSY stall | 1, no scoreboard | decoupled (1 & sb) | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| bsync | BSYNC stall | 5, no scoreboard | decoupled (1 & sb) | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| bra | BRA stall | 5 | decoupled (1 & sb) | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| exit_op | EXIT stall | 5 | decoupled (1 & sb) | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| bar | BAR.SYNC stall | 6 | decoupled AGU (1 & sb) | not generated (NAK class is decoupled, not a fixed GPR RAW) | + +latmeas: taking flock /tmp/aien-gb10.lock for the launch only; timeout 600 +latmeas: estimated chip window 90 s upper bound (printed again by the binary) +latmeas plan: 7 cases, 11 runs, sweep 1..15 (HMMA 1..36), 1408 launches +estimated chip window: 71 s upper bound (1408 launches; 50 ms each covers a cold shader invalidate). Usually shorter. +The runner wraps only the launch with: flock /tmp/aien-gb10.lock timeout 600 +One warp, 32 threads. Clock: CS2R idx 0x50. Mesa 01bab9302a6bf0a082c7ba8c49d74427cf10b0cf +overhead ovh32 run 1/11 rc 0 +overhead ovh32 run 2/11 rc 0 +overhead ovh32 run 3/11 rc 0 +overhead ovh32 run 4/11 rc 0 +overhead ovh32 run 5/11 rc 0 +overhead ovh32 run 6/11 rc 0 +overhead ovh32 run 7/11 rc 0 +overhead ovh32 run 8/11 rc 0 +overhead ovh32 run 9/11 rc 0 +overhead ovh32 run 10/11 rc 0 +overhead ovh32 run 11/11 rc 0 +overhead ovh1 run 1/11 rc 0 +overhead ovh1 run 2/11 rc 0 +overhead ovh1 run 3/11 rc 0 +overhead ovh1 run 4/11 rc 0 +overhead ovh1 run 5/11 rc 0 +overhead ovh1 run 6/11 rc 0 +overhead ovh1 run 7/11 rc 0 +overhead ovh1 run 8/11 rc 0 +overhead ovh1 run 9/11 rc 0 +overhead ovh1 run 10/11 rc 0 +overhead ovh1 run 11/11 rc 0 +case alu_alu run 1/11 +case alu_alu run 2/11 +case alu_alu run 3/11 +case alu_alu run 4/11 +case alu_alu run 5/11 +case alu_alu run 6/11 +case alu_alu run 7/11 +case alu_alu run 8/11 +case alu_alu run 9/11 +case alu_alu run 10/11 +case alu_alu run 11/11 +case fma_fma run 1/11 +case fma_fma run 2/11 +case fma_fma run 3/11 +case fma_fma run 4/11 +case fma_fma run 5/11 +case fma_fma run 6/11 +case fma_fma run 7/11 +case fma_fma run 8/11 +case fma_fma run 9/11 +case fma_fma run 10/11 +case fma_fma run 11/11 +case mixed run 1/11 +case mixed run 2/11 +case mixed run 3/11 +case mixed run 4/11 +case mixed run 5/11 +case mixed run 6/11 +case mixed run 7/11 +case mixed run 8/11 +case mixed run 9/11 +case mixed run 10/11 +case mixed run 11/11 +case pred run 1/11 +case pred run 2/11 +case pred run 3/11 +case pred run 4/11 +case pred run 5/11 +case pred run 6/11 +case pred run 7/11 +case pred run 8/11 +case pred run 9/11 +case pred run 10/11 +case pred run 11/11 +case fmnmx_alu run 1/11 +case fmnmx_alu run 2/11 +case fmnmx_alu run 3/11 +case fmnmx_alu run 4/11 +case fmnmx_alu run 5/11 +case fmnmx_alu run 6/11 +case fmnmx_alu run 7/11 +case fmnmx_alu run 8/11 +case fmnmx_alu run 9/11 +case fmnmx_alu run 10/11 +case fmnmx_alu run 11/11 +case imad_wide_lo run 1/11 +case imad_wide_lo run 2/11 +case imad_wide_lo run 3/11 +case imad_wide_lo run 4/11 +case imad_wide_lo run 5/11 +case imad_wide_lo run 6/11 +case imad_wide_lo run 7/11 +case imad_wide_lo run 8/11 +case imad_wide_lo run 9/11 +case imad_wide_lo run 10/11 +case imad_wide_lo run 11/11 +case hmma_reader run 1/11 +case hmma_reader run 2/11 +case hmma_reader run 3/11 +case hmma_reader run 4/11 +case hmma_reader run 5/11 +case hmma_reader run 6/11 +case hmma_reader run 7/11 +case hmma_reader run 8/11 +case hmma_reader run 9/11 +case hmma_reader run 10/11 +case hmma_reader run 11/11 +wrote latmeas-out/raw.json diff --git a/evidence/LATMEAS-GB10-20261101-allanes/sm121_vs_NAK.md b/evidence/LATMEAS-GB10-20261101-allanes/sm121_vs_NAK.md new file mode 100644 index 00000000..34db984c --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101-allanes/sm121_vs_NAK.md @@ -0,0 +1,42 @@ +# GB10 (sm_121) measured latencies vs NAK + +Chip: NVIDIA GB10 (DGX Spark), sm_121 +Driver: NVRM version: NVIDIA UNIX Open Kernel Module for aarch64 580.173.02 Release Build (dvs-builder@U22-A24-5-4) Tue Jun 23 08:34:19 UTC 2026. No NVIDIA user-space libraries. +Method: dependent-chain timing kernel. One warp (32 threads); every lane stores, and a stall matches only when every lane matches the oracle. +Each fixed-latency producer is issued at a programmed stall. The GB10 column is the smallest stall S such that S and every larger S in the sweep matched the oracle on every run. Cycles are the CS2R clock delta (NAK_SV_CLOCK_LO, 0x50) around that chain, median and max-min. +Source: tests/gpu_latmeas_test.c in aien-dev/CUDApp +How to rerun: sh tools/latmeas/run_spark.sh OUTPUT_DIR + +NAK reference: `nak/sm120_instr_latencies.rs` plus `latencies/sm100/*.csv` at Mesa commit 01bab9302a6bf0a082c7ba8c49d74427cf10b0cf + +## Priority 1: values CUDA++ currently borrows from NAK (needs independent measurement) + +| Writer -> reader | NAK value | NAK source line | GB10 measured | Delta | Runs / spread | Kernel | +|---|---|---|---|---|---|---| +| ALU -> ALU (GPR RAW) | 4 + 1 = 5 | latencies/sm100/reg_raw.csv:2 row alu column alu = 4; sm120_instr_latencies.rs:457 GPR raw adds max(adjust, 1); IAdd3 is Alu at line 44 | 4 | -1 | 11 runs; stall spread 0; cycles at that stall median 2541 spread 0; stall-1 cycles median 2477 spread 0; loop overhead median 1261 | tests/gpu_latmeas_test.c (alu_alu) | +| FMA -> FMA (GPR RAW) | 4 + 1 = 5 | latencies/sm100/reg_raw.csv:5 row fma column fma = 4; sm120_instr_latencies.rs:59 FFma is Fma; line 457 adds 1 | 4 | -1 | 11 runs; stall spread 0; cycles at that stall median 2541 spread 0; stall-1 cycles median 2477 spread 0; loop overhead median 1261 | tests/gpu_latmeas_test.c (fma_fma) | +| Mixed ALU <-> FMA (GPR RAW) | 5 + 1 = 6 | latencies/sm100/reg_raw.csv:2 row alu column fma = 5 and line 5 row fma column alu = 5; sm120_instr_latencies.rs:457 adds 1 | 5 | -1 | 11 runs; stall spread 0; cycles at that stall median 4362 spread 0; stall-1 cycles median 4170 spread 9673; loop overhead median 1261 | tests/gpu_latmeas_test.c (mixed) | +| Predicate write -> guard | 13 + 1 = 14 | latencies/sm100/pred_raw.csv:11 row redirected_fp64 column dualalu = 13; sm120_instr_latencies.rs:474 predicate raw(write, None) uses reader RedirectedFp64 and adds 1; ISetP is Dualalu at line 49 | 13 | -1 | 11 runs; stall spread 0; cycles at that stall median 2714 spread 0; stall-1 cycles median 70 spread 0; loop overhead median 1261 | tests/gpu_latmeas_test.c (pred) | + +## Priority 2: instructions where CUDA++ (nvcc-derived) and NAK disagree + +| Instruction / pair | NAK value | nvcc sm_121 stall | GB10 measured | Agrees with | Kernel | +|---|---|---|---|---|---| +| Predicate write -> guard, attention emitter | 13 + 1 = 14 | 13 | 13 | nvcc | tests/gpu_latmeas_test.c (pred) | +| HMMA -> non-HMMA reader (FADD of d0) | 19 + 9 = 28 | 15 + NOP 14 = 29 | 20 | neither | tests/gpu_latmeas_test.c (hmma_reader) | +| IMAD.WIDE low half -> IADD3 | 3 + 1 = 4 | 6 (blanket K_FIXED; IMAD.WIDE is not in fixed_pair_class) | 3 | neither | tests/gpu_latmeas_test.c (imad_wide_lo) | +| FMNMX (dualalu) -> IADD3 (alu) | 4 + 1 = 5 | 6 (blanket K_FIXED; FMNMX is not in fixed_pair_class) | 4 | neither | tests/gpu_latmeas_test.c (fmnmx_alu) | +| QMMA (IMMA) -> next reader | 19 + 7 = 26 | 15 + NOP 15 = 30 | | | not generated (patched QMMA placeholder; see REPRODUCE.md) | +| Fixed-producer minimum in the numeric checker | 5 + 1 = 6 for mixed ALU/FMA | 5 (any fixed producer; mixed pairs need 6) | 5 | nvcc | tests/gpu_latmeas_test.c (mixed) | +| BSSY stall | decoupled (1 & sb) | 1, no scoreboard | | | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| BSYNC stall | decoupled (1 & sb) | 5, no scoreboard | | | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| BRA stall | decoupled (1 & sb) | 5 | | | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| EXIT stall | decoupled (1 & sb) | 5 | | | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| BAR.SYNC stall | decoupled AGU (1 & sb) | 6 | | | not generated (NAK class is decoupled, not a fixed GPR RAW) | + +## Notes +- Anything not fixed-latency (LDG/LDS, scoreboard waits) is out of scope for this table. +- Decoupled ops (BSSY, BSYNC, BRA, EXIT, BAR) are listed because the encoder gives them a fixed stall and NAK does not. They have no dependent-chain kernel. +- HMMA -> HMMA in NAK is 20 + 9 = 29, the same total the encoder uses. The HMMA row is the non-HMMA reader (28). +- QMMA is not generated here. Its encoder stall is 30 and NAK's IMMA -> ALU total is 26 (IMMA -> IMMA is 27). +- Confidence notes and anomalies go here. From 7ec018409bbc2ca29e51fc88584585fdca3c7231 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 1 Nov 2026 18:45:43 +0000 Subject: [PATCH 07/11] Record the host oracle and which lanes matched. raw.json now has the four oracle words, lanes_matched_min, a lane fail mask, and the first mismatching run, lane, and word. run_spark.sh writes RECEIPT.txt and no longer prints a 90 s bound beside the binary's 71 s. Co-authored-by: aien-spark --- tests/gpu_latmeas_test.c | 197 ++++++++++++++++++++++++++++++++++--- tools/latmeas/REPRODUCE.md | 4 +- tools/latmeas/run_spark.sh | 59 ++++++++++- tools/latmeas/selftest.sh | 5 + 4 files changed, 251 insertions(+), 14 deletions(-) diff --git a/tests/gpu_latmeas_test.c b/tests/gpu_latmeas_test.c index 589aaa0a..86ac4b19 100644 --- a/tests/gpu_latmeas_test.c +++ b/tests/gpu_latmeas_test.c @@ -1081,6 +1081,8 @@ static int sim_self_test(void) { return failed; } +static int lane_field_self_test(void); + static int self_test(void) { int failed = 0; uint8_t code[CODE_CAP]; @@ -1191,6 +1193,7 @@ static int self_test(void) { len = 0; CHECK(build(NULL, 0, 64, 1, code, sizeof code, &len, &gpr) == 0, "overhead kernel"); failed += sim_self_test(); + failed += lane_field_self_test(); int pass[RUNS][SWEEP_MAX]; memset(pass, 0, sizeof pass); @@ -1470,17 +1473,169 @@ static int launch_kern(const Kern *k, OmegaGpuMem *code, OmegaGpuMem *buf, uint3 return 0; } -static int mem_match(const uint32_t *mem, uint32_t result, uint32_t aux, int four) { +/* Host oracle for the four stored result words (lane record words 2..5). + * HMMA: d0..d3, each the 16-term ones dot. Other cases: result, aux, 0, 0. */ +static void oracle_words(const CaseDef *c, uint32_t o[4]) { + uint32_t r = 0, a = 0; + o[0] = o[1] = o[2] = o[3] = 0; + if (!c || expect_abs(c, &r, &a) != 0) return; + int four = !strcmp(c->id, "hmma_reader"); + o[0] = r; + o[1] = a; + o[2] = four ? r : 0; + o[3] = four ? r : 0; +} + +typedef struct { + int run; /* -1 when this stall never mismatched */ + int lane; + int word; /* 2..5, the lane-record index */ + uint32_t got, want; +} LaneMiss; + +/* How many lanes match o[0..3] in words 2..5. The first failure, if any, is written to *miss (lane stays -1 when every lane matches). */ +static int lanes_report(const uint32_t *mem, const uint32_t o[4], LaneMiss *miss) { + int n = 0; + miss->run = -1; + miss->lane = -1; + miss->word = 0; + miss->got = miss->want = 0; for (int lane = 0; lane < LANE_N; lane++) { const uint32_t *w = mem + (size_t)lane * LANE_WORDS; - if (w[2] != result || w[3] != aux) return 0; - if (four) { - if (w[4] != result || w[5] != result) return 0; - } else if (w[4] != 0 || w[5] != 0) { - return 0; + int bad = 0; + for (int k = 0; k < 4; k++) { + if (w[2 + k] == o[k]) continue; + if (miss->lane < 0) { + miss->lane = lane; + miss->word = 2 + k; + miss->got = w[2 + k]; + miss->want = o[k]; + } + bad = 1; } + if (!bad) n++; } - return 1; + return n; +} + +static uint32_t lane_fail_bits(const uint32_t *mem, const uint32_t o[4]) { + uint32_t mask = 0; + for (int lane = 0; lane < LANE_N; lane++) { + const uint32_t *w = mem + (size_t)lane * LANE_WORDS; + for (int k = 0; k < 4; k++) + if (w[2 + k] != o[k]) { + mask |= 1u << lane; + break; + } + } + return mask; +} + +static int mem_match(const uint32_t *mem, uint32_t result, uint32_t aux, int four) { + uint32_t o[4] = {result, aux, four ? result : 0u, four ? result : 0u}; + LaneMiss m; + return lanes_report(mem, o, &m) == LANE_N; +} + +static void fprint_lane_fields(FILE *jf, const uint32_t o[4], const uint8_t *matched_min, const uint32_t *fail_mask, + const LaneMiss *miss, int sweep, int have) { + fprintf(jf, " \"oracle\": [%u, %u, %u, %u],\n", o[0], o[1], o[2], o[3]); + fprintf(jf, " \"reference_result\": %u,\n", o[0]); + fprintf(jf, " \"lanes_matched_min\": ["); + for (int s = 0; s < sweep; s++) { + if (!have || matched_min[s] == 0xff) + fprintf(jf, "%snull", s ? ", " : ""); + else + fprintf(jf, "%s%u", s ? ", " : "", matched_min[s]); + } + fprintf(jf, "],\n \"lane_fail_mask\": ["); + for (int s = 0; s < sweep; s++) { + if (!have || matched_min[s] == 0xff) + fprintf(jf, "%snull", s ? ", " : ""); + else + fprintf(jf, "%s\"0x%08x\"", s ? ", " : "", fail_mask[s]); + } + fprintf(jf, "],\n \"first_mismatch\": ["); + for (int s = 0; s < sweep; s++) { + if (!have || matched_min[s] == 0xff || miss[s].lane < 0) + fprintf(jf, "%snull", s ? ", " : ""); + else + fprintf(jf, + "%s{\"run\": %d, \"lane\": %d, \"word\": %d, \"got\": %u, \"want\": %u}", + s ? ", " : "", miss[s].run, miss[s].lane, miss[s].word, miss[s].got, miss[s].want); + } + fprintf(jf, "],\n"); +} + +static int lane_field_self_test(void) { + int failed = 0; +#define LCHECK(cond, ...) \ + do { \ + if (!(cond)) { \ + failed++; \ + printf("FAIL: "); \ + printf(__VA_ARGS__); \ + printf("\n"); \ + } \ + } while (0) + uint32_t o[4]; + oracle_words(&CASES[6], o); + LCHECK(o[0] == hmma_16816_ones() && o[1] == o[0] && o[2] == o[0] && o[3] == o[0], + "hmma oracle is four destination words"); + oracle_words(&CASES[0], o); + LCHECK(o[0] == (N_FIXED * (N_FIXED + 1u)) / 2u && o[1] == 1u && o[2] == 0u && o[3] == 0u, "alu oracle words"); + + uint32_t mem[LANE_N * LANE_WORDS]; + memset(mem, 0, sizeof mem); + for (int lane = 0; lane < LANE_N; lane++) { + mem[(size_t)lane * LANE_WORDS + 2] = o[0]; + mem[(size_t)lane * LANE_WORDS + 3] = o[1]; + mem[(size_t)lane * LANE_WORDS + 4] = o[2]; + mem[(size_t)lane * LANE_WORDS + 5] = o[3]; + mem[(size_t)lane * LANE_WORDS + 6] = (uint32_t)lane; + } + LaneMiss m; + LCHECK(lanes_report(mem, o, &m) == LANE_N && m.lane < 0, "all lanes match the oracle"); + mem[7u * LANE_WORDS + 4] = 0x11111111u; + LCHECK(lanes_report(mem, o, &m) == LANE_N - 1 && m.lane == 7 && m.word == 4 && m.got == 0x11111111u && m.want == o[2], + "a planted word-4 mismatch names lane 7"); + LCHECK(lane_fail_bits(mem, o) == (1u << 7), "fail mask is lane 7"); + + uint8_t mins[2] = {31, 32}; + uint32_t masks[2] = {1u << 7, 0}; + LaneMiss ms[2]; + ms[0] = m; + ms[0].run = 3; + ms[1].lane = -1; + char *text = NULL; + size_t tlen = 0; + FILE *mf = open_memstream(&text, &tlen); + if (mf) { + fprint_lane_fields(mf, o, mins, masks, ms, 2, 1); + fclose(mf); + } + LCHECK(text && strstr(text, "\"oracle\": [") && strstr(text, "\"lanes_matched_min\": [31, 32]") && + strstr(text, "\"lane_fail_mask\"") && strstr(text, "\"lane\": 7") && strstr(text, "\"word\": 4") && + strstr(text, "\"run\": 3") && strstr(text, ", null]"), + "json lane fields: %s", text ? text : "(none)"); + free(text); + + uint8_t code[CODE_CAP]; + size_t len = 0; + uint32_t gpr = 0, lane0[LANE_WORDS]; + LCHECK(build(&CASES[0], CASES[0].sweep, 0, 0, code, sizeof code, &len, &gpr) == 0, "alu top encode"); + LCHECK(lat_sim(lane0) == 0, "alu top sim"); + oracle_words(&CASES[0], o); + LCHECK(lanes_report(g_sim_mem, o, &m) == LANE_N && m.lane < 0, "sim stall 15 matches on every lane"); + LCHECK(build(&CASES[0], 1, 0, 0, code, sizeof code, &len, &gpr) == 0, "alu stall 1 encode"); + LCHECK(lat_sim(lane0) == 0, "alu stall 1 sim"); + LCHECK(lanes_report(g_sim_mem, o, &m) == 0 && m.lane == 0 && m.word == 2, "sim stall 1 fails word 2 on lane 0"); + oracle_words(&CASES[6], o); + LCHECK(build(&CASES[6], 1, 0, 0, code, sizeof code, &len, &gpr) == 0, "hmma stall 1 encode"); + LCHECK(lat_sim(lane0) == 0, "hmma stall 1 sim"); + LCHECK(lanes_report(g_sim_mem, o, &m) == 0 && m.word >= 2 && m.word <= 5, "hmma stall 1 fails a destination word"); +#undef LCHECK + return failed; } static void json_escape(const char *s, char *dst, size_t n) { @@ -1515,10 +1670,17 @@ static int run_launch(const char *kdir, const char *out_path) { uint64_t cyc[NCASES][RUNS][SWEEP_MAX]; uint64_t ovh_cyc[8][RUNS]; uint32_t run0_res[NCASES][SWEEP_MAX]; + uint8_t lane_min[NCASES][SWEEP_MAX]; + uint32_t fail_mask[NCASES][SWEEP_MAX]; + LaneMiss first_miss[NCASES][SWEEP_MAX]; memset(pass, 0, sizeof pass); memset(cyc, 0, sizeof cyc); memset(ovh_cyc, 0, sizeof ovh_cyc); memset(run0_res, 0, sizeof run0_res); + memset(lane_min, 0xff, sizeof lane_min); + memset(fail_mask, 0, sizeof fail_mask); + for (int ci = 0; ci < NCASES; ci++) + for (int s = 0; s < SWEEP_MAX; s++) first_miss[ci][s].lane = -1; char notes[NCASES][384]; memset(notes, 0, sizeof notes); unsigned ovh_list[8]; @@ -1593,10 +1755,17 @@ static int run_launch(const char *kdir, const char *out_path) { break; } cyc[ci][r][s - 1] = cy; - uint32_t expv = 0, auxv = 0; - int four = !strcmp(c->id, "hmma_reader"); - int ok = expect_abs(c, &expv, &auxv) == 0 && mem_match(mem, expv, auxv, four); - pass[ci][r][s - 1] = ok; + uint32_t oracle[4]; + oracle_words(c, oracle); + LaneMiss one; + int nmatch = lanes_report(mem, oracle, &one); + if (lane_min[ci][s - 1] == 0xff || nmatch < lane_min[ci][s - 1]) lane_min[ci][s - 1] = (uint8_t)nmatch; + fail_mask[ci][s - 1] |= lane_fail_bits(mem, oracle); + if (nmatch < LANE_N && first_miss[ci][s - 1].lane < 0) { + one.run = r; + first_miss[ci][s - 1] = one; + } + pass[ci][r][s - 1] = nmatch == LANE_N; if (r == 0) run0_res[ci][s - 1] = mem[2]; } } @@ -1708,7 +1877,11 @@ write_json:; fprintf(jf, " \"overhead_cycle_median\": null,\n"); else fprintf(jf, " \"overhead_cycle_median\": %llu,\n", (unsigned long long)omed); - fprintf(jf, " \"reference_result\": null,\n"); + { + uint32_t oracle[4]; + oracle_words(c, oracle); + fprint_lane_fields(jf, oracle, lane_min[ci], fail_mask[ci], first_miss[ci], (int)c->sweep, !device_error); + } fprintf(jf, " \"run0_result\": ["); for (unsigned s = 1; s <= c->sweep; s++) fprintf(jf, "%s%u", s == 1 ? "" : ", ", run0_res[ci][s - 1]); diff --git a/tools/latmeas/REPRODUCE.md b/tools/latmeas/REPRODUCE.md index ebe05ba2..22a73cb1 100644 --- a/tools/latmeas/REPRODUCE.md +++ b/tools/latmeas/REPRODUCE.md @@ -40,7 +40,9 @@ The loop increment stalls 15 before `ISETP`. Stall 1 there is shorter than a fix The 2026-11-01 GB10 run of `03cdcc5` merged with PR #251 is `evidence/LATMEAS-GB10-20261101/`. `UNITS.md` there compares the stall field with NAK's `deps.delay`. That run stored lane 0 only, and its HMMA oracle was the stall-36 result. -The kernels after that receipt store eight words from every lane: the two clocks, four result words, and the lane id. HMMA's four result words are `FADD` of `d0` through `d3` against zero. The host value is sixteen products of bf16 1.0, not the stall-36 launch. A stall matches only when all 32 lanes match. Those kernels have not been launched. +The kernels after that receipt store eight words from every lane: the two clocks, four result words, and the lane id. HMMA's four result words are `FADD` of `d0` through `d3` against zero. The host value is sixteen products of bf16 1.0, not the stall-36 launch. A stall matches only when all 32 lanes match. + +`evidence/LATMEAS-GB10-20261101-allanes/` is the GB10 run of that all-lane kernel (`c94c2c9`, merge of `4f3a4d0` and PR #251). Its `raw.json` still has one lane-0 word per stall and no host oracle, so the file alone cannot show that every lane and every HMMA destination matched. The source of that commit did require the full match before it recorded a stall. A later `raw.json` adds `oracle` (four host words), `reference_result` (the first of those), `lanes_matched_min`, `lane_fail_mask`, and `first_mismatch` (run, lane, word, got, want). `run_spark.sh` writes `RECEIPT.txt` after the launch: HEAD, parent SHAs, kernel, driver, device, gcc, UTC time, and the sha256 of `raw.json`. The launch bound is the binary's estimated chip window, not a second number in the script. ## NAK pin diff --git a/tools/latmeas/run_spark.sh b/tools/latmeas/run_spark.sh index f5212bec..5e785e95 100755 --- a/tools/latmeas/run_spark.sh +++ b/tools/latmeas/run_spark.sh @@ -14,6 +14,62 @@ case "$out" in *) out=$root/$out ;; esac cd "$root" + +write_receipt() { + rec=$1/RECEIPT.txt + { + echo "utc: $(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "head: $(git rev-parse HEAD)" + echo "head_parents: $(git rev-parse HEAD^@ 2>/dev/null | tr '\n' ' ')" + if git diff --quiet && git diff --cached --quiet; then + echo "worktree: clean" + else + echo "worktree: dirty" + fi + echo "kernel: $(uname -srvm)" + echo "gcc: $(gcc --version | head -n 1)" + if [ -r /proc/driver/nvidia/version ]; then + echo "driver: $(head -n 1 /proc/driver/nvidia/version)" + else + echo "driver: unread" + fi + found=0 + for info in /proc/driver/nvidia/gpus/*/information; do + [ -f "$info" ] || continue + found=1 + echo "device: $(basename "$(dirname "$info")")" + sed -n 's/^Model:[[:space:]]*/model: /p; s/^Bus Location:[[:space:]]*/bus: /p' "$info" + done + if [ "$found" -eq 0 ]; then + echo "device: none" + fi + if [ -f "$1/raw.json" ]; then + echo "raw_json_sha256: $(sha256sum "$1/raw.json" | awk '{print $1}')" + else + echo "raw_json_sha256: absent" + fi + } > "$rec" + echo "latmeas: wrote $rec" +} + +if [ "$1" = "--receipt-self-test" ]; then + tmp=$(mktemp -d) + printf '{}\n' > "$tmp/raw.json" + write_receipt "$tmp" + got=$(awk '/^raw_json_sha256: / {print $2}' "$tmp/RECEIPT.txt") + want=$(sha256sum "$tmp/raw.json" | awk '{print $1}') + grep -q '^utc: ' "$tmp/RECEIPT.txt" + grep -q '^head: ' "$tmp/RECEIPT.txt" + grep -q '^head_parents: ' "$tmp/RECEIPT.txt" + grep -q '^kernel: ' "$tmp/RECEIPT.txt" + grep -q '^gcc: ' "$tmp/RECEIPT.txt" + grep -q '^device: ' "$tmp/RECEIPT.txt" + test "$got" = "$want" + rm -rf "$tmp" + echo "receipt self-test PASS" + exit 0 +fi + mkdir -p "$out" echo "latmeas: building (outside the chip lock)" make -j"$(nproc)" build/gpu_latmeas_test build/cudapp-doctor @@ -24,7 +80,7 @@ git rev-parse HEAD > "$out/head.txt" ./build/gpu_latmeas_test --emit "$out/kernels" python3 tools/latmeas/diff_nak.py --check --md "$out/diff.md" echo "latmeas: taking flock /tmp/aien-gb10.lock for the launch only; timeout 600" -echo "latmeas: estimated chip window 90 s upper bound (printed again by the binary)" +echo "latmeas: the launch bound is the estimated chip window printed above" # flock waits here without holding the lock. timeout covers only the launch. set +e flock /tmp/aien-gb10.lock timeout 600 ./build/gpu_latmeas_test --launch "$out/kernels" --out "$out/raw.json" @@ -33,4 +89,5 @@ set -e if [ -f "$out/raw.json" ]; then python3 tools/latmeas/fill_template.py --results "$out/raw.json" --md "$out/sm121_vs_NAK.md" fi +write_receipt "$out" exit "$rc" diff --git a/tools/latmeas/selftest.sh b/tools/latmeas/selftest.sh index 907124c8..c80237e3 100755 --- a/tools/latmeas/selftest.sh +++ b/tools/latmeas/selftest.sh @@ -15,4 +15,9 @@ trap 'rm -rf "$dir"' EXIT "$bin" --emit "$dir/kernels" >/dev/null test -f "$dir/kernels/manifest.txt" test -f "$dir/kernels/alu_alu_4.bin" +if grep -q '90 s' tools/latmeas/run_spark.sh; then + echo "FAIL: run_spark.sh still quotes a 90 s bound" >&2 + exit 1 +fi +sh tools/latmeas/run_spark.sh --receipt-self-test echo "latmeas host self-test PASS" From 6e3da800625794424f8af3ed9602ab61db9e49bb Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 1 Nov 2026 18:52:52 +0000 Subject: [PATCH 08/11] Record the GB10 run that stored the host oracle. fb91560 is 7ec0184 merged with PR #251. Passing stalls have lanes_matched_min 32. Every recorded first_mismatch is word 2, so the file does not by itself show a later word failing. Co-authored-by: aien-spark --- .../LATMEAS-GB10-20261101-oracle/README.md | 28 +++ .../LATMEAS-GB10-20261101-oracle/RECEIPT.txt | 11 ++ evidence/LATMEAS-GB10-20261101-oracle/diff.md | 21 +++ .../LATMEAS-GB10-20261101-oracle/head.txt | 1 + .../LATMEAS-GB10-20261101-oracle/raw.json | 142 +++++++++++++++ evidence/LATMEAS-GB10-20261101-oracle/run.log | 165 ++++++++++++++++++ .../sm121_vs_NAK.md | 42 +++++ 7 files changed, 410 insertions(+) create mode 100644 evidence/LATMEAS-GB10-20261101-oracle/README.md create mode 100644 evidence/LATMEAS-GB10-20261101-oracle/RECEIPT.txt create mode 100644 evidence/LATMEAS-GB10-20261101-oracle/diff.md create mode 100644 evidence/LATMEAS-GB10-20261101-oracle/head.txt create mode 100644 evidence/LATMEAS-GB10-20261101-oracle/raw.json create mode 100644 evidence/LATMEAS-GB10-20261101-oracle/run.log create mode 100644 evidence/LATMEAS-GB10-20261101-oracle/sm121_vs_NAK.md diff --git a/evidence/LATMEAS-GB10-20261101-oracle/README.md b/evidence/LATMEAS-GB10-20261101-oracle/README.md new file mode 100644 index 00000000..541485b7 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101-oracle/README.md @@ -0,0 +1,28 @@ +# GB10 latency receipt with the host oracle, 2026-11-01 + +This is the headline chip receipt for the latency kit. `sm121_vs_NAK.md` is the table `fill_template.py` wrote from this directory's `raw.json`, and a fresh fill of that file matches it. Measured cells come from `stall_required` in `raw.json` and from nowhere else. + +## What ran + +- Command: `sh tools/latmeas/run_spark.sh latmeas-out-7ec0184` (`run.log`). +- `RECEIPT.txt` is the file the script wrote. It is copied here unchanged. `utc: 2026-11-01T18:47:06Z`. `head: fb91560dc3492082455576477bffd1794d27ab3c`, parents `7ec018409bbc2ca29e51fc88584585fdca3c7231` and `6a93cb567b33b67bc1185fb45a3c50fc4d88d30e` (PR #254 `7ec0184` merged with PR #251, no other edits; `worktree: clean`). Kernel 7.0.0-1019-nvidia, gcc 13.3.0, open module 580.173.02, device `000f:01:00.0`. +- `raw_json_sha256` in the receipt is `aa3fe3fee12ffe123f572e5a0c9b26b7e7fc82142635ea59442b6a008033534a`, the same as `sha256sum raw.json`. +- Launch rc 0, `"error": ""`, 1408 launches, 11 runs. Upload sha256 `872182a84618fc1c16d14b7ce689e012f36e19515aaadc18c146c7f3115d074c`. Two home-directory paths in `run.log` are replaced with `latmeas-out-7ec0184`. + +`stall_required` and spread, from `raw.json`: alu_alu 4, fma_fma 4, mixed 5, pred 13, fmnmx_alu 4, imad_wide_lo 3, hmma_reader 20, each with spread 0. At that stall and every larger stall, `lanes_matched_min` is 32, `lane_fail_mask` is `0x00000000`, and `first_mismatch` is null. At the stall under it, `lanes_matched_min` is 0, the mask is `0xffffffff`, and `first_mismatch` is run 0, lane 0, word 2. + +## What the file shows about the four words + +`first_mismatch` on every failing stall names word 2 and no other word. That field keeps the first difference in a lane (`tests/gpu_latmeas_test.c` `lanes_report`, the loop at lines 1537-1545: `w[2 + k]` against `o[k]` for `k` from 0 to 3, and it records only the first `k` that differs). A word-2 miss therefore does not say whether words 3, 4, and 5 also differed. HMMA's `oracle` is four copies of `1098907648`. This file has no row whose first difference is word 3, 4, or 5. + +A passing stall is different. `lanes_matched_min` counts a lane only when all four of those words equal the oracle (the same loop, `if (!bad) n++` at line 1547). 32 means every lane matched words 2, 3, 4, and 5. For HMMA that is `d0`..`d3`. That is what 32 at stall 20 and above shows, together with this code. It is not a separate per-word tally. + +The host self-test does plant a miss that is only word 4: `tests/gpu_latmeas_test.c` lines 1648-1650 set lane 7's word 4 and require `lanes_report` to name lane 7 word 4. That test passed on the host (`make test-gpu-latmeas`). It is not a chip result. `word_mismatch_max` (four counts, words 2..5, the maximum across runs) is written by a later commit and is not in this `raw.json`. + +## model: Unknown + +`RECEIPT.txt` says `model: Unknown`. The script copied the proc file's `Model` line. On this machine that line is `Unknown`. The doctor, in `run.log`, names the same PCI id from the chip table: `GPU 000f:01:00.0: NVIDIA GB10 (sm_121)`. The receipt file is the one the script produced and is not rewritten. A later `run_spark.sh` fills `NVIDIA GB10 (sm_121)` from PCI `10de:2e12` when proc `Model` is `Unknown` (`src/omega_gpu_chip.c`). + +## One second and the 71 second bound + +`print_plan` in `tests/gpu_latmeas_test.c` (the `50` in the bound, lines 1269-1272) is `1408` launches times 50 ms, rounded up to whole seconds: 71. The 50 ms is an allowance for a cold shader invalidate. The printed line says the window is usually shorter. The operator reported the launch at about one second. The log has no elapsed-time field, so that duration is the operator's report, not a second clock in this directory. One second is inside the 71 second allowance. diff --git a/evidence/LATMEAS-GB10-20261101-oracle/RECEIPT.txt b/evidence/LATMEAS-GB10-20261101-oracle/RECEIPT.txt new file mode 100644 index 00000000..76664cd2 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101-oracle/RECEIPT.txt @@ -0,0 +1,11 @@ +utc: 2026-11-01T18:47:06Z +head: fb91560dc3492082455576477bffd1794d27ab3c +head_parents: 7ec018409bbc2ca29e51fc88584585fdca3c7231 6a93cb567b33b67bc1185fb45a3c50fc4d88d30e +worktree: clean +kernel: Linux 7.0.0-1019-nvidia #19~24.04.2-Ubuntu SMP PREEMPT_DYNAMIC Sun Sep 6 18:35:11 UTC 2 aarch64 +gcc: gcc (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0 +driver: NVRM version: NVIDIA UNIX Open Kernel Module for aarch64 580.173.02 Release Build (dvs-builder@U22-A24-5-4) Tue Jun 23 08:34:19 UTC 2026 +device: 000f:01:00.0 +model: Unknown +bus: 000f:01:00.0 +raw_json_sha256: aa3fe3fee12ffe123f572e5a0c9b26b7e7fc82142635ea59442b6a008033534a diff --git a/evidence/LATMEAS-GB10-20261101-oracle/diff.md b/evidence/LATMEAS-GB10-20261101-oracle/diff.md new file mode 100644 index 00000000..5ca3637f --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101-oracle/diff.md @@ -0,0 +1,21 @@ +# CUDA++ fixed latencies vs NAK + +Mesa commit `01bab9302a6bf0a082c7ba8c49d74427cf10b0cf`. +Clock: nak_private.h:135 NAK_SV_CLOCK_LO = 0x50; from_nir.rs:3522 SM>=50 uses CS2R; sm70_encode.rs:4278 opcode 0x805, index bits 72..80. +Priority 1 pairs match NAK on paper. They are measured because that match is circular. +Rows below are the ones that differ. A blank NAK stall means NAK calls the op decoupled. + +| id | pair | CUDA++ | NAK | kernel | +|---|---|---|---|---| +| pred_attention | Predicate write -> guard, attention emitter | 13 | 13 + 1 = 14 | tests/gpu_latmeas_test.c (pred) | +| hmma_reader | HMMA -> non-HMMA reader (FADD of d0) | 15 + NOP 14 = 29 | 19 + 9 = 28 | tests/gpu_latmeas_test.c (hmma_reader) | +| imad_wide_lo | IMAD.WIDE low half -> IADD3 | 6 (blanket K_FIXED; IMAD.WIDE is not in fixed_pair_class) | 3 + 1 = 4 | tests/gpu_latmeas_test.c (imad_wide_lo) | +| fmnmx_alu | FMNMX (dualalu) -> IADD3 (alu) | 6 (blanket K_FIXED; FMNMX is not in fixed_pair_class) | 4 + 1 = 5 | tests/gpu_latmeas_test.c (fmnmx_alu) | +| qmma_reader | QMMA (IMMA) -> next reader | 15 + NOP 15 = 30 | 19 + 7 = 26 | not generated (patched QMMA placeholder; see REPRODUCE.md) | +| numeric_min | Fixed-producer minimum in the numeric checker | 5 (any fixed producer; mixed pairs need 6) | 5 + 1 = 6 for mixed ALU/FMA | tests/gpu_latmeas_test.c (mixed) | +| bssy | BSSY stall | 1, no scoreboard | decoupled (1 & sb) | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| bsync | BSYNC stall | 5, no scoreboard | decoupled (1 & sb) | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| bra | BRA stall | 5 | decoupled (1 & sb) | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| exit_op | EXIT stall | 5 | decoupled (1 & sb) | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| bar | BAR.SYNC stall | 6 | decoupled AGU (1 & sb) | not generated (NAK class is decoupled, not a fixed GPR RAW) | + diff --git a/evidence/LATMEAS-GB10-20261101-oracle/head.txt b/evidence/LATMEAS-GB10-20261101-oracle/head.txt new file mode 100644 index 00000000..c2498df8 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101-oracle/head.txt @@ -0,0 +1 @@ +fb91560dc3492082455576477bffd1794d27ab3c diff --git a/evidence/LATMEAS-GB10-20261101-oracle/raw.json b/evidence/LATMEAS-GB10-20261101-oracle/raw.json new file mode 100644 index 00000000..93e0bff2 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101-oracle/raw.json @@ -0,0 +1,142 @@ +{ + "kind": "cudapp-latmeas", + "synthetic": false, + "measured": true, + "mesa_commit": "01bab9302a6bf0a082c7ba8c49d74427cf10b0cf", + "driver": "NVRM version: NVIDIA UNIX Open Kernel Module for aarch64 580.173.02 Release Build (dvs-builder@U22-A24-5-4) Tue Jun 23 08:34:19 UTC 2026", + "machine": "aarch64", + "runs": 11, + "threads": 32, + "warps": 1, + "clock": "CS2R idx 0x50 (NAK_SV_CLOCK_LO)", + "launches": 1408, + "error": "", + "cases": { + "alu_alu": { + "stall_required": 4, + "stall_spread": 0, + "cycle_median": 2541, + "cycle_spread": 0, + "cycle_median_stall1": 2477, + "cycle_spread_stall1": 0, + "overhead_cycle_median": 1261, + "oracle": [528, 1, 0, 0], + "reference_result": 528, + "lanes_matched_min": [0, 0, 0, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32], + "lane_fail_mask": ["0xffffffff", "0xffffffff", "0xffffffff", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000"], + "first_mismatch": [{"run": 0, "lane": 0, "word": 2, "got": 4026531840, "want": 528}, {"run": 0, "lane": 0, "word": 2, "got": 4026531840, "want": 528}, {"run": 0, "lane": 0, "word": 2, "got": 4026531840, "want": 528}, null, null, null, null, null, null, null, null, null, null, null, null], + "run0_result": [4026531840, 4026531840, 4026531840, 528, 528, 528, 528, 528, 528, 528, 528, 528, 528, 528, 528], + "iters": 32, + "ops": 32, + "note": "" + }, + "fma_fma": { + "stall_required": 4, + "stall_spread": 0, + "cycle_median": 2541, + "cycle_spread": 0, + "cycle_median_stall1": 2477, + "cycle_spread_stall1": 0, + "overhead_cycle_median": 1261, + "oracle": [1141112832, 1, 0, 0], + "reference_result": 1141112832, + "lanes_matched_min": [0, 0, 0, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32], + "lane_fail_mask": ["0xffffffff", "0xffffffff", "0xffffffff", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000"], + "first_mismatch": [{"run": 0, "lane": 0, "word": 2, "got": 2139095040, "want": 1141112832}, {"run": 0, "lane": 0, "word": 2, "got": 2139095040, "want": 1141112832}, {"run": 0, "lane": 0, "word": 2, "got": 2139095040, "want": 1141112832}, null, null, null, null, null, null, null, null, null, null, null, null], + "run0_result": [2139095040, 2139095040, 2139095040, 1141112832, 1141112832, 1141112832, 1141112832, 1141112832, 1141112832, 1141112832, 1141112832, 1141112832, 1141112832, 1141112832, 1141112832], + "iters": 32, + "ops": 32, + "note": "" + }, + "mixed": { + "stall_required": 5, + "stall_spread": 0, + "cycle_median": 4362, + "cycle_spread": 0, + "cycle_median_stall1": 4170, + "cycle_spread_stall1": 0, + "overhead_cycle_median": 1261, + "oracle": [704643072, 704643072, 0, 0], + "reference_result": 704643072, + "lanes_matched_min": [0, 0, 0, 0, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32], + "lane_fail_mask": ["0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000"], + "first_mismatch": [{"run": 0, "lane": 0, "word": 2, "got": 4026531840, "want": 704643072}, {"run": 0, "lane": 0, "word": 2, "got": 4026531840, "want": 704643072}, {"run": 0, "lane": 0, "word": 2, "got": 4026531840, "want": 704643072}, {"run": 0, "lane": 0, "word": 2, "got": 1024, "want": 704643072}, null, null, null, null, null, null, null, null, null, null, null], + "run0_result": [4026531840, 4026531840, 4026531840, 1024, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072], + "iters": 32, + "ops": 64, + "note": "" + }, + "pred": { + "stall_required": 13, + "stall_spread": 0, + "cycle_median": 2714, + "cycle_spread": 0, + "cycle_median_stall1": 70, + "cycle_spread_stall1": 0, + "overhead_cycle_median": 1261, + "oracle": [32, 1, 0, 0], + "reference_result": 32, + "lanes_matched_min": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 32, 32], + "lane_fail_mask": ["0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0x00000000", "0x00000000", "0x00000000"], + "first_mismatch": [{"run": 0, "lane": 0, "word": 2, "got": 3735928559, "want": 32}, {"run": 0, "lane": 0, "word": 2, "got": 3735928559, "want": 32}, {"run": 0, "lane": 0, "word": 2, "got": 3735928559, "want": 32}, {"run": 0, "lane": 0, "word": 2, "got": 3735928559, "want": 32}, {"run": 0, "lane": 0, "word": 2, "got": 3735928559, "want": 32}, {"run": 0, "lane": 0, "word": 2, "got": 3735928559, "want": 32}, {"run": 0, "lane": 0, "word": 2, "got": 3735928559, "want": 32}, {"run": 0, "lane": 0, "word": 2, "got": 3735928559, "want": 32}, {"run": 0, "lane": 0, "word": 2, "got": 3735928559, "want": 32}, {"run": 0, "lane": 0, "word": 2, "got": 3735928559, "want": 32}, {"run": 0, "lane": 0, "word": 2, "got": 3735928559, "want": 32}, {"run": 0, "lane": 0, "word": 2, "got": 3735928559, "want": 32}, null, null, null], + "run0_result": [3735928559, 3735928559, 3735928559, 3735928559, 3735928559, 3735928559, 3735928559, 3735928559, 3735928559, 3735928559, 3735928559, 3735928559, 32, 32, 32], + "iters": 32, + "ops": 32, + "note": "" + }, + "fmnmx_alu": { + "stall_required": 4, + "stall_spread": 0, + "cycle_median": 2541, + "cycle_spread": 0, + "cycle_median_stall1": 2507, + "cycle_spread_stall1": 0, + "overhead_cycle_median": 1261, + "oracle": [704643072, 1, 0, 0], + "reference_result": 704643072, + "lanes_matched_min": [0, 0, 0, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32], + "lane_fail_mask": ["0xffffffff", "0xffffffff", "0xffffffff", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000"], + "first_mismatch": [{"run": 0, "lane": 0, "word": 2, "got": 4026531840, "want": 704643072}, {"run": 0, "lane": 0, "word": 2, "got": 4026531840, "want": 704643072}, {"run": 0, "lane": 0, "word": 2, "got": 4026531840, "want": 704643072}, null, null, null, null, null, null, null, null, null, null, null, null], + "run0_result": [4026531840, 4026531840, 4026531840, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072, 704643072], + "iters": 32, + "ops": 32, + "note": "" + }, + "imad_wide_lo": { + "stall_required": 3, + "stall_spread": 0, + "cycle_median": 2768, + "cycle_spread": 0, + "cycle_median_stall1": 2736, + "cycle_spread_stall1": 0, + "overhead_cycle_median": 1261, + "oracle": [528, 1, 0, 0], + "reference_result": 528, + "lanes_matched_min": [0, 0, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32], + "lane_fail_mask": ["0xffffffff", "0xffffffff", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000"], + "first_mismatch": [{"run": 0, "lane": 0, "word": 2, "got": 32, "want": 528}, {"run": 0, "lane": 0, "word": 2, "got": 32, "want": 528}, null, null, null, null, null, null, null, null, null, null, null, null, null], + "run0_result": [32, 32, 528, 528, 528, 528, 528, 528, 528, 528, 528, 528, 528, 528, 528], + "iters": 32, + "ops": 32, + "note": "" + }, + "hmma_reader": { + "stall_required": 20, + "stall_spread": 0, + "cycle_median": 63, + "cycle_spread": 0, + "cycle_median_stall1": 61, + "cycle_spread_stall1": 0, + "overhead_cycle_median": 49, + "oracle": [1098907648, 1098907648, 1098907648, 1098907648], + "reference_result": 1098907648, + "lanes_matched_min": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32], + "lane_fail_mask": ["0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0xffffffff", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000", "0x00000000"], + "first_mismatch": [{"run": 0, "lane": 0, "word": 2, "got": 0, "want": 1098907648}, {"run": 0, "lane": 0, "word": 2, "got": 0, "want": 1098907648}, {"run": 0, "lane": 0, "word": 2, "got": 0, "want": 1098907648}, {"run": 0, "lane": 0, "word": 2, "got": 0, "want": 1098907648}, {"run": 0, "lane": 0, "word": 2, "got": 0, "want": 1098907648}, {"run": 0, "lane": 0, "word": 2, "got": 0, "want": 1098907648}, {"run": 0, "lane": 0, "word": 2, "got": 0, "want": 1098907648}, {"run": 0, "lane": 0, "word": 2, "got": 0, "want": 1098907648}, {"run": 0, "lane": 0, "word": 2, "got": 0, "want": 1098907648}, {"run": 0, "lane": 0, "word": 2, "got": 0, "want": 1098907648}, {"run": 0, "lane": 0, "word": 2, "got": 0, "want": 1098907648}, {"run": 0, "lane": 0, "word": 2, "got": 0, "want": 1098907648}, {"run": 0, "lane": 0, "word": 2, "got": 0, "want": 1098907648}, {"run": 0, "lane": 0, "word": 2, "got": 0, "want": 1098907648}, {"run": 0, "lane": 0, "word": 2, "got": 0, "want": 1098907648}, {"run": 0, "lane": 0, "word": 2, "got": 0, "want": 1098907648}, {"run": 0, "lane": 0, "word": 2, "got": 0, "want": 1098907648}, {"run": 0, "lane": 0, "word": 2, "got": 0, "want": 1098907648}, {"run": 0, "lane": 0, "word": 2, "got": 0, "want": 1098907648}, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null], + "run0_result": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648, 1098907648], + "iters": 1, + "ops": 1, + "note": "" + } + } +} diff --git a/evidence/LATMEAS-GB10-20261101-oracle/run.log b/evidence/LATMEAS-GB10-20261101-oracle/run.log new file mode 100644 index 00000000..0036eeee --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101-oracle/run.log @@ -0,0 +1,165 @@ +latmeas: building (outside the chip lock) +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_ampere_codegen.c -o build/omega_ampere_codegen.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_blackwell_codegen.c -o build/omega_blackwell_codegen.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_cmdlist.c -o build/omega_gpu_cmdlist.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_fp8_api.c -o build/omega_gpu_fp8_api.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_fp8gemm_api.c -o build/omega_gpu_fp8gemm_api.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_q36_graph.c -o build/omega_gpu_q36_graph.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_q38_graph.c -o build/omega_gpu_q38_graph.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_router_api.c -o build/omega_gpu_router_api.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_session.c -o build/omega_gpu_session.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_gpu_tcgemm_api.c -o build/omega_gpu_tcgemm_api.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/omega_cubin.c -o build/omega_cubin.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_amp_runtime.c -o build/cudapp_om_amp_runtime.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -O3 -c src/cudapp_om_closure_reseal.c -o build/cudapp_om_closure_reseal.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_q38_emit.c -o build/cudapp_om_q38_emit.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c src/cudapp_om_q38_runtime.c -o build/cudapp_om_q38_runtime.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -c ./nvrm/nvrm.c -o build/nvrm.o +make: 'build/cudapp-doctor' is up to date. +ar rcs build/libomega_gpu.a build/omega_ampere_codegen.o build/omega_ampere_qmd.o build/omega_blackwell_codegen.o build/omega_blackwell_codegen_attn.o build/omega_blackwell_codegen_attn256.o build/omega_blackwell_codegen_bf16.o build/omega_blackwell_codegen_dec.o build/omega_blackwell_codegen_gdn.o build/omega_blackwell_codegen_gemm.o build/omega_blackwell_codegen_moe.o build/omega_blackwell_codegen_moe2.o build/omega_blackwell_codegen_pattn.o build/omega_blackwell_codegen_pattn256.o build/omega_blackwell_codegen_pf.o build/omega_blackwell_codegen_pfhost.o build/omega_blackwell_codegen_qg.o build/omega_blackwell_codegen_qmma.o build/omega_blackwell_codegen_router.o build/omega_blackwell_codegen_tc.o build/omega_blackwell_codegen_tc2.o build/omega_blackwell_codegen_tern.o build/omega_blackwell_encoder.o build/omega_blackwell_engine.o build/omega_blackwell_matmul.o build/omega_blackwell_qmd.o build/omega_blackwell_realize.o build/omega_blackwell_submit.o build/omega_gpu_attention_api.o build/omega_gpu_bf16_gemv_api.o build/omega_gpu_chip.o build/omega_gpu_cmdlist.o build/omega_gpu_decode_attn256_api.o build/omega_gpu_decode_attn_api.o build/omega_gpu_decode_graph.o build/omega_gpu_elementwise_api.o build/omega_gpu_engine.o build/omega_gpu_fmt.o build/omega_gpu_fp8_api.o build/omega_gpu_fp8_pack.o build/omega_gpu_fp8_slots.o build/omega_gpu_fp8gemm_api.o build/omega_gpu_gdn_api.o build/omega_gpu_gemm.o build/omega_gpu_gemm_ex.o build/omega_gpu_hostmath.o build/omega_gpu_matmul_api.o build/omega_gpu_mem_cached.o build/omega_gpu_moe_api.o build/omega_gpu_pfgemm.o build/omega_gpu_pfhost.o build/omega_gpu_platform.o build/omega_gpu_platform_linux.o build/omega_gpu_prefill.o build/omega_gpu_prefill_attn256_api.o build/omega_gpu_prefill_attn_api.o build/omega_gpu_q36_graph.o build/omega_gpu_q38_graph.o build/omega_gpu_router_api.o build/omega_gpu_scan.o build/omega_gpu_scan_ref.o build/omega_gpu_serving.o build/omega_gpu_session.o build/omega_gpu_tcgemm_api.o build/omega_gpu_tern_api.o build/omega_gpu_wait.o build/omega_fp8_slot_table.o build/omega_cubin.o build/omega_native_artifact.o build/omega_sm121_disasm.o build/sha256.o build/omega_canonical.o build/omega_core.o build/omega_vector.o build/cudapp_om_ampere_enc_sm86.o build/cudapp_om_amp_runtime.o build/cudapp_om_binding.o build/cudapp_om_bw_enc_sm121.o build/cudapp_om_cache_runtime.o build/cudapp_om_closure_alloc.o build/cudapp_om_closure_bind.o build/cudapp_om_closure_blob.o build/cudapp_om_closure_code.o build/cudapp_om_closure_cover.o build/cudapp_om_closure_dir.o build/cudapp_om_closure_intent.o build/cudapp_om_closure_launch.o build/cudapp_om_closure_pack.o build/cudapp_om_closure_reseal.o build/cudapp_om_closure_seal.o build/cudapp_om_closure_series.o build/cudapp_om_code_digest.o build/cudapp_om_command.o build/cudapp_om_compose.o build/cudapp_om_context_lookup.o build/cudapp_om_cubin_elf.o build/cudapp_om_digest.o build/cudapp_om_encode.o build/cudapp_om_graph.o build/cudapp_om_lease.o build/cudapp_om_moe_admission.o build/cudapp_om_moe_group.o build/cudapp_om_moe_ids32.o build/cudapp_om_moe_maps32.o build/cudapp_om_moe_plan.o build/cudapp_om_moe_segs32.o build/cudapp_om_moe_windows.o build/cudapp_om_q38_emit.o build/cudapp_om_q38_runtime.o build/cudapp_om_quant_wire.o build/cudapp_om_repack.o build/cudapp_om_slotrev.o build/cudapp_om_templates.o build/cudapp_om_validate.o build/cudapp_freest_revision.o build/cudapp_freest_epoch.o build/cudapp_freest_f16.o build/m16_native.o build/nvrm.o +cc -std=gnu11 -Wall -Wextra -Werror -MMD -MP -D_GNU_SOURCE -O2 -Isrc -I./m16 -I./nvrm -I./third_party/nvidia-open-580.173.02/src/common/sdk/nvidia/inc -I./third_party/nvidia-open-580.173.02/kernel-open/common/inc -I./third_party/nvidia-open-580.173.02/kernel-open/nvidia-uvm -I./third_party/nvidia-open-580.173.02/src/nvidia/arch/nvalloc/unix/include -DCUDAPP_MIGRATION_PENDING_GEMM_EX=1 -ffp-contract=off -o build/gpu_latmeas_test tests/gpu_latmeas_test.c build/libomega_gpu.a -lpthread -lm +latmeas: doctor (outside the chip lock; NOT READY aborts before flock) +cudapp doctor: is this machine ready for CUDA++? +Diagnostic only: direct library use has no driver-version or PCI-ID admission guard. + info system: Linux 7.0.0-1019-nvidia aarch64 + ok NVIDIA kernel module: open module 580.173.02 (CUDA++ speaks 580.173.02) + ok device files: /dev/nvidiactl /dev/nvidia0 /dev/nvidia-uvm can be opened by this user + ok GPU 000f:01:00.0: NVIDIA GB10 (sm_121), supported target (historical summary: evidence/m16-blackwell-native-path-requalification-receipt.json) + raw M16 evidence unavailable; not current-commit qualification + ok memory: 121.7 GiB shared by the CPU and the GPU, 98.9 GiB available now + info CUDA toolkit: not needed and not used by CUDA++ (one is installed at /usr/local/cuda/bin/nvcc; that is fine) + info compiler: this doctor was built by 13.3.0; CUDA++ needs only make and a C compiler +READY. Next: make hello-gpu +latmeas plan: 7 cases, 11 runs, sweep 1..15 (HMMA 1..36), 1408 launches +estimated chip window: 71 s upper bound (1408 launches; 50 ms each covers a cold shader invalidate). Usually shorter. +The runner wraps only the launch with: flock /tmp/aien-gb10.lock timeout 600 +One warp, 32 threads. Clock: CS2R idx 0x50. Mesa 01bab9302a6bf0a082c7ba8c49d74427cf10b0cf +emitted kernels in latmeas-out-7ec0184/kernels (1408 launches when run) +# CUDA++ fixed latencies vs NAK + +Mesa commit `01bab9302a6bf0a082c7ba8c49d74427cf10b0cf`. +Clock: nak_private.h:135 NAK_SV_CLOCK_LO = 0x50; from_nir.rs:3522 SM>=50 uses CS2R; sm70_encode.rs:4278 opcode 0x805, index bits 72..80. +Priority 1 pairs match NAK on paper. They are measured because that match is circular. +Rows below are the ones that differ. A blank NAK stall means NAK calls the op decoupled. + +| id | pair | CUDA++ | NAK | kernel | +|---|---|---|---|---| +| pred_attention | Predicate write -> guard, attention emitter | 13 | 13 + 1 = 14 | tests/gpu_latmeas_test.c (pred) | +| hmma_reader | HMMA -> non-HMMA reader (FADD of d0) | 15 + NOP 14 = 29 | 19 + 9 = 28 | tests/gpu_latmeas_test.c (hmma_reader) | +| imad_wide_lo | IMAD.WIDE low half -> IADD3 | 6 (blanket K_FIXED; IMAD.WIDE is not in fixed_pair_class) | 3 + 1 = 4 | tests/gpu_latmeas_test.c (imad_wide_lo) | +| fmnmx_alu | FMNMX (dualalu) -> IADD3 (alu) | 6 (blanket K_FIXED; FMNMX is not in fixed_pair_class) | 4 + 1 = 5 | tests/gpu_latmeas_test.c (fmnmx_alu) | +| qmma_reader | QMMA (IMMA) -> next reader | 15 + NOP 15 = 30 | 19 + 7 = 26 | not generated (patched QMMA placeholder; see REPRODUCE.md) | +| numeric_min | Fixed-producer minimum in the numeric checker | 5 (any fixed producer; mixed pairs need 6) | 5 + 1 = 6 for mixed ALU/FMA | tests/gpu_latmeas_test.c (mixed) | +| bssy | BSSY stall | 1, no scoreboard | decoupled (1 & sb) | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| bsync | BSYNC stall | 5, no scoreboard | decoupled (1 & sb) | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| bra | BRA stall | 5 | decoupled (1 & sb) | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| exit_op | EXIT stall | 5 | decoupled (1 & sb) | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| bar | BAR.SYNC stall | 6 | decoupled AGU (1 & sb) | not generated (NAK class is decoupled, not a fixed GPR RAW) | + +latmeas: taking flock /tmp/aien-gb10.lock for the launch only; timeout 600 +latmeas: the launch bound is the estimated chip window printed above +latmeas plan: 7 cases, 11 runs, sweep 1..15 (HMMA 1..36), 1408 launches +estimated chip window: 71 s upper bound (1408 launches; 50 ms each covers a cold shader invalidate). Usually shorter. +The runner wraps only the launch with: flock /tmp/aien-gb10.lock timeout 600 +One warp, 32 threads. Clock: CS2R idx 0x50. Mesa 01bab9302a6bf0a082c7ba8c49d74427cf10b0cf +overhead ovh32 run 1/11 rc 0 +overhead ovh32 run 2/11 rc 0 +overhead ovh32 run 3/11 rc 0 +overhead ovh32 run 4/11 rc 0 +overhead ovh32 run 5/11 rc 0 +overhead ovh32 run 6/11 rc 0 +overhead ovh32 run 7/11 rc 0 +overhead ovh32 run 8/11 rc 0 +overhead ovh32 run 9/11 rc 0 +overhead ovh32 run 10/11 rc 0 +overhead ovh32 run 11/11 rc 0 +overhead ovh1 run 1/11 rc 0 +overhead ovh1 run 2/11 rc 0 +overhead ovh1 run 3/11 rc 0 +overhead ovh1 run 4/11 rc 0 +overhead ovh1 run 5/11 rc 0 +overhead ovh1 run 6/11 rc 0 +overhead ovh1 run 7/11 rc 0 +overhead ovh1 run 8/11 rc 0 +overhead ovh1 run 9/11 rc 0 +overhead ovh1 run 10/11 rc 0 +overhead ovh1 run 11/11 rc 0 +case alu_alu run 1/11 +case alu_alu run 2/11 +case alu_alu run 3/11 +case alu_alu run 4/11 +case alu_alu run 5/11 +case alu_alu run 6/11 +case alu_alu run 7/11 +case alu_alu run 8/11 +case alu_alu run 9/11 +case alu_alu run 10/11 +case alu_alu run 11/11 +case fma_fma run 1/11 +case fma_fma run 2/11 +case fma_fma run 3/11 +case fma_fma run 4/11 +case fma_fma run 5/11 +case fma_fma run 6/11 +case fma_fma run 7/11 +case fma_fma run 8/11 +case fma_fma run 9/11 +case fma_fma run 10/11 +case fma_fma run 11/11 +case mixed run 1/11 +case mixed run 2/11 +case mixed run 3/11 +case mixed run 4/11 +case mixed run 5/11 +case mixed run 6/11 +case mixed run 7/11 +case mixed run 8/11 +case mixed run 9/11 +case mixed run 10/11 +case mixed run 11/11 +case pred run 1/11 +case pred run 2/11 +case pred run 3/11 +case pred run 4/11 +case pred run 5/11 +case pred run 6/11 +case pred run 7/11 +case pred run 8/11 +case pred run 9/11 +case pred run 10/11 +case pred run 11/11 +case fmnmx_alu run 1/11 +case fmnmx_alu run 2/11 +case fmnmx_alu run 3/11 +case fmnmx_alu run 4/11 +case fmnmx_alu run 5/11 +case fmnmx_alu run 6/11 +case fmnmx_alu run 7/11 +case fmnmx_alu run 8/11 +case fmnmx_alu run 9/11 +case fmnmx_alu run 10/11 +case fmnmx_alu run 11/11 +case imad_wide_lo run 1/11 +case imad_wide_lo run 2/11 +case imad_wide_lo run 3/11 +case imad_wide_lo run 4/11 +case imad_wide_lo run 5/11 +case imad_wide_lo run 6/11 +case imad_wide_lo run 7/11 +case imad_wide_lo run 8/11 +case imad_wide_lo run 9/11 +case imad_wide_lo run 10/11 +case imad_wide_lo run 11/11 +case hmma_reader run 1/11 +case hmma_reader run 2/11 +case hmma_reader run 3/11 +case hmma_reader run 4/11 +case hmma_reader run 5/11 +case hmma_reader run 6/11 +case hmma_reader run 7/11 +case hmma_reader run 8/11 +case hmma_reader run 9/11 +case hmma_reader run 10/11 +case hmma_reader run 11/11 +wrote latmeas-out-7ec0184/raw.json +latmeas: wrote latmeas-out-7ec0184/RECEIPT.txt diff --git a/evidence/LATMEAS-GB10-20261101-oracle/sm121_vs_NAK.md b/evidence/LATMEAS-GB10-20261101-oracle/sm121_vs_NAK.md new file mode 100644 index 00000000..283e0e84 --- /dev/null +++ b/evidence/LATMEAS-GB10-20261101-oracle/sm121_vs_NAK.md @@ -0,0 +1,42 @@ +# GB10 (sm_121) measured latencies vs NAK + +Chip: NVIDIA GB10 (DGX Spark), sm_121 +Driver: NVRM version: NVIDIA UNIX Open Kernel Module for aarch64 580.173.02 Release Build (dvs-builder@U22-A24-5-4) Tue Jun 23 08:34:19 UTC 2026. No NVIDIA user-space libraries. +Method: dependent-chain timing kernel. One warp (32 threads); every lane stores, and a stall matches only when every lane matches the oracle. +Each fixed-latency producer is issued at a programmed stall. The GB10 column is the smallest stall S such that S and every larger S in the sweep matched the oracle on every run. Cycles are the CS2R clock delta (NAK_SV_CLOCK_LO, 0x50) around that chain, median and max-min. +Source: tests/gpu_latmeas_test.c in aien-dev/CUDApp +How to rerun: sh tools/latmeas/run_spark.sh OUTPUT_DIR + +NAK reference: `nak/sm120_instr_latencies.rs` plus `latencies/sm100/*.csv` at Mesa commit 01bab9302a6bf0a082c7ba8c49d74427cf10b0cf + +## Priority 1: values CUDA++ currently borrows from NAK (needs independent measurement) + +| Writer -> reader | NAK value | NAK source line | GB10 measured | Delta | Runs / spread | Kernel | +|---|---|---|---|---|---|---| +| ALU -> ALU (GPR RAW) | 4 + 1 = 5 | latencies/sm100/reg_raw.csv:2 row alu column alu = 4; sm120_instr_latencies.rs:457 GPR raw adds max(adjust, 1); IAdd3 is Alu at line 44 | 4 | -1 | 11 runs; stall spread 0; cycles at that stall median 2541 spread 0; stall-1 cycles median 2477 spread 0; loop overhead median 1261 | tests/gpu_latmeas_test.c (alu_alu) | +| FMA -> FMA (GPR RAW) | 4 + 1 = 5 | latencies/sm100/reg_raw.csv:5 row fma column fma = 4; sm120_instr_latencies.rs:59 FFma is Fma; line 457 adds 1 | 4 | -1 | 11 runs; stall spread 0; cycles at that stall median 2541 spread 0; stall-1 cycles median 2477 spread 0; loop overhead median 1261 | tests/gpu_latmeas_test.c (fma_fma) | +| Mixed ALU <-> FMA (GPR RAW) | 5 + 1 = 6 | latencies/sm100/reg_raw.csv:2 row alu column fma = 5 and line 5 row fma column alu = 5; sm120_instr_latencies.rs:457 adds 1 | 5 | -1 | 11 runs; stall spread 0; cycles at that stall median 4362 spread 0; stall-1 cycles median 4170 spread 0; loop overhead median 1261 | tests/gpu_latmeas_test.c (mixed) | +| Predicate write -> guard | 13 + 1 = 14 | latencies/sm100/pred_raw.csv:11 row redirected_fp64 column dualalu = 13; sm120_instr_latencies.rs:474 predicate raw(write, None) uses reader RedirectedFp64 and adds 1; ISetP is Dualalu at line 49 | 13 | -1 | 11 runs; stall spread 0; cycles at that stall median 2714 spread 0; stall-1 cycles median 70 spread 0; loop overhead median 1261 | tests/gpu_latmeas_test.c (pred) | + +## Priority 2: instructions where CUDA++ (nvcc-derived) and NAK disagree + +| Instruction / pair | NAK value | nvcc sm_121 stall | GB10 measured | Agrees with | Kernel | +|---|---|---|---|---|---| +| Predicate write -> guard, attention emitter | 13 + 1 = 14 | 13 | 13 | nvcc | tests/gpu_latmeas_test.c (pred) | +| HMMA -> non-HMMA reader (FADD of d0) | 19 + 9 = 28 | 15 + NOP 14 = 29 | 20 | neither | tests/gpu_latmeas_test.c (hmma_reader) | +| IMAD.WIDE low half -> IADD3 | 3 + 1 = 4 | 6 (blanket K_FIXED; IMAD.WIDE is not in fixed_pair_class) | 3 | neither | tests/gpu_latmeas_test.c (imad_wide_lo) | +| FMNMX (dualalu) -> IADD3 (alu) | 4 + 1 = 5 | 6 (blanket K_FIXED; FMNMX is not in fixed_pair_class) | 4 | neither | tests/gpu_latmeas_test.c (fmnmx_alu) | +| QMMA (IMMA) -> next reader | 19 + 7 = 26 | 15 + NOP 15 = 30 | | | not generated (patched QMMA placeholder; see REPRODUCE.md) | +| Fixed-producer minimum in the numeric checker | 5 + 1 = 6 for mixed ALU/FMA | 5 (any fixed producer; mixed pairs need 6) | 5 | nvcc | tests/gpu_latmeas_test.c (mixed) | +| BSSY stall | decoupled (1 & sb) | 1, no scoreboard | | | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| BSYNC stall | decoupled (1 & sb) | 5, no scoreboard | | | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| BRA stall | decoupled (1 & sb) | 5 | | | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| EXIT stall | decoupled (1 & sb) | 5 | | | not generated (NAK class is decoupled, not a fixed GPR RAW) | +| BAR.SYNC stall | decoupled AGU (1 & sb) | 6 | | | not generated (NAK class is decoupled, not a fixed GPR RAW) | + +## Notes +- Anything not fixed-latency (LDG/LDS, scoreboard waits) is out of scope for this table. +- Decoupled ops (BSSY, BSYNC, BRA, EXIT, BAR) are listed because the encoder gives them a fixed stall and NAK does not. They have no dependent-chain kernel. +- HMMA -> HMMA in NAK is 20 + 9 = 29, the same total the encoder uses. The HMMA row is the non-HMMA reader (28). +- QMMA is not generated here. Its encoder stall is 30 and NAK's IMMA -> ALU total is 26 (IMMA -> IMMA is 27). +- Confidence notes and anomalies go here. From 0c81899c96f16db6525f089c16be5b5bbe11ea2f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 1 Nov 2026 18:52:52 +0000 Subject: [PATCH 09/11] Record per-word mismatch counts and name the GB10 from PCI. word_mismatch_max counts lanes that miss words 2..5. The receipt uses PCI 10de:2e12 when proc Model is Unknown, which is the doctor's GB10 row. Co-authored-by: aien-spark --- tests/gpu_latmeas_test.c | 83 +++++++++++++++++++++++++++++++++----- tools/latmeas/REPRODUCE.md | 2 +- tools/latmeas/run_spark.sh | 41 +++++++++++++++++-- 3 files changed, 112 insertions(+), 14 deletions(-) diff --git a/tests/gpu_latmeas_test.c b/tests/gpu_latmeas_test.c index 86ac4b19..28234da6 100644 --- a/tests/gpu_latmeas_test.c +++ b/tests/gpu_latmeas_test.c @@ -56,12 +56,14 @@ #include "omega_gpu_session.h" #include "omega_sm121_disasm.h" #include +#include #include #include #include #include #include #include +#include #define NOREG (-1) #define SB_NONE 7u @@ -1276,11 +1278,34 @@ static void print_plan(void) { static int mkdir_p(const char *dir); +/* fopen("w") creates the file mode 0666. These outputs are not secrets, and they are not world-writable. */ +static FILE *fopen_0644(const char *path, const char *mode) { + int fd = open(path, O_CREAT | O_TRUNC | O_WRONLY, 0644); + if (fd < 0) return NULL; + FILE *f = fdopen(fd, mode); + if (!f) close(fd); + return f; +} + +/* Manifest ids are our own tokens. Reject anything else before it becomes a path. */ +static int id_token(const char *s, char out[64]) { + size_t n = 0; + if (!s || !s[0]) return 0; + for (const char *p = s; *p; p++) { + char c = *p; + int ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; + if (!ok || n >= 63) return 0; + out[n++] = c; + } + out[n] = 0; + return 1; +} + static int emit_kernels(const char *dir) { if (mkdir_p(dir) != 0) return -1; char path[512], manp[512]; snprintf(manp, sizeof manp, "%s/manifest.txt", dir); - FILE *man = fopen(manp, "w"); + FILE *man = fopen_0644(manp, "w"); if (!man) return -1; uint8_t code[CODE_CAP]; for (int ci = 0; ci < NCASES; ci++) { @@ -1299,7 +1324,7 @@ static int emit_kernels(const char *dir) { return -1; } snprintf(path, sizeof path, "%s/%s_%u.bin", dir, CASES[ci].id, s); - FILE *f = fopen(path, "wb"); + FILE *f = fopen_0644(path, "wb"); if (!f || fwrite(code, 1, len, f) != len) { if (f) fclose(f); fclose(man); @@ -1326,7 +1351,7 @@ static int emit_kernels(const char *dir) { return -1; } snprintf(path, sizeof path, "%s/ovh%u_0.bin", dir, ovh_iters[k]); - FILE *f = fopen(path, "wb"); + FILE *f = fopen_0644(path, "wb"); if (!f || fwrite(code, 1, len, f) != len) { if (f) fclose(f); fclose(man); @@ -1405,10 +1430,16 @@ static Kern *load_manifest(const char *dir, int *nout) { t.ops = ops; t.gpr = gpr; t.nbytes = nbytes; + char safe[64]; + if (!id_token(t.id, safe) || nbytes == 0 || nbytes > CODE_CAP) { + free(k); + fclose(man); + return NULL; + } char path[512]; - snprintf(path, sizeof path, "%s/%s_%u.bin", dir, t.id, t.stall); + snprintf(path, sizeof path, "%s/%s_%u.bin", dir, safe, t.stall); FILE *f = fopen(path, "rb"); - t.code = f ? malloc(nbytes ? nbytes : 1) : NULL; + t.code = malloc(CODE_CAP); if (!f || !t.code || fread(t.code, 1, nbytes, f) != nbytes) { if (f) fclose(f); free(t.code); @@ -1518,6 +1549,16 @@ static int lanes_report(const uint32_t *mem, const uint32_t o[4], LaneMiss *miss return n; } +/* out[0..3] count lanes whose words 2, 3, 4 and 5 differ from the oracle. */ +static void word_fail_counts(const uint32_t *mem, const uint32_t o[4], uint8_t out[4]) { + out[0] = out[1] = out[2] = out[3] = 0; + for (int lane = 0; lane < LANE_N; lane++) { + const uint32_t *w = mem + (size_t)lane * LANE_WORDS; + for (int k = 0; k < 4; k++) + if (w[2 + k] != o[k] && out[k] < 255) out[k]++; + } +} + static uint32_t lane_fail_bits(const uint32_t *mem, const uint32_t o[4]) { uint32_t mask = 0; for (int lane = 0; lane < LANE_N; lane++) { @@ -1538,7 +1579,7 @@ static int mem_match(const uint32_t *mem, uint32_t result, uint32_t aux, int fou } static void fprint_lane_fields(FILE *jf, const uint32_t o[4], const uint8_t *matched_min, const uint32_t *fail_mask, - const LaneMiss *miss, int sweep, int have) { + const uint8_t *word_max, const LaneMiss *miss, int sweep, int have) { fprintf(jf, " \"oracle\": [%u, %u, %u, %u],\n", o[0], o[1], o[2], o[3]); fprintf(jf, " \"reference_result\": %u,\n", o[0]); fprintf(jf, " \"lanes_matched_min\": ["); @@ -1555,6 +1596,14 @@ static void fprint_lane_fields(FILE *jf, const uint32_t o[4], const uint8_t *mat else fprintf(jf, "%s\"0x%08x\"", s ? ", " : "", fail_mask[s]); } + fprintf(jf, "],\n \"word_mismatch_max\": ["); + for (int s = 0; s < sweep; s++) { + const uint8_t *w = word_max + (size_t)s * 4; + if (!have || matched_min[s] == 0xff) + fprintf(jf, "%snull", s ? ", " : ""); + else + fprintf(jf, "%s[%u, %u, %u, %u]", s ? ", " : "", w[0], w[1], w[2], w[3]); + } fprintf(jf, "],\n \"first_mismatch\": ["); for (int s = 0; s < sweep; s++) { if (!have || matched_min[s] == 0xff || miss[s].lane < 0) @@ -1601,8 +1650,12 @@ static int lane_field_self_test(void) { "a planted word-4 mismatch names lane 7"); LCHECK(lane_fail_bits(mem, o) == (1u << 7), "fail mask is lane 7"); + uint8_t wc[4]; + word_fail_counts(mem, o, wc); + LCHECK(wc[0] == 0 && wc[1] == 0 && wc[2] == 1 && wc[3] == 0, "only word 4 mismatches"); uint8_t mins[2] = {31, 32}; uint32_t masks[2] = {1u << 7, 0}; + uint8_t words[2][4] = {{0, 0, 1, 0}, {0, 0, 0, 0}}; LaneMiss ms[2]; ms[0] = m; ms[0].run = 3; @@ -1611,12 +1664,13 @@ static int lane_field_self_test(void) { size_t tlen = 0; FILE *mf = open_memstream(&text, &tlen); if (mf) { - fprint_lane_fields(mf, o, mins, masks, ms, 2, 1); + fprint_lane_fields(mf, o, mins, masks, (const uint8_t *)words, ms, 2, 1); fclose(mf); } LCHECK(text && strstr(text, "\"oracle\": [") && strstr(text, "\"lanes_matched_min\": [31, 32]") && strstr(text, "\"lane_fail_mask\"") && strstr(text, "\"lane\": 7") && strstr(text, "\"word\": 4") && - strstr(text, "\"run\": 3") && strstr(text, ", null]"), + strstr(text, "\"run\": 3") && strstr(text, "\"word_mismatch_max\": [[0, 0, 1, 0], [0, 0, 0, 0]]") && + strstr(text, ", null]"), "json lane fields: %s", text ? text : "(none)"); free(text); @@ -1672,6 +1726,7 @@ static int run_launch(const char *kdir, const char *out_path) { uint32_t run0_res[NCASES][SWEEP_MAX]; uint8_t lane_min[NCASES][SWEEP_MAX]; uint32_t fail_mask[NCASES][SWEEP_MAX]; + uint8_t word_max[NCASES][SWEEP_MAX][4]; LaneMiss first_miss[NCASES][SWEEP_MAX]; memset(pass, 0, sizeof pass); memset(cyc, 0, sizeof cyc); @@ -1679,6 +1734,7 @@ static int run_launch(const char *kdir, const char *out_path) { memset(run0_res, 0, sizeof run0_res); memset(lane_min, 0xff, sizeof lane_min); memset(fail_mask, 0, sizeof fail_mask); + memset(word_max, 0, sizeof word_max); for (int ci = 0; ci < NCASES; ci++) for (int s = 0; s < SWEEP_MAX; s++) first_miss[ci][s].lane = -1; char notes[NCASES][384]; @@ -1761,6 +1817,12 @@ static int run_launch(const char *kdir, const char *out_path) { int nmatch = lanes_report(mem, oracle, &one); if (lane_min[ci][s - 1] == 0xff || nmatch < lane_min[ci][s - 1]) lane_min[ci][s - 1] = (uint8_t)nmatch; fail_mask[ci][s - 1] |= lane_fail_bits(mem, oracle); + { + uint8_t wc[4]; + word_fail_counts(mem, oracle, wc); + for (int k = 0; k < 4; k++) + if (wc[k] > word_max[ci][s - 1][k]) word_max[ci][s - 1][k] = wc[k]; + } if (nmatch < LANE_N && first_miss[ci][s - 1].lane < 0) { one.run = r; first_miss[ci][s - 1] = one; @@ -1778,7 +1840,7 @@ write_json:; int locked = 1; char tmp[512]; snprintf(tmp, sizeof tmp, "%s.tmp", out_path); - FILE *jf = fopen(tmp, "w"); + FILE *jf = fopen_0644(tmp, "w"); if (!jf) { if (locked) { omega_gpu_session_unlock(); @@ -1880,7 +1942,8 @@ write_json:; { uint32_t oracle[4]; oracle_words(c, oracle); - fprint_lane_fields(jf, oracle, lane_min[ci], fail_mask[ci], first_miss[ci], (int)c->sweep, !device_error); + fprint_lane_fields(jf, oracle, lane_min[ci], fail_mask[ci], (const uint8_t *)word_max[ci], first_miss[ci], + (int)c->sweep, !device_error); } fprintf(jf, " \"run0_result\": ["); for (unsigned s = 1; s <= c->sweep; s++) diff --git a/tools/latmeas/REPRODUCE.md b/tools/latmeas/REPRODUCE.md index 22a73cb1..b93881d7 100644 --- a/tools/latmeas/REPRODUCE.md +++ b/tools/latmeas/REPRODUCE.md @@ -42,7 +42,7 @@ The 2026-11-01 GB10 run of `03cdcc5` merged with PR #251 is `evidence/LATMEAS-GB The kernels after that receipt store eight words from every lane: the two clocks, four result words, and the lane id. HMMA's four result words are `FADD` of `d0` through `d3` against zero. The host value is sixteen products of bf16 1.0, not the stall-36 launch. A stall matches only when all 32 lanes match. -`evidence/LATMEAS-GB10-20261101-allanes/` is the GB10 run of that all-lane kernel (`c94c2c9`, merge of `4f3a4d0` and PR #251). Its `raw.json` still has one lane-0 word per stall and no host oracle, so the file alone cannot show that every lane and every HMMA destination matched. The source of that commit did require the full match before it recorded a stall. A later `raw.json` adds `oracle` (four host words), `reference_result` (the first of those), `lanes_matched_min`, `lane_fail_mask`, and `first_mismatch` (run, lane, word, got, want). `run_spark.sh` writes `RECEIPT.txt` after the launch: HEAD, parent SHAs, kernel, driver, device, gcc, UTC time, and the sha256 of `raw.json`. The launch bound is the binary's estimated chip window, not a second number in the script. +`evidence/LATMEAS-GB10-20261101-allanes/` is the GB10 run of that all-lane kernel (`c94c2c9`, merge of `4f3a4d0` and PR #251). Its `raw.json` still has one lane-0 word per stall and no host oracle, so the file alone cannot show that every lane and every HMMA destination matched. The source of that commit did require the full match before it recorded a stall. A later `raw.json` adds `oracle` (four host words), `reference_result` (the first of those), `lanes_matched_min`, `lane_fail_mask`, and `first_mismatch` (run, lane, word, got, want). `run_spark.sh` writes `RECEIPT.txt` after the launch: HEAD, parent SHAs, kernel, driver, device, gcc, UTC time, and the sha256 of `raw.json`. If proc `Model` is `Unknown` and the PCI id is `10de:2e12`, the model line is `NVIDIA GB10 (sm_121)`, the same name the doctor takes from `src/omega_gpu_chip.c`. The launch bound is the binary's estimated chip window: 50 ms per launch, rounded up. 1408 launches print 71 s. A run of about a second is inside that bound. ## NAK pin diff --git a/tools/latmeas/run_spark.sh b/tools/latmeas/run_spark.sh index 5e785e95..ab1095da 100755 --- a/tools/latmeas/run_spark.sh +++ b/tools/latmeas/run_spark.sh @@ -34,11 +34,34 @@ write_receipt() { echo "driver: unread" fi found=0 - for info in /proc/driver/nvidia/gpus/*/information; do + sysroot=${CUDAPP_RECEIPT_SYSROOT:-} + for info in ${CUDAPP_RECEIPT_GPU_INFO:-/proc/driver/nvidia/gpus/*/information}; do [ -f "$info" ] || continue found=1 - echo "device: $(basename "$(dirname "$info")")" - sed -n 's/^Model:[[:space:]]*/model: /p; s/^Bus Location:[[:space:]]*/bus: /p' "$info" + bdf=$(basename "$(dirname "$info")") + echo "device: $bdf" + model=$(sed -n 's/^Model:[[:space:]]*//p' "$info" | head -n 1) + bus=$(sed -n 's/^Bus Location:[[:space:]]*//p' "$info" | head -n 1) + # The open module prints "Model: Unknown" for the GB10. The doctor names + # that chip from PCI id 0x2e12 (src/omega_gpu_chip.c). + if [ -z "$model" ] || [ "$model" = "Unknown" ]; then + ven=$(cat "$sysroot/sys/bus/pci/devices/$bdf/vendor" 2>/dev/null || true) + dev=$(cat "$sysroot/sys/bus/pci/devices/$bdf/device" 2>/dev/null || true) + ven=$(printf '%s' "$ven" | tr -d '[:space:]') + dev=$(printf '%s' "$dev" | tr -d '[:space:]') + if [ "$ven" = "0x10de" ] && [ "$dev" = "0x2e12" ]; then + echo "model: NVIDIA GB10 (sm_121)" + echo "model_source: pci 0x2e12; proc Model was ${model:-empty}" + else + echo "model: Unknown" + echo "model_source: proc Model was ${model:-empty}; pci ${ven:-unread}:${dev:-unread}" + fi + else + echo "model: $model" + fi + if [ -n "$bus" ]; then + echo "bus: $bus" + fi done if [ "$found" -eq 0 ]; then echo "device: none" @@ -66,6 +89,18 @@ if [ "$1" = "--receipt-self-test" ]; then grep -q '^device: ' "$tmp/RECEIPT.txt" test "$got" = "$want" rm -rf "$tmp" + # proc Model "Unknown" plus PCI 10de:2e12 is the GB10 the doctor names. + fake=$(mktemp -d) + mkdir -p "$fake/gpus/000f:01:00.0" "$fake/sys/bus/pci/devices/000f:01:00.0" + printf 'Model: Unknown\nBus Location: 000f:01:00.0\n' > "$fake/gpus/000f:01:00.0/information" + printf '0x10de\n' > "$fake/sys/bus/pci/devices/000f:01:00.0/vendor" + printf '0x2e12\n' > "$fake/sys/bus/pci/devices/000f:01:00.0/device" + printf '{}\n' > "$fake/raw.json" + CUDAPP_RECEIPT_SYSROOT=$fake CUDAPP_RECEIPT_GPU_INFO=$fake/gpus/000f:01:00.0/information \ + write_receipt "$fake" + grep -q '^model: NVIDIA GB10 (sm_121)$' "$fake/RECEIPT.txt" + grep -q '^model_source: pci 0x2e12' "$fake/RECEIPT.txt" + rm -rf "$fake" echo "receipt self-test PASS" exit 0 fi From 4a5184b0df00f06cb117c9745faff0ccf18f37a6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 2 Nov 2026 00:00:28 +0000 Subject: [PATCH 10/11] Load kernel files from the case table, not the manifest text. The path passed to fopen is the case id and a stall from that table. The scanned token is only compared. Emit and the stall sweep are unchanged. Co-authored-by: aien-spark --- tests/gpu_latmeas_test.c | 71 +++++++++++++++++++++++++++++++--------- 1 file changed, 56 insertions(+), 15 deletions(-) diff --git a/tests/gpu_latmeas_test.c b/tests/gpu_latmeas_test.c index 28234da6..e875a001 100644 --- a/tests/gpu_latmeas_test.c +++ b/tests/gpu_latmeas_test.c @@ -1287,18 +1287,31 @@ static FILE *fopen_0644(const char *path, const char *mode) { return f; } -/* Manifest ids are our own tokens. Reject anything else before it becomes a path. */ -static int id_token(const char *s, char out[64]) { - size_t n = 0; - if (!s || !s[0]) return 0; - for (const char *p = s; *p; p++) { - char c = *p; - int ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; - if (!ok || n >= 63) return 0; - out[n++] = c; - } - out[n] = 0; - return 1; +/* The manifest name is chosen from the case table, not copied out of the file. + * The scanned token is only compared. The name and stall written here come from + * that table, so the path passed to fopen is not the fscanf text. */ +static int canonical_kernel(const char *scanned, unsigned stall, char name[64], unsigned *stall_out) { + if (!scanned || !name || !stall_out) return 0; + name[0] = 0; + for (int i = 0; i < NCASES; i++) { + if (strcmp(scanned, CASES[i].id) != 0) continue; + for (unsigned s = 1; s <= CASES[i].sweep; s++) { + if (s != stall) continue; + snprintf(name, 64, "%s", CASES[i].id); + *stall_out = s; + return 1; + } + return 0; + } + for (int i = 0; i < NCASES; i++) { + char ovh[32]; + snprintf(ovh, sizeof ovh, "ovh%u", CASES[i].iters); + if (strcmp(scanned, ovh) != 0 || stall != 0) continue; + snprintf(name, 64, "ovh%u", CASES[i].iters); + *stall_out = 0; + return 1; + } + return 0; } static int emit_kernels(const char *dir) { @@ -1430,14 +1443,17 @@ static Kern *load_manifest(const char *dir, int *nout) { t.ops = ops; t.gpr = gpr; t.nbytes = nbytes; - char safe[64]; - if (!id_token(t.id, safe) || nbytes == 0 || nbytes > CODE_CAP) { + char name[64]; + unsigned stall_used = 0; + if (!canonical_kernel(t.id, stall, name, &stall_used) || nbytes == 0 || nbytes > CODE_CAP) { free(k); fclose(man); return NULL; } + snprintf(t.id, sizeof t.id, "%s", name); + t.stall = stall_used; char path[512]; - snprintf(path, sizeof path, "%s/%s_%u.bin", dir, safe, t.stall); + snprintf(path, sizeof path, "%s/%s_%u.bin", dir, name, stall_used); FILE *f = fopen(path, "rb"); t.code = malloc(CODE_CAP); if (!f || !t.code || fread(t.code, 1, nbytes, f) != nbytes) { @@ -1688,6 +1704,31 @@ static int lane_field_self_test(void) { LCHECK(build(&CASES[6], 1, 0, 0, code, sizeof code, &len, &gpr) == 0, "hmma stall 1 encode"); LCHECK(lat_sim(lane0) == 0, "hmma stall 1 sim"); LCHECK(lanes_report(g_sim_mem, o, &m) == 0 && m.word >= 2 && m.word <= 5, "hmma stall 1 fails a destination word"); + char tmpl[] = "/tmp/latmeas-load-XXXXXX"; + char *dump = mkdtemp(tmpl); + LCHECK(dump != NULL, "load-check directory"); + if (dump) { + int nk = 0; + LCHECK(emit_kernels(dump) == 0, "emit for the load check"); + Kern *kk = load_manifest(dump, &nk); + int files = 0, ns = 0; + unsigned seen[8]; + for (int i = 0; i < NCASES; i++) { + files += (int)CASES[i].sweep; + int have = 0; + for (int j = 0; j < ns; j++) + if (seen[j] == CASES[i].iters) have = 1; + if (!have && ns < 8) seen[ns++] = CASES[i].iters; + } + files += ns; + LCHECK(kk && nk == files, "manifest load count %d want %d", nk, files); + LCHECK(kk && find_kern(kk, nk, "alu_alu", 1) && find_kern(kk, nk, "hmma_reader", 20) && find_kern(kk, nk, "ovh32", 0), + "loaded kernel names"); + if (kk) { + for (int i = 0; i < nk; i++) free(kk[i].code); + free(kk); + } + } #undef LCHECK return failed; } From 23f266313e95f8bae2ce8880e914537685b61bf1 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 2 Nov 2026 00:13:35 +0000 Subject: [PATCH 11/11] Record the launch command and name the predicate clock. The receipt prints the exact flock launch and its exit code. git status --porcelain marks a tree dirty when it has untracked files. Programmed stall 1 cycles stay in the table as a raw clock, not as the pass bit: the stall is the oracle sweep, and neither clock column subtracts the loop overhead. The host self-test removes the kernel dump it creates. Co-authored-by: aien-spark --- .../sm121_vs_NAK.md | 9 ++-- tests/gpu_latmeas_test.c | 13 ++++++ tools/latmeas/REPRODUCE.md | 2 +- tools/latmeas/fill_template.py | 2 +- tools/latmeas/run_spark.sh | 42 +++++++++++++++---- 5 files changed, 55 insertions(+), 13 deletions(-) diff --git a/evidence/LATMEAS-GB10-20261101-oracle/sm121_vs_NAK.md b/evidence/LATMEAS-GB10-20261101-oracle/sm121_vs_NAK.md index 283e0e84..ddfee9cc 100644 --- a/evidence/LATMEAS-GB10-20261101-oracle/sm121_vs_NAK.md +++ b/evidence/LATMEAS-GB10-20261101-oracle/sm121_vs_NAK.md @@ -13,10 +13,10 @@ NAK reference: `nak/sm120_instr_latencies.rs` plus `latencies/sm100/*.csv` at Me | Writer -> reader | NAK value | NAK source line | GB10 measured | Delta | Runs / spread | Kernel | |---|---|---|---|---|---|---| -| ALU -> ALU (GPR RAW) | 4 + 1 = 5 | latencies/sm100/reg_raw.csv:2 row alu column alu = 4; sm120_instr_latencies.rs:457 GPR raw adds max(adjust, 1); IAdd3 is Alu at line 44 | 4 | -1 | 11 runs; stall spread 0; cycles at that stall median 2541 spread 0; stall-1 cycles median 2477 spread 0; loop overhead median 1261 | tests/gpu_latmeas_test.c (alu_alu) | -| FMA -> FMA (GPR RAW) | 4 + 1 = 5 | latencies/sm100/reg_raw.csv:5 row fma column fma = 4; sm120_instr_latencies.rs:59 FFma is Fma; line 457 adds 1 | 4 | -1 | 11 runs; stall spread 0; cycles at that stall median 2541 spread 0; stall-1 cycles median 2477 spread 0; loop overhead median 1261 | tests/gpu_latmeas_test.c (fma_fma) | -| Mixed ALU <-> FMA (GPR RAW) | 5 + 1 = 6 | latencies/sm100/reg_raw.csv:2 row alu column fma = 5 and line 5 row fma column alu = 5; sm120_instr_latencies.rs:457 adds 1 | 5 | -1 | 11 runs; stall spread 0; cycles at that stall median 4362 spread 0; stall-1 cycles median 4170 spread 0; loop overhead median 1261 | tests/gpu_latmeas_test.c (mixed) | -| Predicate write -> guard | 13 + 1 = 14 | latencies/sm100/pred_raw.csv:11 row redirected_fp64 column dualalu = 13; sm120_instr_latencies.rs:474 predicate raw(write, None) uses reader RedirectedFp64 and adds 1; ISetP is Dualalu at line 49 | 13 | -1 | 11 runs; stall spread 0; cycles at that stall median 2714 spread 0; stall-1 cycles median 70 spread 0; loop overhead median 1261 | tests/gpu_latmeas_test.c (pred) | +| ALU -> ALU (GPR RAW) | 4 + 1 = 5 | latencies/sm100/reg_raw.csv:2 row alu column alu = 4; sm120_instr_latencies.rs:457 GPR raw adds max(adjust, 1); IAdd3 is Alu at line 44 | 4 | -1 | 11 runs; stall spread 0; cycles at that stall median 2541 spread 0; programmed stall 1 cycles median 2477 spread 0; loop overhead median 1261 | tests/gpu_latmeas_test.c (alu_alu) | +| FMA -> FMA (GPR RAW) | 4 + 1 = 5 | latencies/sm100/reg_raw.csv:5 row fma column fma = 4; sm120_instr_latencies.rs:59 FFma is Fma; line 457 adds 1 | 4 | -1 | 11 runs; stall spread 0; cycles at that stall median 2541 spread 0; programmed stall 1 cycles median 2477 spread 0; loop overhead median 1261 | tests/gpu_latmeas_test.c (fma_fma) | +| Mixed ALU <-> FMA (GPR RAW) | 5 + 1 = 6 | latencies/sm100/reg_raw.csv:2 row alu column fma = 5 and line 5 row fma column alu = 5; sm120_instr_latencies.rs:457 adds 1 | 5 | -1 | 11 runs; stall spread 0; cycles at that stall median 4362 spread 0; programmed stall 1 cycles median 4170 spread 0; loop overhead median 1261 | tests/gpu_latmeas_test.c (mixed) | +| Predicate write -> guard | 13 + 1 = 14 | latencies/sm100/pred_raw.csv:11 row redirected_fp64 column dualalu = 13; sm120_instr_latencies.rs:474 predicate raw(write, None) uses reader RedirectedFp64 and adds 1; ISetP is Dualalu at line 49 | 13 | -1 | 11 runs; stall spread 0; cycles at that stall median 2714 spread 0; programmed stall 1 cycles median 70 spread 0; loop overhead median 1261 | tests/gpu_latmeas_test.c (pred) | ## Priority 2: instructions where CUDA++ (nvcc-derived) and NAK disagree @@ -40,3 +40,4 @@ NAK reference: `nak/sm120_instr_latencies.rs` plus `latencies/sm100/*.csv` at Me - HMMA -> HMMA in NAK is 20 + 9 = 29, the same total the encoder uses. The HMMA row is the non-HMMA reader (28). - QMMA is not generated here. Its encoder stall is 30 and NAK's IMMA -> ALU total is 26 (IMMA -> IMMA is 27). - Confidence notes and anomalies go here. +- The predicate row's "programmed stall 1 cycles median 70" is `cycle_median_stall1`, the raw clock at programmed stall 1 (`tests/gpu_latmeas_test.c` lines 1965 and 1987). It is not stall 13 minus one, and it is not overhead-subtracted. At stall 1 the predicate guard is still the previous true `P0`, the poison branch leaves the loop, and the clocks cover that exit, so 70 is below the 1261-cycle empty loop. The pass criterion is the correctness sweep: `stall_required` 13 is `reported_threshold` (line 1223) of lanes matching the oracle (line 1884). The clock at that stall is `cycle_median` 2714 (lines 1946 and 1981), also raw, and it is above the overhead. Neither clock decides the stall. diff --git a/tests/gpu_latmeas_test.c b/tests/gpu_latmeas_test.c index e875a001..56f94229 100644 --- a/tests/gpu_latmeas_test.c +++ b/tests/gpu_latmeas_test.c @@ -55,6 +55,7 @@ #include "omega_blackwell_codegen.h" #include "omega_gpu_session.h" #include "omega_sm121_disasm.h" +#include #include #include #include @@ -1728,6 +1729,18 @@ static int lane_field_self_test(void) { for (int i = 0; i < nk; i++) free(kk[i].code); free(kk); } + DIR *dd = opendir(dump); + if (dd) { + struct dirent *de; + while ((de = readdir(dd)) != NULL) { + if (de->d_name[0] == '.') continue; + char p[768]; + snprintf(p, sizeof p, "%s/%s", dump, de->d_name); + unlink(p); + } + closedir(dd); + } + rmdir(dump); } #undef LCHECK return failed; diff --git a/tools/latmeas/REPRODUCE.md b/tools/latmeas/REPRODUCE.md index b93881d7..3ace2ca1 100644 --- a/tools/latmeas/REPRODUCE.md +++ b/tools/latmeas/REPRODUCE.md @@ -30,7 +30,7 @@ One warp, 32 threads, all of them executing the same chain. Lane 0 stores four w The clock is `CS2R` with index `0x50`. NAK's `nak_private.h` names that `NAK_SV_CLOCK_LO`. On SM 50 and later, `from_nir.rs` emits `CS2R` rather than `S2R` (`sm70_encode.rs`: opcode `0x805`, index at bits 72..80). CUDA++'s encoder has `S2R` only, so the test emits `S2R` immediate `0x50` and rewrites the opcode to `0x805`. -The runner tries stall 1 through 15 (1 through 36 for HMMA; the stall field is four bits, and anything past 15 is `NOP`s). `stall_required` is the smallest S above 1 such that S and every larger S matched on all 11 runs. If stall 1 already matches, `stall_required` is null: that is not a measured latency of 1. `cycle_median_stall1` is the clock delta at programmed stall 1 whether or not the oracle matched. `cycle_median` is the delta at `stall_required` when that exists. `run0_result` is lane 0's result word at each stall of the first run. For HMMA, `reference_result` is `d0` at stall 36; a zero reference leaves `stall_required` null. +The runner tries stall 1 through 15 (1 through 36 for HMMA; the stall field is four bits, and anything past 15 is `NOP`s). `stall_required` is the smallest S above 1 such that S and every larger S matched on all 11 runs (`reported_threshold`, `tests/gpu_latmeas_test.c:1223`, from `pass[][]` at line 1884). If stall 1 already matches, `stall_required` is null: that is not a measured latency of 1. `cycle_median_stall1` is the raw clock delta at programmed stall 1 (`cyc[][][0]`, lines 1965 and 1987), whether or not the oracle matched. It is not stall S minus one. `cycle_median` is the raw delta at `stall_required` (lines 1946 and 1981). Neither column subtracts `overhead_cycle_median` (line 1995). The clock is not the pass criterion. `run0_result` is lane 0's result word at each stall of the first run. For HMMA, `reference_result` is `d0` at stall 36; a zero reference leaves `stall_required` null. The fixed-latency oracle arms the destination to `0x7f800000` and the producer does not read it. The next instruction does. A stale read adds that sentinel. The host self-test checks the fresh and stale totals differ, and that the stall-1 SASS really is that pair with no scoreboard and no yield. HMMA is unchanged: `d` starts at 0, the consumer is `FADD` of `d0`, and stall 19 is HMMA stall 15 plus a `NOP` of stall 4. diff --git a/tools/latmeas/fill_template.py b/tools/latmeas/fill_template.py index 80cf61a8..0de2666c 100755 --- a/tools/latmeas/fill_template.py +++ b/tools/latmeas/fill_template.py @@ -47,7 +47,7 @@ def runs_text(case, runs): if case.get("cycle_median") is not None: parts.append("cycles at that stall median %s spread %s" % (case["cycle_median"], case.get("cycle_spread"))) if case.get("cycle_median_stall1") is not None: - parts.append("stall-1 cycles median %s spread %s" % (case["cycle_median_stall1"], case.get("cycle_spread_stall1"))) + parts.append("programmed stall 1 cycles median %s spread %s (not the stall under the threshold)" % (case["cycle_median_stall1"], case.get("cycle_spread_stall1"))) if case.get("overhead_cycle_median") is not None: parts.append("loop overhead median %s" % case["overhead_cycle_median"]) if case.get("stall_required") is None and case.get("cycle_median_stall1") is None and case.get("overhead_cycle_median") is None: diff --git a/tools/latmeas/run_spark.sh b/tools/latmeas/run_spark.sh index ab1095da..31fdbb14 100755 --- a/tools/latmeas/run_spark.sh +++ b/tools/latmeas/run_spark.sh @@ -17,14 +17,21 @@ cd "$root" write_receipt() { rec=$1/RECEIPT.txt + gitdir=${CUDAPP_RECEIPT_GIT:-.} + launch_cmd=${2:-} + launch_rc=${3:-} { echo "utc: $(date -u +%Y-%m-%dT%H:%M:%SZ)" - echo "head: $(git rev-parse HEAD)" - echo "head_parents: $(git rev-parse HEAD^@ 2>/dev/null | tr '\n' ' ')" - if git diff --quiet && git diff --cached --quiet; then - echo "worktree: clean" - else + echo "head: $(git -C "$gitdir" rev-parse HEAD)" + echo "head_parents: $(git -C "$gitdir" rev-parse HEAD^@ 2>/dev/null | tr '\n' ' ')" + if [ -n "$(git -C "$gitdir" status --porcelain)" ]; then echo "worktree: dirty" + else + echo "worktree: clean" + fi + if [ -n "$launch_cmd" ]; then + echo "launch_command: $launch_cmd" + echo "launch_rc: $launch_rc" fi echo "kernel: $(uname -srvm)" echo "gcc: $(gcc --version | head -n 1)" @@ -78,7 +85,7 @@ write_receipt() { if [ "$1" = "--receipt-self-test" ]; then tmp=$(mktemp -d) printf '{}\n' > "$tmp/raw.json" - write_receipt "$tmp" + write_receipt "$tmp" "flock /tmp/aien-gb10.lock timeout 600 ./build/gpu_latmeas_test --launch $tmp/kernels --out $tmp/raw.json" 0 got=$(awk '/^raw_json_sha256: / {print $2}' "$tmp/RECEIPT.txt") want=$(sha256sum "$tmp/raw.json" | awk '{print $1}') grep -q '^utc: ' "$tmp/RECEIPT.txt" @@ -88,6 +95,8 @@ if [ "$1" = "--receipt-self-test" ]; then grep -q '^gcc: ' "$tmp/RECEIPT.txt" grep -q '^device: ' "$tmp/RECEIPT.txt" test "$got" = "$want" + grep -q '^launch_command: ' "$tmp/RECEIPT.txt" + grep -q '^launch_rc: 0$' "$tmp/RECEIPT.txt" rm -rf "$tmp" # proc Model "Unknown" plus PCI 10de:2e12 is the GB10 the doctor names. fake=$(mktemp -d) @@ -101,6 +110,24 @@ if [ "$1" = "--receipt-self-test" ]; then grep -q '^model: NVIDIA GB10 (sm_121)$' "$fake/RECEIPT.txt" grep -q '^model_source: pci 0x2e12' "$fake/RECEIPT.txt" rm -rf "$fake" + # An untracked file must mark the tree dirty. git diff --quiet would miss it. + repo=$(mktemp -d) + rout=$(mktemp -d) + git -C "$repo" init -q + git -C "$repo" config user.email receipt@example.com + git -C "$repo" config user.name receipt + echo tracked > "$repo/a" + git -C "$repo" add a + git -C "$repo" commit -q -m init + printf '{}\n' > "$rout/raw.json" + echo untracked > "$repo/untracked" + CUDAPP_RECEIPT_GIT=$repo write_receipt "$rout" "launch-cmd" 7 + grep -q '^worktree: dirty$' "$rout/RECEIPT.txt" + grep -q '^launch_rc: 7$' "$rout/RECEIPT.txt" + rm "$repo/untracked" + CUDAPP_RECEIPT_GIT=$repo write_receipt "$rout" "launch-cmd" 0 + grep -q '^worktree: clean$' "$rout/RECEIPT.txt" + rm -rf "$repo" "$rout" echo "receipt self-test PASS" exit 0 fi @@ -118,11 +145,12 @@ echo "latmeas: taking flock /tmp/aien-gb10.lock for the launch only; timeout 600 echo "latmeas: the launch bound is the estimated chip window printed above" # flock waits here without holding the lock. timeout covers only the launch. set +e +launch_cmd="flock /tmp/aien-gb10.lock timeout 600 ./build/gpu_latmeas_test --launch $out/kernels --out $out/raw.json" flock /tmp/aien-gb10.lock timeout 600 ./build/gpu_latmeas_test --launch "$out/kernels" --out "$out/raw.json" rc=$? set -e if [ -f "$out/raw.json" ]; then python3 tools/latmeas/fill_template.py --results "$out/raw.json" --md "$out/sm121_vs_NAK.md" fi -write_receipt "$out" +write_receipt "$out" "$launch_cmd" "$rc" exit "$rc"