diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index 8c735a045b35..ed98b262cc08 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -82,6 +82,19 @@ function(ggml_add_cpu_backend_variant_impl tag_name) list(APPEND GGML_CPU_SOURCES ggml-cpu/llamafile/sgemm.cpp + ggml-cpu/llamafile/qbit_ppc_mma.cpp + ggml-cpu/llamafile/q4k_ppc_mma.cpp + ggml-cpu/llamafile/q5k_ppc_mma.cpp + ggml-cpu/llamafile/q6k_ppc_mma.cpp + ggml-cpu/llamafile/q2k_ppc_mma.cpp + ggml-cpu/llamafile/q3k_ppc_mma.cpp + ggml-cpu/llamafile/iq4_ppc_mma.cpp + ggml-cpu/llamafile/legacy_ppc_mma.cpp + ggml-cpu/llamafile/iq_grid_ppc_mma.cpp + ggml-cpu/llamafile/ppc_pack_cache.cpp + ggml-cpu/llamafile/iq_grids_ppc.h + ggml-cpu/llamafile/kquants_ppc_mma.h + ggml-cpu/llamafile/qbit_ppc_mma.h ggml-cpu/llamafile/sgemm.h) endif() diff --git a/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp new file mode 100644 index 000000000000..bc5d15127811 --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/iq4_ppc_mma.cpp @@ -0,0 +1,448 @@ +// iq4_ppc_mma.cpp - imported from github.com/mavin2009/ppc-mma-kernels +// (standalone-verified vs exact double references under qemu -cpu power10). +// IQ4_NL x Q8_0, IQ4_XS x Q8_K, MXFP4 x Q8_0 on POWER10/POWER11 MMA. + +#include "ggml-impl.h" +#include "ggml-cpu-impl.h" +#include "ggml-quants.h" +#include "kquants_ppc_mma.h" + +#include +#include +#include +#include +#include + + + + + + + + + +static_assert(sizeof(block_iq4_nl) == sizeof(ggml_half) + QK4_NL/2, "bad iq4_nl"); +static_assert(sizeof(block_mxfp4) == 1 + QK_MXFP4/2, "bad mxfp4"); +static_assert(sizeof(block_iq4_xs) == sizeof(ggml_half) + 2 + QK_K/64 + QK_K/2, "bad iq4_xs"); + +static const int8_t iq4_kvalues_local[16] = { + -127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113, +}; +// MXFP4 e2m1 values, pre-doubled to integers (the E8M0 "half" scale +// supplies the /2): {0,1,2,3,4,6,8,12} with sign. +static const int8_t kvalues_mxfp4_[16] = { + 0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12, +}; +#include +static inline float e8m0_half_to_fp32(uint8_t e) { return ldexpf(1.0f, (int)e - 128); } + + +#if defined(__MMA__) && defined(__powerpc64__) + +typedef vector unsigned char vuc; +typedef vector signed char vsc; +typedef vector unsigned int vui; +typedef vector signed int vsi; +typedef vector float vfl; + +// Unaligned 16-byte load via memcpy: compiles to a single lxv (which is +// alignment-agnostic on POWER) while staying well-defined C++ for any +// source alignment -- several block structs place qs at odd offsets. +static inline vuc load16u(const void * p) { vuc v; memcpy(&v, p, 16); return v; } + + +#define KC_ELEMS 2048 // K slab +#define KC_CH (KC_ELEMS / 32) // 32-element chunks per slab +#define MR 8 +#define NR 8 + +// packed weights: per chunk, 8 depth-steps x 2 rowgroups of signed values +typedef struct { + vuc v[KC_CH][16]; + vfl sA [KC_CH][2]; // per-row scale (dA or d*(ls-32)) + vfl C128[KC_CH][2]; // 128 * W * scale, pre-folded +} aiq4_t; + +// packed activations: per chunk, 8 depth-steps x 2 colgroups, flipped +typedef struct { + vuc v[KC_CH][16]; + vfl dB[KC_CH][2]; // per-chunk column scales +} biq4_t; + +static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { + vui t0 = vec_mergeh(rows[0], rows[1]); + vui t1 = vec_mergel(rows[0], rows[1]); + vui t2 = vec_mergeh(rows[2], rows[3]); + vui t3 = vec_mergel(rows[2], rows[3]); + out[0*stride] = (vuc)vec_xxpermdi(t0, t2, 0); + out[1*stride] = (vuc)vec_xxpermdi(t0, t2, 3); + out[2*stride] = (vuc)vec_xxpermdi(t1, t3, 0); + out[3*stride] = (vuc)vec_xxpermdi(t1, t3, 3); +} + +// 32 signed codebook values from 16 packed bytes; elements j / j+16 +// come from qs[j] low / high nibble. One vec_perm each. +static inline void iq4_lookup32t(const uint8_t * qs, const int8_t * table, vsc v[2]) { + vsc tbl; memcpy(&tbl, table, 16); + vuc raw = load16u((const uint8_t *)(qs) + (0)); + vuc lo = vec_and(raw, vec_splats((unsigned char)0xF)); + vuc hi = vec_sr(raw, vec_splats((unsigned char)4)); + v[0] = vec_perm(tbl, tbl, lo); + v[1] = vec_perm(tbl, tbl, hi); +} + +static inline int hsum(vsi s) { return s[0] + s[1] + s[2] + s[3]; } + +// pack one chunk's 8 weight rows into T at chunk ch; w[r] = row values, +// scale[r] = per-row scale for this chunk. +static void iq4_place_chunk(aiq4_t * T, int64_t ch, + const vsc w[MR][2], const float scale[MR]) { + for (int g = 0; g < 2; g++) { + float W[4]; + vui rows4[4]; + for (int h = 0; h < 2; h++) { + for (int r = 0; r < 4; r++) rows4[r] = (vui)w[4*g + r][h]; + mma_transpose4(rows4, &T->v[ch][8*h + g], 2); + } + for (int r = 0; r < 4; r++) { + vsi z = vec_splats(0); + vsi s = vec_sum4s(w[4*g + r][0], z); + s = vec_sum4s(w[4*g + r][1], s); + W[r] = (float)hsum(s); + } + T->sA [ch][g] = (vfl){ scale[4*g], scale[4*g+1], scale[4*g+2], scale[4*g+3] }; + T->C128[ch][g] = (vfl){ 128.0f*W[0], 128.0f*W[1], + 128.0f*W[2], 128.0f*W[3] }; // exact int; see field FAIL 2026-07-21 + } +} + +static inline int64_t rt8(int64_t m) { return (m + MR - 1) / MR; } +static inline int64_t ct8(int64_t n) { return (n + NR - 1) / NR; } +static inline int64_t sl32(int64_t k) { return (k/32 + KC_CH - 1) / KC_CH; } + +extern "C" size_t iq4_apack_size(int64_t m, int64_t k) { + return (((size_t)(rt8(m) * sl32(k)) * sizeof(aiq4_t)) + 63) & ~(size_t)63; +} +extern "C" size_t iq4_bpack_size(int64_t n, int64_t k) { + return (((size_t)(ct8(n) * sl32(k)) * sizeof(biq4_t)) + 63) & ~(size_t)63; +} + +// ---- weight repack: IQ4_NL ---- +extern "C" void iq4nl_repack_a(const block_iq4_nl * A, int64_t lda, + int64_t m, int64_t k, void * packed) { + aiq4_t * P = (aiq4_t *)packed; + const int64_t kb = k/32, ns = sl32(k); + for (int64_t it = 0; it < rt8(m); it++) + for (int64_t s = 0; s < ns; s++) { + aiq4_t * T = &P[it*ns + s]; + const int64_t b0 = s*KC_CH; + const int64_t nb = (kb - b0) < KC_CH ? (kb - b0) : KC_CH; + for (int64_t b = 0; b < nb; b++) { + vsc w[MR][2]; float sc[MR]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + const block_iq4_nl * bp = &A[rr*lda + b0 + b]; + sc[r] = GGML_FP16_TO_FP32(bp->d); + iq4_lookup32t(bp->qs, iq4_kvalues_local, w[r]); + } + iq4_place_chunk(T, b, w, sc); + } + } +} + +// ---- weight repack: Q8_0 (signed int8 weights, identity decode) ---- +extern "C" void q8_0_repack_a(const block_q8_0 * A, int64_t lda, + int64_t m, int64_t k, void * packed) { + aiq4_t * P = (aiq4_t *)packed; + const int64_t kb = k/32, ns = sl32(k); + for (int64_t it = 0; it < rt8(m); it++) + for (int64_t s = 0; s < ns; s++) { + aiq4_t * T = &P[it*ns + s]; + const int64_t b0 = s*KC_CH; + const int64_t nb = (kb - b0) < KC_CH ? (kb - b0) : KC_CH; + for (int64_t b = 0; b < nb; b++) { + vsc w[MR][2]; float sc[MR]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + const block_q8_0 * bp = &A[rr*lda + b0 + b]; + sc[r] = GGML_FP16_TO_FP32(bp->d); + w[r][0] = (vsc)load16u(bp->qs); + w[r][1] = (vsc)load16u(bp->qs + 16); + } + iq4_place_chunk(T, b, w, sc); + } + } +} + +// ---- weight repack: MXFP4 (same shape as IQ4_NL, different table+scale) ---- +extern "C" void mxfp4_repack_a(const block_mxfp4 * A, int64_t lda, + int64_t m, int64_t k, void * packed) { + aiq4_t * P = (aiq4_t *)packed; + const int64_t kb = k/32, ns = sl32(k); + for (int64_t it = 0; it < rt8(m); it++) + for (int64_t s = 0; s < ns; s++) { + aiq4_t * T = &P[it*ns + s]; + const int64_t b0 = s*KC_CH; + const int64_t nb = (kb - b0) < KC_CH ? (kb - b0) : KC_CH; + for (int64_t b = 0; b < nb; b++) { + vsc w[MR][2]; float sc[MR]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + const block_mxfp4 * bp = &A[rr*lda + b0 + b]; + sc[r] = e8m0_half_to_fp32(bp->e); + iq4_lookup32t(bp->qs, kvalues_mxfp4_, w[r]); + } + iq4_place_chunk(T, b, w, sc); + } + } +} + +// ---- weight repack: IQ4_XS ---- +extern "C" void iq4xs_repack_a(const block_iq4_xs * A, int64_t lda, + int64_t m, int64_t k, void * packed) { + aiq4_t * P = (aiq4_t *)packed; + const int64_t nsb = k/QK_K, ns = sl32(k); + for (int64_t it = 0; it < rt8(m); it++) + for (int64_t s = 0; s < ns; s++) { + aiq4_t * T = &P[it*ns + s]; + const int64_t sb0 = (s*KC_CH)/8; + const int64_t nsl = (nsb - sb0) < KC_CH/8 ? (nsb - sb0) : KC_CH/8; + for (int64_t sb = 0; sb < nsl; sb++) { + const block_iq4_xs * bp[MR]; float d[MR]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + bp[r] = &A[rr*lda + sb0 + sb]; + d[r] = GGML_FP16_TO_FP32(bp[r]->d); + } + for (int ib = 0; ib < 8; ib++) { // 32-groups + vsc w[MR][2]; float sc[MR]; + for (int r = 0; r < MR; r++) { + const int ls = ((bp[r]->scales_l[ib/2] >> 4*(ib%2)) & 0xF) + | (((bp[r]->scales_h >> 2*ib) & 3) << 4); + sc[r] = d[r] * (float)(ls - 32); + iq4_lookup32t(bp[r]->qs + 16*ib, iq4_kvalues_local, w[r]); + } + iq4_place_chunk(T, 8*sb + ib, w, sc); + } + } + } +} + +// ---- activation packs ---- +extern "C" void iq4_pack_b_q8_0(const block_q8_0 * B, int64_t ldb, + int64_t n, int64_t k, void * packed) { + biq4_t * P = (biq4_t *)packed; + const int64_t kb = k/32, ns = sl32(k); + const vuc flip = vec_splats((unsigned char)0x80); + for (int64_t jt = 0; jt < ct8(n); jt++) + for (int64_t s = 0; s < ns; s++) { + biq4_t * T = &P[jt*ns + s]; + const int64_t b0 = s*KC_CH; + const int64_t nb = (kb - b0) < KC_CH ? (kb - b0) : KC_CH; + for (int64_t b = 0; b < nb; b++) { + const block_q8_0 * yb[NR]; float dB[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = jt*NR + j; if (jj >= n) jj = n - 1; + yb[j] = &B[jj*ldb + b0 + b]; + dB[j] = GGML_FP16_TO_FP32(yb[j]->d); + } + T->dB[b][0] = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + T->dB[b][1] = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + vui rows4[4]; + for (int a = 0; a < 2; a++) + for (int h = 0; h < 2; h++) { + for (int j = 0; j < 4; j++) + rows4[j] = (vui)vec_xor( + load16u((const uint8_t *)(yb[4*a + j]->qs) + (16*h)), flip); + mma_transpose4(rows4, &T->v[b][8*h + a], 2); + } + } + } +} + +extern "C" void iq4_pack_b_q8_K(const block_q8_K * B, int64_t ldb, + int64_t n, int64_t k, void * packed) { + biq4_t * P = (biq4_t *)packed; + const int64_t nsb = k/QK_K, ns = sl32(k); + const vuc flip = vec_splats((unsigned char)0x80); + for (int64_t jt = 0; jt < ct8(n); jt++) + for (int64_t s = 0; s < ns; s++) { + biq4_t * T = &P[jt*ns + s]; + const int64_t sb0 = (s*KC_CH)/8; + const int64_t nsl = (nsb - sb0) < KC_CH/8 ? (nsb - sb0) : KC_CH/8; + for (int64_t sb = 0; sb < nsl; sb++) { + const block_q8_K * yb[NR]; float dB[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = jt*NR + j; if (jj >= n) jj = n - 1; + yb[j] = &B[jj*ldb + sb0 + sb]; + dB[j] = yb[j]->d; + } + for (int ib = 0; ib < 8; ib++) { + const int64_t ch = 8*sb + ib; + T->dB[ch][0] = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + T->dB[ch][1] = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + vui rows4[4]; + for (int a = 0; a < 2; a++) + for (int h = 0; h < 2; h++) { + for (int j = 0; j < 4; j++) + rows4[j] = (vui)vec_xor( + load16u((const uint8_t *)(yb[4*a + j]->qs) + (32*ib + 16*h)), flip); + mma_transpose4(rows4, &T->v[ch][8*h + a], 2); + } + } + } + } +} + +// ---- 8x8 microkernel on 4 accumulators (weights on the signed operand; +// acc rows = weight rows) ---- +static void kernel_iq4_8x8(const aiq4_t * PA, const biq4_t * PB, + int64_t nch, vfl fin[MR][2]) { + for (int64_t ch = 0; ch < nch; ch++) { + const vuc * a = PA->v[ch]; + const vuc * y = PB->v[ch]; + if (ch + 1 < nch) { + // each chunk's packed panel spans two 128B lines; touch both +#ifdef PPC_DCBT_LINES + __builtin_prefetch(PA->v[ch + 1], 0, 3); + __builtin_prefetch((const char *)PA->v[ch + 1] + 128, 0, 3); + __builtin_prefetch(PB->v[ch + 1], 0, 3); + __builtin_prefetch((const char *)PB->v[ch + 1] + 128, 0, 3); +#else + __asm__ volatile("dcbt 0,%0,8" :: "r"(PA->v[ch + 1])); + __asm__ volatile("dcbt 0,%0,8" :: "r"(PB->v[ch + 1])); +#endif + } + __vector_quad acc[2][2]; + for (int g = 0; g < 2; g++) + for (int cgi = 0; cgi < 2; cgi++) + __builtin_mma_xxsetaccz(&acc[g][cgi]); + for (int x = 0; x < 8; x++) { + const vuc w0 = a[2*x], w1 = a[2*x + 1]; + const vuc y0 = y[2*x], y1 = y[2*x + 1]; + __builtin_mma_xvi8ger4pp(&acc[0][0], w0, y0); + __builtin_mma_xvi8ger4pp(&acc[0][1], w0, y1); + __builtin_mma_xvi8ger4pp(&acc[1][0], w1, y0); + __builtin_mma_xvi8ger4pp(&acc[1][1], w1, y1); + } + for (int g = 0; g < 2; g++) { + const vfl sA = PA->sA [ch][g]; + const vfl C128 = PA->C128[ch][g]; + for (int cgi = 0; cgi < 2; cgi++) { + vsi rowsP[4]; + __builtin_mma_disassemble_acc(rowsP, &acc[g][cgi]); + const vfl dB = PB->dB[ch][cgi]; + // fin += dB ⊙ (P*sA_r - C128_r) per weight row r + for (int r = 0; r < 4; r++) { + vfl t = vec_sub(vec_ctf(rowsP[r],0), vec_splats(C128[r])); + vfl sc = vec_mul(vec_splats(sA[r]), dB); + fin[4*g + r][cgi] = vec_madd(t, sc, fin[4*g + r][cgi]); + } + } + } + } +} + +static void iq4_gemm_core(int64_t m, int64_t n, int64_t k, + const aiq4_t * PA, const biq4_t * PB, + float * C, int64_t ldc, int ith, int nth) { + const int64_t kb = k/32, ns = sl32(k), mt = rt8(m), njt = ct8(n); + const int64_t tpt = (mt + nth - 1) / nth; + const int64_t t0 = ith*tpt, t1 = (ith+1)*tpt < mt ? (ith+1)*tpt : mt; + vfl fin[MR][2]; + for (int64_t it = t0; it < t1; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + for (int64_t jt = 0; jt < njt; jt++) { + for (int r = 0; r < MR; r++) fin[r][0] = fin[r][1] = vec_splats(0.0f); + for (int64_t s = 0; s < ns; s++) { + const int64_t b0 = s*KC_CH; + const int64_t nch = (kb - b0) < KC_CH ? (kb - b0) : KC_CH; + kernel_iq4_8x8(&PA[it*ns + s], &PB[jt*ns + s], nch, fin); + } + const int64_t j0 = jt*NR; + const int64_t cols = (n - j0) < NR ? (n - j0) : NR; + for (int64_t r = 0; r < rows; r++) + for (int64_t cj = 0; cj < cols; cj++) + C[(i + r) + (j0 + cj)*ldc] = fin[r][cj >> 2][cj & 3]; + } + } +} + +extern "C" void iq4_gemm_packed(int64_t m, int64_t n, int64_t k, + const void * packedA, const void * packedB, + float * C, int64_t ldc, int ith, int nth) { + iq4_gemm_core(m, n, k, (const aiq4_t *)packedA, (const biq4_t *)packedB, + C, ldc, ith, nth); +} + + +#define IQ4_ONESHOT(NAME, BLKA, REPACK, YBLK, PACKB, VARIANT) \ +extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ + const void * Av, int64_t lda, const void * Bv, int64_t ldb, \ + float * C, int64_t ldc, int ith, int nth) { \ + const BLKA * A = (const BLKA *)Av; \ + const YBLK * B = (const YBLK *)Bv; \ + int fresh = 0; \ + void * PA = ppc_apack_cache_acquire_par(Av, m, k, VARIANT, \ + iq4_apack_size(m, k), nth, &fresh); \ + if (PA) { \ + if (fresh) { \ + int64_t i0_, rows_; \ + ppc_apack_slice(m, MR, ith, nth, &i0_, &rows_); \ + if (rows_ > 0) REPACK(A + i0_*lda, lda, rows_, k, \ + (void *)((aiq4_t *)PA + (i0_/MR)*sl32(k))); \ + ppc_apack_cache_slice_done(Av, m, k, VARIANT); \ + } \ + const int64_t njt = (n + NR - 1) / NR; \ + if (njt < nth) { \ + /* n too small to feed every thread by columns (worst \ + case n == 1 generation: one column, nth-1 idle threads \ + while scalar row-partitions). Row-partition with the \ + cached pack instead; the full activation pack is tiny \ + at these n. Field regression, Q4_K tg32, 2026-07-21. */ \ + void * PBs = aligned_alloc(64, iq4_bpack_size(n, k)); \ + if (!PBs) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + PACKB(B, ldb, n, k, PBs); \ + iq4_gemm_packed(m, n, k, PA, PBs, C, ldc, ith, nth); \ + free(PBs); \ + } else { \ + const int64_t jpt = (njt + nth - 1) / nth; \ + const int64_t jt0 = (int64_t)ith*jpt; \ + const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; \ + if (jt0 < jt1) { \ + const int64_t j0 = jt0*NR; \ + const int64_t nc = (n - j0) < (jt1 - jt0)*NR ? (n - j0) \ + : (jt1 - jt0)*NR; \ + void * PBl = aligned_alloc(64, iq4_bpack_size(nc, k)); \ + if (!PBl) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + PACKB(B + j0*ldb, ldb, nc, k, PBl); \ + iq4_gemm_packed(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); \ + free(PBl); \ + } \ + } \ + } else { \ + void * PB = aligned_alloc(64, iq4_bpack_size(n, k)); \ + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + PACKB(B, ldb, n, k, PB); \ + void * PT = aligned_alloc(64, iq4_apack_size(MR, k)); \ + if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + const int64_t mt = (m + MR - 1) / MR; \ + const int64_t tpt = (mt + nth - 1) / nth; \ + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { \ + const int64_t i = it*MR; \ + const int64_t rows = (m - i) < MR ? (m - i) : MR; \ + REPACK(A + i*lda, lda, rows, k, PT); \ + iq4_gemm_packed(rows, n, k, PT, PB, C + i, ldc, 0, 1); \ + } \ + free(PT); \ + free(PB); \ + } \ +} +IQ4_ONESHOT(gemm_iq4_nl_q8_0_ppc, block_iq4_nl, iq4nl_repack_a, block_q8_0, iq4_pack_b_q8_0, 1) +IQ4_ONESHOT(gemm_iq4_xs_q8_K_ppc, block_iq4_xs, iq4xs_repack_a, block_q8_K, iq4_pack_b_q8_K, 2) +IQ4_ONESHOT(gemm_mxfp4_q8_0_ppc, block_mxfp4, mxfp4_repack_a, block_q8_0, iq4_pack_b_q8_0, 3) +IQ4_ONESHOT(gemm_q8_0_q8_0_ppc, block_q8_0, q8_0_repack_a, block_q8_0, iq4_pack_b_q8_0, 4) + +#endif // __MMA__ + diff --git a/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp new file mode 100644 index 000000000000..e9cc8545d4c0 --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/iq_grid_ppc_mma.cpp @@ -0,0 +1,1159 @@ +// iq_grid_ppc_mma.cpp - imported from github.com/mavin2009/ppc-mma-kernels +// (standalone-verified under qemu -cpu power10). Grid-codebook, ternary, +// and NVFP4 formats via decode-at-repack + shared signed kernels (32- +// and 16-deep chunk variants). Local grid tables in iq_grids_ppc.h. + +#include "ggml-impl.h" +#include "ggml-cpu-impl.h" +#include "kquants_ppc_mma.h" + +#include +#include +#include +#include +#include +#include "iq_grids_ppc.h" + +#define QK_K 256 +#define IQ1S_DELTA 0.125f + +typedef uint16_t ppc_half; + +typedef struct { uint8_t qs[(QK_K - 4*QK_K/64)/5]; uint8_t qh[QK_K/64]; ppc_half d; } block_tq1_0_ppc; +typedef struct { uint8_t qs[QK_K/4]; ppc_half d; } block_tq2_0_ppc; +typedef struct { ppc_half d; uint16_t qs[QK_K/8]; } block_iq2_xxs_ppc; +typedef struct { ppc_half d; uint8_t qs[3*QK_K/8]; } block_iq3_xxs_ppc; +typedef struct { ppc_half d; uint8_t qs[QK_K/4]; uint8_t qh[QK_K/32]; + uint8_t signs[QK_K/8]; uint8_t scales[QK_K/64]; } block_iq3_s_ppc; +typedef struct { ppc_half d; uint8_t qs[QK_K/8]; uint16_t qh[QK_K/32]; } block_iq1_s_ppc; +typedef struct { ppc_half d; uint16_t qs[QK_K/8]; uint8_t scales[QK_K/32]; } block_iq2_xs_ppc; +typedef struct { ppc_half d; uint8_t qs[QK_K/4]; uint8_t qh[QK_K/32]; uint8_t scales[QK_K/32]; } block_iq2_s_ppc; +typedef struct { uint8_t qs[QK_K/8]; uint8_t qh[QK_K/16]; uint8_t scales[QK_K/32]; } block_iq1_m_ppc; +typedef struct { float d; int8_t qs[QK_K]; int16_t bsums[QK_K/16]; } block_q8_K_ppc; +#define QK_NVFP4 64 +#define QK_NVFP4_SUB 16 +typedef struct { uint8_t d[QK_NVFP4/QK_NVFP4_SUB]; uint8_t qs[QK_NVFP4/2]; } block_nvfp4_ppc; +#define QK8_0 32 +typedef struct { ppc_half d; int8_t qs[QK8_0]; } block_q8_0_ppc; + +static_assert(sizeof(block_tq1_0_ppc) == 2 + 4 + 48, "bad tq1_0"); +static_assert(sizeof(block_tq2_0_ppc) == 2 + 64, "bad tq2_0"); +static_assert(sizeof(block_iq2_xxs_ppc) == 2 + QK_K/4, "bad iq2_xxs"); +static_assert(sizeof(block_iq3_xxs_ppc) == 2 + 3*QK_K/8, "bad iq3_xxs"); +static_assert(sizeof(block_iq3_s_ppc) == 2 + 13*(QK_K/32) + QK_K/64, "bad iq3_s"); +static_assert(sizeof(block_iq1_s_ppc) == 2 + QK_K/8 + QK_K/16, "bad iq1_s"); +static_assert(sizeof(block_iq2_xs_ppc) == 2 + QK_K/4 + QK_K/32, "bad iq2_xs"); +static_assert(sizeof(block_iq2_s_ppc) == 2 + QK_K/4 + QK_K/32 + QK_K/32, "bad iq2_s"); +static_assert(sizeof(block_iq1_m_ppc) == QK_K/8 + QK_K/16 + QK_K/32, "bad iq1_m"); +static_assert(sizeof(block_nvfp4_ppc) == 4 + QK_NVFP4/2, "bad nvfp4"); + +static inline float fp16_to_fp32(ppc_half h) { + const uint32_t sign = (uint32_t)(h >> 15) << 31; + uint32_t exp = (h >> 10) & 0x1f; + uint32_t mant = h & 0x3ff; + uint32_t bits; + if (exp == 0) { + if (mant == 0) bits = sign; + else { + exp = 127 - 15 + 1; + while (!(mant & 0x400)) { mant <<= 1; exp--; } + mant &= 0x3ff; + bits = sign | (exp << 23) | (mant << 13); + } + } else if (exp == 0x1f) bits = sign | 0x7f800000u | (mant << 13); + else bits = sign | ((exp + 127 - 15) << 23) | (mant << 13); + float f; memcpy(&f, &bits, 4); + return f; +} + +// ---- scalar decoders: superblock -> 256 signed int8 codes + 8 scales ---- +// (one per 32-chunk). Direct ports of dequantize_row semantics. + +static void dec_tq2_0(const block_tq2_0_ppc * b, int8_t code[QK_K], float sc[8]) { + const float d = fp16_to_fp32(b->d); + int8_t * y = code; + for (size_t j = 0; j < sizeof(b->qs); j += 32) + for (size_t l = 0; l < 4; ++l) + for (size_t m = 0; m < 32; ++m) + *y++ = (int8_t)(((b->qs[j + m] >> (l*2)) & 3) - 1); + for (int c = 0; c < 8; c++) sc[c] = d; +} + +static void dec_tq1_0(const block_tq1_0_ppc * b, int8_t code[QK_K], float sc[8]) { + static const uint8_t pow3[6] = { 1, 3, 9, 27, 81, 243 }; + const float d = fp16_to_fp32(b->d); + int8_t * y = code; + for (size_t j = 0; j < sizeof(b->qs) - sizeof(b->qs) % 32; j += 32) + for (size_t n = 0; n < 5; ++n) + for (size_t m = 0; m < 32; ++m) { + uint8_t q = (uint8_t)(b->qs[j + m] * pow3[n]); + *y++ = (int8_t)((((uint16_t)q * 3) >> 8) - 1); + } + for (size_t j = sizeof(b->qs) - sizeof(b->qs) % 32; j < sizeof(b->qs); j += 16) + for (size_t n = 0; n < 5; ++n) + for (size_t m = 0; m < 16; ++m) { + uint8_t q = (uint8_t)(b->qs[j + m] * pow3[n]); + *y++ = (int8_t)((((uint16_t)q * 3) >> 8) - 1); + } + for (size_t n = 0; n < 4; ++n) + for (size_t j = 0; j < sizeof(b->qh); ++j) { + uint8_t q = (uint8_t)(b->qh[j] * pow3[n]); + *y++ = (int8_t)((((uint16_t)q * 3) >> 8) - 1); + } + for (int c = 0; c < 8; c++) sc[c] = d; +} + +static void dec_iq2_xxs(const block_iq2_xxs_ppc * b, int8_t code[QK_K], float sc[8]) { + const float d = fp16_to_fp32(b->d); + uint32_t aux32[2]; + const uint8_t * aux8 = (const uint8_t *)aux32; + for (int ib32 = 0; ib32 < 8; ++ib32) { + memcpy(aux32, b->qs + 4*ib32, 8); + sc[ib32] = d * (0.5f + (aux32[1] >> 28)) * 0.25f; + int8_t * y = code + 32*ib32; + for (int l = 0; l < 4; ++l) { + const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]); + const uint8_t signs = ksigns_iq2xs[(aux32[1] >> 7*l) & 127]; + for (int j = 0; j < 8; ++j) + y[8*l + j] = (int8_t)((signs & kmask_iq2xs[j]) ? -(int)grid[j] : (int)grid[j]); + } + } +} + +static void dec_iq3_xxs(const block_iq3_xxs_ppc * b, int8_t code[QK_K], float sc[8]) { + const float d = fp16_to_fp32(b->d); + const uint8_t * qs = b->qs; + const uint8_t * sas = b->qs + QK_K/4; + uint32_t aux32; + for (int ib32 = 0; ib32 < 8; ++ib32) { + memcpy(&aux32, sas + 4*ib32, 4); + sc[ib32] = d * (0.5f + (aux32 >> 28)) * 0.5f; + int8_t * y = code + 32*ib32; + for (int l = 0; l < 4; ++l) { + const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*l) & 127]; + const uint8_t * g1 = (const uint8_t *)(iq3xxs_grid + qs[8*ib32 + 2*l + 0]); + const uint8_t * g2 = (const uint8_t *)(iq3xxs_grid + qs[8*ib32 + 2*l + 1]); + for (int j = 0; j < 4; ++j) { + y[8*l + j + 0] = (int8_t)((signs & kmask_iq2xs[j+0]) ? -(int)g1[j] : (int)g1[j]); + y[8*l + j + 4] = (int8_t)((signs & kmask_iq2xs[j+4]) ? -(int)g2[j] : (int)g2[j]); + } + } + } +} + +static void dec_iq3_s(const block_iq3_s_ppc * b, int8_t code[QK_K], float sc[8]) { + const float d = fp16_to_fp32(b->d); + const uint8_t * qs = b->qs; + const uint8_t * qh = b->qh; + const uint8_t * signs = b->signs; + for (int ib32 = 0; ib32 < 8; ib32 += 2) { + sc[ib32 + 0] = d * (1 + 2*(b->scales[ib32/2] & 0xF)); + sc[ib32 + 1] = d * (1 + 2*(b->scales[ib32/2] >> 4)); + for (int half = 0; half < 2; half++) { + int8_t * y = code + 32*(ib32 + half); + for (int l = 0; l < 4; ++l) { + const uint8_t * g1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[half] << (8-2*l)) & 256))); + const uint8_t * g2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[half] << (7-2*l)) & 256))); + for (int j = 0; j < 4; ++j) { + y[8*l + j + 0] = (int8_t)((signs[l] & kmask_iq2xs[j+0]) ? -(int)g1[j] : (int)g1[j]); + y[8*l + j + 4] = (int8_t)((signs[l] & kmask_iq2xs[j+4]) ? -(int)g2[j] : (int)g2[j]); + } + } + qs += 8; signs += 4; + } + qh += 2; + } +} + +// IQ1_S: value = dl*(g + delta), g in {-1,+1}, delta = +/-1/8 per 32. +// Exact integer form: code = 8*g + (delta<0 ? -1 : +1), scale = dl/8. +static void dec_iq1_s(const block_iq1_s_ppc * b, int8_t code[QK_K], float sc[8]) { + const float d = fp16_to_fp32(b->d); + const uint8_t * qs = b->qs; + for (int ib = 0; ib < 8; ++ib) { + const float dl = d * (2*((b->qh[ib] >> 12) & 7) + 1); + sc[ib] = dl * 0.125f; + const int dsign = (b->qh[ib] & 0x8000) ? -1 : +1; + int8_t * y = code + 32*ib; + for (int l = 0; l < 4; ++l) { + const int8_t * grid = (const int8_t *)(iq1s_grid + (qs[l] | (((b->qh[ib] >> 3*l) & 7) << 8))); + for (int j = 0; j < 8; ++j) + y[8*l + j] = (int8_t)(8*grid[j] + dsign); + } + qs += 4; + } +} + + +// NVFP4 (64-element blocks, four 16-element sub-blocks with UE4M3 +// scales, MXFP4's pre-doubled e2m1 codebook -- the *0.5 inside the +// UE4M3 conversion compensates the doubling, matching ggml's dequant). +static const int8_t nvfp4_kvalues[16] = { + 0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12, +}; +#include +static inline float ue4m3_to_fp32(uint8_t x) { + if (x == 0 || x == 0x7F) return 0.0f; + const int exp = (x >> 3) & 0xF; + const int man = x & 0x7; + float raw = (exp == 0) ? ldexpf((float)man, -9) + : ldexpf(1.0f + (float)man / 8.0f, exp - 7); + return raw * 0.5f; +} + +// one 64-block -> 64 signed codes + 4 per-16 scales +static void dec_nvfp4(const block_nvfp4_ppc * b, int8_t code[QK_NVFP4], float sc[4]) { + for (int s = 0; s < 4; s++) { + sc[s] = ue4m3_to_fp32(b->d[s]); + for (int j = 0; j < 8; j++) { + code[16*s + j + 0] = nvfp4_kvalues[b->qs[8*s + j] & 0xF]; + code[16*s + j + 8] = nvfp4_kvalues[b->qs[8*s + j] >> 4]; + } + } +} + +// ---- per-16-scale decoders (codes + 16 scales per superblock) ---- + +static void dec16_iq2_xs(const block_iq2_xs_ppc * b, int8_t code[QK_K], float sc[16]) { + const float d = fp16_to_fp32(b->d); + for (int ib32 = 0; ib32 < 8; ++ib32) { + sc[2*ib32 + 0] = d * (0.5f + (b->scales[ib32] & 0xF)) * 0.25f; + sc[2*ib32 + 1] = d * (0.5f + (b->scales[ib32] >> 4)) * 0.25f; + int8_t * y = code + 32*ib32; + for (int l = 0; l < 4; ++l) { + const uint16_t w = b->qs[4*ib32 + l]; + const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (w & 511)); + const uint8_t signs = ksigns_iq2xs[w >> 9]; + for (int j = 0; j < 8; ++j) + y[8*l + j] = (int8_t)((signs & kmask_iq2xs[j]) ? -(int)grid[j] : (int)grid[j]); + } + } +} + +static void dec16_iq2_s(const block_iq2_s_ppc * b, int8_t code[QK_K], float sc[16]) { + const float d = fp16_to_fp32(b->d); + const uint8_t * qs = b->qs; + const uint8_t * signs = b->qs + QK_K/8; + for (int ib32 = 0; ib32 < 8; ++ib32) { + sc[2*ib32 + 0] = d * (0.5f + (b->scales[ib32] & 0xF)) * 0.25f; + sc[2*ib32 + 1] = d * (0.5f + (b->scales[ib32] >> 4)) * 0.25f; + int8_t * y = code + 32*ib32; + for (int l = 0; l < 4; ++l) { + const uint8_t * grid = (const uint8_t *)(iq2s_grid + + (qs[l] | ((b->qh[ib32] << (8 - 2*l)) & 0x300))); + for (int j = 0; j < 8; ++j) + y[8*l + j] = (int8_t)((signs[l] & kmask_iq2xs[j]) ? -(int)grid[j] : (int)grid[j]); + } + qs += 4; signs += 4; + } +} + +// IQ1_M: fp16 superscale reassembled from scale-nibble high bits; +// per-16 3-bit scales, per-8 deltas. Exact integer form as IQ1_S: +// codes = 8*grid + sign(delta), scale = dl/8. +static void dec16_iq1_m(const block_iq1_m_ppc * b, int8_t code[QK_K], float sc[16]) { + const uint16_t * scw = (const uint16_t *)b->scales; + uint16_t du16 = (uint16_t)((scw[0] >> 12) | ((scw[1] >> 8) & 0x00f0) | + ((scw[2] >> 4) & 0x0f00) | (scw[3] & 0xf000)); + const float d = fp16_to_fp32(du16); + const uint8_t * qs = b->qs; + const uint8_t * qh = b->qh; + for (int ib = 0; ib < 8; ++ib) { + const float dl1 = d * (2*((scw[ib/2] >> (6*(ib%2)+0)) & 0x7) + 1); + const float dl2 = d * (2*((scw[ib/2] >> (6*(ib%2)+3)) & 0x7) + 1); + sc[2*ib + 0] = dl1 * 0.125f; + sc[2*ib + 1] = dl2 * 0.125f; + uint16_t idx[4]; int dsg[4]; + idx[0] = (uint16_t)(qs[0] | ((qh[0] << 8) & 0x700)); + idx[1] = (uint16_t)(qs[1] | ((qh[0] << 4) & 0x700)); + idx[2] = (uint16_t)(qs[2] | ((qh[1] << 8) & 0x700)); + idx[3] = (uint16_t)(qs[3] | ((qh[1] << 4) & 0x700)); + dsg[0] = (qh[0] & 0x08) ? -1 : +1; + dsg[1] = (qh[0] & 0x80) ? -1 : +1; + dsg[2] = (qh[1] & 0x08) ? -1 : +1; + dsg[3] = (qh[1] & 0x80) ? -1 : +1; + int8_t * y = code + 32*ib; + for (int l = 0; l < 4; ++l) { + const int8_t * grid = (const int8_t *)(iq1s_grid + idx[l]); + for (int j = 0; j < 8; ++j) + y[8*l + j] = (int8_t)(8*grid[j] + dsg[l]); + } + qs += 4; qh += 2; + } +} + +#if defined(__MMA__) && defined(__powerpc64__) + +typedef vector unsigned char vuc; +typedef vector signed char vsc; +typedef vector unsigned int vui; +typedef vector signed int vsi; +typedef vector float vfl; + +// Unaligned 16-byte load via memcpy: compiles to a single lxv (which is +// alignment-agnostic on POWER) while staying well-defined C++ for any +// source alignment -- several block structs place qs at odd offsets. +static inline vuc load16u(const void * p) { vuc v; memcpy(&v, p, 16); return v; } + + +#define KC_ELEMS 2048 +#define KC_CH (KC_ELEMS / 32) +#define MR 8 +#define NR 8 + +typedef struct { + vuc v[KC_CH][16]; + vfl sA [KC_CH][2]; + vfl C128[KC_CH][2]; +} agrid_t; + +typedef struct { + vuc v[KC_CH][16]; + vfl dB[KC_CH][2]; +} bgrid_t; + +static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { + vui t0 = vec_mergeh(rows[0], rows[1]); + vui t1 = vec_mergel(rows[0], rows[1]); + vui t2 = vec_mergeh(rows[2], rows[3]); + vui t3 = vec_mergel(rows[2], rows[3]); + out[0*stride] = (vuc)vec_xxpermdi(t0, t2, 0); + out[1*stride] = (vuc)vec_xxpermdi(t0, t2, 3); + out[2*stride] = (vuc)vec_xxpermdi(t1, t3, 0); + out[3*stride] = (vuc)vec_xxpermdi(t1, t3, 3); +} + +static inline int hsum(vsi s) { return s[0] + s[1] + s[2] + s[3]; } + +#ifdef IQGRID_LXVP +static inline void load_pair(const vuc * p, vuc out[2]) { + __vector_pair vp = *(const __vector_pair *)p; + __builtin_vsx_disassemble_pair((void *)out, &vp); +} +#endif + +static void grid_place_chunk(agrid_t * T, int64_t ch, + const vsc w[MR][2], const float scale[MR]) { + for (int g = 0; g < 2; g++) { + float W[4]; + vui rows4[4]; + for (int h = 0; h < 2; h++) { + for (int r = 0; r < 4; r++) rows4[r] = (vui)w[4*g + r][h]; + mma_transpose4(rows4, &T->v[ch][8*h + g], 2); + } + for (int r = 0; r < 4; r++) { + vsi z = vec_splats(0); + vsi s = vec_sum4s(w[4*g + r][0], z); + s = vec_sum4s(w[4*g + r][1], s); + W[r] = (float)hsum(s); + } + T->sA [ch][g] = (vfl){ scale[4*g], scale[4*g+1], scale[4*g+2], scale[4*g+3] }; + T->C128[ch][g] = (vfl){ 128.0f*W[0], 128.0f*W[1], + 128.0f*W[2], 128.0f*W[3] }; // exact int; see field FAIL 2026-07-21 + } +} + +static inline int64_t rt(int64_t m) { return (m + MR - 1) / MR; } +static inline int64_t ct(int64_t n) { return (n + NR - 1) / NR; } +static inline int64_t sl(int64_t k) { return (k/32 + KC_CH - 1) / KC_CH; } + +extern "C" size_t grid_apack_size(int64_t m, int64_t k) { + return (((size_t)(rt(m) * sl(k)) * sizeof(agrid_t)) + 63) & ~(size_t)63; +} +extern "C" size_t grid_bpack_size(int64_t n, int64_t k) { + return (((size_t)(ct(n) * sl(k)) * sizeof(bgrid_t)) + 63) & ~(size_t)63; +} + +// generic repack over any superblock decoder +template +static void repack_grid(const BLK * A, int64_t lda, int64_t m, int64_t k, agrid_t * P) { + const int64_t nsb = k/QK_K, ns = sl(k); + for (int64_t it = 0; it < rt(m); it++) + for (int64_t s = 0; s < ns; s++) { + agrid_t * T = &P[it*ns + s]; + const int64_t sb0 = (s*KC_CH)/8; + const int64_t nsl = (nsb - sb0) < KC_CH/8 ? (nsb - sb0) : KC_CH/8; + for (int64_t sb = 0; sb < nsl; sb++) { + int8_t code[MR][QK_K]; float sc[MR][8]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + DEC(&A[rr*lda + sb0 + sb], code[r], sc[r]); + } + for (int c = 0; c < 8; c++) { + vsc w[MR][2]; float scale[MR]; + for (int r = 0; r < MR; r++) { + memcpy(&w[r][0], code[r] + 32*c, 16); + memcpy(&w[r][1], code[r] + 32*c + 16, 16); + scale[r] = sc[r][c]; + } + grid_place_chunk(T, 8*sb + c, w, scale); + } + } + } +} + +extern "C" void grid_repack_tq2_0(const block_tq2_0_ppc * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_grid(A, lda, m, k, (agrid_t *)p); } +extern "C" void grid_repack_tq1_0(const block_tq1_0_ppc * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_grid(A, lda, m, k, (agrid_t *)p); } +extern "C" void grid_repack_iq2_xxs(const block_iq2_xxs_ppc * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_grid(A, lda, m, k, (agrid_t *)p); } +extern "C" void grid_repack_iq3_xxs(const block_iq3_xxs_ppc * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_grid(A, lda, m, k, (agrid_t *)p); } +extern "C" void grid_repack_iq3_s(const block_iq3_s_ppc * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_grid(A, lda, m, k, (agrid_t *)p); } +extern "C" void grid_repack_iq1_s(const block_iq1_s_ppc * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_grid(A, lda, m, k, (agrid_t *)p); } + +extern "C" void grid_pack_b_q8_K(const block_q8_K_ppc * B, int64_t ldb, + int64_t n, int64_t k, void * packed) { + bgrid_t * P = (bgrid_t *)packed; + const int64_t nsb = k/QK_K, ns = sl(k); + const vuc flip = vec_splats((unsigned char)0x80); + for (int64_t jt = 0; jt < ct(n); jt++) + for (int64_t s = 0; s < ns; s++) { + bgrid_t * T = &P[jt*ns + s]; + const int64_t sb0 = (s*KC_CH)/8; + const int64_t nsl = (nsb - sb0) < KC_CH/8 ? (nsb - sb0) : KC_CH/8; + for (int64_t sb = 0; sb < nsl; sb++) { + const block_q8_K_ppc * yb[NR]; float dB[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = jt*NR + j; if (jj >= n) jj = n - 1; + yb[j] = &B[jj*ldb + sb0 + sb]; + dB[j] = yb[j]->d; + } + for (int ib = 0; ib < 8; ib++) { + const int64_t ch = 8*sb + ib; + T->dB[ch][0] = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + T->dB[ch][1] = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + vui rows4[4]; + for (int a = 0; a < 2; a++) + for (int h = 0; h < 2; h++) { + for (int j = 0; j < 4; j++) + rows4[j] = (vui)vec_xor( + load16u((const uint8_t *)(yb[4*a + j]->qs) + (32*ib + 16*h)), flip); + mma_transpose4(rows4, &T->v[ch][8*h + a], 2); + } + } + } + } +} + +__attribute__((unused)) static void kernel_grid_8x8(const agrid_t * PA, const bgrid_t * PB, + int64_t nch, vfl fin[MR][2]) { + for (int64_t ch = 0; ch < nch; ch++) { + const vuc * a = PA->v[ch]; + const vuc * y = PB->v[ch]; + if (ch + 1 < nch) { + // each chunk's packed panel spans two 128B lines; touch both + __builtin_prefetch(PA->v[ch + 1], 0, 3); + __builtin_prefetch((const char *)PA->v[ch + 1] + 128, 0, 3); + __builtin_prefetch(PB->v[ch + 1], 0, 3); + __builtin_prefetch((const char *)PB->v[ch + 1] + 128, 0, 3); + } + __vector_quad acc[2][2]; + for (int g = 0; g < 2; g++) + for (int cgi = 0; cgi < 2; cgi++) + __builtin_mma_xxsetaccz(&acc[g][cgi]); + for (int x = 0; x < 8; x++) { +#ifdef IQGRID_LXVP + vuc wv[2], yv[2]; + load_pair(a + 2*x, wv); + load_pair(y + 2*x, yv); + const vuc w0 = wv[0], w1 = wv[1]; + const vuc y0 = yv[0], y1 = yv[1]; +#else + const vuc w0 = a[2*x], w1 = a[2*x + 1]; + const vuc y0 = y[2*x], y1 = y[2*x + 1]; +#endif + __builtin_mma_xvi8ger4pp(&acc[0][0], w0, y0); + __builtin_mma_xvi8ger4pp(&acc[0][1], w0, y1); + __builtin_mma_xvi8ger4pp(&acc[1][0], w1, y0); + __builtin_mma_xvi8ger4pp(&acc[1][1], w1, y1); + } + for (int g = 0; g < 2; g++) { + const vfl sA = PA->sA [ch][g]; + const vfl C128 = PA->C128[ch][g]; + for (int cgi = 0; cgi < 2; cgi++) { + vsi rowsP[4]; + __builtin_mma_disassemble_acc(rowsP, &acc[g][cgi]); + const vfl dB = PB->dB[ch][cgi]; + for (int r = 0; r < 4; r++) { + vfl t = vec_sub(vec_ctf(rowsP[r],0), vec_splats(C128[r])); + vfl sc = vec_mul(vec_splats(sA[r]), dB); + fin[4*g + r][cgi] = vec_madd(t, sc, fin[4*g + r][cgi]); + } + } + } + } +} + +// --------------------------------------------------------------------------- +// Accumulator ping-pong variant (-DIQGRID_PINGPONG), 32-deep kernel. +// +// Microarchitectural rationale (Power10 MMA): the eight accumulators +// are physically resident in the MMA engine; xxmfacc drains engine +// state to the VRF and serializes against outstanding GERs on that +// accumulator. The default kernel therefore idles the engine through +// every per-chunk fixup. This variant runs two 4-accumulator sets in +// alternation: chunk c+1's GER stream is issued on set B before set A +// is drained, so the drain + VSU fixup of chunk c overlap live GER +// execution. Cost: all 8 accumulators alias VSRs 0-31, raising spill +// pressure (visible in the static count); benefit: engine-idle removal +// (invisible to any static count). Only silicon can arbitrate -- +// this is hardware experiment #1 in docs/BENCHMARKS-QEMU.md. + +static inline void grid_pp_compute(const vuc * a, const vuc * y, + __vector_quad acc[2][2]) { + for (int g = 0; g < 2; g++) + for (int cgi = 0; cgi < 2; cgi++) + __builtin_mma_xxsetaccz(&acc[g][cgi]); + for (int x = 0; x < 8; x++) { +#ifdef IQGRID_LXVP + vuc wv[2], yv[2]; + load_pair(a + 2*x, wv); + load_pair(y + 2*x, yv); + const vuc w0 = wv[0], w1 = wv[1]; + const vuc y0 = yv[0], y1 = yv[1]; +#else + const vuc w0 = a[2*x], w1 = a[2*x + 1]; + const vuc y0 = y[2*x], y1 = y[2*x + 1]; +#endif + __builtin_mma_xvi8ger4pp(&acc[0][0], w0, y0); + __builtin_mma_xvi8ger4pp(&acc[0][1], w0, y1); + __builtin_mma_xvi8ger4pp(&acc[1][0], w1, y0); + __builtin_mma_xvi8ger4pp(&acc[1][1], w1, y1); + } +} + +static inline void grid_pp_fixup(const agrid_t * PA, const bgrid_t * PB, + int64_t ch, __vector_quad acc[2][2], + vfl fin[MR][2]) { + for (int g = 0; g < 2; g++) { + const vfl sA = PA->sA [ch][g]; + const vfl C128 = PA->C128[ch][g]; + for (int cgi = 0; cgi < 2; cgi++) { + vsi rowsP[4]; + __builtin_mma_disassemble_acc(rowsP, &acc[g][cgi]); + const vfl dB = PB->dB[ch][cgi]; + for (int r = 0; r < 4; r++) { + vfl t = vec_sub(vec_ctf(rowsP[r],0), vec_splats(C128[r])); + vfl sc = vec_mul(vec_splats(sA[r]), dB); + fin[4*g + r][cgi] = vec_madd(t, sc, fin[4*g + r][cgi]); + } + } + } +} + +__attribute__((unused)) static void kernel_grid_8x8_pp(const agrid_t * PA, const bgrid_t * PB, + int64_t nch, vfl fin[MR][2]) { + if (nch <= 0) return; + __vector_quad accA[2][2], accB[2][2]; + grid_pp_compute(PA->v[0], PB->v[0], accA); + int64_t ch = 1; + int aLive = 1; // which set holds chunk ch-1 + for (; ch < nch; ch++) { + if (aLive) { grid_pp_compute(PA->v[ch], PB->v[ch], accB); + grid_pp_fixup(PA, PB, ch - 1, accA, fin); } + else { grid_pp_compute(PA->v[ch], PB->v[ch], accA); + grid_pp_fixup(PA, PB, ch - 1, accB, fin); } + aLive ^= 1; + } + grid_pp_fixup(PA, PB, nch - 1, aLive ? accA : accB, fin); +} + +extern "C" void grid_gemm_packed(int64_t m, int64_t n, int64_t k, + const void * packedA, const void * packedB, + float * C, int64_t ldc, int ith, int nth) { + const agrid_t * PA = (const agrid_t *)packedA; + const bgrid_t * PB = (const bgrid_t *)packedB; + const int64_t kb = k/32, ns = sl(k), mt = rt(m), njt = ct(n); + const int64_t tpt = (mt + nth - 1) / nth; + const int64_t t0 = ith*tpt, t1 = (ith+1)*tpt < mt ? (ith+1)*tpt : mt; + vfl fin[MR][2]; + for (int64_t it = t0; it < t1; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + for (int64_t jt = 0; jt < njt; jt++) { + for (int r = 0; r < MR; r++) fin[r][0] = fin[r][1] = vec_splats(0.0f); + for (int64_t s = 0; s < ns; s++) { + const int64_t b0 = s*KC_CH; + const int64_t nch = (kb - b0) < KC_CH ? (kb - b0) : KC_CH; +#ifdef IQGRID_PINGPONG + kernel_grid_8x8_pp(&PA[it*ns + s], &PB[jt*ns + s], nch, fin); +#else + kernel_grid_8x8(&PA[it*ns + s], &PB[jt*ns + s], nch, fin); +#endif + } + const int64_t j0 = jt*NR; + const int64_t cols = (n - j0) < NR ? (n - j0) : NR; + for (int64_t r = 0; r < rows; r++) + for (int64_t cj = 0; cj < cols; cj++) + C[(i + r) + (j0 + cj)*ldc] = fin[r][cj >> 2][cj & 3]; + } + } +} + + +// ---- 16-deep chunk variant for per-16-scale formats ---- + +#define KC_CH16 (KC_ELEMS / 16) + +typedef struct { + vuc v[KC_CH16][8]; // 4 depth-steps x 2 rowgroups + vfl sA [KC_CH16][2]; + vfl C128[KC_CH16][2]; +} agrid16_t; + +typedef struct { + vuc v[KC_CH16][8]; // 4 depth-steps x 2 colgroups + vfl dB[KC_CH16][2]; +} bgrid16_t; + +extern "C" size_t grid16_apack_size(int64_t m, int64_t k) { + return (((size_t)(rt(m) * sl(k)) * sizeof(agrid16_t)) + 63) & ~(size_t)63; +} +extern "C" size_t grid16_bpack_size(int64_t n, int64_t k) { + return (((size_t)(ct(n) * sl(k)) * sizeof(bgrid16_t)) + 63) & ~(size_t)63; +} + +static void grid16_place_chunk(agrid16_t * T, int64_t ch, + const vsc w[MR], const float scale[MR]) { + for (int g = 0; g < 2; g++) { + float W[4]; + vui rows4[4]; + for (int r = 0; r < 4; r++) rows4[r] = (vui)w[4*g + r]; + mma_transpose4(rows4, &T->v[ch][g], 2); + for (int r = 0; r < 4; r++) { + vsi z = vec_splats(0); + vsi sm = vec_sum4s(w[4*g + r], z); + W[r] = (float)hsum(sm); + } + T->sA [ch][g] = (vfl){ scale[4*g], scale[4*g+1], scale[4*g+2], scale[4*g+3] }; + T->C128[ch][g] = (vfl){ 128.0f*W[0], 128.0f*W[1], + 128.0f*W[2], 128.0f*W[3] }; // exact int; see field FAIL 2026-07-21 + } +} + +template +static void repack_grid16(const BLK * A, int64_t lda, int64_t m, int64_t k, agrid16_t * P) { + const int64_t nsb = k/QK_K, ns = sl(k); + for (int64_t it = 0; it < rt(m); it++) + for (int64_t s = 0; s < ns; s++) { + agrid16_t * T = &P[it*ns + s]; + const int64_t sb0 = (s*KC_CH)/8; + const int64_t nsl = (nsb - sb0) < KC_CH/8 ? (nsb - sb0) : KC_CH/8; + for (int64_t sb = 0; sb < nsl; sb++) { + int8_t code[MR][QK_K]; float sc[MR][16]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + DEC(&A[rr*lda + sb0 + sb], code[r], sc[r]); + } + for (int c = 0; c < 16; c++) { + vsc w[MR]; float scale[MR]; + for (int r = 0; r < MR; r++) { + memcpy(&w[r], code[r] + 16*c, 16); + scale[r] = sc[r][c]; + } + grid16_place_chunk(T, 16*sb + c, w, scale); + } + } + } +} + +extern "C" void grid16_repack_iq2_xs(const block_iq2_xs_ppc * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_grid16(A, lda, m, k, (agrid16_t *)p); } +extern "C" void grid16_repack_iq2_s(const block_iq2_s_ppc * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_grid16(A, lda, m, k, (agrid16_t *)p); } +extern "C" void grid16_repack_iq1_m(const block_iq1_m_ppc * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_grid16(A, lda, m, k, (agrid16_t *)p); } + +extern "C" void grid16_repack_nvfp4(const block_nvfp4_ppc * A, int64_t lda, + int64_t m, int64_t k, void * p) { + agrid16_t * P = (agrid16_t *)p; + const int64_t nb = k/QK_NVFP4, ns = sl(k); + for (int64_t it = 0; it < rt(m); it++) + for (int64_t s = 0; s < ns; s++) { + agrid16_t * T = &P[it*ns + s]; + const int64_t b0 = (s*KC_CH16)/4; // 64-blocks per slab start + const int64_t nbl = (nb - b0) < KC_CH16/4 ? (nb - b0) : KC_CH16/4; + for (int64_t b = 0; b < nbl; b++) { + int8_t code[MR][QK_NVFP4]; float sc[MR][4]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + dec_nvfp4(&A[rr*lda + b0 + b], code[r], sc[r]); + } + for (int c = 0; c < 4; c++) { + vsc w[MR]; float scale[MR]; + for (int r = 0; r < MR; r++) { + memcpy(&w[r], code[r] + 16*c, 16); + scale[r] = sc[r][c]; + } + grid16_place_chunk(T, 4*b + c, w, scale); + } + } + } +} + +// q8_0 activations for the 16-deep framework: each 32-block feeds two +// chunks; its dB is replicated to both. +extern "C" void grid16_pack_b_q8_0(const block_q8_0_ppc * B, int64_t ldb, + int64_t n, int64_t k, void * packed) { + bgrid16_t * P = (bgrid16_t *)packed; + const int64_t kb = k/32, ns = sl(k); + const vuc flip = vec_splats((unsigned char)0x80); + for (int64_t jt = 0; jt < ct(n); jt++) + for (int64_t s = 0; s < ns; s++) { + bgrid16_t * T = &P[jt*ns + s]; + const int64_t b0 = (s*KC_CH16)/2; + const int64_t nbl = (kb - b0) < KC_CH16/2 ? (kb - b0) : KC_CH16/2; + for (int64_t b = 0; b < nbl; b++) { + const block_q8_0_ppc * yb[NR]; float dB[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = jt*NR + j; if (jj >= n) jj = n - 1; + yb[j] = &B[jj*ldb + b0 + b]; + dB[j] = fp16_to_fp32(yb[j]->d); + } + for (int h = 0; h < 2; h++) { + const int64_t ch = 2*b + h; + T->dB[ch][0] = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + T->dB[ch][1] = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + vui rows4[4]; + for (int a = 0; a < 2; a++) { + for (int j = 0; j < 4; j++) + rows4[j] = (vui)vec_xor( + load16u((const uint8_t *)(yb[4*a + j]->qs) + 16*h), flip); + mma_transpose4(rows4, &T->v[ch][a], 2); + } + } + } + } +} + +extern "C" void grid16_pack_b_q8_K(const block_q8_K_ppc * B, int64_t ldb, + int64_t n, int64_t k, void * packed) { + bgrid16_t * P = (bgrid16_t *)packed; + const int64_t nsb = k/QK_K, ns = sl(k); + const vuc flip = vec_splats((unsigned char)0x80); + for (int64_t jt = 0; jt < ct(n); jt++) + for (int64_t s = 0; s < ns; s++) { + bgrid16_t * T = &P[jt*ns + s]; + const int64_t sb0 = (s*KC_CH)/8; + const int64_t nsl = (nsb - sb0) < KC_CH/8 ? (nsb - sb0) : KC_CH/8; + for (int64_t sb = 0; sb < nsl; sb++) { + const block_q8_K_ppc * yb[NR]; float dB[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = jt*NR + j; if (jj >= n) jj = n - 1; + yb[j] = &B[jj*ldb + sb0 + sb]; + dB[j] = yb[j]->d; + } + for (int c = 0; c < 16; c++) { + const int64_t ch = 16*sb + c; + T->dB[ch][0] = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + T->dB[ch][1] = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + vui rows4[4]; + for (int a = 0; a < 2; a++) { + for (int j = 0; j < 4; j++) + rows4[j] = (vui)vec_xor( + load16u((const uint8_t *)(yb[4*a + j]->qs) + 16*c), flip); + mma_transpose4(rows4, &T->v[ch][a], 2); + } + } + } + } +} + +static void kernel_grid16_8x8(const agrid16_t * PA, const bgrid16_t * PB, + int64_t nch, vfl fin[MR][2]) { + for (int64_t ch = 0; ch < nch; ch++) { + const vuc * a = PA->v[ch]; + const vuc * y = PB->v[ch]; + if (ch + 1 < nch) { + // each chunk's packed panel spans two 128B lines; touch both + __builtin_prefetch(PA->v[ch + 1], 0, 3); + __builtin_prefetch((const char *)PA->v[ch + 1] + 128, 0, 3); + __builtin_prefetch(PB->v[ch + 1], 0, 3); + __builtin_prefetch((const char *)PB->v[ch + 1] + 128, 0, 3); + } + __vector_quad acc[2][2]; + for (int g = 0; g < 2; g++) + for (int cgi = 0; cgi < 2; cgi++) + __builtin_mma_xxsetaccz(&acc[g][cgi]); + for (int x = 0; x < 4; x++) { + const vuc w0 = a[2*x], w1 = a[2*x + 1]; + const vuc y0 = y[2*x], y1 = y[2*x + 1]; + __builtin_mma_xvi8ger4pp(&acc[0][0], w0, y0); + __builtin_mma_xvi8ger4pp(&acc[0][1], w0, y1); + __builtin_mma_xvi8ger4pp(&acc[1][0], w1, y0); + __builtin_mma_xvi8ger4pp(&acc[1][1], w1, y1); + } + for (int g = 0; g < 2; g++) { + const vfl sA = PA->sA [ch][g]; + const vfl C128 = PA->C128[ch][g]; + for (int cgi = 0; cgi < 2; cgi++) { + vsi rowsP[4]; + __builtin_mma_disassemble_acc(rowsP, &acc[g][cgi]); + const vfl dB = PB->dB[ch][cgi]; + for (int r = 0; r < 4; r++) { + vfl t = vec_sub(vec_ctf(rowsP[r],0), vec_splats(C128[r])); + vfl sc = vec_mul(vec_splats(sA[r]), dB); + fin[4*g + r][cgi] = vec_madd(t, sc, fin[4*g + r][cgi]); + } + } + } + } +} + +extern "C" void grid16_gemm_packed(int64_t m, int64_t n, int64_t k, + const void * packedA, const void * packedB, + float * C, int64_t ldc, int ith, int nth) { + const agrid16_t * PA = (const agrid16_t *)packedA; + const bgrid16_t * PB = (const bgrid16_t *)packedB; + const int64_t kc16 = k/16, ns = sl(k), mt = rt(m), njt = ct(n); + const int64_t tpt = (mt + nth - 1) / nth; + const int64_t t0 = ith*tpt, t1 = (ith+1)*tpt < mt ? (ith+1)*tpt : mt; + vfl fin[MR][2]; + for (int64_t it = t0; it < t1; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + for (int64_t jt = 0; jt < njt; jt++) { + for (int r = 0; r < MR; r++) fin[r][0] = fin[r][1] = vec_splats(0.0f); + for (int64_t s = 0; s < ns; s++) { + const int64_t c0 = s*KC_CH16; + const int64_t nch = (kc16 - c0) < KC_CH16 ? (kc16 - c0) : KC_CH16; + kernel_grid16_8x8(&PA[it*ns + s], &PB[jt*ns + s], nch, fin); + } + const int64_t j0 = jt*NR; + const int64_t cols = (n - j0) < NR ? (n - j0) : NR; + for (int64_t r = 0; r < rows; r++) + for (int64_t cj = 0; cj < cols; cj++) + C[(i + r) + (j0 + cj)*ldc] = fin[r][cj >> 2][cj & 3]; + } + } +} + + +// ---- packed GEMV, n = 1: GER over the cached tiles -------------------- +// +// The two-row VSX GEMV lost to vec_dot because n = 1 on this machine +// is issue-rate bound (VALIDATION-POWER10.md s9.2). This kernel +// attacks that constraint with the one unit that crushes instructions +// per byte: each xvi8ger4pp retires 4 rows x 4 depth of dot product, +// and the A-side tiles, per-chunk scales and 128-offset corrections +// already sit decoded in the pack cache. The activation group is +// XOR-flipped and replicated across the GER's four columns (one perm +// per depth group), so every accumulator lane holds the same answer; +// the fixup gathers the four rows into lane form with two merges and +// a permdi. Roughly 62 instructions per 8-row x 32-deep chunk versus +// ~190 for the same work through the per-format vec_dot. +extern "C" void grid_gemv_packed(int64_t m, int64_t k, const void * PAv, + const block_q8_K_ppc * y, float * C, + int ith, int nth) { + const agrid_t * PA = (const agrid_t *)PAv; + const int64_t nt = rt(m), ns = sl(k), ncht = k/32; + const vuc flip = vec_splats((unsigned char)0x80); + const vuc zero = vec_splats((unsigned char)0); + static const vuc repl[4] = { + (vuc){ 0,1,2,3, 0,1,2,3, 0,1,2,3, 0,1,2,3 }, + (vuc){ 4,5,6,7, 4,5,6,7, 4,5,6,7, 4,5,6,7 }, + (vuc){ 8,9,10,11, 8,9,10,11, 8,9,10,11, 8,9,10,11 }, + (vuc){ 12,13,14,15, 12,13,14,15, 12,13,14,15, 12,13,14,15 }, + }; + const int64_t tpt = (nt + nth - 1) / nth; + const int64_t t0 = (int64_t)ith * tpt; + const int64_t t1 = (t0 + tpt) < nt ? (t0 + tpt) : nt; + + for (int64_t it = t0; it < t1; it++) { + vfl fin0 = vec_splats(0.0f); // rows it*MR+0..3, lane-per-row + vfl fin1 = vec_splats(0.0f); // rows it*MR+4..7 + for (int64_t s = 0; s < ns; s++) { + const agrid_t * T = &PA[it*ns + s]; + const int64_t c0 = s*KC_CH; + const int64_t nch = (ncht - c0) < KC_CH ? (ncht - c0) : KC_CH; + for (int64_t ch = 0; ch < nch; ch++) { + if (ch + 1 < nch) { + __builtin_prefetch(T->v[ch + 1], 0, 3); + __builtin_prefetch((const char *)T->v[ch + 1] + 128, 0, 3); + } + const int64_t gc = c0 + ch; + const int8_t * q8 = y[gc/8].qs + 32*(gc%8); + const vfl vdB = vec_splats(y[gc/8].d); + const vuc ylo = vec_xor(load16u(q8), flip); + const vuc yhi = vec_xor(load16u(q8 + 16), flip); + __vector_quad acc0, acc1; + __builtin_mma_xxsetaccz(&acc0); + __builtin_mma_xxsetaccz(&acc1); + const vuc * a = T->v[ch]; + for (int x = 0; x < 8; x++) { + const vuc yg = vec_perm(x < 4 ? ylo : yhi, zero, repl[x & 3]); + __builtin_mma_xvi8ger4pp(&acc0, a[2*x], yg); + __builtin_mma_xvi8ger4pp(&acc1, a[2*x + 1], yg); + } + { + vsi rp[4]; + __builtin_mma_disassemble_acc(rp, &acc0); + vui m01 = vec_mergeh((vui)rp[0], (vui)rp[1]); + vui m23 = vec_mergeh((vui)rp[2], (vui)rp[3]); + vsi lanes = (vsi)vec_xxpermdi(m01, m23, 0); + vfl t = vec_sub(vec_ctf(lanes, 0), T->C128[ch][0]); + fin0 = vec_madd(t, vec_mul(T->sA[ch][0], vdB), fin0); + } + { + vsi rp[4]; + __builtin_mma_disassemble_acc(rp, &acc1); + vui m01 = vec_mergeh((vui)rp[0], (vui)rp[1]); + vui m23 = vec_mergeh((vui)rp[2], (vui)rp[3]); + vsi lanes = (vsi)vec_xxpermdi(m01, m23, 0); + vfl t = vec_sub(vec_ctf(lanes, 0), T->C128[ch][1]); + fin1 = vec_madd(t, vec_mul(T->sA[ch][1], vdB), fin1); + } + } + } + const int64_t r0 = it*MR; + for (int r = 0; r < 4; r++) { + if (r0 + r < m) C[r0 + r] = fin0[r]; + if (r0 + 4 + r < m) C[r0 + 4 + r] = fin1[r]; + } + } +} + + + +// ---- packed GEMV for the 16-deep family (see grid_gemv_packed) -------- +// +// The 16-deep chunks give each accumulator only FOUR GERs before the +// fixup wants it drained, and xxmfacc serializes against that +// accumulator's GER stream -- the engine never reaches a deep +// pipeline (first cut measured 10 t/s where vec_dot does 33). So +// this kernel ping-pongs two accumulator sets across chunks: chunk +// c+1's GERs issue on the idle set while chunk c drains and fixes up. +// The same trade the 8x8 kernels measured at n = 8 is decisive here. +static inline void grid16_gemv_issue(const agrid16_t * T, int64_t ch, + const block_q8_K_ppc * y, int64_t gc, + const vuc repl[4], vuc flip, vuc zero, + __vector_quad * a0, __vector_quad * a1) { + const int8_t * q8 = y[gc/16].qs + 16*(gc%16); + const vuc yv = vec_xor(load16u(q8), flip); + __builtin_mma_xxsetaccz(a0); + __builtin_mma_xxsetaccz(a1); + const vuc * a = T->v[ch]; + for (int t = 0; t < 4; t++) { + const vuc yg = vec_perm(yv, zero, repl[t]); + __builtin_mma_xvi8ger4pp(a0, a[2*t], yg); + __builtin_mma_xvi8ger4pp(a1, a[2*t + 1], yg); + } +} + +static inline void grid16_gemv_fixup(const agrid16_t * T, int64_t ch, + const block_q8_K_ppc * y, int64_t gc, + __vector_quad * a0, __vector_quad * a1, + vfl * fin0, vfl * fin1) { + const vfl vdB = vec_splats(y[gc/16].d); + { + vsi rp[4]; + __builtin_mma_disassemble_acc(rp, a0); + vui m01 = vec_mergeh((vui)rp[0], (vui)rp[1]); + vui m23 = vec_mergeh((vui)rp[2], (vui)rp[3]); + vsi lanes = (vsi)vec_xxpermdi(m01, m23, 0); + vfl t = vec_sub(vec_ctf(lanes, 0), T->C128[ch][0]); + *fin0 = vec_madd(t, vec_mul(T->sA[ch][0], vdB), *fin0); + } + { + vsi rp[4]; + __builtin_mma_disassemble_acc(rp, a1); + vui m01 = vec_mergeh((vui)rp[0], (vui)rp[1]); + vui m23 = vec_mergeh((vui)rp[2], (vui)rp[3]); + vsi lanes = (vsi)vec_xxpermdi(m01, m23, 0); + vfl t = vec_sub(vec_ctf(lanes, 0), T->C128[ch][1]); + *fin1 = vec_madd(t, vec_mul(T->sA[ch][1], vdB), *fin1); + } +} + +extern "C" void grid16_gemv_packed(int64_t m, int64_t k, const void * PAv, + const block_q8_K_ppc * y, float * C, + int ith, int nth) { + const agrid16_t * PA = (const agrid16_t *)PAv; + const int64_t nt = rt(m), ns = sl(k), ncht = k/16; + const vuc flip = vec_splats((unsigned char)0x80); + const vuc zero = vec_splats((unsigned char)0); + static const vuc repl[4] = { + (vuc){ 0,1,2,3, 0,1,2,3, 0,1,2,3, 0,1,2,3 }, + (vuc){ 4,5,6,7, 4,5,6,7, 4,5,6,7, 4,5,6,7 }, + (vuc){ 8,9,10,11, 8,9,10,11, 8,9,10,11, 8,9,10,11 }, + (vuc){ 12,13,14,15, 12,13,14,15, 12,13,14,15, 12,13,14,15 }, + }; + const int64_t tpt = (nt + nth - 1) / nth; + const int64_t t0 = (int64_t)ith * tpt; + const int64_t t1 = (t0 + tpt) < nt ? (t0 + tpt) : nt; + + for (int64_t it = t0; it < t1; it++) { + vfl fin0 = vec_splats(0.0f); + vfl fin1 = vec_splats(0.0f); + for (int64_t s = 0; s < ns; s++) { + const agrid16_t * T = &PA[it*ns + s]; + const int64_t c0 = s*KC_CH16; + const int64_t nch = (ncht - c0) < KC_CH16 ? (ncht - c0) : KC_CH16; + if (nch <= 0) continue; + __vector_quad accP0, accP1, accQ0, accQ1; + grid16_gemv_issue(T, 0, y, c0, repl, flip, zero, &accP0, &accP1); + int p = 1; + for (int64_t ch = 0; ch < nch; ch++) { + if (ch + 2 < nch) __builtin_prefetch(T->v[ch + 2], 0, 3); + if (ch + 1 < nch) { + if (p) grid16_gemv_issue(T, ch + 1, y, c0 + ch + 1, repl, flip, zero, &accQ0, &accQ1); + else grid16_gemv_issue(T, ch + 1, y, c0 + ch + 1, repl, flip, zero, &accP0, &accP1); + } + if (p) grid16_gemv_fixup(T, ch, y, c0 + ch, &accP0, &accP1, &fin0, &fin1); + else grid16_gemv_fixup(T, ch, y, c0 + ch, &accQ0, &accQ1, &fin0, &fin1); + p ^= 1; + } + } + const int64_t r0 = it*MR; + for (int r = 0; r < 4; r++) { + if (r0 + r < m) C[r0 + r] = fin0[r]; + if (r0 + 4 + r < m) C[r0 + 4 + r] = fin1[r]; + } + } +} + + + +#define GRID_ONESHOT_C(NAME, BLKA, REPACK, GEMM, GEMV, ASZ, BSZ, PACKB, VARIANT) \ +extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ + const void * Av, int64_t lda, const void * Bv, int64_t ldb, \ + float * C, int64_t ldc, int ith, int nth) { \ + const BLKA * A = (const BLKA *)Av; \ + const block_q8_K_ppc * B = (const block_q8_K_ppc *)Bv; \ + int fresh = 0; \ + void * PA = ppc_apack_cache_acquire_par(Av, m, k, VARIANT, \ + ASZ(m, k), nth, &fresh); \ + if (PA) { \ + if (fresh) { \ + int64_t i0_, rows_; \ + ppc_apack_slice(m, MR, ith, nth, &i0_, &rows_); \ + if (rows_ > 0) REPACK(A + i0_*lda, lda, rows_, k, \ + (void *)((agrid_t *)PA + (i0_/MR)*sl(k))); \ + ppc_apack_cache_slice_done(Av, m, k, VARIANT); \ + } \ + const int64_t njt = (n + NR - 1) / NR; \ + if (njt < nth) { \ + /* n too small to feed every thread by columns (worst \ + case n == 1 generation: one column, nth-1 idle threads \ + while scalar row-partitions). Row-partition with the \ + cached pack instead; the full activation pack is tiny \ + at these n. Field regression, Q4_K tg32, 2026-07-21. */ \ + if (n == 1) { \ + GEMV(m, k, PA, B, (float *)C, ith, nth); \ + } else { \ + void * PBs = aligned_alloc(64, BSZ(n, k)); \ + if (!PBs) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + PACKB(B, ldb, n, k, PBs); \ + GEMM(m, n, k, PA, PBs, C, ldc, ith, nth); \ + free(PBs); \ + } \ + } else { \ + const int64_t jpt = (njt + nth - 1) / nth; \ + const int64_t jt0 = (int64_t)ith*jpt; \ + const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; \ + if (jt0 < jt1) { \ + const int64_t j0 = jt0*NR; \ + const int64_t nc = (n - j0) < (jt1 - jt0)*NR ? (n - j0) \ + : (jt1 - jt0)*NR; \ + void * PBl = aligned_alloc(64, BSZ(nc, k)); \ + if (!PBl) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + PACKB(B + j0*ldb, ldb, nc, k, PBl); \ + GEMM(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); \ + free(PBl); \ + } \ + } \ + } else { \ + void * PB = aligned_alloc(64, BSZ(n, k)); \ + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + PACKB(B, ldb, n, k, PB); \ + void * PT = aligned_alloc(64, ASZ(MR, k)); \ + if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + const int64_t mt = (m + MR - 1) / MR; \ + const int64_t tpt = (mt + nth - 1) / nth; \ + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { \ + const int64_t i = it*MR; \ + const int64_t rows = (m - i) < MR ? (m - i) : MR; \ + REPACK(A + i*lda, lda, rows, k, PT); \ + GEMM(rows, n, k, PT, PB, C + i, ldc, 0, 1); \ + } \ + free(PT); \ + free(PB); \ + } \ +} +GRID_ONESHOT_C(gemm_tq2_0_q8_K_ppc, block_tq2_0_ppc, grid_repack_tq2_0, grid_gemm_packed, grid_gemv_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 10) +GRID_ONESHOT_C(gemm_tq1_0_q8_K_ppc, block_tq1_0_ppc, grid_repack_tq1_0, grid_gemm_packed, grid_gemv_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 11) +GRID_ONESHOT_C(gemm_iq2_xxs_q8_K_ppc, block_iq2_xxs_ppc, grid_repack_iq2_xxs, grid_gemm_packed, grid_gemv_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 12) +GRID_ONESHOT_C(gemm_iq3_xxs_q8_K_ppc, block_iq3_xxs_ppc, grid_repack_iq3_xxs, grid_gemm_packed, grid_gemv_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 13) +GRID_ONESHOT_C(gemm_iq3_s_q8_K_ppc, block_iq3_s_ppc, grid_repack_iq3_s, grid_gemm_packed, grid_gemv_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 14) +GRID_ONESHOT_C(gemm_iq1_s_q8_K_ppc, block_iq1_s_ppc, grid_repack_iq1_s, grid_gemm_packed, grid_gemv_packed, grid_apack_size, grid_bpack_size, grid_pack_b_q8_K, 15) +GRID_ONESHOT_C(gemm_iq2_xs_q8_K_ppc, block_iq2_xs_ppc, grid16_repack_iq2_xs, grid16_gemm_packed, grid16_gemv_packed, grid16_apack_size, grid16_bpack_size, grid16_pack_b_q8_K, 16) +GRID_ONESHOT_C(gemm_iq2_s_q8_K_ppc, block_iq2_s_ppc, grid16_repack_iq2_s, grid16_gemm_packed, grid16_gemv_packed, grid16_apack_size, grid16_bpack_size, grid16_pack_b_q8_K, 17) +GRID_ONESHOT_C(gemm_iq1_m_q8_K_ppc, block_iq1_m_ppc, grid16_repack_iq1_m, grid16_gemm_packed, grid16_gemv_packed, grid16_apack_size, grid16_bpack_size, grid16_pack_b_q8_K, 18) + +extern "C" void gemm_nvfp4_q8_0_ppc(int64_t m, int64_t n, int64_t k, + const void * Av, int64_t lda, const void * Bv, int64_t ldb, + float * C, int64_t ldc, int ith, int nth) { + const block_nvfp4_ppc * A = (const block_nvfp4_ppc *)Av; + const block_q8_0_ppc * B = (const block_q8_0_ppc *)Bv; + int fresh = 0; + void * PA = ppc_apack_cache_acquire_par(Av, m, k, 19, + grid16_apack_size(m, k), nth, &fresh); + if (PA) { + if (fresh) { + int64_t i0_, rows_; + ppc_apack_slice(m, MR, ith, nth, &i0_, &rows_); + if (rows_ > 0) grid16_repack_nvfp4(A + i0_*lda, lda, rows_, k, + (void *)((agrid16_t *)PA + (i0_/MR)*sl(k))); + ppc_apack_cache_slice_done(Av, m, k, 19); + } + const int64_t njt = (n + NR - 1) / NR; + if (njt < nth) { + /* n too small to feed every thread by columns (worst + case n == 1 generation: one column, nth-1 idle threads + while scalar row-partitions). Row-partition with the + cached pack instead; the full activation pack is tiny + at these n. Field regression, Q4_K tg32, 2026-07-21. */ + void * PBs = aligned_alloc(64, grid16_bpack_size(n, k)); + if (!PBs) { GGML_ABORT("ppc-mma: pack alloc failed"); } + grid16_pack_b_q8_0(B, ldb, n, k, PBs); + grid16_gemm_packed(m, n, k, PA, PBs, C, ldc, ith, nth); + free(PBs); + } else { + const int64_t jpt = (njt + nth - 1) / nth; + const int64_t jt0 = (int64_t)ith*jpt; + const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; + if (jt0 < jt1) { + const int64_t j0 = jt0*NR; + const int64_t nc = (n - j0) < (jt1 - jt0)*NR ? (n - j0) : (jt1 - jt0)*NR; + void * PBl = aligned_alloc(64, grid16_bpack_size(nc, k)); + if (!PBl) { GGML_ABORT("ppc-mma: pack alloc failed"); } + grid16_pack_b_q8_0(B + j0*ldb, ldb, nc, k, PBl); + grid16_gemm_packed(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); + free(PBl); + } + } + } else { + void * PB = aligned_alloc(64, grid16_bpack_size(n, k)); + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } + grid16_pack_b_q8_0(B, ldb, n, k, PB); + void * PT = aligned_alloc(64, grid16_apack_size(MR, k)); + if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } + const int64_t mt = (m + MR - 1) / MR; + const int64_t tpt = (mt + nth - 1) / nth; + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + grid16_repack_nvfp4(A + i*lda, lda, rows, k, PT); + grid16_gemm_packed(rows, n, k, PT, PB, C + i, ldc, 0, 1); + } + free(PT); + free(PB); + } +} + +#endif // __MMA__ + diff --git a/ggml/src/ggml-cpu/llamafile/iq_grids_ppc.h b/ggml/src/ggml-cpu/llamafile/iq_grids_ppc.h new file mode 100644 index 000000000000..917f4ed0f487 --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/iq_grids_ppc.h @@ -0,0 +1,1094 @@ +// iq_grids.h - grid/sign tables extracted from ggml-common.h (MIT, ggml authors) +#pragma once +#include + +static const uint64_t iq2xxs_grid[256] = { + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, + 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808, + 0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819, + 0x0808080819081908, 0x0808080819190808, 0x0808080819192b08, 0x08080808192b0819, + 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b, 0x080808082b082b2b, + 0x080808082b2b082b, 0x0808081908080819, 0x0808081908081908, 0x0808081908190808, + 0x0808081908191919, 0x0808081919080808, 0x080808192b081908, 0x080808192b192b08, + 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b082b082b, 0x0808082b2b08082b, + 0x0808190808080819, 0x0808190808081908, 0x0808190808190808, 0x08081908082b0819, + 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819082b08, + 0x08081908192b0808, 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, + 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08, + 0x08081919082b0808, 0x080819191908192b, 0x08081919192b2b19, 0x080819192b080808, + 0x080819192b190819, 0x0808192b08082b19, 0x0808192b08190808, 0x0808192b19080808, + 0x0808192b2b081908, 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919, + 0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, 0x08082b0819080819, + 0x08082b0819081908, 0x08082b0819190808, 0x08082b081919082b, 0x08082b082b082b08, + 0x08082b1908081908, 0x08082b1919080808, 0x08082b2b0808082b, 0x08082b2b08191908, + 0x0819080808080819, 0x0819080808081908, 0x0819080808190808, 0x08190808082b0819, + 0x0819080819080808, 0x08190808192b0808, 0x081908082b081908, 0x081908082b190808, + 0x081908082b191919, 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808, + 0x0819081919190808, 0x0819081919192b2b, 0x081908192b080808, 0x0819082b082b1908, + 0x0819082b19081919, 0x0819190808080808, 0x0819190808082b08, 0x08191908082b0808, + 0x08191908082b1919, 0x0819190819082b19, 0x081919082b080808, 0x0819191908192b08, + 0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, 0x08192b0808080819, + 0x08192b0808081908, 0x08192b0808190808, 0x08192b0819080808, 0x08192b082b080819, + 0x08192b1908080808, 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819, + 0x082b080808080808, 0x082b08080808082b, 0x082b080808082b2b, 0x082b080819081908, + 0x082b0808192b0819, 0x082b08082b080808, 0x082b08082b08082b, 0x082b0819082b2b19, + 0x082b081919082b08, 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819, + 0x082b190808081908, 0x082b190808190808, 0x082b190819080808, 0x082b19081919192b, + 0x082b191908080808, 0x082b191919080819, 0x082b1919192b1908, 0x082b192b2b190808, + 0x082b2b0808082b08, 0x082b2b08082b0808, 0x082b2b082b191908, 0x082b2b2b19081908, + 0x1908080808080819, 0x1908080808081908, 0x1908080808190808, 0x1908080808192b08, + 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808, 0x1908080819082b08, + 0x190808081919192b, 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908, + 0x190808082b190808, 0x1908081908080808, 0x19080819082b0808, 0x19080819192b0819, + 0x190808192b080808, 0x190808192b081919, 0x1908082b08080819, 0x1908082b08190808, + 0x1908082b19082b08, 0x1908082b1919192b, 0x1908082b192b2b08, 0x1908190808080808, + 0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, 0x190819082b192b19, + 0x190819190819082b, 0x19081919082b1908, 0x1908192b08080808, 0x19082b0808080819, + 0x19082b0808081908, 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, + 0x19082b1908080808, 0x19082b1919192b08, 0x19082b19192b0819, 0x19082b192b08082b, + 0x19082b2b19081919, 0x19082b2b2b190808, 0x1919080808080808, 0x1919080808082b08, + 0x1919080808190819, 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808, + 0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, 0x191908192b2b1908, + 0x1919082b2b190819, 0x191919082b190808, 0x191919082b19082b, 0x1919191908082b2b, + 0x1919192b08080819, 0x1919192b19191908, 0x19192b0808080808, 0x19192b0808190819, + 0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808, 0x19192b2b08082b08, + 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, 0x192b0808192b2b08, + 0x192b081908080808, 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808, + 0x192b190808080808, 0x192b190808081919, 0x192b191908190808, 0x192b19190819082b, + 0x192b19192b081908, 0x192b2b081908082b, 0x2b08080808080808, 0x2b0808080808082b, + 0x2b08080808082b2b, 0x2b08080819080819, 0x2b0808082b08082b, 0x2b08081908081908, + 0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, 0x2b08190808080819, + 0x2b08190808081908, 0x2b08190808190808, 0x2b08190808191919, 0x2b08190819080808, + 0x2b081908192b0808, 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908, + 0x2b08192b08082b19, 0x2b08192b19080808, 0x2b08192b192b0808, 0x2b082b080808082b, + 0x2b082b1908081908, 0x2b082b2b08190819, 0x2b19080808081908, 0x2b19080808190808, + 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b, + 0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, 0x2b191908082b082b, + 0x2b19190819081908, 0x2b19191919190819, 0x2b192b082b080819, 0x2b192b19082b0808, + 0x2b2b08080808082b, 0x2b2b080819190808, 0x2b2b08082b081919, 0x2b2b081908082b19, + 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908, +}; + +static const uint32_t iq3xxs_grid[256] = { + 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414, + 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14, + 0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404, + 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e, + 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c, + 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c, + 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34, + 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c, + 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c, + 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04, + 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c, + 0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414, + 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434, + 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c, + 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e, + 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24, + 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24, + 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c, + 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c, + 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14, + 0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414, + 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e, + 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404, + 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c, + 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c, + 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14, + 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c, + 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c, + 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14, + 0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14, + 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c, + 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04, +}; + +static const uint32_t iq3s_grid[512] = { + 0x01010101, 0x01010103, 0x01010105, 0x0101010b, 0x0101010f, 0x01010301, 0x01010303, 0x01010305, + 0x01010309, 0x0101030d, 0x01010501, 0x01010503, 0x0101050b, 0x01010707, 0x01010901, 0x01010905, + 0x0101090b, 0x0101090f, 0x01010b03, 0x01010b07, 0x01010d01, 0x01010d05, 0x01010f03, 0x01010f09, + 0x01010f0f, 0x01030101, 0x01030103, 0x01030105, 0x01030109, 0x01030301, 0x01030303, 0x0103030b, + 0x01030501, 0x01030507, 0x0103050f, 0x01030703, 0x0103070b, 0x01030909, 0x01030d03, 0x01030d0b, + 0x01030f05, 0x01050101, 0x01050103, 0x0105010b, 0x0105010f, 0x01050301, 0x01050307, 0x0105030d, + 0x01050503, 0x0105050b, 0x01050701, 0x01050709, 0x01050905, 0x0105090b, 0x0105090f, 0x01050b03, + 0x01050b07, 0x01050f01, 0x01050f07, 0x01070107, 0x01070303, 0x0107030b, 0x01070501, 0x01070505, + 0x01070703, 0x01070707, 0x0107070d, 0x01070909, 0x01070b01, 0x01070b05, 0x01070d0f, 0x01070f03, + 0x01070f0b, 0x01090101, 0x01090307, 0x0109030f, 0x01090503, 0x01090509, 0x01090705, 0x01090901, + 0x01090907, 0x01090b03, 0x01090f01, 0x010b0105, 0x010b0109, 0x010b0501, 0x010b0505, 0x010b050d, + 0x010b0707, 0x010b0903, 0x010b090b, 0x010b090f, 0x010b0d0d, 0x010b0f07, 0x010d010d, 0x010d0303, + 0x010d0307, 0x010d0703, 0x010d0b05, 0x010d0f03, 0x010f0101, 0x010f0105, 0x010f0109, 0x010f0501, + 0x010f0505, 0x010f050d, 0x010f0707, 0x010f0b01, 0x010f0b09, 0x03010101, 0x03010103, 0x03010105, + 0x03010109, 0x03010301, 0x03010303, 0x03010307, 0x0301030b, 0x0301030f, 0x03010501, 0x03010505, + 0x03010703, 0x03010709, 0x0301070d, 0x03010b09, 0x03010b0d, 0x03010d03, 0x03010f05, 0x03030101, + 0x03030103, 0x03030107, 0x0303010d, 0x03030301, 0x03030309, 0x03030503, 0x03030701, 0x03030707, + 0x03030903, 0x03030b01, 0x03030b05, 0x03030f01, 0x03030f0d, 0x03050101, 0x03050305, 0x0305030b, + 0x0305030f, 0x03050501, 0x03050509, 0x03050705, 0x03050901, 0x03050907, 0x03050b0b, 0x03050d01, + 0x03050f05, 0x03070103, 0x03070109, 0x0307010f, 0x03070301, 0x03070307, 0x03070503, 0x0307050f, + 0x03070701, 0x03070709, 0x03070903, 0x03070d05, 0x03070f01, 0x03090107, 0x0309010b, 0x03090305, + 0x03090309, 0x03090703, 0x03090707, 0x03090905, 0x0309090d, 0x03090b01, 0x03090b09, 0x030b0103, + 0x030b0301, 0x030b0307, 0x030b0503, 0x030b0701, 0x030b0705, 0x030b0b03, 0x030d0501, 0x030d0509, + 0x030d050f, 0x030d0909, 0x030d090d, 0x030f0103, 0x030f0107, 0x030f0301, 0x030f0305, 0x030f0503, + 0x030f070b, 0x030f0903, 0x030f0d05, 0x030f0f01, 0x05010101, 0x05010103, 0x05010107, 0x0501010b, + 0x0501010f, 0x05010301, 0x05010305, 0x05010309, 0x0501030d, 0x05010503, 0x05010507, 0x0501050f, + 0x05010701, 0x05010705, 0x05010903, 0x05010907, 0x0501090b, 0x05010b01, 0x05010b05, 0x05010d0f, + 0x05010f01, 0x05010f07, 0x05010f0b, 0x05030101, 0x05030105, 0x05030301, 0x05030307, 0x0503030f, + 0x05030505, 0x0503050b, 0x05030703, 0x05030709, 0x05030905, 0x05030b03, 0x05050103, 0x05050109, + 0x0505010f, 0x05050503, 0x05050507, 0x05050701, 0x0505070f, 0x05050903, 0x05050b07, 0x05050b0f, + 0x05050f03, 0x05050f09, 0x05070101, 0x05070105, 0x0507010b, 0x05070303, 0x05070505, 0x05070509, + 0x05070703, 0x05070707, 0x05070905, 0x05070b01, 0x05070d0d, 0x05090103, 0x0509010f, 0x05090501, + 0x05090507, 0x05090705, 0x0509070b, 0x05090903, 0x05090f05, 0x05090f0b, 0x050b0109, 0x050b0303, + 0x050b0505, 0x050b070f, 0x050b0901, 0x050b0b07, 0x050b0f01, 0x050d0101, 0x050d0105, 0x050d010f, + 0x050d0503, 0x050d0b0b, 0x050d0d03, 0x050f010b, 0x050f0303, 0x050f050d, 0x050f0701, 0x050f0907, + 0x050f0b01, 0x07010105, 0x07010303, 0x07010307, 0x0701030b, 0x0701030f, 0x07010505, 0x07010703, + 0x07010707, 0x0701070b, 0x07010905, 0x07010909, 0x0701090f, 0x07010b03, 0x07010d07, 0x07010f03, + 0x07030103, 0x07030107, 0x0703010b, 0x07030309, 0x07030503, 0x07030507, 0x07030901, 0x07030d01, + 0x07030f05, 0x07030f0d, 0x07050101, 0x07050305, 0x07050501, 0x07050705, 0x07050709, 0x07050b01, + 0x07070103, 0x07070301, 0x07070309, 0x07070503, 0x07070507, 0x0707050f, 0x07070701, 0x07070903, + 0x07070907, 0x0707090f, 0x07070b0b, 0x07070f07, 0x07090107, 0x07090303, 0x0709030d, 0x07090505, + 0x07090703, 0x07090b05, 0x07090d01, 0x07090d09, 0x070b0103, 0x070b0301, 0x070b0305, 0x070b050b, + 0x070b0705, 0x070b0909, 0x070b0b0d, 0x070b0f07, 0x070d030d, 0x070d0903, 0x070f0103, 0x070f0107, + 0x070f0501, 0x070f0505, 0x070f070b, 0x09010101, 0x09010109, 0x09010305, 0x09010501, 0x09010509, + 0x0901050f, 0x09010705, 0x09010903, 0x09010b01, 0x09010f01, 0x09030105, 0x0903010f, 0x09030303, + 0x09030307, 0x09030505, 0x09030701, 0x0903070b, 0x09030907, 0x09030b03, 0x09030b0b, 0x09050103, + 0x09050107, 0x09050301, 0x0905030b, 0x09050503, 0x09050707, 0x09050901, 0x09050b0f, 0x09050d05, + 0x09050f01, 0x09070109, 0x09070303, 0x09070307, 0x09070501, 0x09070505, 0x09070703, 0x0907070b, + 0x09090101, 0x09090105, 0x09090509, 0x0909070f, 0x09090901, 0x09090f03, 0x090b010b, 0x090b010f, + 0x090b0503, 0x090b0d05, 0x090d0307, 0x090d0709, 0x090d0d01, 0x090f0301, 0x090f030b, 0x090f0701, + 0x090f0907, 0x090f0b03, 0x0b010105, 0x0b010301, 0x0b010309, 0x0b010505, 0x0b010901, 0x0b010909, + 0x0b01090f, 0x0b010b05, 0x0b010d0d, 0x0b010f09, 0x0b030103, 0x0b030107, 0x0b03010b, 0x0b030305, + 0x0b030503, 0x0b030705, 0x0b030f05, 0x0b050101, 0x0b050303, 0x0b050507, 0x0b050701, 0x0b05070d, + 0x0b050b07, 0x0b070105, 0x0b07010f, 0x0b070301, 0x0b07050f, 0x0b070909, 0x0b070b03, 0x0b070d0b, + 0x0b070f07, 0x0b090103, 0x0b090109, 0x0b090501, 0x0b090705, 0x0b09090d, 0x0b0b0305, 0x0b0b050d, + 0x0b0b0b03, 0x0b0b0b07, 0x0b0d0905, 0x0b0f0105, 0x0b0f0109, 0x0b0f0505, 0x0d010303, 0x0d010307, + 0x0d01030b, 0x0d010703, 0x0d010707, 0x0d010d01, 0x0d030101, 0x0d030501, 0x0d03050f, 0x0d030d09, + 0x0d050305, 0x0d050709, 0x0d050905, 0x0d050b0b, 0x0d050d05, 0x0d050f01, 0x0d070101, 0x0d070309, + 0x0d070503, 0x0d070901, 0x0d09050b, 0x0d090907, 0x0d090d05, 0x0d0b0101, 0x0d0b0107, 0x0d0b0709, + 0x0d0b0d01, 0x0d0d010b, 0x0d0d0901, 0x0d0f0303, 0x0d0f0307, 0x0f010101, 0x0f010109, 0x0f01010f, + 0x0f010501, 0x0f010505, 0x0f01070d, 0x0f010901, 0x0f010b09, 0x0f010d05, 0x0f030105, 0x0f030303, + 0x0f030509, 0x0f030907, 0x0f03090b, 0x0f050103, 0x0f050109, 0x0f050301, 0x0f05030d, 0x0f050503, + 0x0f050701, 0x0f050b03, 0x0f070105, 0x0f070705, 0x0f07070b, 0x0f070b07, 0x0f090103, 0x0f09010b, + 0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101, +}; + +static const uint8_t ksigns_iq2xs[128] = { + 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15, + 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159, + 160, 33, 34, 163, 36, 165, 166, 39, 40, 169, 170, 43, 172, 45, 46, 175, + 48, 177, 178, 51, 180, 53, 54, 183, 184, 57, 58, 187, 60, 189, 190, 63, + 192, 65, 66, 195, 68, 197, 198, 71, 72, 201, 202, 75, 204, 77, 78, 207, + 80, 209, 210, 83, 212, 85, 86, 215, 216, 89, 90, 219, 92, 221, 222, 95, + 96, 225, 226, 99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237, 238, 111, + 240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255, +}; + +static const uint8_t kmask_iq2xs[8] = { + 1, 2, 4, 8, 16, 32, 64, 128 +}; +static const uint64_t iq1s_grid[2048] = { + 0xffffffffffffffff, 0xffffffffffffff01, 0xffffffffffff0000, 0xffffffffffff01ff, + 0xffffffffffff0101, 0xffffffffff00ff00, 0xffffffffff000000, 0xffffffffff01ffff, + 0xffffffffff01ff01, 0xffffffffff0101ff, 0xffffffffff010101, 0xffffffff00ff0000, + 0xffffffff0000ff00, 0xffffffff000000ff, 0xffffffff00000001, 0xffffffff00010000, + 0xffffffff01ffffff, 0xffffffff01ffff01, 0xffffffff01ff01ff, 0xffffffff01ff0101, + 0xffffffff01000000, 0xffffffff0101ffff, 0xffffffff0101ff01, 0xffffffff010101ff, + 0xffffffff01010101, 0xffffff00ffff00ff, 0xffffff00ffff0000, 0xffffff00ff00ff00, + 0xffffff00ff0000ff, 0xffffff00ff000001, 0xffffff00ff000100, 0xffffff00ff000101, + 0xffffff00ff010000, 0xffffff0000ffff00, 0xffffff0000ff0001, 0xffffff0000ff0100, + 0xffffff000000ff01, 0xffffff0000000000, 0xffffff0000000101, 0xffffff000001ff00, + 0xffffff00000100ff, 0xffffff0000010001, 0xffffff00000101ff, 0xffffff0001ff0000, + 0xffffff000100ff00, 0xffffff00010000ff, 0xffffff0001000001, 0xffffff0001010000, + 0xffffff01ffffffff, 0xffffff01ffffff01, 0xffffff01ffff01ff, 0xffffff01ffff0101, + 0xffffff01ff000000, 0xffffff01ff01ffff, 0xffffff01ff01ff01, 0xffffff01ff0101ff, + 0xffffff01ff010101, 0xffffff0100ff0000, 0xffffff010000ff00, 0xffffff0100000100, + 0xffffff01000100ff, 0xffffff0100010100, 0xffffff0101ffffff, 0xffffff0101ffff01, + 0xffffff0101ff01ff, 0xffffff0101ff0101, 0xffffff010100ff00, 0xffffff0101000000, + 0xffffff0101000100, 0xffffff010101ffff, 0xffffff010101ff01, 0xffffff01010101ff, + 0xffffff0101010101, 0xffff00ffff00ff00, 0xffff00ffff0000ff, 0xffff00ffff000001, + 0xffff00ffff010000, 0xffff00ff00ffff00, 0xffff00ff00ff0100, 0xffff00ff00000000, + 0xffff00ff00000101, 0xffff00ff000100ff, 0xffff00ff00010000, 0xffff00ff0100ff00, + 0xffff00ff01000100, 0xffff00ff01010000, 0xffff0000ffffff00, 0xffff0000ffff00ff, + 0xffff0000ffff0000, 0xffff0000ffff0001, 0xffff0000ff000000, 0xffff0000ff0001ff, + 0xffff0000ff000101, 0xffff0000ff010100, 0xffff000000ffffff, 0xffff000000ff0000, + 0xffff000000ff0101, 0xffff00000000ffff, 0xffff00000000ff00, 0xffff0000000000ff, + 0xffff000000000000, 0xffff000000000001, 0xffff000000000100, 0xffff00000001ffff, + 0xffff00000001ff01, 0xffff000000010000, 0xffff0000000101ff, 0xffff000000010101, + 0xffff000001ffff00, 0xffff00000100ff00, 0xffff000001000000, 0xffff0000010001ff, + 0xffff000001000101, 0xffff00000101ff00, 0xffff0000010100ff, 0xffff000001010000, + 0xffff000001010001, 0xffff000001010100, 0xffff0001ff0000ff, 0xffff0001ff000100, + 0xffff000100ffff00, 0xffff000100ff00ff, 0xffff00010000ffff, 0xffff00010000ff01, + 0xffff000100000000, 0xffff0001000001ff, 0xffff00010001ffff, 0xffff00010001ff00, + 0xffff000100010001, 0xffff000100010100, 0xffff000101ff0000, 0xffff00010100ff00, + 0xffff0001010000ff, 0xffff000101000100, 0xffff01ffffffffff, 0xffff01ffffffff01, + 0xffff01ffffff01ff, 0xffff01ffffff0101, 0xffff01ffff000000, 0xffff01ffff01ffff, + 0xffff01ffff01ff01, 0xffff01ffff0101ff, 0xffff01ffff010101, 0xffff01ff00ff0000, + 0xffff01ff0000ff00, 0xffff01ff00000001, 0xffff01ff00010000, 0xffff01ff01ffffff, + 0xffff01ff01ffff01, 0xffff01ff01ff01ff, 0xffff01ff01ff0101, 0xffff01ff01000000, + 0xffff01ff0101ffff, 0xffff01ff0101ff01, 0xffff01ff010101ff, 0xffff01ff01010101, + 0xffff0100ffff0000, 0xffff0100ff00ff00, 0xffff0100ff0000ff, 0xffff0100ff000100, + 0xffff0100ff0100ff, 0xffff0100ff010000, 0xffff010000ffff00, 0xffff01000000ffff, + 0xffff01000000ff00, 0xffff010000000000, 0xffff01000001ff00, 0xffff0100000100ff, + 0xffff010000010100, 0xffff01000100ff00, 0xffff0100010000ff, 0xffff010001000001, + 0xffff010001000100, 0xffff010001010000, 0xffff0101ffffffff, 0xffff0101ffffff01, + 0xffff0101ffff01ff, 0xffff0101ffff0101, 0xffff0101ff000000, 0xffff0101ff01ffff, + 0xffff0101ff01ff01, 0xffff0101ff0101ff, 0xffff0101ff010101, 0xffff010100ff0000, + 0xffff01010000ff00, 0xffff010100000100, 0xffff01010001ff00, 0xffff010100010000, + 0xffff010101ffffff, 0xffff010101ffff01, 0xffff010101ff0000, 0xffff010101ff01ff, + 0xffff010101ff0101, 0xffff010101000000, 0xffff01010101ffff, 0xffff01010101ff01, + 0xffff0101010101ff, 0xffff010101010101, 0xff00ffffff00ffff, 0xff00ffffff00ff00, + 0xff00ffffff0000ff, 0xff00ffffff000100, 0xff00ffffff0100ff, 0xff00ffffff010000, + 0xff00ffff00ffff00, 0xff00ffff00ff00ff, 0xff00ffff0000ffff, 0xff00ffff00000000, + 0xff00ffff000001ff, 0xff00ffff0001ff00, 0xff00ffff000100ff, 0xff00ffff00010000, + 0xff00ffff00010100, 0xff00ffff0100ff00, 0xff00ffff010000ff, 0xff00ffff01000001, + 0xff00ffff0101ff00, 0xff00ffff01010000, 0xff00ff00ffffff00, 0xff00ff00ffff00ff, + 0xff00ff00ffff0001, 0xff00ff00ffff0100, 0xff00ff00ff00ffff, 0xff00ff00ff00ff01, + 0xff00ff00ff000000, 0xff00ff00ff0001ff, 0xff00ff00ff01ff00, 0xff00ff00ff0100ff, + 0xff00ff00ff010100, 0xff00ff0000ff0000, 0xff00ff0000ff0101, 0xff00ff000000ffff, + 0xff00ff000000ff00, 0xff00ff000000ff01, 0xff00ff00000000ff, 0xff00ff0000000000, + 0xff00ff0000000001, 0xff00ff0000000100, 0xff00ff000001ffff, 0xff00ff0000010000, + 0xff00ff0001ff00ff, 0xff00ff000100ff01, 0xff00ff0001000000, 0xff00ff000101ff00, + 0xff00ff00010100ff, 0xff00ff01ff00ff00, 0xff00ff01ff0000ff, 0xff00ff01ff000001, + 0xff00ff01ff010000, 0xff00ff0100ffffff, 0xff00ff0100ff0001, 0xff00ff0100ff0100, + 0xff00ff010000ff01, 0xff00ff0100000000, 0xff00ff01000001ff, 0xff00ff0100000101, + 0xff00ff01000100ff, 0xff00ff0100010001, 0xff00ff0101ff0000, 0xff00ff010100ff00, + 0xff00ff01010000ff, 0xff00ff0101000001, 0xff00ff0101010000, 0xff0000ffffffff00, + 0xff0000ffffff0001, 0xff0000ffffff0100, 0xff0000ffff0000ff, 0xff0000ffff000000, + 0xff0000ffff0001ff, 0xff0000ffff000100, 0xff0000ffff01ff00, 0xff0000ffff010001, + 0xff0000ff00ffff00, 0xff0000ff00ff0000, 0xff0000ff00ff0001, 0xff0000ff00ff01ff, + 0xff0000ff00ff0101, 0xff0000ff0000ff00, 0xff0000ff000000ff, 0xff0000ff00000000, + 0xff0000ff00000001, 0xff0000ff00000100, 0xff0000ff0001ff01, 0xff0000ff00010000, + 0xff0000ff000101ff, 0xff0000ff01ff00ff, 0xff0000ff01ff0100, 0xff0000ff0100ffff, + 0xff0000ff010000ff, 0xff0000ff01000000, 0xff0000ff010001ff, 0xff0000ff01000100, + 0xff0000ff01000101, 0xff0000ff0101ff00, 0xff0000ff010100ff, 0xff0000ff01010000, + 0xff0000ff01010100, 0xff000000ffffff01, 0xff000000ffff0000, 0xff000000ffff0101, + 0xff000000ff00ff00, 0xff000000ff0000ff, 0xff000000ff000000, 0xff000000ff000001, + 0xff000000ff000100, 0xff000000ff01ffff, 0xff000000ff01ff01, 0xff000000ff010000, + 0xff000000ff0101ff, 0xff000000ff010101, 0xff00000000ffff00, 0xff00000000ff00ff, + 0xff00000000ff0000, 0xff00000000ff0001, 0xff0000000000ff00, 0xff0000000000ff01, + 0xff000000000000ff, 0xff00000000000000, 0xff00000000000001, 0xff00000000000100, + 0xff00000000000101, 0xff0000000001ff00, 0xff000000000100ff, 0xff00000000010000, + 0xff00000000010001, 0xff00000000010100, 0xff00000001ffffff, 0xff00000001ffff01, + 0xff00000001ff00ff, 0xff00000001ff0000, 0xff00000001ff01ff, 0xff00000001ff0101, + 0xff0000000100ffff, 0xff0000000100ff00, 0xff000000010000ff, 0xff00000001000000, + 0xff00000001000001, 0xff00000001000100, 0xff00000001000101, 0xff0000000101ffff, + 0xff0000000101ff01, 0xff00000001010000, 0xff000001ffffff00, 0xff000001ffff00ff, + 0xff000001ffff0000, 0xff000001ffff0001, 0xff000001ff000000, 0xff000001ff000001, + 0xff000001ff0001ff, 0xff000001ff000101, 0xff000001ff01ff00, 0xff000001ff010001, + 0xff00000100ffffff, 0xff00000100ffff01, 0xff00000100ff00ff, 0xff00000100ff0000, + 0xff00000100ff01ff, 0xff00000100ff0101, 0xff0000010000ff00, 0xff00000100000000, + 0xff00000100000001, 0xff000001000001ff, 0xff00000100000100, 0xff0000010001ff00, + 0xff000001000100ff, 0xff00000100010000, 0xff000001000101ff, 0xff00000100010100, + 0xff00000100010101, 0xff00000101ff0001, 0xff00000101ff0101, 0xff0000010100ff01, + 0xff00000101000000, 0xff000001010100ff, 0xff00000101010100, 0xff0001ffff00ff00, + 0xff0001ffff000001, 0xff0001ffff010000, 0xff0001ff00ffff00, 0xff0001ff00ff00ff, + 0xff0001ff00ff0001, 0xff0001ff00ff0100, 0xff0001ff0000ffff, 0xff0001ff00000000, + 0xff0001ff000001ff, 0xff0001ff00000101, 0xff0001ff0001ffff, 0xff0001ff0001ff00, + 0xff0001ff000100ff, 0xff0001ff00010001, 0xff0001ff00010100, 0xff0001ff01ff0000, + 0xff0001ff0100ff00, 0xff0001ff010000ff, 0xff0001ff01010000, 0xff000100ff00ffff, + 0xff000100ff00ff01, 0xff000100ff000000, 0xff000100ff000101, 0xff000100ff01ff00, + 0xff000100ff010000, 0xff00010000ffff01, 0xff00010000ff00ff, 0xff00010000ff0000, + 0xff00010000ff01ff, 0xff0001000000ff00, 0xff000100000000ff, 0xff00010000000000, + 0xff00010000000001, 0xff00010000000100, 0xff00010000000101, 0xff0001000001ffff, + 0xff00010000010000, 0xff00010000010101, 0xff00010001ff0100, 0xff0001000100ff00, + 0xff0001000100ff01, 0xff00010001000000, 0xff000100010001ff, 0xff0001000101ff00, + 0xff00010001010001, 0xff00010001010100, 0xff000101ffff0100, 0xff000101ff000001, + 0xff000101ff0100ff, 0xff000101ff010001, 0xff00010100ff00ff, 0xff00010100ff0001, + 0xff00010100ff0100, 0xff0001010000ffff, 0xff0001010000ff01, 0xff00010100000000, + 0xff000101000001ff, 0xff0001010001ff00, 0xff00010100010001, 0xff00010100010100, + 0xff00010101ff0000, 0xff0001010100ff00, 0xff00010101000001, 0xff00010101000101, + 0xff01ffffffffffff, 0xff01ffffffffff01, 0xff01ffffffff01ff, 0xff01ffffffff0101, + 0xff01ffffff000000, 0xff01ffffff01ffff, 0xff01ffffff01ff01, 0xff01ffffff010000, + 0xff01ffffff0101ff, 0xff01ffffff010101, 0xff01ffff00ff0000, 0xff01ffff0000ff00, + 0xff01ffff00000100, 0xff01ffff0001ff00, 0xff01ffff00010000, 0xff01ffff01ffffff, + 0xff01ffff01ffff01, 0xff01ffff01ff01ff, 0xff01ffff01ff0101, 0xff01ffff01000000, + 0xff01ffff0101ffff, 0xff01ffff0101ff01, 0xff01ffff01010000, 0xff01ffff010101ff, + 0xff01ffff01010101, 0xff01ff00ffff0000, 0xff01ff00ff00ff00, 0xff01ff00ff0000ff, + 0xff01ff00ff000100, 0xff01ff00ff010000, 0xff01ff0000ffff01, 0xff01ff0000ff00ff, + 0xff01ff0000ff0100, 0xff01ff0000000000, 0xff01ff00000001ff, 0xff01ff0000000101, + 0xff01ff000001ff00, 0xff01ff00000100ff, 0xff01ff0000010000, 0xff01ff0000010001, + 0xff01ff0001ff0000, 0xff01ff000100ffff, 0xff01ff0001000001, 0xff01ff0001000100, + 0xff01ff0001010000, 0xff01ff01ffffff00, 0xff01ff01ffff01ff, 0xff01ff01ffff0101, + 0xff01ff01ff00ff00, 0xff01ff01ff000000, 0xff01ff01ff01ffff, 0xff01ff01ff01ff01, + 0xff01ff01ff0101ff, 0xff01ff01ff010101, 0xff01ff0100ff0000, 0xff01ff010000ff00, + 0xff01ff0100000001, 0xff01ff0100000100, 0xff01ff0100010000, 0xff01ff0101ffff00, + 0xff01ff0101ff01ff, 0xff01ff0101ff0101, 0xff01ff010100ff00, 0xff01ff0101000000, + 0xff01ff010101ffff, 0xff01ff010101ff01, 0xff01ff01010101ff, 0xff01ff0101010101, + 0xff0100ffffff0000, 0xff0100ffff0000ff, 0xff0100ffff000001, 0xff0100ffff000100, + 0xff0100ffff010000, 0xff0100ff00ff00ff, 0xff0100ff00ff0000, 0xff0100ff00ff0001, + 0xff0100ff00ff0100, 0xff0100ff0000ff01, 0xff0100ff00000000, 0xff0100ff000001ff, + 0xff0100ff00000101, 0xff0100ff00010001, 0xff0100ff01ff0000, 0xff0100ff0100ff00, + 0xff0100ff010000ff, 0xff0100ff01000100, 0xff0100ff0101ff00, 0xff0100ff01010000, + 0xff010000ffff0100, 0xff010000ff000000, 0xff010000ff01ff00, 0xff010000ff010100, + 0xff01000000ffffff, 0xff01000000ff0000, 0xff01000000ff01ff, 0xff0100000000ff00, + 0xff010000000000ff, 0xff01000000000000, 0xff01000000000100, 0xff0100000001ff01, + 0xff01000000010000, 0xff010000000101ff, 0xff01000001ff0100, 0xff0100000100ffff, + 0xff010000010000ff, 0xff01000001000000, 0xff010000010001ff, 0xff01000001000101, + 0xff0100000101ff00, 0xff010000010100ff, 0xff01000001010001, 0xff01000001010100, + 0xff010001ffff0000, 0xff010001ff00ffff, 0xff010001ff00ff01, 0xff010001ff000100, + 0xff010001ff010000, 0xff01000100ffff00, 0xff01000100ff0100, 0xff01000100000000, + 0xff0100010001ffff, 0xff0100010001ff00, 0xff01000100010100, 0xff01000101ff00ff, + 0xff01000101ff0001, 0xff0100010100ffff, 0xff01000101000101, 0xff0101ffffffffff, + 0xff0101ffffffff01, 0xff0101ffffff01ff, 0xff0101ffffff0101, 0xff0101ffff000000, + 0xff0101ffff01ffff, 0xff0101ffff01ff01, 0xff0101ffff0101ff, 0xff0101ffff010101, + 0xff0101ff00ff0000, 0xff0101ff0000ff00, 0xff0101ff000000ff, 0xff0101ff00010000, + 0xff0101ff01ffffff, 0xff0101ff01ffff01, 0xff0101ff01ff01ff, 0xff0101ff01ff0101, + 0xff0101ff0101ffff, 0xff0101ff0101ff01, 0xff0101ff010101ff, 0xff0101ff01010101, + 0xff010100ffff0100, 0xff010100ff00ff00, 0xff010100ff0000ff, 0xff010100ff000100, + 0xff010100ff010000, 0xff01010000ff0001, 0xff01010000ff0100, 0xff0101000000ff01, + 0xff01010000000000, 0xff0101000001ff00, 0xff010100000100ff, 0xff01010000010001, + 0xff01010000010100, 0xff01010001ff0000, 0xff0101000100ffff, 0xff01010001000001, + 0xff01010001000100, 0xff010100010100ff, 0xff01010001010000, 0xff010101ffffffff, + 0xff010101ffffff01, 0xff010101ffff01ff, 0xff010101ffff0101, 0xff010101ff01ffff, + 0xff010101ff01ff01, 0xff010101ff0101ff, 0xff010101ff010101, 0xff01010100ff0000, + 0xff0101010000ff00, 0xff01010100000001, 0xff01010100000100, 0xff01010100010000, + 0xff01010101ffffff, 0xff01010101ffff01, 0xff01010101ff01ff, 0xff01010101ff0101, + 0xff01010101000000, 0xff0101010101ffff, 0xff0101010101ff01, 0xff010101010101ff, + 0xff01010101010101, 0x00ffffffffff0000, 0x00ffffffff00ff00, 0x00ffffffff000001, + 0x00ffffffff010000, 0x00ffffff00ff0100, 0x00ffffff0000ff01, 0x00ffffff00000000, + 0x00ffffff000001ff, 0x00ffffff00000101, 0x00ffffff0001ff00, 0x00ffffff000100ff, + 0x00ffffff00010001, 0x00ffffff010000ff, 0x00ffffff01000100, 0x00ffffff0101ff00, + 0x00ffffff01010001, 0x00ffff00ffffffff, 0x00ffff00ffffff00, 0x00ffff00ffff00ff, + 0x00ffff00ffff0001, 0x00ffff00ffff0100, 0x00ffff00ff00ff01, 0x00ffff00ff000000, + 0x00ffff00ff000001, 0x00ffff00ff0001ff, 0x00ffff00ff000101, 0x00ffff00ff01ff00, + 0x00ffff00ff010001, 0x00ffff00ff010100, 0x00ffff0000ff0000, 0x00ffff0000ff01ff, + 0x00ffff0000ff0101, 0x00ffff000000ff00, 0x00ffff00000000ff, 0x00ffff0000000000, + 0x00ffff0000000001, 0x00ffff0000000100, 0x00ffff0000000101, 0x00ffff0000010000, + 0x00ffff00000101ff, 0x00ffff0000010101, 0x00ffff0001ffff00, 0x00ffff0001ff00ff, + 0x00ffff0001ff0001, 0x00ffff000100ffff, 0x00ffff000100ff01, 0x00ffff0001000000, + 0x00ffff000101ffff, 0x00ffff000101ff00, 0x00ffff000101ff01, 0x00ffff01ffff0000, + 0x00ffff01ff00ff00, 0x00ffff01ff0000ff, 0x00ffff01ff000001, 0x00ffff01ff010000, + 0x00ffff0100ffff00, 0x00ffff010000ff01, 0x00ffff0100000000, 0x00ffff0100000101, + 0x00ffff01000100ff, 0x00ffff0100010100, 0x00ffff0101ff0100, 0x00ffff01010000ff, + 0x00ffff0101010000, 0x00ff00ffffffff00, 0x00ff00ffff000000, 0x00ff00ffff000100, + 0x00ff00ffff010100, 0x00ff00ff00ff0000, 0x00ff00ff00ff01ff, 0x00ff00ff00ff0101, + 0x00ff00ff0000ff00, 0x00ff00ff000000ff, 0x00ff00ff00000000, 0x00ff00ff00000001, + 0x00ff00ff0001ff00, 0x00ff00ff0001ff01, 0x00ff00ff00010000, 0x00ff00ff000101ff, + 0x00ff00ff00010101, 0x00ff00ff01ffff00, 0x00ff00ff01ff0001, 0x00ff00ff01ff0100, + 0x00ff00ff0100ffff, 0x00ff00ff0100ff01, 0x00ff00ff01000000, 0x00ff00ff0101ffff, + 0x00ff00ff0101ff00, 0x00ff00ff01010100, 0x00ff0000ffffff00, 0x00ff0000ffffff01, + 0x00ff0000ffff0000, 0x00ff0000ffff0101, 0x00ff0000ff00ff00, 0x00ff0000ff0000ff, + 0x00ff0000ff000000, 0x00ff0000ff000001, 0x00ff0000ff000100, 0x00ff0000ff01ffff, + 0x00ff0000ff010000, 0x00ff0000ff010101, 0x00ff000000ffff00, 0x00ff000000ff00ff, + 0x00ff000000ff0000, 0x00ff000000ff0001, 0x00ff000000ff0100, 0x00ff00000000ffff, + 0x00ff00000000ff00, 0x00ff0000000000ff, 0x00ff000000000000, 0x00ff000000000001, + 0x00ff0000000001ff, 0x00ff000000000100, 0x00ff00000001ff00, 0x00ff0000000100ff, + 0x00ff000000010000, 0x00ff000000010001, 0x00ff000000010100, 0x00ff000001ffff01, + 0x00ff000001ff00ff, 0x00ff000001ff0000, 0x00ff000001ff01ff, 0x00ff00000100ff00, + 0x00ff0000010000ff, 0x00ff000001000000, 0x00ff000001000001, 0x00ff000001000100, + 0x00ff000001000101, 0x00ff000001010000, 0x00ff0000010101ff, 0x00ff000001010101, + 0x00ff0001ffffff00, 0x00ff0001ffff0000, 0x00ff0001ffff0100, 0x00ff0001ff0000ff, + 0x00ff0001ff000000, 0x00ff0001ff0001ff, 0x00ff0001ff000101, 0x00ff0001ff01ff00, + 0x00ff0001ff0100ff, 0x00ff0001ff010100, 0x00ff000100ffffff, 0x00ff000100ffff01, + 0x00ff000100ff0000, 0x00ff000100ff01ff, 0x00ff00010000ffff, 0x00ff00010000ff00, + 0x00ff00010000ff01, 0x00ff000100000000, 0x00ff000100000001, 0x00ff000100000100, + 0x00ff00010001ff01, 0x00ff000100010000, 0x00ff0001000101ff, 0x00ff000101ffff00, + 0x00ff000101ff0000, 0x00ff000101ff0101, 0x00ff0001010000ff, 0x00ff000101000000, + 0x00ff00010101ff00, 0x00ff0001010100ff, 0x00ff000101010001, 0x00ff01ffffff0000, + 0x00ff01ffff00ff00, 0x00ff01ffff000000, 0x00ff01ffff000101, 0x00ff01ffff010000, + 0x00ff01ff00ffff01, 0x00ff01ff00ff0100, 0x00ff01ff0000ffff, 0x00ff01ff00000000, + 0x00ff01ff000001ff, 0x00ff01ff0001ff00, 0x00ff01ff000100ff, 0x00ff01ff00010001, + 0x00ff01ff00010100, 0x00ff01ff01ff0000, 0x00ff01ff0100ff00, 0x00ff01ff010000ff, + 0x00ff01ff01000001, 0x00ff01ff01000100, 0x00ff01ff01010000, 0x00ff0100ffffff00, + 0x00ff0100ffff0000, 0x00ff0100ffff0001, 0x00ff0100ffff0101, 0x00ff0100ff00ffff, + 0x00ff0100ff0000ff, 0x00ff0100ff000000, 0x00ff0100ff0001ff, 0x00ff0100ff01ff00, + 0x00ff0100ff0100ff, 0x00ff0100ff010001, 0x00ff010000ffffff, 0x00ff010000ff0000, + 0x00ff010000ff0101, 0x00ff01000000ff00, 0x00ff01000000ff01, 0x00ff0100000000ff, + 0x00ff010000000000, 0x00ff010000000001, 0x00ff010000000100, 0x00ff01000001ffff, + 0x00ff01000001ff01, 0x00ff010000010000, 0x00ff010000010001, 0x00ff010000010101, + 0x00ff010001ff0001, 0x00ff010001ff0100, 0x00ff01000100ff01, 0x00ff010001000000, + 0x00ff010001000001, 0x00ff0100010001ff, 0x00ff01000101ff00, 0x00ff0100010100ff, + 0x00ff010001010001, 0x00ff010001010100, 0x00ff0101ff000001, 0x00ff010100ff00ff, + 0x00ff010100ff0001, 0x00ff010100ff0100, 0x00ff010100000000, 0x00ff0101000001ff, + 0x00ff010100000101, 0x00ff0101000100ff, 0x00ff010100010100, 0x00ff0101010000ff, + 0x00ff010101010000, 0x0000ffffffffff00, 0x0000ffffffff00ff, 0x0000ffffffff0000, + 0x0000ffffffff0001, 0x0000ffffffff0100, 0x0000ffffff00ff01, 0x0000ffffff000000, + 0x0000ffffff000101, 0x0000ffffff01ff00, 0x0000ffffff0100ff, 0x0000ffffff010100, + 0x0000ffff00ffffff, 0x0000ffff00ff0000, 0x0000ffff00ff01ff, 0x0000ffff0000ff00, + 0x0000ffff000000ff, 0x0000ffff00000000, 0x0000ffff00000001, 0x0000ffff00000100, + 0x0000ffff00010000, 0x0000ffff000101ff, 0x0000ffff01ff0001, 0x0000ffff01ff0100, + 0x0000ffff01000000, 0x0000ffff010001ff, 0x0000ffff0101ffff, 0x0000ffff0101ff00, + 0x0000ffff01010001, 0x0000ffff01010100, 0x0000ff00ffff0000, 0x0000ff00ffff01ff, + 0x0000ff00ffff0100, 0x0000ff00ffff0101, 0x0000ff00ff00ff00, 0x0000ff00ff0000ff, + 0x0000ff00ff000000, 0x0000ff00ff000001, 0x0000ff00ff0001ff, 0x0000ff00ff000100, + 0x0000ff00ff01ffff, 0x0000ff00ff010000, 0x0000ff00ff010001, 0x0000ff00ff0101ff, + 0x0000ff00ff010101, 0x0000ff0000ffff00, 0x0000ff0000ff00ff, 0x0000ff0000ff0000, + 0x0000ff0000ff0001, 0x0000ff0000ff0100, 0x0000ff000000ffff, 0x0000ff000000ff00, + 0x0000ff000000ff01, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001, + 0x0000ff00000001ff, 0x0000ff0000000100, 0x0000ff0000000101, 0x0000ff000001ff00, + 0x0000ff00000100ff, 0x0000ff0000010000, 0x0000ff0000010001, 0x0000ff0000010100, + 0x0000ff0001ffff01, 0x0000ff0001ff0000, 0x0000ff000100ff00, 0x0000ff00010000ff, + 0x0000ff0001000000, 0x0000ff0001000001, 0x0000ff0001000100, 0x0000ff000101ffff, + 0x0000ff0001010000, 0x0000ff0001010101, 0x0000ff01ffffff00, 0x0000ff01ffff0001, + 0x0000ff01ff00ff01, 0x0000ff01ff000000, 0x0000ff01ff000101, 0x0000ff01ff01ff00, + 0x0000ff01ff0100ff, 0x0000ff0100ffff01, 0x0000ff0100ff0000, 0x0000ff0100ff0101, + 0x0000ff010000ff00, 0x0000ff01000000ff, 0x0000ff0100000000, 0x0000ff0100000001, + 0x0000ff0100000100, 0x0000ff010001ff01, 0x0000ff0100010000, 0x0000ff0101ff0000, + 0x0000ff010100ffff, 0x0000ff010100ff01, 0x0000ff0101000000, 0x0000ff0101000100, + 0x0000ff0101000101, 0x0000ff01010100ff, 0x000000ffffff00ff, 0x000000ffffff0000, + 0x000000ffff00ff00, 0x000000ffff0000ff, 0x000000ffff000000, 0x000000ffff000001, + 0x000000ffff0001ff, 0x000000ffff000100, 0x000000ffff01ff00, 0x000000ffff010000, + 0x000000ffff0101ff, 0x000000ffff010101, 0x000000ff00ffff00, 0x000000ff00ff00ff, + 0x000000ff00ff0000, 0x000000ff00ff0001, 0x000000ff00ff0100, 0x000000ff00ff0101, + 0x000000ff0000ffff, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000, + 0x000000ff00000001, 0x000000ff000001ff, 0x000000ff00000100, 0x000000ff00000101, + 0x000000ff0001ff00, 0x000000ff0001ff01, 0x000000ff000100ff, 0x000000ff00010000, + 0x000000ff00010001, 0x000000ff00010100, 0x000000ff01ffffff, 0x000000ff01ff01ff, + 0x000000ff01ff0101, 0x000000ff0100ff00, 0x000000ff010000ff, 0x000000ff01000000, + 0x000000ff01000001, 0x000000ff01000100, 0x000000ff0101ff00, 0x000000ff010100ff, + 0x000000ff01010000, 0x000000ff01010101, 0x00000000ffffff00, 0x00000000ffffff01, + 0x00000000ffff00ff, 0x00000000ffff0000, 0x00000000ffff0001, 0x00000000ffff0100, + 0x00000000ff00ffff, 0x00000000ff00ff00, 0x00000000ff00ff01, 0x00000000ff0000ff, + 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff000101, + 0x00000000ff01ff00, 0x00000000ff0100ff, 0x00000000ff010000, 0x00000000ff010001, + 0x00000000ff010100, 0x0000000000ffffff, 0x0000000000ffff00, 0x0000000000ffff01, + 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001, 0x0000000000ff01ff, + 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01, + 0x00000000000000ff, 0x0000000000000000, 0x0000000000000001, 0x00000000000001ff, + 0x0000000000000100, 0x0000000000000101, 0x000000000001ffff, 0x000000000001ff00, + 0x00000000000100ff, 0x0000000000010000, 0x0000000000010001, 0x00000000000101ff, + 0x0000000000010100, 0x0000000000010101, 0x0000000001ffff00, 0x0000000001ff00ff, + 0x0000000001ff0000, 0x0000000001ff0100, 0x0000000001ff0101, 0x000000000100ffff, + 0x000000000100ff00, 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001, + 0x00000000010001ff, 0x0000000001000100, 0x000000000101ff00, 0x00000000010100ff, + 0x0000000001010000, 0x0000000001010001, 0x0000000001010100, 0x00000001ffffffff, + 0x00000001ffffff00, 0x00000001ffffff01, 0x00000001ffff00ff, 0x00000001ffff0001, + 0x00000001ffff01ff, 0x00000001ffff0100, 0x00000001ff00ff00, 0x00000001ff0000ff, + 0x00000001ff000000, 0x00000001ff0001ff, 0x00000001ff000100, 0x00000001ff01ffff, + 0x00000001ff01ff00, 0x00000001ff01ff01, 0x00000001ff0100ff, 0x00000001ff010000, + 0x00000001ff010001, 0x00000001ff0101ff, 0x00000001ff010100, 0x0000000100ffff00, + 0x0000000100ff0000, 0x0000000100ff0001, 0x0000000100ff01ff, 0x0000000100ff0100, + 0x0000000100ff0101, 0x000000010000ffff, 0x000000010000ff00, 0x000000010000ff01, + 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001, 0x00000001000001ff, + 0x0000000100000100, 0x0000000100000101, 0x000000010001ff00, 0x00000001000100ff, + 0x0000000100010000, 0x0000000100010100, 0x0000000101ffff01, 0x0000000101ff0000, + 0x0000000101ff0001, 0x0000000101ff01ff, 0x0000000101ff0100, 0x0000000101ff0101, + 0x000000010100ff00, 0x0000000101000000, 0x0000000101000101, 0x000000010101ff01, + 0x0000000101010000, 0x0000000101010001, 0x00000001010101ff, 0x0000000101010100, + 0x000001ffffff00ff, 0x000001ffffff0000, 0x000001ffffff0001, 0x000001ffffff0100, + 0x000001ffff00ffff, 0x000001ffff000000, 0x000001ffff0001ff, 0x000001ffff01ff00, + 0x000001ffff010101, 0x000001ff00ff0000, 0x000001ff00ff01ff, 0x000001ff00ff0101, + 0x000001ff0000ff00, 0x000001ff000000ff, 0x000001ff00000000, 0x000001ff00000001, + 0x000001ff000001ff, 0x000001ff00000100, 0x000001ff0001ffff, 0x000001ff0001ff01, + 0x000001ff000100ff, 0x000001ff00010000, 0x000001ff01ffff01, 0x000001ff01ff0100, + 0x000001ff0100ffff, 0x000001ff0100ff01, 0x000001ff01000000, 0x000001ff010001ff, + 0x000001ff0101ff00, 0x000001ff01010100, 0x00000100ffffff00, 0x00000100ffffff01, + 0x00000100ffff0000, 0x00000100ffff0101, 0x00000100ff00ff00, 0x00000100ff0000ff, + 0x00000100ff000000, 0x00000100ff000001, 0x00000100ff000100, 0x00000100ff010000, + 0x0000010000ffff00, 0x0000010000ff00ff, 0x0000010000ff0000, 0x0000010000ff0001, + 0x0000010000ff0100, 0x000001000000ffff, 0x000001000000ff00, 0x000001000000ff01, + 0x00000100000000ff, 0x0000010000000000, 0x0000010000000001, 0x00000100000001ff, + 0x0000010000000100, 0x0000010000000101, 0x000001000001ff00, 0x00000100000100ff, + 0x0000010000010000, 0x0000010000010001, 0x0000010000010100, 0x0000010001ffff00, + 0x0000010001ff0000, 0x0000010001ff0100, 0x000001000100ff00, 0x00000100010000ff, + 0x0000010001000000, 0x0000010001000001, 0x00000100010001ff, 0x0000010001000100, + 0x0000010001010000, 0x00000101ffff00ff, 0x00000101ffff01ff, 0x00000101ff000000, + 0x00000101ff000101, 0x00000101ff01ffff, 0x00000101ff010000, 0x00000101ff010001, + 0x00000101ff010100, 0x0000010100ff0000, 0x0000010100ff01ff, 0x0000010100ff0100, + 0x000001010000ff00, 0x0000010100000000, 0x0000010100000001, 0x00000101000001ff, + 0x0000010100000100, 0x000001010001ff01, 0x0000010100010000, 0x00000101000101ff, + 0x0000010100010101, 0x0000010101ffff00, 0x0000010101ff0101, 0x000001010100ff01, + 0x0000010101000000, 0x0000010101000001, 0x00000101010001ff, 0x0000010101000101, + 0x000001010101ff00, 0x0001ffffffff0000, 0x0001ffffff0000ff, 0x0001ffffff000001, + 0x0001ffffff000100, 0x0001ffffff010000, 0x0001ffff00ff00ff, 0x0001ffff0000ffff, + 0x0001ffff00000000, 0x0001ffff00000001, 0x0001ffff000001ff, 0x0001ffff00000101, + 0x0001ffff0001ff00, 0x0001ffff000100ff, 0x0001ffff00010001, 0x0001ffff00010100, + 0x0001ffff01ffff00, 0x0001ffff01000001, 0x0001ffff01010000, 0x0001ff00ffffff00, + 0x0001ff00ffff00ff, 0x0001ff00ffff0001, 0x0001ff00ffff0100, 0x0001ff00ff00ff01, + 0x0001ff00ff000000, 0x0001ff00ff01ff00, 0x0001ff00ff01ff01, 0x0001ff00ff010001, + 0x0001ff00ff010100, 0x0001ff0000ff0000, 0x0001ff0000ff0100, 0x0001ff000000ff00, + 0x0001ff0000000000, 0x0001ff0000000001, 0x0001ff0000000100, 0x0001ff0000010000, + 0x0001ff0000010001, 0x0001ff0000010101, 0x0001ff0001ff00ff, 0x0001ff0001ff0101, + 0x0001ff000100ff01, 0x0001ff0001000000, 0x0001ff000101ff00, 0x0001ff0001010001, + 0x0001ff0001010100, 0x0001ff01ff00ff00, 0x0001ff01ff000001, 0x0001ff01ff000100, + 0x0001ff0100ffffff, 0x0001ff0100ffff00, 0x0001ff0100ff0001, 0x0001ff0100000000, + 0x0001ff0100000001, 0x0001ff01000001ff, 0x0001ff010001ffff, 0x0001ff0101ff0000, + 0x0001ff010100ff00, 0x0001ff0101000001, 0x0001ff0101010000, 0x000100ffff00ff00, + 0x000100ffff00ff01, 0x000100ffff000000, 0x000100ffff000001, 0x000100ffff000101, + 0x000100ffff01ff00, 0x000100ffff010001, 0x000100ffff010100, 0x000100ff00ffffff, + 0x000100ff00ffff01, 0x000100ff00ff0000, 0x000100ff00ff01ff, 0x000100ff00ff0101, + 0x000100ff0000ff00, 0x000100ff000000ff, 0x000100ff00000000, 0x000100ff00000001, + 0x000100ff00000100, 0x000100ff00000101, 0x000100ff0001ffff, 0x000100ff0001ff01, + 0x000100ff00010000, 0x000100ff01ff00ff, 0x000100ff01ff0000, 0x000100ff01ff0100, + 0x000100ff0100ffff, 0x000100ff0100ff01, 0x000100ff010000ff, 0x000100ff01000000, + 0x000100ff01000001, 0x000100ff010001ff, 0x000100ff01000101, 0x000100ff0101ff00, + 0x000100ff010100ff, 0x000100ff01010100, 0x00010000ffff0000, 0x00010000ffff01ff, + 0x00010000ffff0101, 0x00010000ff00ff00, 0x00010000ff000000, 0x00010000ff000001, + 0x00010000ff000100, 0x0001000000ff00ff, 0x0001000000ff0000, 0x0001000000ff0001, + 0x0001000000ff0100, 0x000100000000ffff, 0x000100000000ff00, 0x00010000000000ff, + 0x0001000000000000, 0x0001000000000001, 0x0001000000000100, 0x000100000001ff00, + 0x00010000000100ff, 0x0001000000010000, 0x0001000000010001, 0x0001000000010100, + 0x0001000001ff0001, 0x0001000001ff0100, 0x0001000001ff0101, 0x000100000100ff00, + 0x0001000001000000, 0x0001000001000001, 0x0001000001000100, 0x0001000001000101, + 0x000100000101ff01, 0x0001000001010000, 0x0001000001010001, 0x00010000010101ff, + 0x00010001ffffff01, 0x00010001ffff0100, 0x00010001ff000000, 0x00010001ff01ffff, + 0x00010001ff010001, 0x00010001ff0101ff, 0x00010001ff010100, 0x0001000100ffffff, + 0x0001000100ff0000, 0x0001000100ff01ff, 0x0001000100ff0101, 0x000100010000ff00, + 0x00010001000000ff, 0x0001000100000000, 0x0001000100000001, 0x00010001000001ff, + 0x0001000100000101, 0x000100010001ffff, 0x0001000100010000, 0x00010001000101ff, + 0x0001000101ffffff, 0x0001000101ffff01, 0x0001000101ff0000, 0x0001000101ff0101, + 0x00010001010000ff, 0x0001000101000001, 0x00010001010001ff, 0x0001000101000100, + 0x000100010101ffff, 0x00010001010100ff, 0x0001000101010001, 0x0001000101010101, + 0x000101ffff000001, 0x000101ffff000100, 0x000101ffff010000, 0x000101ff00ffff00, + 0x000101ff0000ff01, 0x000101ff00000000, 0x000101ff00000101, 0x000101ff0001ff00, + 0x000101ff00010100, 0x000101ff01ff0000, 0x000101ff0100ff00, 0x000101ff010001ff, + 0x000101ff01010001, 0x00010100ffffff00, 0x00010100ffff00ff, 0x00010100ff00ffff, + 0x00010100ff000000, 0x00010100ff01ff00, 0x00010100ff0100ff, 0x00010100ff010001, + 0x00010100ff010100, 0x0001010000ffffff, 0x0001010000ffff00, 0x0001010000ff0000, + 0x0001010000ff0001, 0x0001010000ff01ff, 0x000101000000ff00, 0x00010100000000ff, + 0x0001010000000000, 0x0001010000000001, 0x0001010000000100, 0x000101000001ffff, + 0x0001010000010000, 0x0001010000010101, 0x0001010001ffff01, 0x0001010001ff00ff, + 0x0001010001ff0101, 0x0001010001000000, 0x000101000101ff00, 0x00010100010100ff, + 0x0001010001010000, 0x0001010001010100, 0x00010101ff00ff00, 0x00010101ff000001, + 0x00010101ff0001ff, 0x0001010100ffff00, 0x0001010100ff00ff, 0x0001010100ff0100, + 0x000101010000ffff, 0x0001010100000000, 0x00010101000001ff, 0x0001010100000101, + 0x00010101000100ff, 0x0001010100010000, 0x0001010100010100, 0x0001010101ff0001, + 0x00010101010000ff, 0x00010101010001ff, 0x0001010101000101, 0x0001010101010001, + 0x01ffffffffffffff, 0x01ffffffffffff01, 0x01ffffffffff01ff, 0x01ffffffffff0101, + 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff, 0x01ffffffff010101, + 0x01ffffff00ff0000, 0x01ffffff0000ffff, 0x01ffffff0000ff00, 0x01ffffff000000ff, + 0x01ffffff00000001, 0x01ffffff00000100, 0x01ffffff00010000, 0x01ffffff01ffffff, + 0x01ffffff01ffff01, 0x01ffffff01ff01ff, 0x01ffffff01ff0101, 0x01ffffff01000000, + 0x01ffffff0101ffff, 0x01ffffff0101ff01, 0x01ffffff010101ff, 0x01ffffff01010101, + 0x01ffff00ffff0000, 0x01ffff00ff00ff00, 0x01ffff00ff0000ff, 0x01ffff00ff000001, + 0x01ffff00ff000100, 0x01ffff00ff010000, 0x01ffff0000ffff00, 0x01ffff0000ff00ff, + 0x01ffff0000ff0100, 0x01ffff000000ffff, 0x01ffff000000ff01, 0x01ffff0000000000, + 0x01ffff0000000001, 0x01ffff00000001ff, 0x01ffff0000000100, 0x01ffff00000100ff, + 0x01ffff0000010001, 0x01ffff0000010100, 0x01ffff0001ff0000, 0x01ffff0001ff0100, + 0x01ffff00010000ff, 0x01ffff0001000001, 0x01ffff0001000100, 0x01ffff0001010000, + 0x01ffff01ffffffff, 0x01ffff01ffffff01, 0x01ffff01ffff01ff, 0x01ffff01ffff0101, + 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff01ff01, 0x01ffff01ff0101ff, + 0x01ffff01ff010101, 0x01ffff010000ff00, 0x01ffff01000000ff, 0x01ffff0100000100, + 0x01ffff0100010000, 0x01ffff0101ffffff, 0x01ffff0101ffff01, 0x01ffff0101ff01ff, + 0x01ffff0101ff0101, 0x01ffff0101000000, 0x01ffff010101ffff, 0x01ffff010101ff01, + 0x01ffff01010101ff, 0x01ffff0101010101, 0x01ff00ffff0000ff, 0x01ff00ffff000100, + 0x01ff00ff00ffff00, 0x01ff00ff00ff00ff, 0x01ff00ff0000ff00, 0x01ff00ff00000000, + 0x01ff00ff00000101, 0x01ff00ff0001ff00, 0x01ff00ff000100ff, 0x01ff00ff00010100, + 0x01ff00ff010000ff, 0x01ff00ff01000100, 0x01ff0000ffffff00, 0x01ff0000ffff0100, + 0x01ff0000ff00ff01, 0x01ff0000ff000000, 0x01ff0000ff000101, 0x01ff0000ff010001, + 0x01ff0000ff010100, 0x01ff000000ffffff, 0x01ff000000ffff00, 0x01ff000000ff0000, + 0x01ff000000ff01ff, 0x01ff00000000ff00, 0x01ff0000000000ff, 0x01ff000000000000, + 0x01ff000000000001, 0x01ff000000000100, 0x01ff000000000101, 0x01ff000000010000, + 0x01ff000000010001, 0x01ff0000000101ff, 0x01ff000000010101, 0x01ff000001ffff00, + 0x01ff000001ff00ff, 0x01ff000001ff0001, 0x01ff000001ff0100, 0x01ff00000100ffff, + 0x01ff00000100ff01, 0x01ff000001000000, 0x01ff0000010001ff, 0x01ff000001010001, + 0x01ff0001ff00ff00, 0x01ff0001ff000001, 0x01ff0001ff000100, 0x01ff0001ff010000, + 0x01ff000100ffff00, 0x01ff000100ff00ff, 0x01ff000100ff0100, 0x01ff000100ff0101, + 0x01ff00010000ffff, 0x01ff000100000000, 0x01ff000100000100, 0x01ff000100000101, + 0x01ff00010001ff00, 0x01ff000100010001, 0x01ff000100010101, 0x01ff000101ff0000, + 0x01ff00010100ff00, 0x01ff000101000101, 0x01ff0001010100ff, 0x01ff01ffffffffff, + 0x01ff01ffffffff01, 0x01ff01ffffff01ff, 0x01ff01ffffff0101, 0x01ff01ffff000000, + 0x01ff01ffff01ffff, 0x01ff01ffff01ff01, 0x01ff01ffff0101ff, 0x01ff01ffff010101, + 0x01ff01ff00ffff00, 0x01ff01ff00ff0000, 0x01ff01ff0000ff00, 0x01ff01ff000000ff, + 0x01ff01ff00000100, 0x01ff01ff00010000, 0x01ff01ff00010100, 0x01ff01ff01ffffff, + 0x01ff01ff01ffff01, 0x01ff01ff01ff01ff, 0x01ff01ff01ff0101, 0x01ff01ff01000000, + 0x01ff01ff0101ffff, 0x01ff01ff0101ff01, 0x01ff01ff010101ff, 0x01ff01ff01010101, + 0x01ff0100ffff0000, 0x01ff0100ffff0001, 0x01ff0100ff00ff00, 0x01ff0100ff0000ff, + 0x01ff0100ff000001, 0x01ff0100ff010000, 0x01ff010000ffff00, 0x01ff010000ff00ff, + 0x01ff010000ff0001, 0x01ff010000ff0100, 0x01ff01000000ffff, 0x01ff01000000ff01, + 0x01ff010000000000, 0x01ff010000000101, 0x01ff01000001ff00, 0x01ff0100000100ff, + 0x01ff010001ff0000, 0x01ff010001000001, 0x01ff010001000100, 0x01ff010001010000, + 0x01ff0101ffffffff, 0x01ff0101ffffff01, 0x01ff0101ffff01ff, 0x01ff0101ffff0101, + 0x01ff0101ff000000, 0x01ff0101ff01ffff, 0x01ff0101ff01ff01, 0x01ff0101ff0101ff, + 0x01ff0101ff010101, 0x01ff010100ff0000, 0x01ff01010000ff00, 0x01ff0101000000ff, + 0x01ff010100000001, 0x01ff010101ffffff, 0x01ff010101ffff01, 0x01ff010101ff01ff, + 0x01ff010101ff0101, 0x01ff010101000000, 0x01ff01010101ffff, 0x01ff01010101ff01, + 0x01ff0101010101ff, 0x01ff010101010101, 0x0100ffffffff0000, 0x0100ffffff00ff00, + 0x0100ffffff000001, 0x0100ffffff0001ff, 0x0100ffffff000100, 0x0100ffffff010000, + 0x0100ffff00ffff00, 0x0100ffff00ff0001, 0x0100ffff00ff0100, 0x0100ffff00000000, + 0x0100ffff000001ff, 0x0100ffff00000101, 0x0100ffff00010100, 0x0100ffff00010101, + 0x0100ffff01ff0000, 0x0100ffff0100ff00, 0x0100ffff010000ff, 0x0100ffff01000001, + 0x0100ffff01000100, 0x0100ffff01010000, 0x0100ff00ffffff00, 0x0100ff00ffff00ff, + 0x0100ff00ffff0001, 0x0100ff00ffff0100, 0x0100ff00ff00ffff, 0x0100ff00ff000000, + 0x0100ff00ff0001ff, 0x0100ff00ff000101, 0x0100ff00ff01ff00, 0x0100ff00ff0100ff, + 0x0100ff00ff010001, 0x0100ff00ff010100, 0x0100ff0000ffffff, 0x0100ff0000ff0000, + 0x0100ff000000ffff, 0x0100ff000000ff00, 0x0100ff00000000ff, 0x0100ff0000000000, + 0x0100ff0000000001, 0x0100ff0000000100, 0x0100ff000001ff01, 0x0100ff0000010000, + 0x0100ff0001ff00ff, 0x0100ff0001ff0001, 0x0100ff000100ff01, 0x0100ff0001000000, + 0x0100ff00010001ff, 0x0100ff000101ff00, 0x0100ff00010100ff, 0x0100ff0001010001, + 0x0100ff0001010100, 0x0100ff01ffff0000, 0x0100ff01ff00ff00, 0x0100ff01ff0000ff, + 0x0100ff01ff000100, 0x0100ff01ff010000, 0x0100ff0100ff00ff, 0x0100ff0100ff0001, + 0x0100ff0100ff0100, 0x0100ff010000ffff, 0x0100ff010000ff01, 0x0100ff0100000000, + 0x0100ff01000001ff, 0x0100ff0100010001, 0x0100ff0100010100, 0x0100ff0101ff0000, + 0x0100ff01010000ff, 0x0100ff0101000001, 0x0100ff0101010100, 0x010000ffffffff00, + 0x010000ffffff00ff, 0x010000ffffff0001, 0x010000ffff00ffff, 0x010000ffff000000, + 0x010000ffff0001ff, 0x010000ffff010001, 0x010000ff00ffffff, 0x010000ff00ff0101, + 0x010000ff0000ff00, 0x010000ff000000ff, 0x010000ff00000000, 0x010000ff00000001, + 0x010000ff000001ff, 0x010000ff00000100, 0x010000ff0001ffff, 0x010000ff0001ff00, + 0x010000ff0001ff01, 0x010000ff00010000, 0x010000ff01ff00ff, 0x010000ff01ff0001, + 0x010000ff0100ff01, 0x010000ff010000ff, 0x010000ff01000000, 0x010000ff010001ff, + 0x010000ff0101ff00, 0x010000ff01010100, 0x01000000ffffffff, 0x01000000ffff0000, + 0x01000000ffff01ff, 0x01000000ffff0101, 0x01000000ff00ffff, 0x01000000ff00ff00, + 0x01000000ff0000ff, 0x01000000ff000000, 0x01000000ff000001, 0x01000000ff000100, + 0x01000000ff01ff00, 0x01000000ff010000, 0x01000000ff010100, 0x01000000ff010101, + 0x0100000000ffff00, 0x0100000000ff00ff, 0x0100000000ff0000, 0x0100000000ff0001, + 0x0100000000ff0100, 0x010000000000ffff, 0x010000000000ff00, 0x010000000000ff01, + 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001, 0x01000000000001ff, + 0x0100000000000100, 0x0100000000000101, 0x010000000001ff00, 0x01000000000100ff, + 0x0100000000010000, 0x0100000000010001, 0x0100000000010100, 0x0100000001ffff00, + 0x0100000001ff0000, 0x0100000001ff01ff, 0x010000000100ff00, 0x010000000100ff01, + 0x01000000010000ff, 0x0100000001000000, 0x0100000001000001, 0x0100000001000100, + 0x0100000001000101, 0x010000000101ffff, 0x010000000101ff01, 0x0100000001010000, + 0x01000000010101ff, 0x0100000001010101, 0x01000001ffffff00, 0x01000001ffff00ff, + 0x01000001ff00ffff, 0x01000001ff000000, 0x01000001ff000100, 0x01000001ff01ffff, + 0x01000001ff010001, 0x01000001ff010100, 0x0100000100ff0000, 0x0100000100ff01ff, + 0x0100000100ff0100, 0x010000010000ff00, 0x010000010000ff01, 0x0100000100000000, + 0x0100000100000001, 0x0100000100000100, 0x0100000100010000, 0x01000001000101ff, + 0x0100000101ffff01, 0x0100000101ff00ff, 0x0100000101ff0100, 0x0100000101ff0101, + 0x010000010100ff01, 0x01000001010000ff, 0x0100000101000000, 0x01000001010100ff, + 0x0100000101010001, 0x0100000101010100, 0x010001ffffff0000, 0x010001ffff000001, + 0x010001ffff000100, 0x010001ffff010000, 0x010001ff00ffff00, 0x010001ff00ff0001, + 0x010001ff0000ffff, 0x010001ff0000ff01, 0x010001ff00000000, 0x010001ff00000001, + 0x010001ff00000101, 0x010001ff000100ff, 0x010001ff00010000, 0x010001ff01ff0000, + 0x010001ff0100ff00, 0x010001ff01000001, 0x010001ff01000100, 0x010001ff01010000, + 0x01000100ffff00ff, 0x01000100ffff0001, 0x01000100ffff0100, 0x01000100ff00ffff, + 0x01000100ff00ff01, 0x01000100ff000000, 0x01000100ff0001ff, 0x01000100ff000101, + 0x01000100ff01ffff, 0x01000100ff01ff00, 0x01000100ff0100ff, 0x01000100ff010001, + 0x0100010000ffffff, 0x0100010000ffff01, 0x0100010000ff0000, 0x0100010000ff01ff, + 0x0100010000ff0101, 0x010001000000ff00, 0x01000100000000ff, 0x0100010000000000, + 0x0100010000000001, 0x0100010000000100, 0x010001000001ff01, 0x0100010000010000, + 0x0100010000010001, 0x0100010000010101, 0x0100010001ffff00, 0x0100010001ff00ff, + 0x010001000100ffff, 0x010001000100ff01, 0x0100010001000000, 0x0100010001000101, + 0x010001000101ff00, 0x0100010001010001, 0x01000101ffff0000, 0x01000101ff000000, + 0x01000101ff010000, 0x0100010100ff00ff, 0x0100010100ff0001, 0x0100010100ff0100, + 0x010001010000ffff, 0x0100010100000000, 0x01000101000001ff, 0x010001010001ff00, + 0x0100010101ff0000, 0x010001010100ff00, 0x01000101010000ff, 0x0100010101000000, + 0x0100010101000001, 0x0101ffffffffffff, 0x0101ffffffffff01, 0x0101ffffffff01ff, + 0x0101ffffffff0101, 0x0101ffffff000000, 0x0101ffffff01ffff, 0x0101ffffff01ff01, + 0x0101ffffff0101ff, 0x0101ffffff010101, 0x0101ffff00ff0000, 0x0101ffff0000ff00, + 0x0101ffff000000ff, 0x0101ffff00000001, 0x0101ffff00000100, 0x0101ffff01ffffff, + 0x0101ffff01ffff01, 0x0101ffff01ff01ff, 0x0101ffff01ff0101, 0x0101ffff01000000, + 0x0101ffff0101ffff, 0x0101ffff0101ff01, 0x0101ffff010101ff, 0x0101ffff01010101, + 0x0101ff00ffff0000, 0x0101ff00ffff0100, 0x0101ff00ff00ff00, 0x0101ff00ff0000ff, + 0x0101ff00ff000001, 0x0101ff00ff000100, 0x0101ff00ff000101, 0x0101ff0000ff0001, + 0x0101ff0000ff0100, 0x0101ff000000ff00, 0x0101ff0000000000, 0x0101ff00000001ff, + 0x0101ff0000000101, 0x0101ff000001ff00, 0x0101ff00000100ff, 0x0101ff0001ff0000, + 0x0101ff000100ffff, 0x0101ff000100ff01, 0x0101ff0001000001, 0x0101ff0001000100, + 0x0101ff01ffffff01, 0x0101ff01ffff01ff, 0x0101ff01ffff0101, 0x0101ff01ff00ffff, + 0x0101ff01ff000100, 0x0101ff01ff01ff01, 0x0101ff01ff0101ff, 0x0101ff01ff010101, + 0x0101ff0100ff0000, 0x0101ff010000ff00, 0x0101ff0100000001, 0x0101ff0100000100, + 0x0101ff0100010000, 0x0101ff0101ffffff, 0x0101ff0101ffff01, 0x0101ff0101ff01ff, + 0x0101ff0101ff0101, 0x0101ff0101000000, 0x0101ff010101ffff, 0x0101ff010101ff01, + 0x0101ff01010101ff, 0x0101ff0101010101, 0x010100ffff000100, 0x010100ffff010000, + 0x010100ff00ffff00, 0x010100ff00ff00ff, 0x010100ff0000ffff, 0x010100ff000000ff, + 0x010100ff00000000, 0x010100ff000001ff, 0x010100ff00000101, 0x010100ff0001ff00, + 0x010100ff00010000, 0x010100ff00010001, 0x010100ff000101ff, 0x010100ff00010100, + 0x010100ff01ff0000, 0x01010000ffff0001, 0x01010000ffff0100, 0x01010000ff00ffff, + 0x01010000ff00ff01, 0x01010000ff000000, 0x01010000ff0001ff, 0x01010000ff010001, + 0x01010000ff010100, 0x0101000000ffff01, 0x0101000000ff0000, 0x010100000000ff00, + 0x01010000000000ff, 0x0101000000000000, 0x0101000000000001, 0x0101000000000100, + 0x0101000000010000, 0x0101000000010101, 0x0101000001ffff00, 0x0101000001ff00ff, + 0x0101000001ff0000, 0x0101000001ff0001, 0x0101000001ff0100, 0x010100000100ff01, + 0x0101000001000000, 0x01010000010001ff, 0x01010001ffff0000, 0x01010001ff00ff00, + 0x01010001ff000001, 0x01010001ff000101, 0x01010001ff01ff00, 0x01010001ff010000, + 0x0101000100ff00ff, 0x0101000100ff0001, 0x0101000100ff0101, 0x010100010000ff01, + 0x0101000100000000, 0x0101000100000001, 0x01010001000001ff, 0x010100010001ffff, + 0x010100010001ff01, 0x0101000101ff0001, 0x010100010100ffff, 0x0101000101000000, + 0x0101000101000001, 0x0101000101000100, 0x010100010101ff00, 0x01010001010100ff, + 0x0101000101010001, 0x010101ffffffffff, 0x010101ffffffff01, 0x010101ffffff01ff, + 0x010101ffffff0101, 0x010101ffff01ffff, 0x010101ffff01ff01, 0x010101ffff0101ff, + 0x010101ffff010101, 0x010101ff0000ff00, 0x010101ff000000ff, 0x010101ff00000001, + 0x010101ff00000100, 0x010101ff01ffffff, 0x010101ff01ffff01, 0x010101ff01ff01ff, + 0x010101ff01ff0101, 0x010101ff01000000, 0x010101ff0101ffff, 0x010101ff0101ff01, + 0x010101ff010101ff, 0x010101ff01010101, 0x01010100ffff0000, 0x01010100ff0000ff, + 0x01010100ff000100, 0x01010100ff01ff00, 0x01010100ff010000, 0x0101010000ffff00, + 0x010101000000ffff, 0x0101010000000000, 0x0101010000000101, 0x010101000001ff00, + 0x0101010000010001, 0x0101010000010100, 0x010101000100ffff, 0x0101010001000001, + 0x01010101ffffffff, 0x01010101ffffff01, 0x01010101ffff01ff, 0x01010101ffff0101, + 0x01010101ff01ffff, 0x01010101ff01ff01, 0x01010101ff0101ff, 0x01010101ff010101, + 0x010101010000ff00, 0x01010101000000ff, 0x0101010100000001, 0x0101010101ffffff, + 0x0101010101ffff01, 0x0101010101ff01ff, 0x0101010101ff0101, 0x0101010101000000, + 0x010101010101ffff, 0x010101010101ff01, 0x01010101010101ff, 0x0101010101010101, +}; +static const uint64_t iq2xs_grid[512] = { + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, + 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, + 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919, + 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b, + 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919, + 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808, + 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819, + 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819, + 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, + 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b, + 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b, + 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908, + 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908, + 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919, + 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808, + 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919, + 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908, + 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, + 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, + 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08, + 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808, + 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808, + 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819, + 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908, + 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819, + 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808, + 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b, + 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819, + 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819, + 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808, + 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908, + 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19, + 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b, + 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b, + 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919, + 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808, + 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819, + 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819, + 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b, + 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908, + 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808, + 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819, + 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808, + 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, + 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808, + 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808, + 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908, + 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908, + 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808, + 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b, + 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819, + 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, + 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908, + 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808, + 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908, + 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919, + 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08, + 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19, + 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b, + 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b, + 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808, + 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08, + 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b, + 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908, + 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b, + 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908, + 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, + 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808, + 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808, + 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08, + 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819, + 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919, + 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808, + 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808, + 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819, + 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819, + 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908, + 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908, + 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b, + 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908, + 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908, + 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908, + 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808, + 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, + 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819, + 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819, + 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808, + 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b, + 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819, + 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819, + 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08, + 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808, + 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19, + 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919, + 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, + 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19, + 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b, + 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808, + 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b, + 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b, + 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, + 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b, + 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808, + 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819, + 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808, + 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808, + 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08, + 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b, + 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19, + 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08, + 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919, + 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08, + 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08, + 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908, + 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908, + 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b, + 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908, + 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808, + 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b, + 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808, + 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808, + 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19, + 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08, + 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808, + 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b, + 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808, + 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b, + 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b, +}; + +static const uint64_t iq2s_grid[1024] = { + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, + 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, + 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919, + 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b, + 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919, + 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b, + 0x08080808192b2b19, 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919, + 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b2b0808, + 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819, 0x0808081908081908, + 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b, + 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908, + 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08, + 0x0808081919190819, 0x0808081919191908, 0x080808191919192b, 0x0808081919192b19, + 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08, 0x080808192b080819, + 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919, + 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b, + 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, + 0x0808082b082b0808, 0x0808082b082b2b2b, 0x0808082b19080819, 0x0808082b19081908, + 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808, 0x0808082b19191919, + 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908, + 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b, + 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, 0x0808190808191919, + 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, 0x08081908082b192b, + 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, + 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908, + 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b, + 0x08081908192b1919, 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b, + 0x080819082b082b19, 0x080819082b190808, 0x080819082b191919, 0x080819082b192b08, + 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, + 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819, + 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808, + 0x08081919082b1919, 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908, + 0x080819191908192b, 0x0808191919082b19, 0x0808191919190808, 0x080819191919082b, + 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819, 0x08081919192b1908, + 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08, + 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819, + 0x0808192b08081908, 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808, + 0x0808192b08191919, 0x0808192b19080808, 0x0808192b19081919, 0x0808192b19082b08, + 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808, 0x0808192b2b080819, + 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b, + 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908, + 0x08082b080819192b, 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919, + 0x08082b08082b2b2b, 0x08082b0819080819, 0x08082b0819081908, 0x08082b081908192b, + 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b, 0x08082b0819191919, + 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808, + 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819, + 0x08082b1908081908, 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919, + 0x08082b1908192b08, 0x08082b19082b0819, 0x08082b1919080808, 0x08082b1919081919, + 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908, 0x08082b19192b0808, + 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819, + 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b, + 0x08082b2b19190808, 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908, + 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, + 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, + 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919, + 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b, + 0x0819080819192b19, 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919, + 0x08190808192b2b08, 0x081908082b080819, 0x081908082b081908, 0x081908082b08192b, + 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08, 0x081908082b2b0819, + 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919, + 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908, + 0x081908190819192b, 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b, + 0x08190819082b1919, 0x08190819082b2b08, 0x0819081919080819, 0x0819081919081908, + 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808, 0x081908191919082b, + 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908, + 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08, + 0x081908192b190819, 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908, + 0x0819082b08082b19, 0x0819082b08190808, 0x0819082b08191919, 0x0819082b082b0819, + 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919, 0x0819082b19190819, + 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808, + 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08, + 0x0819190808190819, 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19, + 0x08191908082b0808, 0x08191908082b1919, 0x08191908082b2b08, 0x0819190819080819, + 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19, 0x0819190819190808, + 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819, + 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919, + 0x081919082b082b08, 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808, + 0x0819191908080819, 0x0819191908081908, 0x081919190808192b, 0x0819191908082b19, + 0x0819191908190808, 0x081919190819082b, 0x0819191908191919, 0x0819191908192b08, + 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b, + 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908, + 0x08191919192b0808, 0x081919192b080819, 0x081919192b081908, 0x081919192b190808, + 0x0819192b08080808, 0x0819192b08081919, 0x0819192b08082b08, 0x0819192b08190819, + 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819, 0x0819192b19081908, + 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819, + 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808, + 0x08192b0808191919, 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808, + 0x08192b081908082b, 0x08192b0819081919, 0x08192b0819082b08, 0x08192b0819190819, + 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819, 0x08192b082b081908, + 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08, + 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819, + 0x08192b1919081908, 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b, + 0x08192b2b08081908, 0x08192b2b08190808, 0x08192b2b19080808, 0x08192b2b1919192b, + 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08, + 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19, + 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819, + 0x082b080819081908, 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919, + 0x082b0808192b1908, 0x082b08082b080808, 0x082b08082b082b2b, 0x082b08082b191908, + 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908, 0x082b081908190808, + 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808, + 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908, + 0x082b0819192b0808, 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808, + 0x082b082b08080808, 0x082b082b08082b2b, 0x082b082b082b082b, 0x082b082b082b2b08, + 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808, 0x082b082b2b082b08, + 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908, + 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919, + 0x082b190808192b08, 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808, + 0x082b19081908082b, 0x082b190819081919, 0x082b190819082b08, 0x082b190819190819, + 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819, 0x082b19082b081908, + 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08, + 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819, + 0x082b191919081908, 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808, + 0x082b192b08080819, 0x082b192b08081908, 0x082b192b08190808, 0x082b192b19080808, + 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919, 0x082b2b0808190819, + 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808, + 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908, + 0x082b2b1908190808, 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b, + 0x082b2b2b192b1908, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, + 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, + 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b, + 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808, + 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b, + 0x1908080819190819, 0x1908080819191908, 0x190808081919192b, 0x1908080819192b19, + 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919, 0x190808082b080819, + 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08, + 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b, + 0x1908081908081919, 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908, + 0x190808190819192b, 0x1908081908192b19, 0x19080819082b0808, 0x19080819082b082b, + 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908, 0x190808191908192b, + 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919, + 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808, + 0x190808192b08082b, 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819, + 0x190808192b191908, 0x190808192b2b0808, 0x1908082b08080819, 0x1908082b08081908, + 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919, 0x1908082b08192b08, + 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08, + 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819, + 0x1908082b2b081908, 0x1908190808080808, 0x190819080808082b, 0x1908190808081919, + 0x1908190808082b08, 0x1908190808082b2b, 0x1908190808190819, 0x1908190808191908, + 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808, 0x19081908082b082b, + 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908, + 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b, + 0x1908190819191919, 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908, + 0x190819082b080808, 0x190819082b08082b, 0x190819082b081919, 0x190819082b082b08, + 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808, 0x1908191908080819, + 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808, + 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819, + 0x19081919082b1908, 0x1908191919080808, 0x190819191908082b, 0x1908191919081919, + 0x1908191919082b08, 0x1908191919190819, 0x1908191919191908, 0x19081919192b0808, + 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908, 0x190819192b190808, + 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08, + 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819, + 0x1908192b19081908, 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919, + 0x19082b0808080819, 0x19082b0808081908, 0x19082b0808082b19, 0x19082b0808190808, + 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08, 0x19082b08082b0819, + 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919, + 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808, + 0x19082b082b081908, 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b, + 0x19082b1908081919, 0x19082b1908082b08, 0x19082b1908190819, 0x19082b1908191908, + 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908, 0x19082b1919190808, + 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908, + 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b, + 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908, + 0x191908080819192b, 0x1919080808192b19, 0x19190808082b0808, 0x19190808082b082b, + 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819, 0x1919080819081908, + 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b, + 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908, + 0x191908082b080808, 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08, + 0x191908082b190819, 0x191908082b191908, 0x1919081908080819, 0x1919081908081908, + 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808, 0x191908190819082b, + 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908, + 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08, + 0x1919081919190819, 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819, + 0x191908192b081908, 0x191908192b190808, 0x1919082b08080808, 0x1919082b08081919, + 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908, 0x1919082b082b0808, + 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19, + 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b, + 0x1919190808082b19, 0x1919190808190808, 0x191919080819082b, 0x1919190808191919, + 0x1919190808192b08, 0x19191908082b0819, 0x19191908082b1908, 0x1919190819080808, + 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08, 0x1919190819190819, + 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908, + 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919, + 0x1919191908082b08, 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808, + 0x1919191919080819, 0x1919191919081908, 0x1919191919190808, 0x191919192b080808, + 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808, 0x1919192b082b192b, + 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919, + 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808, + 0x19192b0819080819, 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b, + 0x19192b082b080808, 0x19192b1908080819, 0x19192b1908081908, 0x19192b1908190808, + 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19, 0x19192b2b2b081919, + 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b, + 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08, + 0x192b0808082b0819, 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919, + 0x192b080819082b08, 0x192b080819190819, 0x192b080819191908, 0x192b0808192b0808, + 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808, 0x192b08190808082b, + 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908, + 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808, + 0x192b08192b080808, 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808, + 0x192b082b19080808, 0x192b082b1919192b, 0x192b082b2b2b0819, 0x192b190808080808, + 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819, 0x192b190808191908, + 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808, + 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808, + 0x192b191919080808, 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b, + 0x192b192b08080808, 0x192b192b2b191908, 0x192b2b0808080819, 0x192b2b0808081908, + 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08, 0x192b2b1908080808, + 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808, + 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819, + 0x2b08080808191908, 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919, + 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808081919082b, + 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819, 0x2b0808082b080808, + 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819, + 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b, + 0x2b08081908191919, 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908, + 0x2b08081919080808, 0x2b0808191908082b, 0x2b08081919081919, 0x2b08081919082b08, + 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819, 0x2b0808192b081908, + 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919, + 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819, + 0x2b08082b19081908, 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908, + 0x2b0819080808192b, 0x2b08190808082b19, 0x2b08190808190808, 0x2b0819080819082b, + 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819, 0x2b08190819080808, + 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819, + 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908, + 0x2b0819082b190808, 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919, + 0x2b08191908082b08, 0x2b08191908190819, 0x2b08191908191908, 0x2b081919082b0808, + 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808, 0x2b0819192b080808, + 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808, + 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919, + 0x2b082b0808190819, 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908, + 0x2b082b0819190808, 0x2b082b082b2b082b, 0x2b082b1908080819, 0x2b082b1908081908, + 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b, 0x2b082b2b19192b08, + 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819, + 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b, + 0x2b19080808191919, 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808, + 0x2b1908081908082b, 0x2b19080819081919, 0x2b19080819082b08, 0x2b19080819190819, + 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819, 0x2b1908082b081908, + 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819, + 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808, + 0x2b19081919192b2b, 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808, + 0x2b19082b19080808, 0x2b19082b2b2b192b, 0x2b19190808080808, 0x2b1919080808082b, + 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819, 0x2b19190808191908, + 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808, + 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908, + 0x2b19191908190808, 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819, + 0x2b19192b08080808, 0x2b19192b1908192b, 0x2b19192b192b1908, 0x2b192b0808080819, + 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b, 0x2b192b0819080808, + 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b, + 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b, + 0x2b2b080808191908, 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819, + 0x2b2b080819081908, 0x2b2b080819190808, 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b, + 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b, 0x2b2b082b08082b2b, + 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b, + 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819, + 0x2b2b190808081908, 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19, + 0x2b2b19082b2b1908, 0x2b2b191908080808, 0x2b2b191908192b19, 0x2b2b192b19190819, + 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, 0x2b2b2b1919191908, + 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808, + 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b, +}; + +static const int8_t kvalues_mxfp4[16] = { + 0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12, +}; diff --git a/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h b/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h new file mode 100644 index 000000000000..ee238855524d --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/kquants_ppc_mma.h @@ -0,0 +1,92 @@ +#pragma once +// POWER10/POWER11 MMA GEMM one-shot entries for K-quant weights x Q8_K +// activations. k in ELEMENTS (multiple of 256); lda/ldb in superblocks; +// C column-major float. From github.com/mavin2009/ppc-mma-kernels. +#include +#ifdef __cplusplus +extern "C" { +#endif +void gemm_q4_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_q5_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_q6_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_q2_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_q3_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_iq4_nl_q8_0_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_iq4_xs_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_q4_1_q8_1_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_q5_0_q8_0_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_q5_1_q8_1_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_tq2_0_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_tq1_0_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_iq2_xxs_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_iq3_xxs_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_iq3_s_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_iq1_s_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_mxfp4_q8_0_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_iq2_xs_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_iq2_s_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_iq1_m_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_nvfp4_q8_0_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_q8_0_q8_0_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_q4_0_q8_0_ppc(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +// tensor-keyed cache for repacked weights (ppc_pack_cache.cpp) +#include +void * ppc_apack_cache_acquire(const void * key, int64_t m, int64_t k, + int variant, size_t bytes, int * fresh); +void ppc_apack_cache_publish(const void * key, int64_t m, int64_t k, int variant); +void * ppc_apack_cache_acquire_par(const void * key, int64_t m, int64_t k, + int variant, size_t bytes, int nth, int * fresh); +void ppc_apack_cache_slice_done(const void * key, int64_t m, int64_t k, int variant); +void ppc_apack_slice(int64_t m, int mr, int ith, int nth, int64_t * i0, int64_t * rows); +void gemv2_q4_K_q8_K_ppc(int64_t m, int64_t k, const void * Av, int64_t lda, + const void * Bv, float * C, int ith, int nth); +void ppc_apack_cache_clear(void); +#ifdef __cplusplus +} +#endif diff --git a/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp new file mode 100644 index 000000000000..29db954d621e --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/legacy_ppc_mma.cpp @@ -0,0 +1,366 @@ +// legacy_ppc_mma.cpp - imported from github.com/mavin2009/ppc-mma-kernels +// (standalone-verified vs exact double references under qemu -cpu power10). +// Q4_1 x Q8_1, Q5_0 x Q8_0, Q5_1 x Q8_1 on POWER10/POWER11 MMA. + +#include "ggml-impl.h" +#include "ggml-cpu-impl.h" +#include "ggml-quants.h" +#include "kquants_ppc_mma.h" + +#include +#include +#include +#include +#include + + + + + + + + + +static_assert(sizeof(block_q4_1) == 4 + 16, "bad q4_1"); +static_assert(sizeof(block_q5_0) == 2 + 4 + 16, "bad q5_0"); +static_assert(sizeof(block_q5_1) == 4 + 4 + 16, "bad q5_1"); +static_assert(sizeof(block_q8_1) == 4 + 32, "bad q8_1"); + + +#if defined(__MMA__) && defined(__powerpc64__) + +typedef vector unsigned char vuc; +typedef vector signed char vsc; +typedef vector unsigned int vui; +typedef vector signed int vsi; +typedef vector float vfl; + +// Unaligned 16-byte load via memcpy: single lxv, well-defined for the +// odd struct offsets several block formats use. +static inline vuc load16u(const void * p) { vuc v; memcpy(&v, p, 16); return v; } + + +#define KC_CH 64 // 32-elem chunks per slab (2048) +#define MR 16 +#define NR 8 + +typedef struct { + vuc v[KC_CH][32]; // 8 depth-steps x 4 rowgroups + vfl dA[KC_CH][4]; // per-block scale per rowgroup + vfl mA[KC_CH][4]; // per-block min (Q4_1/Q5_1; 0 for Q5_0) +} aleg_t; + +typedef struct { + vuc v[KC_CH][16]; // 8 depth-steps x 2 colgroups + vfl dB[KC_CH][2]; // per-block activation scale + vfl SB[KC_CH][2]; // Q8_1: s = dB*sum(y); Q8_0: 16*dB*sum(y) +} bleg_t; + +static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { + vui t0 = vec_mergeh(rows[0], rows[1]); + vui t1 = vec_mergel(rows[0], rows[1]); + vui t2 = vec_mergeh(rows[2], rows[3]); + vui t3 = vec_mergel(rows[2], rows[3]); + out[0*stride] = (vuc)vec_xxpermdi(t0, t2, 0); + out[1*stride] = (vuc)vec_xxpermdi(t0, t2, 3); + out[2*stride] = (vuc)vec_xxpermdi(t1, t3, 0); + out[3*stride] = (vuc)vec_xxpermdi(t1, t3, 3); +} + +static inline void nibbles32(const uint8_t * qs, vuc v[2]) { + vuc raw = load16u((const uint8_t *)(qs) + (0)); + v[0] = vec_and(raw, vec_splats((unsigned char)0xF)); + v[1] = vec_sr(raw, vec_splats((unsigned char)4)); +} + +// merge qh bits: elems 0..15 use bits 0..15 (bytes 0-1), elems 16..31 +// use bits 16..31 (bytes 2-3); bit -> position 4. +static inline void qh_merge(const uint8_t * qh, vuc v[2]) { + vuc raw = load16u((const uint8_t *)(qh) + (0)); // first 4 bytes used + const vuc repL = { 0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1 }; + const vuc repH = { 2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3 }; + const vuc sh = { 0,1,2,3,4,5,6,7, 0,1,2,3,4,5,6,7 }; + const vuc one = vec_splats((unsigned char)1); + const vuc four = vec_splats((unsigned char)4); + vuc b0 = vec_sl(vec_and(vec_sr(vec_perm(raw, raw, repL), sh), one), four); + vuc b1 = vec_sl(vec_and(vec_sr(vec_perm(raw, raw, repH), sh), one), four); + v[0] = vec_or(v[0], b0); + v[1] = vec_or(v[1], b1); +} + +static inline int64_t rt(int64_t m) { return (m + MR - 1) / MR; } +static inline int64_t ct(int64_t n) { return (n + NR - 1) / NR; } +static inline int64_t sl(int64_t k) { return (k/32 + KC_CH - 1) / KC_CH; } + +extern "C" size_t leg_apack_size(int64_t m, int64_t k) { + return (((size_t)(rt(m) * sl(k)) * sizeof(aleg_t)) + 63) & ~(size_t)63; +} +extern "C" size_t leg_bpack_size(int64_t n, int64_t k) { + return (((size_t)(ct(n) * sl(k)) * sizeof(bleg_t)) + 63) & ~(size_t)63; +} + +// generic weight repack over a per-block "codes + d + m" extractor +template +static void repack_generic(const BLK * A, int64_t lda, int64_t m, int64_t k, aleg_t * P) { + const int64_t kb = k/32, ns = sl(k); + for (int64_t it = 0; it < rt(m); it++) + for (int64_t s = 0; s < ns; s++) { + aleg_t * T = &P[it*ns + s]; + const int64_t b0 = s*KC_CH; + const int64_t nb = (kb - b0) < KC_CH ? (kb - b0) : KC_CH; + for (int64_t b = 0; b < nb; b++) { + vuc t[MR][2]; float d[MR], mm[MR]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + const BLK * bp = &A[rr*lda + b0 + b]; + d[r] = GGML_FP16_TO_FP32(bp->d); + mm[r] = HAS_M ? GGML_FP16_TO_FP32(((const ggml_half *)bp)[1]) : 0.0f; + CODES(bp, t[r]); + } + for (int g = 0; g < 4; g++) { + T->dA[b][g] = (vfl){ d[4*g], d[4*g+1], d[4*g+2], d[4*g+3] }; + T->mA[b][g] = (vfl){ mm[4*g], mm[4*g+1], mm[4*g+2], mm[4*g+3] }; + } + vui rows4[4]; + for (int g = 0; g < 4; g++) + for (int h = 0; h < 2; h++) { + for (int r = 0; r < 4; r++) rows4[r] = (vui)t[4*g + r][h]; + mma_transpose4(rows4, &T->v[b][16*h + g], 4); + } + } + } +} + +static void codes_q4_1(const block_q4_1 * bp, vuc v[2]) { nibbles32(bp->qs, v); } +static void codes_q4_0(const block_q4_0 * bp, vuc v[2]) { nibbles32(bp->qs, v); } +static void codes_q5_0(const block_q5_0 * bp, vuc v[2]) { nibbles32(bp->qs, v); qh_merge(bp->qh, v); } +static void codes_q5_1(const block_q5_1 * bp, vuc v[2]) { nibbles32(bp->qs, v); qh_merge(bp->qh, v); } + +extern "C" void leg_repack_q4_1(const block_q4_1 * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_generic(A, lda, m, k, (aleg_t *)p); +} +extern "C" void leg_repack_q4_0(const block_q4_0 * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_generic(A, lda, m, k, (aleg_t *)p); +} +extern "C" void leg_repack_q5_0(const block_q5_0 * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_generic(A, lda, m, k, (aleg_t *)p); +} +extern "C" void leg_repack_q5_1(const block_q5_1 * A, int64_t lda, int64_t m, int64_t k, void * p) { + repack_generic(A, lda, m, k, (aleg_t *)p); +} + +// activation packs: SB = dB*sum(y) computed (Q8_0, x16 for the Q5_0 +// offset) or read from the precomputed s field (Q8_1). +template +static void pack_b_generic(const YBLK * B, int64_t ldb, int64_t n, int64_t k, bleg_t * P) { + const int64_t kb = k/32, ns = sl(k); + for (int64_t jt = 0; jt < ct(n); jt++) + for (int64_t s = 0; s < ns; s++) { + bleg_t * T = &P[jt*ns + s]; + const int64_t b0 = s*KC_CH; + const int64_t nb = (kb - b0) < KC_CH ? (kb - b0) : KC_CH; + for (int64_t b = 0; b < nb; b++) { + const YBLK * yb[NR]; float dB[NR], SB[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = jt*NR + j; if (jj >= n) jj = n - 1; + yb[j] = &B[jj*ldb + b0 + b]; + dB[j] = GGML_FP16_TO_FP32(yb[j]->d); + if (READ_S) { + SB[j] = GGML_FP16_TO_FP32(((const ggml_half *)yb[j])[1]); + } else { + vsi z = vec_splats(0); + vsi sm = vec_sum4s((vsc)load16u((const uint8_t *)(yb[j]->qs) + (0)), z); + sm = vec_sum4s((vsc)load16u((const uint8_t *)(yb[j]->qs) + (16)), sm); + SB[j] = (float)SFACT * dB[j] * (float)(sm[0]+sm[1]+sm[2]+sm[3]); + } + } + T->dB[b][0] = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + T->dB[b][1] = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + T->SB[b][0] = (vfl){ SB[0], SB[1], SB[2], SB[3] }; + T->SB[b][1] = (vfl){ SB[4], SB[5], SB[6], SB[7] }; + vui rows4[4]; + for (int a = 0; a < 2; a++) + for (int h = 0; h < 2; h++) { + for (int j = 0; j < 4; j++) + rows4[j] = (vui)load16u((const uint8_t *)(yb[4*a + j]->qs) + (16*h)); + mma_transpose4(rows4, &T->v[b][8*h + a], 2); + } + } + } +} + +extern "C" void leg_pack_b_q8_0(const block_q8_0 * B, int64_t ldb, int64_t n, int64_t k, void * p) { + pack_b_generic(B, ldb, n, k, (bleg_t *)p); +} +// SFACT 8 variant for Q4_0's offset (t = q - 8) +extern "C" void leg_pack_b_q8_0_o8(const block_q8_0 * B, int64_t ldb, int64_t n, int64_t k, void * p) { + pack_b_generic(B, ldb, n, k, (bleg_t *)p); +} +extern "C" void leg_pack_b_q8_1(const block_q8_1 * B, int64_t ldb, int64_t n, int64_t k, void * p) { + pack_b_generic(B, ldb, n, k, (bleg_t *)p); +} + +// MODE 0 (offset, Q5_0): fin += dA ⊙ (dB_j*P - SB_j) [SB = 16*dB*S] +// MODE 1 (affine, Q4_1/Q5_1): fin += dA*(dB_j*P) + mA*SB_j [SB = dB*sum(y)] +template +static void kernel_leg_16x8(const aleg_t * PA, const bleg_t * PB, + int64_t nb, vfl fin[NR][4]) { + for (int64_t b = 0; b < nb; b++) { + const vuc * a = PA->v[b]; + const vuc * y = PB->v[b]; + if (b + 1 < nb) { + __builtin_prefetch(PA->v[b + 1], 0, 3); + __builtin_prefetch((const char *)PA->v[b + 1] + 256, 0, 3); + __builtin_prefetch(PB->v[b + 1], 0, 3); + } + __vector_quad acc[2][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_xxsetaccz(&acc[i][g]); + for (int x = 0; x < 8; x++) { + const vuc y0 = y[2*x], y1 = y[2*x + 1]; + const vuc w0 = a[4*x], w1 = a[4*x+1], w2 = a[4*x+2], w3 = a[4*x+3]; + __builtin_mma_xvi8ger4pp(&acc[0][0], y0, w0); + __builtin_mma_xvi8ger4pp(&acc[0][1], y0, w1); + __builtin_mma_xvi8ger4pp(&acc[0][2], y0, w2); + __builtin_mma_xvi8ger4pp(&acc[0][3], y0, w3); + __builtin_mma_xvi8ger4pp(&acc[1][0], y1, w0); + __builtin_mma_xvi8ger4pp(&acc[1][1], y1, w1); + __builtin_mma_xvi8ger4pp(&acc[1][2], y1, w2); + __builtin_mma_xvi8ger4pp(&acc[1][3], y1, w3); + } + vsi pr[2][4][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_disassemble_acc(pr[i][g], &acc[i][g]); + for (int i = 0; i < 2; i++) { + const vfl dB = PB->dB[b][i]; + const vfl SB = PB->SB[b][i]; + for (int g = 0; g < 4; g++) { + const vfl dA = PA->dA[b][g]; + const vfl mA = PA->mA[b][g]; + for (int r = 0; r < 4; r++) { + const vfl res = vec_ctf(pr[i][g][r], 0); + if (MODE == 0) { + vfl t = vec_msub(res, vec_splat(dB, r), vec_splat(SB, r)); + fin[4*i + r][g] = vec_madd(t, dA, fin[4*i + r][g]); + } else { + vfl vs = vec_mul(vec_splat(dB, r), dA); + fin[4*i + r][g] = vec_madd(res, vs, fin[4*i + r][g]); + fin[4*i + r][g] = vec_madd(mA, vec_splat(SB, r), fin[4*i + r][g]); + } + } + } + } + } +} + +template +static void leg_gemm_core(int64_t m, int64_t n, int64_t k, + const aleg_t * PA, const bleg_t * PB, + float * C, int64_t ldc, int ith, int nth) { + const int64_t kb = k/32, ns = sl(k), mt = rt(m), njt = ct(n); + const int64_t tpt = (mt + nth - 1) / nth; + const int64_t t0 = ith*tpt, t1 = (ith+1)*tpt < mt ? (ith+1)*tpt : mt; + vfl fin[NR][4]; + for (int64_t it = t0; it < t1; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + for (int64_t jt = 0; jt < njt; jt++) { + for (int j = 0; j < NR; j++) + for (int g = 0; g < 4; g++) fin[j][g] = vec_splats(0.0f); + for (int64_t s = 0; s < ns; s++) { + const int64_t b0 = s*KC_CH; + const int64_t nb = (kb - b0) < KC_CH ? (kb - b0) : KC_CH; + kernel_leg_16x8(&PA[it*ns + s], &PB[jt*ns + s], nb, fin); + } + const int64_t j0 = jt*NR; + const int64_t cols = (n - j0) < NR ? (n - j0) : NR; + for (int64_t cj = 0; cj < cols; cj++) { + float * dst = C + i + (j0 + cj)*ldc; + if (rows == MR) { + for (int g = 0; g < 4; g++) vec_xst(fin[cj][g], 16*g, dst); + } else { + for (int64_t r = 0; r < rows; r++) + dst[r] = fin[cj][r >> 2][r & 3]; + } + } + } + } +} + +extern "C" void leg_gemm_offset(int64_t m, int64_t n, int64_t k, + const void * PA, const void * PB, float * C, int64_t ldc, int ith, int nth) { + leg_gemm_core<0>(m, n, k, (const aleg_t *)PA, (const bleg_t *)PB, C, ldc, ith, nth); +} +extern "C" void leg_gemm_affine(int64_t m, int64_t n, int64_t k, + const void * PA, const void * PB, float * C, int64_t ldc, int ith, int nth) { + leg_gemm_core<1>(m, n, k, (const aleg_t *)PA, (const bleg_t *)PB, C, ldc, ith, nth); +} + + +// one-shot drivers (pack per call; see K-quant note in INTEGRATION.md) +#define LEG_ONESHOT(NAME, BLKA, REPACK, YBLK, PACKB, GEMM, VARIANT) \ +extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ + const void * Av, int64_t lda, const void * Bv, int64_t ldb, \ + float * C, int64_t ldc, int ith, int nth) { \ + const BLKA * A = (const BLKA *)Av; \ + const YBLK * B = (const YBLK *)Bv; \ + int fresh = 0; \ + void * PA = ppc_apack_cache_acquire(Av, m, k, VARIANT, \ + leg_apack_size(m, k), &fresh); \ + if (PA) { \ + if (fresh) { REPACK(A, lda, m, k, PA); \ + ppc_apack_cache_publish(Av, m, k, VARIANT); } \ + const int64_t njt = (n + NR - 1) / NR; \ + if (njt < nth) { \ + /* n too small to feed every thread by columns (worst \ + case n == 1 generation: one column, nth-1 idle threads \ + while scalar row-partitions). Row-partition with the \ + cached pack instead; the full activation pack is tiny \ + at these n. Field regression, Q4_K tg32, 2026-07-21. */ \ + void * PBs = aligned_alloc(64, leg_bpack_size(n, k)); \ + if (!PBs) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + PACKB(B, ldb, n, k, PBs); \ + GEMM(m, n, k, PA, PBs, C, ldc, ith, nth); \ + free(PBs); \ + } else { \ + const int64_t jpt = (njt + nth - 1) / nth; \ + const int64_t jt0 = (int64_t)ith*jpt; \ + const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; \ + if (jt0 < jt1) { \ + const int64_t j0 = jt0*NR; \ + const int64_t nc = (n - j0) < (jt1 - jt0)*NR ? (n - j0) \ + : (jt1 - jt0)*NR; \ + void * PBl = aligned_alloc(64, leg_bpack_size(nc, k)); \ + if (!PBl) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + PACKB(B + j0*ldb, ldb, nc, k, PBl); \ + GEMM(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); \ + free(PBl); \ + } \ + } \ + } else { \ + void * PB = aligned_alloc(64, leg_bpack_size(n, k)); \ + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + PACKB(B, ldb, n, k, PB); \ + void * PT = aligned_alloc(64, leg_apack_size(MR, k)); \ + if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + const int64_t mt = (m + MR - 1) / MR; \ + const int64_t tpt = (mt + nth - 1) / nth; \ + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { \ + const int64_t i = it*MR; \ + const int64_t rows = (m - i) < MR ? (m - i) : MR; \ + REPACK(A + i*lda, lda, rows, k, PT); \ + GEMM(rows, n, k, PT, PB, C + i, ldc, 0, 1); \ + } \ + free(PT); \ + free(PB); \ + } \ +} +LEG_ONESHOT(gemm_q4_1_q8_1_ppc, block_q4_1, leg_repack_q4_1, block_q8_1, leg_pack_b_q8_1, leg_gemm_affine, 25) +LEG_ONESHOT(gemm_q5_0_q8_0_ppc, block_q5_0, leg_repack_q5_0, block_q8_0, leg_pack_b_q8_0, leg_gemm_offset, 26) +LEG_ONESHOT(gemm_q5_1_q8_1_ppc, block_q5_1, leg_repack_q5_1, block_q8_1, leg_pack_b_q8_1, leg_gemm_affine, 27) +LEG_ONESHOT(gemm_q4_0_q8_0_ppc, block_q4_0, leg_repack_q4_0, block_q8_0, leg_pack_b_q8_0_o8, leg_gemm_offset, 28) + +#endif // __MMA__ + diff --git a/ggml/src/ggml-cpu/llamafile/ppc_pack_cache.cpp b/ggml/src/ggml-cpu/llamafile/ppc_pack_cache.cpp new file mode 100644 index 000000000000..2d01e5f1facc --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/ppc_pack_cache.cpp @@ -0,0 +1,279 @@ +// ppc_pack_cache.cpp — tensor-keyed weight-pack cache. +// +// CANONICAL COPY: this file lives in ppc-mma-kernels/src/ and is +// carried into ggml verbatim by patch 0016. Edit here, regenerate the +// patch; the unit test (xcheck_cache.cpp, `make test`) runs against +// this exact code, including the N > capacity regime — the regime both +// prior cache defects lived in (REVIEW.md defect log: round-robin +// thrash, then 128-slot admission exhaustion). +// +// Policy, stated for review: +// * Key = (data pointer, m, k, variant) + a 64-byte content +// fingerprint that detects model reloads at the same address. +// * Admission-only, no eviction: an admitted pack lives until +// ppc_apack_cache_clear(). The cache's effect is monotonic. +// * The slot table GROWS (doubling, starting at 256): slot count is +// never the binding constraint. The byte capacity is the only +// bound (default 2048 MiB, PPC_MMA_PACK_CACHE_MB overrides, 0 +// disables). +// * A refusal is NEVER silent: the first refusal and every 64th +// after it print to stderr with running totals, and the totals are +// queryable via ppc_apack_cache_stats(). A tensor that packs per +// call is a performance defect (measured 2-8x tg loss on POWER10, +// VALIDATION-POWER10.md D3); it must be visible. + +#if defined(__has_include) +# if __has_include("kquants_ppc_mma.h") +# include "kquants_ppc_mma.h" // prototypes, when compiled inside ggml +# endif +#endif + +#include +#include +#include +#include +#include + +typedef struct { + const void * key; + uint64_t fp; + int64_t m, k; + int variant; + void * buf; + size_t bytes; + int ready; // 0 = packing in progress, 1 = usable + int pending; // outstanding parallel pack slices + int used; +} slot_t; + +static slot_t * g_slots = NULL; +static int g_nslots = 0; +static size_t g_total = 0; +static size_t g_cap = (size_t)2048 * 1024 * 1024; +static int g_cap_init = 0; +static size_t g_admitted = 0; +static size_t g_refused = 0; +static pthread_mutex_t g_mu = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t g_cv = PTHREAD_COND_INITIALIZER; + +static uint64_t fingerprint(const void * p, int64_t m, int64_t k) { + const uint8_t * b = (const uint8_t *)p; + const size_t approx = (size_t)m * (size_t)(k/8); + const size_t tail = approx >= 32 ? approx - 32 : 0; + uint64_t h = 1469598103934665603ull; + for (int i = 0; i < 32; i++) { h ^= b[i]; h *= 1099511628211ull; } + for (int i = 0; i < 32; i++) { h ^= b[tail + i]; h *= 1099511628211ull; } + return h; +} + +static void cap_init_locked(void) { + if (g_cap_init) return; + g_cap_init = 1; + const char * e = getenv("PPC_MMA_PACK_CACHE_MB"); + if (e) g_cap = (size_t)strtoull(e, NULL, 10) * 1024 * 1024; +} + +static int grow_locked(void) { + const int n = g_nslots ? g_nslots * 2 : 256; + slot_t * s = (slot_t *)realloc(g_slots, (size_t)n * sizeof(slot_t)); + if (!s) return 0; + memset(s + g_nslots, 0, (size_t)(n - g_nslots) * sizeof(slot_t)); + g_slots = s; + g_nslots = n; + return 1; +} + +static void refuse_locked(size_t bytes) { + g_refused++; + if (g_refused == 1 || g_refused % 64 == 0) { + fprintf(stderr, + "ppc-mma pack cache: capacity refusal #%zu (%zu MiB requested, " + "%zu/%zu MiB resident, %zu packs admitted) -- this tensor will " + "re-pack EVERY call; raise PPC_MMA_PACK_CACHE_MB\n", + g_refused, bytes >> 20, g_total >> 20, g_cap >> 20, g_admitted); + } +} + +extern "C" void * ppc_apack_cache_acquire(const void * key, int64_t m, int64_t k, + int variant, size_t bytes, int * fresh) { + *fresh = 0; + pthread_mutex_lock(&g_mu); + cap_init_locked(); + if (g_cap == 0 || bytes > g_cap) { + if (g_cap != 0) refuse_locked(bytes); + pthread_mutex_unlock(&g_mu); + return NULL; + } + const uint64_t fp = fingerprint(key, m, k); + for (;;) { + slot_t * hit = NULL; + slot_t * stale = NULL; + for (int i = 0; i < g_nslots; i++) + if (g_slots[i].used && g_slots[i].key == key && g_slots[i].m == m && + g_slots[i].k == k && g_slots[i].variant == variant) { + if (g_slots[i].fp == fp) { hit = &g_slots[i]; } + else { stale = &g_slots[i]; } // same address, new contents + break; + } + if (stale && stale->ready) { // reload detected: retire it + free(stale->buf); + g_total -= stale->bytes; + memset(stale, 0, sizeof(*stale)); + } else if (stale) { // being packed by another thread + while (!stale->ready) pthread_cond_wait(&g_cv, &g_mu); + continue; + } + if (hit) { + while (!hit->ready) pthread_cond_wait(&g_cv, &g_mu); + void * b = hit->buf; + pthread_mutex_unlock(&g_mu); + return b; + } + // admission only -- no eviction (see header) + slot_t * dst = NULL; + for (int i = 0; i < g_nslots; i++) + if (!g_slots[i].used) { dst = &g_slots[i]; break; } + if (!dst) { + const int before = g_nslots; + if (!grow_locked()) { refuse_locked(bytes); pthread_mutex_unlock(&g_mu); return NULL; } + dst = &g_slots[before]; // first slot the growth added + } + if (g_total + bytes > g_cap) { refuse_locked(bytes); pthread_mutex_unlock(&g_mu); return NULL; } + void * buf = aligned_alloc(64, bytes); + if (!buf) { refuse_locked(bytes); pthread_mutex_unlock(&g_mu); return NULL; } + dst->key = key; dst->fp = fp; dst->m = m; dst->k = k; dst->variant = variant; + dst->buf = buf; dst->bytes = bytes; dst->ready = 0; dst->used = 1; + g_total += bytes; + g_admitted++; + *fresh = 1; + pthread_mutex_unlock(&g_mu); + return buf; + } +} + +extern "C" void ppc_apack_cache_publish(const void * key, int64_t m, int64_t k, int variant) { + pthread_mutex_lock(&g_mu); + for (int i = 0; i < g_nslots; i++) + if (g_slots[i].used && g_slots[i].key == key && g_slots[i].m == m && + g_slots[i].k == k && g_slots[i].variant == variant) { g_slots[i].ready = 1; break; } + pthread_cond_broadcast(&g_cv); + pthread_mutex_unlock(&g_mu); +} + +// explicit invalidation for embedders that unload models +extern "C" void ppc_apack_cache_clear(void) { + pthread_mutex_lock(&g_mu); + for (int i = 0; i < g_nslots; i++) + if (g_slots[i].used && g_slots[i].ready) { + free(g_slots[i].buf); + g_total -= g_slots[i].bytes; + memset(&g_slots[i], 0, sizeof(g_slots[i])); + } + pthread_mutex_unlock(&g_mu); +} + +// Parallel first touch. All nth threads of one ggml op call +// acquire_par with the same (key, m, k, variant, nth); the first +// arrival creates the slot with pending = nth, and EVERY caller that +// sees the slot unready gets the buffer with *fresh = 1: each packs +// its disjoint row-tile slice, then calls slice_done, which counts +// down and blocks until the pack is whole. Later ops hit ready +// slots and take the plain path (*fresh = 0). Single-threaded cold +// start was measured at ~0.5 GB/s for grid decode -- 3 s of one +// thread working and seven waiting on a 1.5B model, worse for bigger +// ones (VALIDATION-POWER10.md, remaining-work item 2). +extern "C" void * ppc_apack_cache_acquire_par(const void * key, int64_t m, int64_t k, + int variant, size_t bytes, int nth, + int * fresh) { + *fresh = 0; + pthread_mutex_lock(&g_mu); + cap_init_locked(); + if (g_cap == 0 || bytes > g_cap) { + if (g_cap != 0) refuse_locked(bytes); + pthread_mutex_unlock(&g_mu); + return NULL; + } + const uint64_t fp = fingerprint(key, m, k); + for (;;) { + slot_t * hit = NULL; + slot_t * stale = NULL; + for (int i = 0; i < g_nslots; i++) + if (g_slots[i].used && g_slots[i].key == key && g_slots[i].m == m && + g_slots[i].k == k && g_slots[i].variant == variant) { + if (g_slots[i].fp == fp) { hit = &g_slots[i]; } + else { stale = &g_slots[i]; } + break; + } + if (stale && stale->ready) { + free(stale->buf); + g_total -= stale->bytes; + memset(stale, 0, sizeof(*stale)); + } else if (stale) { + while (!stale->ready) pthread_cond_wait(&g_cv, &g_mu); + continue; + } + if (hit) { + void * b = hit->buf; + if (!hit->ready) *fresh = 1; // join the parallel fill + pthread_mutex_unlock(&g_mu); + return b; + } + slot_t * dst = NULL; + for (int i = 0; i < g_nslots; i++) + if (!g_slots[i].used) { dst = &g_slots[i]; break; } + if (!dst) { + const int before = g_nslots; + if (!grow_locked()) { refuse_locked(bytes); pthread_mutex_unlock(&g_mu); return NULL; } + dst = &g_slots[before]; + } + if (g_total + bytes > g_cap) { refuse_locked(bytes); pthread_mutex_unlock(&g_mu); return NULL; } + void * buf = aligned_alloc(64, bytes); + if (!buf) { refuse_locked(bytes); pthread_mutex_unlock(&g_mu); return NULL; } + dst->key = key; dst->fp = fp; dst->m = m; dst->k = k; dst->variant = variant; + dst->buf = buf; dst->bytes = bytes; dst->ready = 0; dst->pending = nth; dst->used = 1; + g_total += bytes; + g_admitted++; + *fresh = 1; + pthread_mutex_unlock(&g_mu); + return buf; + } +} + +extern "C" void ppc_apack_cache_slice_done(const void * key, int64_t m, int64_t k, int variant) { + pthread_mutex_lock(&g_mu); + slot_t * s = NULL; + for (int i = 0; i < g_nslots; i++) + if (g_slots[i].used && g_slots[i].key == key && g_slots[i].m == m && + g_slots[i].k == k && g_slots[i].variant == variant) { s = &g_slots[i]; break; } + if (s) { + if (s->pending > 0 && --s->pending == 0) { + s->ready = 1; + pthread_cond_broadcast(&g_cv); + } + while (!s->ready) pthread_cond_wait(&g_cv, &g_mu); + } + pthread_mutex_unlock(&g_mu); +} + +// Row-tile slice assignment for the parallel fill: MR-aligned, evenly +// spread, empty for surplus threads. +extern "C" void ppc_apack_slice(int64_t m, int mr, int ith, int nth, + int64_t * i0, int64_t * rows) { + const int64_t tiles = (m + mr - 1) / mr; + const int64_t tpt = (tiles + nth - 1) / nth; + int64_t t0 = (int64_t)ith * tpt, t1 = t0 + tpt; + if (t1 > tiles) t1 = tiles; + if (t0 >= t1) { *i0 = 0; *rows = 0; return; } + *i0 = t0 * mr; + *rows = (t1 * mr < m ? t1 * mr : m) - *i0; +} + +// observability: totals since process start +extern "C" void ppc_apack_cache_stats(size_t * admitted, size_t * refused, + size_t * resident_bytes) { + pthread_mutex_lock(&g_mu); + if (admitted) *admitted = g_admitted; + if (refused) *refused = g_refused; + if (resident_bytes) *resident_bytes = g_total; + pthread_mutex_unlock(&g_mu); +} diff --git a/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp new file mode 100644 index 000000000000..cac025d2e3ac --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/q2k_ppc_mma.cpp @@ -0,0 +1,325 @@ +// q2k_ppc_mma.cpp - POWER10/POWER11 MMA GEMM for block_q2_K x Q8_K. +// Imported from github.com/mavin2009/ppc-mma-kernels (standalone-verified +// against exact double references under qemu -cpu power10; see that +// repo's docs/DESIGN.md). This TU adds a one-shot driver that packs the +// calling thread's row tiles and (per thread) the activations, then runs +// the packed GEMM -- a first integration; a repack.cpp-based load-time +// weight pack is the follow-up. + +#include "ggml-impl.h" +#include "ggml-cpu-impl.h" +#include "ggml-quants.h" +#include "kquants_ppc_mma.h" + +#include +#include +#include +#include +#include + + + + + + +static_assert(sizeof(block_q2_K) == QK_K/16 + QK_K/4 + 2*sizeof(ggml_half), "bad q2_K"); + + +#if defined(__MMA__) && defined(__powerpc64__) + +typedef vector unsigned char vuc; +typedef vector signed char vsc; +typedef vector unsigned int vui; +typedef vector signed int vsi; +typedef vector float vfl; + +// Unaligned 16-byte load via memcpy: single lxv, well-defined for the +// odd struct offsets several block formats use. +static inline vuc load16u(const void * p) { vuc v; memcpy(&v, p, 16); return v; } + + +#define KC_SB 8 // superblocks per slab (2048 elems) +#define KC_CH16 (KC_SB * 16) // 16-element chunks per slab +#define MR 16 +#define NR 8 + +typedef struct { + vuc v[KC_CH16][16]; // per chunk: 4 depth-steps x 4 rowgroups + vfl dsc[KC_CH16][4]; // d*(sc&0xF) per chunk per rowgroup + vfl dm [KC_CH16][4]; // dmin*(sc>>4) +} a2k_t; + +typedef struct { + vuc v[KC_CH16][8]; // per chunk: 4 depth-steps x 2 colgroups + vfl dB[KC_SB][2]; + vfl TS[KC_CH16][2]; // dB * bsums[c] per column +} b2k_t; + +static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { + vui t0 = vec_mergeh(rows[0], rows[1]); + vui t1 = vec_mergel(rows[0], rows[1]); + vui t2 = vec_mergeh(rows[2], rows[3]); + vui t3 = vec_mergel(rows[2], rows[3]); + out[0*stride] = (vuc)vec_xxpermdi(t0, t2, 0); + out[1*stride] = (vuc)vec_xxpermdi(t0, t2, 3); + out[2*stride] = (vuc)vec_xxpermdi(t1, t3, 0); + out[3*stride] = (vuc)vec_xxpermdi(t1, t3, 3); +} + +// 16 unsigned 2-bit codes of chunk c (0..15) of one superblock. +static inline vuc q2k_codes16(const uint8_t * qs, int c) { + const int h = c >> 3, idx = c & 7, j = idx >> 1, lo = idx & 1; + vuc v = load16u((const uint8_t *)(qs) + (32*h + 16*lo)); + return vec_and(vec_sr(v, vec_splats((unsigned char)(2*j))), + vec_splats((unsigned char)3)); +} + +static inline int64_t rt(int64_t m) { return (m + MR - 1) / MR; } +static inline int64_t ct(int64_t n) { return (n + NR - 1) / NR; } +static inline int64_t sl(int64_t k) { return (k/QK_K + KC_SB - 1) / KC_SB; } + +extern "C" size_t q2k_apack_size(int64_t m, int64_t k) { + return (((size_t)(rt(m) * sl(k)) * sizeof(a2k_t)) + 63) & ~(size_t)63; +} +extern "C" size_t q2k_bpack_size(int64_t n, int64_t k) { + return (((size_t)(ct(n) * sl(k)) * sizeof(b2k_t)) + 63) & ~(size_t)63; +} + +extern "C" void q2k_repack_a(const block_q2_K * A, int64_t lda, + int64_t m, int64_t k, void * packed) { + a2k_t * P = (a2k_t *)packed; + const int64_t nsb = k/QK_K, ns = sl(k); + for (int64_t it = 0; it < rt(m); it++) + for (int64_t s = 0; s < ns; s++) { + a2k_t * T = &P[it*ns + s]; + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + for (int64_t b = 0; b < nsl; b++) { + const block_q2_K * bp[MR]; float d[MR], dmin[MR]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + bp[r] = &A[rr*lda + sb0 + b]; + d[r] = GGML_FP16_TO_FP32(bp[r]->d); + dmin[r] = GGML_FP16_TO_FP32(bp[r]->dmin); + } + for (int c = 0; c < 16; c++) { + const int64_t ch = 16*b + c; + for (int g = 0; g < 4; g++) { + float sc[4], mn[4]; + for (int r = 0; r < 4; r++) { + const uint8_t s8 = bp[4*g + r]->scales[c]; + sc[r] = d[4*g + r] * (float)(s8 & 0xF); + mn[r] = dmin[4*g + r] * (float)(s8 >> 4); + } + T->dsc[ch][g] = (vfl){ sc[0], sc[1], sc[2], sc[3] }; + T->dm [ch][g] = (vfl){ mn[0], mn[1], mn[2], mn[3] }; + } + vui rows4[4]; + for (int g = 0; g < 4; g++) { + for (int r = 0; r < 4; r++) + rows4[r] = (vui)q2k_codes16(bp[4*g + r]->qs, c); + mma_transpose4(rows4, &T->v[ch][g], 4); + } + } + } + } +} + +extern "C" void q2k_pack_b(const block_q8_K * B, int64_t ldb, + int64_t n, int64_t k, void * packed) { + b2k_t * P = (b2k_t *)packed; + const int64_t nsb = k/QK_K, ns = sl(k); + for (int64_t jt = 0; jt < ct(n); jt++) + for (int64_t s = 0; s < ns; s++) { + b2k_t * T = &P[jt*ns + s]; + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + for (int64_t b = 0; b < nsl; b++) { + const block_q8_K * yb[NR]; float dB[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = jt*NR + j; if (jj >= n) jj = n - 1; + yb[j] = &B[jj*ldb + sb0 + b]; + dB[j] = yb[j]->d; + } + T->dB[b][0] = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + T->dB[b][1] = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + for (int c = 0; c < 16; c++) { + const int64_t ch = 16*b + c; + float TS[NR]; + for (int j = 0; j < NR; j++) + TS[j] = dB[j] * (float)yb[j]->bsums[c]; + T->TS[ch][0] = (vfl){ TS[0], TS[1], TS[2], TS[3] }; + T->TS[ch][1] = (vfl){ TS[4], TS[5], TS[6], TS[7] }; + vui rows4[4]; + for (int a = 0; a < 2; a++) { + for (int j = 0; j < 4; j++) + rows4[j] = (vui)load16u((const uint8_t *)(yb[4*a + j]->qs) + (16*c)); + mma_transpose4(rows4, &T->v[ch][a], 2); + } + } + } + } +} + +static void kernel2k_16x8(const a2k_t * PA, const b2k_t * PB, + int64_t nsl, vfl fin[NR][4]) { + for (int64_t b = 0; b < nsl; b++) { + const vfl dB0 = PB->dB[b][0], dB1 = PB->dB[b][1]; + for (int c = 0; c < 16; c++) { + const int64_t ch = 16*b + c; + const vuc * a = PA->v[ch]; + const vuc * y = PB->v[ch]; + if (ch + 1 < 16*nsl) { +#ifdef PPC_DCBT_STREAM + __asm__ volatile(\"dcbt 0,%0,8\" :: \"r\"(PA->v[ch + 1])); + __asm__ volatile(\"dcbt 0,%0,8\" :: \"r\"(PB->v[ch + 1])); +#else + __builtin_prefetch(PA->v[ch + 1], 0, 3); + __builtin_prefetch(PB->v[ch + 1], 0, 3); +#endif + } + __vector_quad acc[2][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_xxsetaccz(&acc[i][g]); + for (int x = 0; x < 4; x++) { + const vuc y0 = y[2*x], y1 = y[2*x + 1]; + const vuc w0 = a[4*x], w1 = a[4*x+1], w2 = a[4*x+2], w3 = a[4*x+3]; + __builtin_mma_xvi8ger4pp(&acc[0][0], y0, w0); + __builtin_mma_xvi8ger4pp(&acc[0][1], y0, w1); + __builtin_mma_xvi8ger4pp(&acc[0][2], y0, w2); + __builtin_mma_xvi8ger4pp(&acc[0][3], y0, w3); + __builtin_mma_xvi8ger4pp(&acc[1][0], y1, w0); + __builtin_mma_xvi8ger4pp(&acc[1][1], y1, w1); + __builtin_mma_xvi8ger4pp(&acc[1][2], y1, w2); + __builtin_mma_xvi8ger4pp(&acc[1][3], y1, w3); + } + // Disassemble all accumulators to a stack buffer first: + // frees the acc-aliased VSRs (0-31) before the fixup runs, so + // fin + scale vectors fit the register file without spills. + vsi pr[2][4][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_disassemble_acc(pr[i][g], &acc[i][g]); + // fin += dsc*(dB_j*P) ; fin -= dm*TS_j (mins term) + for (int i = 0; i < 2; i++) { + const vfl dB = i ? dB1 : dB0; + const vfl TS = PB->TS[ch][i]; + for (int g = 0; g < 4; g++) { + const vsi * rowsP = pr[i][g]; + const vfl dsc = PA->dsc[ch][g]; + const vfl dm = PA->dm [ch][g]; + const vfl s0 = vec_mul(vec_splat(dB, 0), dsc); + const vfl s1 = vec_mul(vec_splat(dB, 1), dsc); + const vfl s2 = vec_mul(vec_splat(dB, 2), dsc); + const vfl s3 = vec_mul(vec_splat(dB, 3), dsc); + fin[4*i+0][g] = vec_madd(vec_ctf(rowsP[0],0), s0, fin[4*i+0][g]); + fin[4*i+1][g] = vec_madd(vec_ctf(rowsP[1],0), s1, fin[4*i+1][g]); + fin[4*i+2][g] = vec_madd(vec_ctf(rowsP[2],0), s2, fin[4*i+2][g]); + fin[4*i+3][g] = vec_madd(vec_ctf(rowsP[3],0), s3, fin[4*i+3][g]); + fin[4*i+0][g] = vec_nmsub(dm, vec_splat(TS, 0), fin[4*i+0][g]); + fin[4*i+1][g] = vec_nmsub(dm, vec_splat(TS, 1), fin[4*i+1][g]); + fin[4*i+2][g] = vec_nmsub(dm, vec_splat(TS, 2), fin[4*i+2][g]); + fin[4*i+3][g] = vec_nmsub(dm, vec_splat(TS, 3), fin[4*i+3][g]); + } + } + } + } +} + +extern "C" void q2k_gemm_packed(int64_t m, int64_t n, int64_t k, + const void * packedA, const void * packedB, + float * C, int64_t ldc, int ith, int nth) { + const a2k_t * PA = (const a2k_t *)packedA; + const b2k_t * PB = (const b2k_t *)packedB; + const int64_t nsb = k/QK_K, ns = sl(k), mt = rt(m), njt = ct(n); + const int64_t tpt = (mt + nth - 1) / nth; + const int64_t t0 = ith*tpt, t1 = (ith+1)*tpt < mt ? (ith+1)*tpt : mt; + + vfl fin[NR][4]; + for (int64_t it = t0; it < t1; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + for (int64_t jt = 0; jt < njt; jt++) { + for (int j = 0; j < NR; j++) + for (int g = 0; g < 4; g++) fin[j][g] = vec_splats(0.0f); + for (int64_t s = 0; s < ns; s++) { + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + kernel2k_16x8(&PA[it*ns + s], &PB[jt*ns + s], nsl, fin); + } + const int64_t j0 = jt*NR; + const int64_t cols = (n - j0) < NR ? (n - j0) : NR; + for (int64_t cj = 0; cj < cols; cj++) { + float * dst = C + i + (j0 + cj)*ldc; + if (rows == MR) { + for (int g = 0; g < 4; g++) vec_xst(fin[cj][g], 16*g, dst); + } else { + for (int64_t r = 0; r < rows; r++) + dst[r] = fin[cj][r >> 2][r & 3]; + } + } + } + } +} + + +// One-shot driver: packs this thread's row tiles (and, per thread, the +// activations) then runs the packed GEMM. +extern "C" void gemm_q2_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * Av, int64_t lda, const void * Bv, int64_t ldb, + float * C, int64_t ldc, int ith, int nth) { + const block_q2_K * A = (const block_q2_K *)Av; + const block_q8_K * B = (const block_q8_K *)Bv; + int fresh = 0; /* B is packed per-branch below */ + void * PA = ppc_apack_cache_acquire(Av, m, k, 23, q2k_apack_size(m, k), &fresh); + if (PA) { + if (fresh) { q2k_repack_a(A, lda, m, k, PA); + ppc_apack_cache_publish(Av, m, k, 23); } + const int64_t njt = (n + NR - 1) / NR; + if (njt < nth) { + /* n too small to feed every thread by columns (worst + case n == 1 generation: one column, nth-1 idle threads + while scalar row-partitions). Row-partition with the + cached pack instead; the full activation pack is tiny + at these n. Field regression, Q4_K tg32, 2026-07-21. */ + void * PBs = aligned_alloc(64, q2k_bpack_size(n, k)); + if (!PBs) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q2k_pack_b(B, ldb, n, k, PBs); + q2k_gemm_packed(m, n, k, PA, PBs, C, ldc, ith, nth); + free(PBs); + } else { + const int64_t jpt = (njt + nth - 1) / nth; + const int64_t jt0 = (int64_t)ith*jpt; + const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; + if (jt0 < jt1) { + const int64_t j0 = jt0*NR; + const int64_t nc = (n - j0) < (jt1 - jt0)*NR ? (n - j0) : (jt1 - jt0)*NR; + void * PBl = aligned_alloc(64, q2k_bpack_size(nc, k)); + if (!PBl) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q2k_pack_b(B + j0*ldb, ldb, nc, k, PBl); + q2k_gemm_packed(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); + free(PBl); + } + } + } else { + void * PB = aligned_alloc(64, q2k_bpack_size(n, k)); + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q2k_pack_b(B, ldb, n, k, PB); + void * PT = aligned_alloc(64, q2k_apack_size(MR, k)); + if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } + const int64_t mt = (m + MR - 1) / MR; + const int64_t tpt = (mt + nth - 1) / nth; + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + q2k_repack_a(A + i*lda, lda, rows, k, PT); + q2k_gemm_packed(rows, n, k, PT, PB, C + i, ldc, 0, 1); + } + free(PT); + free(PB); + } +} + +#endif // __MMA__ + diff --git a/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp new file mode 100644 index 000000000000..b1c774b5d041 --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/q3k_ppc_mma.cpp @@ -0,0 +1,321 @@ +// q3k_ppc_mma.cpp - imported from github.com/mavin2009/ppc-mma-kernels +// (standalone-verified vs exact double references under qemu -cpu power10). + +#include "ggml-impl.h" +#include "ggml-cpu-impl.h" +#include "ggml-quants.h" +#include "kquants_ppc_mma.h" + +#include +#include +#include +#include +#include + + + + + + + +static_assert(sizeof(block_q3_K) == QK_K/8 + QK_K/4 + 12 + sizeof(ggml_half), "bad q3_K"); + + +#if defined(__MMA__) && defined(__powerpc64__) + +typedef vector unsigned char vuc; +typedef vector signed char vsc; +typedef vector unsigned int vui; +typedef vector signed int vsi; +typedef vector float vfl; + +// Unaligned 16-byte load via memcpy: single lxv, well-defined for the +// odd struct offsets several block formats use. +static inline vuc load16u(const void * p) { vuc v; memcpy(&v, p, 16); return v; } + + +#define KC_SB 8 // superblocks per slab (2048 elems) +#define KC_CH16 (KC_SB * 16) // 16-element chunks per slab +#define MR 16 +#define NR 8 + +typedef struct { + vuc v[KC_CH16][16]; // per chunk: 4 depth-steps x 4 rowgroups + vfl dsc[KC_CH16][4]; // d*sc per chunk per rowgroup +} a3k_t; + +typedef struct { + vuc v[KC_CH16][8]; // per chunk: 4 depth-steps x 2 colgroups + vfl dB[KC_SB][2]; + vfl TS[KC_CH16][2]; // dB * 4 * bsums[c] per column +} b3k_t; + +static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { + vui t0 = vec_mergeh(rows[0], rows[1]); + vui t1 = vec_mergel(rows[0], rows[1]); + vui t2 = vec_mergeh(rows[2], rows[3]); + vui t3 = vec_mergel(rows[2], rows[3]); + out[0*stride] = (vuc)vec_xxpermdi(t0, t2, 0); + out[1*stride] = (vuc)vec_xxpermdi(t0, t2, 3); + out[2*stride] = (vuc)vec_xxpermdi(t1, t3, 0); + out[3*stride] = (vuc)vec_xxpermdi(t1, t3, 3); +} + +// 16 unsigned 3-bit codes q' = code | (hbit << 2) of chunk c (0..15). +static inline vuc q3k_codes16(const uint8_t * qs, const uint8_t * hmask, int c) { + const int h = c >> 3, idx = c & 7, j = idx >> 1, lo = idx & 1; + vuc code = vec_and(vec_sr(load16u((const uint8_t *)(qs) + (32*h + 16*lo)), + vec_splats((unsigned char)(2*j))), + vec_splats((unsigned char)3)); + vuc hb = vec_sl(vec_and(vec_sr(load16u((const uint8_t *)(hmask) + (16*lo)), + vec_splats((unsigned char)(4*h + j))), + vec_splats((unsigned char)1)), + vec_splats((unsigned char)2)); + return vec_or(code, hb); +} + +static inline int64_t rt(int64_t m) { return (m + MR - 1) / MR; } +static inline int64_t ct(int64_t n) { return (n + NR - 1) / NR; } +static inline int64_t sl(int64_t k) { return (k/QK_K + KC_SB - 1) / KC_SB; } + +extern "C" size_t q3k_apack_size(int64_t m, int64_t k) { + return (((size_t)(rt(m) * sl(k)) * sizeof(a3k_t)) + 63) & ~(size_t)63; +} +extern "C" size_t q3k_bpack_size(int64_t n, int64_t k) { + return (((size_t)(ct(n) * sl(k)) * sizeof(b3k_t)) + 63) & ~(size_t)63; +} + +extern "C" void q3k_repack_a(const block_q3_K * A, int64_t lda, + int64_t m, int64_t k, void * packed) { + a3k_t * P = (a3k_t *)packed; + const int64_t nsb = k/QK_K, ns = sl(k); + for (int64_t it = 0; it < rt(m); it++) + for (int64_t s = 0; s < ns; s++) { + a3k_t * T = &P[it*ns + s]; + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + for (int64_t b = 0; b < nsl; b++) { + const block_q3_K * bp[MR]; float d[MR]; int8_t sc6[MR][16]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + bp[r] = &A[rr*lda + sb0 + b]; + d[r] = GGML_FP16_TO_FP32(bp[r]->d); + // decode 16 x 6-bit scales from the 12-byte packing + const uint32_t kmask1 = 0x03030303, kmask2 = 0x0f0f0f0f; + uint32_t aux[4]; + memcpy(aux, bp[r]->scales, 12); + uint32_t tmp = aux[2]; + aux[2] = ((aux[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4); + aux[3] = ((aux[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4); + aux[0] = (aux[0] & kmask2) | (((tmp >> 0) & kmask1) << 4); + aux[1] = (aux[1] & kmask2) | (((tmp >> 2) & kmask1) << 4); + memcpy(sc6[r], aux, 16); + } + for (int c = 0; c < 16; c++) { + const int64_t ch = 16*b + c; + for (int g = 0; g < 4; g++) + T->dsc[ch][g] = (vfl){ + d[4*g+0]*(sc6[4*g+0][c]-32), d[4*g+1]*(sc6[4*g+1][c]-32), + d[4*g+2]*(sc6[4*g+2][c]-32), d[4*g+3]*(sc6[4*g+3][c]-32) }; + vui rows4[4]; + for (int g = 0; g < 4; g++) { + for (int r = 0; r < 4; r++) + rows4[r] = (vui)q3k_codes16(bp[4*g + r]->qs, bp[4*g + r]->hmask, c); + mma_transpose4(rows4, &T->v[ch][g], 4); + } + } + } + } +} + +extern "C" void q3k_pack_b(const block_q8_K * B, int64_t ldb, + int64_t n, int64_t k, void * packed) { + b3k_t * P = (b3k_t *)packed; + const int64_t nsb = k/QK_K, ns = sl(k); + for (int64_t jt = 0; jt < ct(n); jt++) + for (int64_t s = 0; s < ns; s++) { + b3k_t * T = &P[jt*ns + s]; + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + for (int64_t b = 0; b < nsl; b++) { + const block_q8_K * yb[NR]; float dB[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = jt*NR + j; if (jj >= n) jj = n - 1; + yb[j] = &B[jj*ldb + sb0 + b]; + dB[j] = yb[j]->d; + } + T->dB[b][0] = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + T->dB[b][1] = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + for (int c = 0; c < 16; c++) { + const int64_t ch = 16*b + c; + float TS[NR]; + for (int j = 0; j < NR; j++) + TS[j] = dB[j] * 4.0f * (float)yb[j]->bsums[c]; + T->TS[ch][0] = (vfl){ TS[0], TS[1], TS[2], TS[3] }; + T->TS[ch][1] = (vfl){ TS[4], TS[5], TS[6], TS[7] }; + vui rows4[4]; + for (int a = 0; a < 2; a++) { + for (int j = 0; j < 4; j++) + rows4[j] = (vui)load16u((const uint8_t *)(yb[4*a + j]->qs) + (16*c)); + mma_transpose4(rows4, &T->v[ch][a], 2); + } + } + } + } +} + +static void kernel3k_16x8(const a3k_t * PA, const b3k_t * PB, + int64_t nsl, vfl fin[NR][4]) { + for (int64_t b = 0; b < nsl; b++) { + const vfl dB0 = PB->dB[b][0], dB1 = PB->dB[b][1]; + for (int c = 0; c < 16; c++) { + const int64_t ch = 16*b + c; + const vuc * a = PA->v[ch]; + const vuc * y = PB->v[ch]; + if (ch + 1 < 16*nsl) { +#ifdef PPC_DCBT_STREAM + __asm__ volatile(\"dcbt 0,%0,8\" :: \"r\"(PA->v[ch + 1])); + __asm__ volatile(\"dcbt 0,%0,8\" :: \"r\"(PB->v[ch + 1])); +#else + __builtin_prefetch(PA->v[ch + 1], 0, 3); + __builtin_prefetch(PB->v[ch + 1], 0, 3); +#endif + } + __vector_quad acc[2][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_xxsetaccz(&acc[i][g]); + for (int x = 0; x < 4; x++) { + const vuc y0 = y[2*x], y1 = y[2*x + 1]; + const vuc w0 = a[4*x], w1 = a[4*x+1], w2 = a[4*x+2], w3 = a[4*x+3]; + __builtin_mma_xvi8ger4pp(&acc[0][0], y0, w0); + __builtin_mma_xvi8ger4pp(&acc[0][1], y0, w1); + __builtin_mma_xvi8ger4pp(&acc[0][2], y0, w2); + __builtin_mma_xvi8ger4pp(&acc[0][3], y0, w3); + __builtin_mma_xvi8ger4pp(&acc[1][0], y1, w0); + __builtin_mma_xvi8ger4pp(&acc[1][1], y1, w1); + __builtin_mma_xvi8ger4pp(&acc[1][2], y1, w2); + __builtin_mma_xvi8ger4pp(&acc[1][3], y1, w3); + } + // Disassemble all accumulators to a stack buffer first: + // frees the acc-aliased VSRs (0-31) before the fixup runs, so + // fin + scale vectors fit the register file without spills. + vsi pr[2][4][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_disassemble_acc(pr[i][g], &acc[i][g]); + // fin += dsc * (dB_j*P - TS_j) (offset correction folded) + for (int i = 0; i < 2; i++) { + const vfl dB = i ? dB1 : dB0; + const vfl TS = PB->TS[ch][i]; + for (int g = 0; g < 4; g++) { + const vsi * rowsP = pr[i][g]; + const vfl dsc = PA->dsc[ch][g]; + vfl t0 = vec_msub(vec_ctf(rowsP[0],0), vec_splat(dB,0), vec_splat(TS,0)); + vfl t1 = vec_msub(vec_ctf(rowsP[1],0), vec_splat(dB,1), vec_splat(TS,1)); + vfl t2 = vec_msub(vec_ctf(rowsP[2],0), vec_splat(dB,2), vec_splat(TS,2)); + vfl t3 = vec_msub(vec_ctf(rowsP[3],0), vec_splat(dB,3), vec_splat(TS,3)); + fin[4*i+0][g] = vec_madd(t0, dsc, fin[4*i+0][g]); + fin[4*i+1][g] = vec_madd(t1, dsc, fin[4*i+1][g]); + fin[4*i+2][g] = vec_madd(t2, dsc, fin[4*i+2][g]); + fin[4*i+3][g] = vec_madd(t3, dsc, fin[4*i+3][g]); + } + } + } + } +} + +extern "C" void q3k_gemm_packed(int64_t m, int64_t n, int64_t k, + const void * packedA, const void * packedB, + float * C, int64_t ldc, int ith, int nth) { + const a3k_t * PA = (const a3k_t *)packedA; + const b3k_t * PB = (const b3k_t *)packedB; + const int64_t nsb = k/QK_K, ns = sl(k), mt = rt(m), njt = ct(n); + const int64_t tpt = (mt + nth - 1) / nth; + const int64_t t0 = ith*tpt, t1 = (ith+1)*tpt < mt ? (ith+1)*tpt : mt; + + vfl fin[NR][4]; + for (int64_t it = t0; it < t1; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + for (int64_t jt = 0; jt < njt; jt++) { + for (int j = 0; j < NR; j++) + for (int g = 0; g < 4; g++) fin[j][g] = vec_splats(0.0f); + for (int64_t s = 0; s < ns; s++) { + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + kernel3k_16x8(&PA[it*ns + s], &PB[jt*ns + s], nsl, fin); + } + const int64_t j0 = jt*NR; + const int64_t cols = (n - j0) < NR ? (n - j0) : NR; + for (int64_t cj = 0; cj < cols; cj++) { + float * dst = C + i + (j0 + cj)*ldc; + if (rows == MR) { + for (int g = 0; g < 4; g++) vec_xst(fin[cj][g], 16*g, dst); + } else { + for (int64_t r = 0; r < rows; r++) + dst[r] = fin[cj][r >> 2][r & 3]; + } + } + } + } +} + + +extern "C" void gemm_q3_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * Av, int64_t lda, const void * Bv, int64_t ldb, + float * C, int64_t ldc, int ith, int nth) { + const block_q3_K * A = (const block_q3_K *)Av; + const block_q8_K * B = (const block_q8_K *)Bv; + int fresh = 0; /* B is packed per-branch below */ + void * PA = ppc_apack_cache_acquire(Av, m, k, 24, q3k_apack_size(m, k), &fresh); + if (PA) { + if (fresh) { q3k_repack_a(A, lda, m, k, PA); + ppc_apack_cache_publish(Av, m, k, 24); } + const int64_t njt = (n + NR - 1) / NR; + if (njt < nth) { + /* n too small to feed every thread by columns (worst + case n == 1 generation: one column, nth-1 idle threads + while scalar row-partitions). Row-partition with the + cached pack instead; the full activation pack is tiny + at these n. Field regression, Q4_K tg32, 2026-07-21. */ + void * PBs = aligned_alloc(64, q3k_bpack_size(n, k)); + if (!PBs) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q3k_pack_b(B, ldb, n, k, PBs); + q3k_gemm_packed(m, n, k, PA, PBs, C, ldc, ith, nth); + free(PBs); + } else { + const int64_t jpt = (njt + nth - 1) / nth; + const int64_t jt0 = (int64_t)ith*jpt; + const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; + if (jt0 < jt1) { + const int64_t j0 = jt0*NR; + const int64_t nc = (n - j0) < (jt1 - jt0)*NR ? (n - j0) : (jt1 - jt0)*NR; + void * PBl = aligned_alloc(64, q3k_bpack_size(nc, k)); + if (!PBl) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q3k_pack_b(B + j0*ldb, ldb, nc, k, PBl); + q3k_gemm_packed(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); + free(PBl); + } + } + } else { + void * PB = aligned_alloc(64, q3k_bpack_size(n, k)); + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q3k_pack_b(B, ldb, n, k, PB); + void * PT = aligned_alloc(64, q3k_apack_size(MR, k)); + if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } + const int64_t mt = (m + MR - 1) / MR; + const int64_t tpt = (mt + nth - 1) / nth; + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + q3k_repack_a(A + i*lda, lda, rows, k, PT); + q3k_gemm_packed(rows, n, k, PT, PB, C + i, ldc, 0, 1); + } + free(PT); + free(PB); + } +} + +#endif // __MMA__ + diff --git a/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp new file mode 100644 index 000000000000..dc04870c1a34 --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/q4k_ppc_mma.cpp @@ -0,0 +1,595 @@ +// q4k_ppc_mma.cpp - POWER10/POWER11 MMA GEMM for block_q4_K x Q8_K. +// Imported from github.com/mavin2009/ppc-mma-kernels (standalone-verified +// against exact double references under qemu -cpu power10; see that +// repo's docs/DESIGN.md). This TU adds a one-shot driver that packs the +// calling thread's row tiles and (per thread) the activations, then runs +// the packed GEMM -- a first integration; a repack.cpp-based load-time +// weight pack is the follow-up. + +#include "ggml-impl.h" +#include "ggml-cpu-impl.h" +#include "ggml-quants.h" +#include "kquants_ppc_mma.h" + +#include +#include +#include +#include +#include + + + + + + +static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/2, "bad q4_K"); +static_assert(sizeof(block_q8_K) == sizeof(float) + QK_K + (QK_K/16)*sizeof(int16_t), "bad q8_K"); + + +// ggml's 6-bit scale/min decode +static inline void q4k_get_scale_min(int j, const uint8_t * q, uint8_t * d, uint8_t * m) { + if (j < 4) { + *d = q[j] & 63; *m = q[j + 4] & 63; + } else { + *d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4); + *m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4); + } +} + +#if defined(__MMA__) && defined(__powerpc64__) + +typedef vector unsigned char vuc; +typedef vector signed char vsc; +typedef vector unsigned int vui; +typedef vector signed int vsi; +typedef vector float vfl; + +// Unaligned 16-byte load via memcpy: single lxv, well-defined for the +// odd struct offsets several block formats use. +static inline vuc load16u(const void * p) { vuc v; memcpy(&v, p, 16); return v; } + + +#define KC_SB 8 // superblocks per K slab (2048 elems) +#define KC_CHUNKS (KC_SB * 8) // 32-element chunks per slab +#define MR 16 +#define NR 8 + +typedef struct { + vuc v[KC_CHUNKS][32]; // MMA-layout unsigned nibbles + vfl dsc[KC_CHUNKS][4]; // d*sc per chunk, 4 rows per vfl + vfl dm [KC_CHUNKS][4]; // dmin*m per chunk +} a4k_t; + +typedef struct { + vuc v[KC_CHUNKS][16]; // signed activations + vfl dB[KC_SB][2]; // Q8_K d per column, per superblock + vfl TS[KC_CHUNKS][2]; // dB * S_sub per (col, chunk) +} b4k_t; + +static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { + vui t0 = vec_mergeh(rows[0], rows[1]); + vui t1 = vec_mergel(rows[0], rows[1]); + vui t2 = vec_mergeh(rows[2], rows[3]); + vui t3 = vec_mergel(rows[2], rows[3]); + out[0*stride] = (vuc)vec_xxpermdi(t0, t2, 0); + out[1*stride] = (vuc)vec_xxpermdi(t0, t2, 3); + out[2*stride] = (vuc)vec_xxpermdi(t1, t3, 0); + out[3*stride] = (vuc)vec_xxpermdi(t1, t3, 3); +} + +// nibble codes of sub-block `sub` (0..7) of one superblock +static inline void q4k_codes32(const uint8_t * qs, int sub, vuc v[2]) { + const int g = sub >> 1; + vuc lo = load16u((const uint8_t *)(qs) + (32*g)); + vuc hi = load16u((const uint8_t *)(qs) + (32*g + 16)); + if ((sub & 1) == 0) { + const vuc mF = vec_splats((unsigned char)0xF); + v[0] = vec_and(lo, mF); + v[1] = vec_and(hi, mF); + } else { + v[0] = vec_sr(lo, vec_splats((unsigned char)4)); + v[1] = vec_sr(hi, vec_splats((unsigned char)4)); + } +} + +// ---- one-time packing ---- + +static inline int64_t rt4k(int64_t m) { return (m + MR - 1) / MR; } +static inline int64_t ct4k(int64_t n) { return (n + NR - 1) / NR; } +static inline int64_t sl4k(int64_t k) { return (k/QK_K + KC_SB - 1) / KC_SB; } + +extern "C" size_t q4k_apack_size(int64_t m, int64_t k) { + return (((size_t)(rt4k(m) * sl4k(k)) * sizeof(a4k_t)) + 63) & ~(size_t)63; +} +extern "C" size_t q4k_bpack_size(int64_t n, int64_t k) { + return (((size_t)(ct4k(n) * sl4k(k)) * sizeof(b4k_t)) + 63) & ~(size_t)63; +} + +extern "C" void q4k_repack_a(const block_q4_K * A, int64_t lda, + int64_t m, int64_t k, void * packed) { + a4k_t * P = (a4k_t *)packed; + const int64_t nsb = k/QK_K, ns = sl4k(k); + for (int64_t it = 0; it < rt4k(m); it++) + for (int64_t s = 0; s < ns; s++) { + a4k_t * T = &P[it*ns + s]; + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + for (int64_t b = 0; b < nsl; b++) { + const block_q4_K * bp[MR]; + float dsc[MR][8], dm[MR][8]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + bp[r] = &A[rr*lda + sb0 + b]; + const float d = GGML_FP16_TO_FP32(bp[r]->d); + const float dmin = GGML_FP16_TO_FP32(bp[r]->dmin); + for (int sub = 0; sub < 8; sub++) { + uint8_t sc, mn; + q4k_get_scale_min(sub, bp[r]->scales, &sc, &mn); + dsc[r][sub] = d * sc; + dm [r][sub] = dmin * mn; + } + } + for (int sub = 0; sub < 8; sub++) { + const int64_t ch = 8*b + sub; + for (int g = 0; g < 4; g++) { + T->dsc[ch][g] = (vfl){ dsc[4*g][sub], dsc[4*g+1][sub], + dsc[4*g+2][sub], dsc[4*g+3][sub] }; + T->dm [ch][g] = (vfl){ dm[4*g][sub], dm[4*g+1][sub], + dm[4*g+2][sub], dm[4*g+3][sub] }; + } + vuc t[MR][2]; + for (int r = 0; r < MR; r++) q4k_codes32(bp[r]->qs, sub, t[r]); + vui rows4[4]; + for (int g = 0; g < 4; g++) + for (int h = 0; h < 2; h++) { + for (int r = 0; r < 4; r++) rows4[r] = (vui)t[4*g + r][h]; + mma_transpose4(rows4, &T->v[ch][16*h + g], 4); + } + } + } + } +} + +extern "C" void q4k_pack_b(const block_q8_K * B, int64_t ldb, + int64_t n, int64_t k, void * packed) { + b4k_t * P = (b4k_t *)packed; + const int64_t nsb = k/QK_K, ns = sl4k(k); + for (int64_t jt = 0; jt < ct4k(n); jt++) + for (int64_t s = 0; s < ns; s++) { + b4k_t * T = &P[jt*ns + s]; + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + for (int64_t b = 0; b < nsl; b++) { + const block_q8_K * yb[NR]; + float dB[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = jt*NR + j; if (jj >= n) jj = n - 1; + yb[j] = &B[jj*ldb + sb0 + b]; + dB[j] = yb[j]->d; + } + T->dB[b][0] = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + T->dB[b][1] = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + for (int sub = 0; sub < 8; sub++) { + const int64_t ch = 8*b + sub; + float TS[NR]; + for (int j = 0; j < NR; j++) + TS[j] = dB[j] * (float)(yb[j]->bsums[2*sub] + yb[j]->bsums[2*sub + 1]); + T->TS[ch][0] = (vfl){ TS[0], TS[1], TS[2], TS[3] }; + T->TS[ch][1] = (vfl){ TS[4], TS[5], TS[6], TS[7] }; + vui rows4[4]; + for (int a = 0; a < 2; a++) { + vuc q[4][2]; + for (int j = 0; j < 4; j++) { + q[j][0] = load16u((const uint8_t *)(yb[4*a + j]->qs) + (32*sub)); + q[j][1] = load16u((const uint8_t *)(yb[4*a + j]->qs) + (32*sub + 16)); + } + for (int h = 0; h < 2; h++) { + for (int j = 0; j < 4; j++) rows4[j] = (vui)q[j][h]; + mma_transpose4(rows4, &T->v[ch][8*h + a], 2); + } + } + } + } + } +} + +// ---- 16x8 microkernel ---- +static void kernel4k_16x8(const a4k_t * PA, const b4k_t * PB, + int64_t nsl, vfl fin[NR][4]) { + for (int64_t b = 0; b < nsl; b++) { + for (int sub = 0; sub < 8; sub++) { + const int64_t ch = 8*b + sub; + const vuc * a = PA->v[ch]; + const vuc * y = PB->v[ch]; + if (ch + 1 < 8*nsl) { +#ifdef PPC_DCBT_STREAM + __asm__ volatile(\"dcbt 0,%0,8\" :: \"r\"(PA->v[ch + 1])); + __asm__ volatile(\"dcbt 0,%0,8\" :: \"r\"(PB->v[ch + 1])); +#else + __builtin_prefetch(PA->v[ch + 1], 0, 3); + __builtin_prefetch((const char *)PA->v[ch + 1] + 256, 0, 3); + __builtin_prefetch(PB->v[ch + 1], 0, 3); +#endif + } + __vector_quad acc[2][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_xxsetaccz(&acc[i][g]); + for (int x = 0; x < 8; x++) { + const vuc y0 = y[2*x], y1 = y[2*x + 1]; + const vuc w0 = a[4*x], w1 = a[4*x+1], w2 = a[4*x+2], w3 = a[4*x+3]; + __builtin_mma_xvi8ger4pp(&acc[0][0], y0, w0); + __builtin_mma_xvi8ger4pp(&acc[0][1], y0, w1); + __builtin_mma_xvi8ger4pp(&acc[0][2], y0, w2); + __builtin_mma_xvi8ger4pp(&acc[0][3], y0, w3); + __builtin_mma_xvi8ger4pp(&acc[1][0], y1, w0); + __builtin_mma_xvi8ger4pp(&acc[1][1], y1, w1); + __builtin_mma_xvi8ger4pp(&acc[1][2], y1, w2); + __builtin_mma_xvi8ger4pp(&acc[1][3], y1, w3); + } + // Stage all accumulators to the stack first (frees VSRs 0-31 + // for the fixup; see DESIGN.md register-pressure note). + vsi pr[2][4][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_disassemble_acc(pr[i][g], &acc[i][g]); + // fin += P * (dB_j * dsc_g) ; then fin -= TS_j * dm_g + const vfl dB0 = PB->dB[b][0], dB1 = PB->dB[b][1]; + for (int i = 0; i < 2; i++) { + const vfl dB = i ? dB1 : dB0; + const vfl TS = PB->TS[ch][i]; + for (int g = 0; g < 4; g++) { + const vsi * rowsP = pr[i][g]; + const vfl dsc = PA->dsc[ch][g]; + const vfl dm = PA->dm [ch][g]; + const vfl s0 = vec_mul(vec_splat(dB, 0), dsc); + const vfl s1 = vec_mul(vec_splat(dB, 1), dsc); + const vfl s2 = vec_mul(vec_splat(dB, 2), dsc); + const vfl s3 = vec_mul(vec_splat(dB, 3), dsc); + fin[4*i+0][g] = vec_madd(vec_ctf(rowsP[0],0), s0, fin[4*i+0][g]); + fin[4*i+1][g] = vec_madd(vec_ctf(rowsP[1],0), s1, fin[4*i+1][g]); + fin[4*i+2][g] = vec_madd(vec_ctf(rowsP[2],0), s2, fin[4*i+2][g]); + fin[4*i+3][g] = vec_madd(vec_ctf(rowsP[3],0), s3, fin[4*i+3][g]); + fin[4*i+0][g] = vec_nmsub(dm, vec_splat(TS, 0), fin[4*i+0][g]); + fin[4*i+1][g] = vec_nmsub(dm, vec_splat(TS, 1), fin[4*i+1][g]); + fin[4*i+2][g] = vec_nmsub(dm, vec_splat(TS, 2), fin[4*i+2][g]); + fin[4*i+3][g] = vec_nmsub(dm, vec_splat(TS, 3), fin[4*i+3][g]); + } + } + } + } +} + +extern "C" void q4k_gemm_packed(int64_t m, int64_t n, int64_t k, + const void * packedA, const void * packedB, + float * C, int64_t ldc, int ith, int nth) { + const a4k_t * PA = (const a4k_t *)packedA; + const b4k_t * PB = (const b4k_t *)packedB; + const int64_t nsb = k/QK_K, ns = sl4k(k), mt = rt4k(m), njt = ct4k(n); + const int64_t tpt = (mt + nth - 1) / nth; + const int64_t t0 = ith*tpt, t1 = (ith+1)*tpt < mt ? (ith+1)*tpt : mt; + + vfl fin[NR][4]; + for (int64_t it = t0; it < t1; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + for (int64_t jt = 0; jt < njt; jt++) { + for (int j = 0; j < NR; j++) + for (int g = 0; g < 4; g++) fin[j][g] = vec_splats(0.0f); + for (int64_t s = 0; s < ns; s++) { + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + kernel4k_16x8(&PA[it*ns + s], &PB[jt*ns + s], nsl, fin); + } + const int64_t j0 = jt*NR; + const int64_t cols = (n - j0) < NR ? (n - j0) : NR; + for (int64_t cj = 0; cj < cols; cj++) { + float * dst = C + i + (j0 + cj)*ldc; + if (rows == MR) { + for (int g = 0; g < 4; g++) vec_xst(fin[cj][g], 16*g, dst); + } else { + for (int64_t r = 0; r < rows; r++) + dst[r] = fin[cj][r >> 2][r & 3]; + } + } + } + } +} + + +// One-shot driver: packs this thread's row tiles (and, per thread, the +// activations) then runs the packed GEMM. +extern "C" void gemm_q4_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * Av, int64_t lda, const void * Bv, int64_t ldb, + float * C, int64_t ldc, int ith, int nth) { + const block_q4_K * A = (const block_q4_K *)Av; + const block_q8_K * B = (const block_q8_K *)Bv; + int fresh = 0; /* B is packed per-branch below */ + void * PA = ppc_apack_cache_acquire(Av, m, k, 20, q4k_apack_size(m, k), &fresh); + if (PA) { + if (fresh) { q4k_repack_a(A, lda, m, k, PA); + ppc_apack_cache_publish(Av, m, k, 20); } + const int64_t njt = (n + NR - 1) / NR; + if (njt < nth) { + /* n too small to feed every thread by columns (worst + case n == 1 generation: one column, nth-1 idle threads + while scalar row-partitions). Row-partition with the + cached pack instead; the full activation pack is tiny + at these n. Field regression, Q4_K tg32, 2026-07-21. */ + void * PBs = aligned_alloc(64, q4k_bpack_size(n, k)); + if (!PBs) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q4k_pack_b(B, ldb, n, k, PBs); + q4k_gemm_packed(m, n, k, PA, PBs, C, ldc, ith, nth); + free(PBs); + } else { + const int64_t jpt = (njt + nth - 1) / nth; + const int64_t jt0 = (int64_t)ith*jpt; + const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; + if (jt0 < jt1) { + const int64_t j0 = jt0*NR; + const int64_t nc = (n - j0) < (jt1 - jt0)*NR ? (n - j0) : (jt1 - jt0)*NR; + void * PBl = aligned_alloc(64, q4k_bpack_size(nc, k)); + if (!PBl) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q4k_pack_b(B + j0*ldb, ldb, nc, k, PBl); + q4k_gemm_packed(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); + free(PBl); + } + } + } else { + void * PB = aligned_alloc(64, q4k_bpack_size(n, k)); + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q4k_pack_b(B, ldb, n, k, PB); + void * PT = aligned_alloc(64, q4k_apack_size(MR, k)); + if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } + const int64_t mt = (m + MR - 1) / MR; + const int64_t tpt = (mt + nth - 1) / nth; + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + q4k_repack_a(A + i*lda, lda, rows, k, PT); + q4k_gemm_packed(rows, n, k, PT, PB, C + i, ldc, 0, 1); + } + free(PT); + free(PB); + } +} + +#endif // __MMA__ + +// ---- two-row GEMV, n = 1 (the near-tie squeeze) ----------------------- +// +// Generation cannot use the MMA tiles profitably (the packed operand +// rides at 1.8x native bytes; VALIDATION-POWER10.md D3), and ggml's +// vec_dot keeps one weight stream per thread alive -- measured at 49% +// of the machine's bandwidth on a 27B (s9.1). This kernel walks two +// rows per pass: twice the memory-level parallelism on the weight +// side, and every activation load shared between the rows. Per-row +// arithmetic order is copied exactly from ggml's POWER9 vec_dot, so +// each row's result is bit-identical to the single-row path -- the +// speedup must come from the memory system alone, and correctness +// comparisons stay trivial. +extern "C" void gemv2_q4_K_q8_K_ppc(int64_t m, int64_t k, + const void * Av, int64_t lda, + const void * Bv, float * C, + int ith, int nth) { + const block_q4_K * __restrict A = (const block_q4_K *)Av; + const block_q8_K * __restrict y = (const block_q8_K *)Bv; + const int64_t nb = k / QK_K; + + const vector signed char lowMask = vec_splats((signed char)0xF); + const vector signed char lowMask1 = vec_splats((int8_t)0x3f); + const vector signed char lowMask2 = vec_splats((int8_t)0x30); + const vector int v0 = vec_splats((int32_t)0); + const vector unsigned char v2 = vec_splats((uint8_t)2); + const vector unsigned char v4 = vec_splats((unsigned char)0x4); + + const int64_t rpt = (m + nth - 1) / nth; + const int64_t lo = (int64_t)ith * rpt; + const int64_t hi = (lo + rpt) < m ? (lo + rpt) : m; + + for (int64_t r = lo; r < hi; r += 2) { + const int two = (r + 1 < hi); + const block_q4_K * xa = A + r*lda; + const block_q4_K * xb = A + (two ? (r + 1)*lda : r*lda); + + vector float sfa0 = vec_splats(0.0f), sfa1 = vec_splats(0.0f); + vector float sfa2 = vec_splats(0.0f), sfa3 = vec_splats(0.0f); + vector float sfb0 = vec_splats(0.0f), sfb1 = vec_splats(0.0f); + vector float sfb2 = vec_splats(0.0f), sfb3 = vec_splats(0.0f); + + for (int64_t i = 0; i < nb; ++i) { + const vector float vyd = vec_splats(y[i].d); + const vector signed short q8ysums0 = vec_xl( 0, y[i].bsums); + const vector signed short q8ysums1 = vec_xl(16, y[i].bsums); + + // ---- per-row scale/min setup (identical to ggml's vec_dot) ---- + vector float vda, vdmina, vdb, vdminb; + vector signed short vscalesa, vscalesb; + vector signed short q4xmins0a, q4xmins1a, q4xmins0b, q4xmins1b; + { + const vector float vxd = vec_splats(ggml_fp16_to_fp32(xa[i].d)); + vda = vec_mul(vxd, vyd); + const vector float vxmin = vec_splats(ggml_fp16_to_fp32(xa[i].dmin)); + vdmina = vec_mul(vxmin, vyd); + vector signed char u0 = (vector signed char)vec_xl_len((unsigned char *)xa[i].scales, 8); + vector signed char u1 = vec_and(vec_sr(u0, v2), lowMask2); + vector signed char u2 = (vector signed char)vec_xl_len((unsigned char *)xa[i].scales + 8, 4); + vector signed char u3 = vec_sr(u2, v4); + vector signed char u30 = u1; + vector signed char u31 = (vector signed char)vec_mergeh((vector signed int)vec_and(u2, lowMask), (vector signed int)u3); + u1 = vec_and(u0, lowMask1); + u2 = vec_or(u30, u31); + vector signed char utmps = (vector signed char)vec_mergeh((vector signed int)u1, (vector signed int)u2); + vscalesa = vec_unpackh(utmps); + vector signed short q4xmins = vec_unpackl(utmps); + q4xmins0a = vec_mergeh(q4xmins, q4xmins); + q4xmins1a = vec_mergel(q4xmins, q4xmins); + } + { + const vector float vxd = vec_splats(ggml_fp16_to_fp32(xb[i].d)); + vdb = vec_mul(vxd, vyd); + const vector float vxmin = vec_splats(ggml_fp16_to_fp32(xb[i].dmin)); + vdminb = vec_mul(vxmin, vyd); + vector signed char u0 = (vector signed char)vec_xl_len((unsigned char *)xb[i].scales, 8); + vector signed char u1 = vec_and(vec_sr(u0, v2), lowMask2); + vector signed char u2 = (vector signed char)vec_xl_len((unsigned char *)xb[i].scales + 8, 4); + vector signed char u3 = vec_sr(u2, v4); + vector signed char u30 = u1; + vector signed char u31 = (vector signed char)vec_mergeh((vector signed int)vec_and(u2, lowMask), (vector signed int)u3); + u1 = vec_and(u0, lowMask1); + u2 = vec_or(u30, u31); + vector signed char utmps = (vector signed char)vec_mergeh((vector signed int)u1, (vector signed int)u2); + vscalesb = vec_unpackh(utmps); + vector signed short q4xmins = vec_unpackl(utmps); + q4xmins0b = vec_mergeh(q4xmins, q4xmins); + q4xmins1b = vec_mergel(q4xmins, q4xmins); + } + + { + vector signed int prod0 = vec_mule(q4xmins0a, q8ysums0); + vector signed int prod1 = vec_mule(q4xmins1a, q8ysums1); + vector signed int prod2 = vec_mulo(q4xmins0a, q8ysums0); + vector signed int prod3 = vec_mulo(q4xmins1a, q8ysums1); + sfa0 = vec_nmsub(vec_ctf(prod0, 0), vdmina, sfa0); + sfa1 = vec_nmsub(vec_ctf(prod1, 0), vdmina, sfa1); + sfa2 = vec_nmsub(vec_ctf(prod2, 0), vdmina, sfa2); + sfa3 = vec_nmsub(vec_ctf(prod3, 0), vdmina, sfa3); + } + { + vector signed int prod0 = vec_mule(q4xmins0b, q8ysums0); + vector signed int prod1 = vec_mule(q4xmins1b, q8ysums1); + vector signed int prod2 = vec_mulo(q4xmins0b, q8ysums0); + vector signed int prod3 = vec_mulo(q4xmins1b, q8ysums1); + sfb0 = vec_nmsub(vec_ctf(prod0, 0), vdminb, sfb0); + sfb1 = vec_nmsub(vec_ctf(prod1, 0), vdminb, sfb1); + sfb2 = vec_nmsub(vec_ctf(prod2, 0), vdminb, sfb2); + sfb3 = vec_nmsub(vec_ctf(prod3, 0), vdminb, sfb3); + } + + vector signed int sia0 = v0, sia1 = v0, sia2 = v0, sia3 = v0; + vector signed int sib0 = v0, sib1 = v0, sib2 = v0, sib3 = v0; + + const uint8_t * q4a = xa[i].qs; + const uint8_t * q4b = xb[i].qs; + const int8_t * q8 = y[i].qs; + + for (int j = 0; j < QK_K/64; j += 2) { + __builtin_prefetch(q4a, 0, 1); + __builtin_prefetch(q4b, 0, 1); + __builtin_prefetch(q8, 0, 1); + const vector signed char qxa0 = (vector signed char)vec_xl( 0, q4a); + const vector signed char qxa1 = (vector signed char)vec_xl(16, q4a); + const vector signed char qxa2 = (vector signed char)vec_xl(32, q4a); + const vector signed char qxa3 = (vector signed char)vec_xl(48, q4a); + q4a += 64; + const vector signed char qxb0 = (vector signed char)vec_xl( 0, q4b); + const vector signed char qxb1 = (vector signed char)vec_xl(16, q4b); + const vector signed char qxb2 = (vector signed char)vec_xl(32, q4b); + const vector signed char qxb3 = (vector signed char)vec_xl(48, q4b); + q4b += 64; + + const vector signed char q8y00 = vec_xl( 0, q8); + const vector signed char q8y10 = vec_xl( 16, q8); + const vector signed char q8y01 = vec_xl( 32, q8); + const vector signed char q8y11 = vec_xl( 48, q8); + const vector signed char q8y20 = vec_xl( 64, q8); + const vector signed char q8y30 = vec_xl( 80, q8); + const vector signed char q8y21 = vec_xl( 96, q8); + const vector signed char q8y31 = vec_xl(112, q8); + q8 += 128; + + { + vector unsigned char x00 = (vector unsigned char)vec_and(qxa0, lowMask); + vector unsigned char x01 = (vector unsigned char)vec_sr(qxa0, v4); + vector unsigned char x10 = (vector unsigned char)vec_and(qxa1, lowMask); + vector unsigned char x11 = (vector unsigned char)vec_sr(qxa1, v4); + vector unsigned char x20 = (vector unsigned char)vec_and(qxa2, lowMask); + vector unsigned char x21 = (vector unsigned char)vec_sr(qxa2, v4); + vector unsigned char x30 = (vector unsigned char)vec_and(qxa3, lowMask); + vector unsigned char x31 = (vector unsigned char)vec_sr(qxa3, v4); + vector signed int qv00 = vec_msum(q8y00, x00, v0); + vector signed int qv01 = vec_msum(q8y01, x01, v0); + vector signed int qv10 = vec_msum(q8y10, x10, v0); + vector signed int qv11 = vec_msum(q8y11, x11, v0); + vector signed int qv20 = vec_msum(q8y20, x20, v0); + vector signed int qv21 = vec_msum(q8y21, x21, v0); + vector signed int qv30 = vec_msum(q8y30, x30, v0); + vector signed int qv31 = vec_msum(q8y31, x31, v0); + vector signed int vscales_h = vec_unpackh(vscalesa); + vector signed int vs0 = vec_splat(vscales_h, 0); + vector signed int vs1 = vec_splat(vscales_h, 1); + vector signed int vs2 = vec_splat(vscales_h, 2); + vector signed int vs3 = vec_splat(vscales_h, 3); + vscalesa = vec_sld(vscalesa, vscalesa, 8); + sia0 = vec_add(vec_mul(qv00, vs0), sia0); + sia1 = vec_add(vec_mul(qv01, vs1), sia1); + sia2 = vec_add(vec_mul(qv20, vs2), sia2); + sia3 = vec_add(vec_mul(qv21, vs3), sia3); + sia0 = vec_add(vec_mul(qv10, vs0), sia0); + sia1 = vec_add(vec_mul(qv11, vs1), sia1); + sia2 = vec_add(vec_mul(qv30, vs2), sia2); + sia3 = vec_add(vec_mul(qv31, vs3), sia3); + } + { + vector unsigned char x00 = (vector unsigned char)vec_and(qxb0, lowMask); + vector unsigned char x01 = (vector unsigned char)vec_sr(qxb0, v4); + vector unsigned char x10 = (vector unsigned char)vec_and(qxb1, lowMask); + vector unsigned char x11 = (vector unsigned char)vec_sr(qxb1, v4); + vector unsigned char x20 = (vector unsigned char)vec_and(qxb2, lowMask); + vector unsigned char x21 = (vector unsigned char)vec_sr(qxb2, v4); + vector unsigned char x30 = (vector unsigned char)vec_and(qxb3, lowMask); + vector unsigned char x31 = (vector unsigned char)vec_sr(qxb3, v4); + vector signed int qv00 = vec_msum(q8y00, x00, v0); + vector signed int qv01 = vec_msum(q8y01, x01, v0); + vector signed int qv10 = vec_msum(q8y10, x10, v0); + vector signed int qv11 = vec_msum(q8y11, x11, v0); + vector signed int qv20 = vec_msum(q8y20, x20, v0); + vector signed int qv21 = vec_msum(q8y21, x21, v0); + vector signed int qv30 = vec_msum(q8y30, x30, v0); + vector signed int qv31 = vec_msum(q8y31, x31, v0); + vector signed int vscales_h = vec_unpackh(vscalesb); + vector signed int vs0 = vec_splat(vscales_h, 0); + vector signed int vs1 = vec_splat(vscales_h, 1); + vector signed int vs2 = vec_splat(vscales_h, 2); + vector signed int vs3 = vec_splat(vscales_h, 3); + vscalesb = vec_sld(vscalesb, vscalesb, 8); + sib0 = vec_add(vec_mul(qv00, vs0), sib0); + sib1 = vec_add(vec_mul(qv01, vs1), sib1); + sib2 = vec_add(vec_mul(qv20, vs2), sib2); + sib3 = vec_add(vec_mul(qv21, vs3), sib3); + sib0 = vec_add(vec_mul(qv10, vs0), sib0); + sib1 = vec_add(vec_mul(qv11, vs1), sib1); + sib2 = vec_add(vec_mul(qv30, vs2), sib2); + sib3 = vec_add(vec_mul(qv31, vs3), sib3); + } + } + + sfa0 = vec_madd(vec_ctf(sia0, 0), vda, sfa0); + sfa1 = vec_madd(vec_ctf(sia1, 0), vda, sfa1); + sfa2 = vec_madd(vec_ctf(sia2, 0), vda, sfa2); + sfa3 = vec_madd(vec_ctf(sia3, 0), vda, sfa3); + sfb0 = vec_madd(vec_ctf(sib0, 0), vdb, sfb0); + sfb1 = vec_madd(vec_ctf(sib1, 0), vdb, sfb1); + sfb2 = vec_madd(vec_ctf(sib2, 0), vdb, sfb2); + sfb3 = vec_madd(vec_ctf(sib3, 0), vdb, sfb3); + } + + sfa0 = vec_add(sfa0, sfa2); + sfa1 = vec_add(sfa1, sfa3); + sfa0 = vec_add(sfa0, sfa1); + sfa0 = vec_add(sfa0, vec_sld(sfa0, sfa0, 4)); + sfa0 = vec_add(sfa0, vec_sld(sfa0, sfa0, 8)); + C[r] = vec_extract(sfa0, 0); + + if (two) { + sfb0 = vec_add(sfb0, sfb2); + sfb1 = vec_add(sfb1, sfb3); + sfb0 = vec_add(sfb0, sfb1); + sfb0 = vec_add(sfb0, vec_sld(sfb0, sfb0, 4)); + sfb0 = vec_add(sfb0, vec_sld(sfb0, sfb0, 8)); + C[r + 1] = vec_extract(sfb0, 0); + } + } +} + + diff --git a/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp new file mode 100644 index 000000000000..774f27a456aa --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/q5k_ppc_mma.cpp @@ -0,0 +1,366 @@ +// q5k_ppc_mma.cpp - POWER10/POWER11 MMA GEMM for block_q5_K x Q8_K. +// Imported from github.com/mavin2009/ppc-mma-kernels (standalone-verified +// against exact double references under qemu -cpu power10; see that +// repo's docs/DESIGN.md). This TU adds a one-shot driver that packs the +// calling thread's row tiles and (per thread) the activations, then runs +// the packed GEMM -- a first integration; a repack.cpp-based load-time +// weight pack is the follow-up. + +#include "ggml-impl.h" +#include "ggml-cpu-impl.h" +#include "ggml-quants.h" +#include "kquants_ppc_mma.h" + +#include +#include +#include +#include +#include + + + + + + +static_assert(sizeof(block_q5_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/8 + QK_K/2, "bad q5_K"); +static_assert(sizeof(block_q8_K) == sizeof(float) + QK_K + (QK_K/16)*sizeof(int16_t), "bad q8_K"); + + +// ggml's 6-bit scale/min decode +static inline void q5k_get_scale_min(int j, const uint8_t * q, uint8_t * d, uint8_t * m) { + if (j < 4) { + *d = q[j] & 63; *m = q[j + 4] & 63; + } else { + *d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4); + *m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4); + } +} + +#if defined(__MMA__) && defined(__powerpc64__) + +typedef vector unsigned char vuc; +typedef vector signed char vsc; +typedef vector unsigned int vui; +typedef vector signed int vsi; +typedef vector float vfl; + +// Unaligned 16-byte load via memcpy: single lxv, well-defined for the +// odd struct offsets several block formats use. +static inline vuc load16u(const void * p) { vuc v; memcpy(&v, p, 16); return v; } + + +#define KC_SB 8 // superblocks per K slab (2048 elems) +#define KC_CHUNKS (KC_SB * 8) // 32-element chunks per slab +#define MR 16 +#define NR 8 + +typedef struct { + vuc v[KC_CHUNKS][32]; // MMA-layout unsigned nibbles + vfl dsc[KC_CHUNKS][4]; // d*sc per chunk, 4 rows per vfl + vfl dm [KC_CHUNKS][4]; // dmin*m per chunk +} a4k_t; + +typedef struct { + vuc v[KC_CHUNKS][16]; // signed activations + vfl dB[KC_SB][2]; // Q8_K d per column, per superblock + vfl TS[KC_CHUNKS][2]; // dB * S_sub per (col, chunk) +} b4k_t; + +static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { + vui t0 = vec_mergeh(rows[0], rows[1]); + vui t1 = vec_mergel(rows[0], rows[1]); + vui t2 = vec_mergeh(rows[2], rows[3]); + vui t3 = vec_mergel(rows[2], rows[3]); + out[0*stride] = (vuc)vec_xxpermdi(t0, t2, 0); + out[1*stride] = (vuc)vec_xxpermdi(t0, t2, 3); + out[2*stride] = (vuc)vec_xxpermdi(t1, t3, 0); + out[3*stride] = (vuc)vec_xxpermdi(t1, t3, 3); +} + +// 5-bit codes of sub-block `sub` (0..7): Q4_K nibble + qh bit `sub`. +static inline void q5k_codes32(const uint8_t * qs, const uint8_t * qh, + int sub, vuc v[2]) { + const int g = sub >> 1; + vuc lo = load16u((const uint8_t *)(qs) + (32*g)); + vuc hi = load16u((const uint8_t *)(qs) + (32*g + 16)); + if ((sub & 1) == 0) { + const vuc mF = vec_splats((unsigned char)0xF); + lo = vec_and(lo, mF); + hi = vec_and(hi, mF); + } else { + lo = vec_sr(lo, vec_splats((unsigned char)4)); + hi = vec_sr(hi, vec_splats((unsigned char)4)); + } + const vuc hsh = vec_splats((unsigned char)sub); + const vuc one = vec_splats((unsigned char)1); + const vuc four = vec_splats((unsigned char)4); + vuc h0 = vec_sl(vec_and(vec_sr(load16u((const uint8_t *)(qh) + (0)), hsh), one), four); + vuc h1 = vec_sl(vec_and(vec_sr(load16u((const uint8_t *)(qh) + (16)), hsh), one), four); + v[0] = vec_or(lo, h0); + v[1] = vec_or(hi, h1); +} + +// ---- one-time packing ---- + +static inline int64_t rt4k(int64_t m) { return (m + MR - 1) / MR; } +static inline int64_t ct4k(int64_t n) { return (n + NR - 1) / NR; } +static inline int64_t sl4k(int64_t k) { return (k/QK_K + KC_SB - 1) / KC_SB; } + +extern "C" size_t q5k_apack_size(int64_t m, int64_t k) { + return (((size_t)(rt4k(m) * sl4k(k)) * sizeof(a4k_t)) + 63) & ~(size_t)63; +} +extern "C" size_t q5k_bpack_size(int64_t n, int64_t k) { + return (((size_t)(ct4k(n) * sl4k(k)) * sizeof(b4k_t)) + 63) & ~(size_t)63; +} + +extern "C" void q5k_repack_a(const block_q5_K * A, int64_t lda, + int64_t m, int64_t k, void * packed) { + a4k_t * P = (a4k_t *)packed; + const int64_t nsb = k/QK_K, ns = sl4k(k); + for (int64_t it = 0; it < rt4k(m); it++) + for (int64_t s = 0; s < ns; s++) { + a4k_t * T = &P[it*ns + s]; + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + for (int64_t b = 0; b < nsl; b++) { + const block_q5_K * bp[MR]; + float dsc[MR][8], dm[MR][8]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + bp[r] = &A[rr*lda + sb0 + b]; + const float d = GGML_FP16_TO_FP32(bp[r]->d); + const float dmin = GGML_FP16_TO_FP32(bp[r]->dmin); + for (int sub = 0; sub < 8; sub++) { + uint8_t sc, mn; + q5k_get_scale_min(sub, bp[r]->scales, &sc, &mn); + dsc[r][sub] = d * sc; + dm [r][sub] = dmin * mn; + } + } + for (int sub = 0; sub < 8; sub++) { + const int64_t ch = 8*b + sub; + for (int g = 0; g < 4; g++) { + T->dsc[ch][g] = (vfl){ dsc[4*g][sub], dsc[4*g+1][sub], + dsc[4*g+2][sub], dsc[4*g+3][sub] }; + T->dm [ch][g] = (vfl){ dm[4*g][sub], dm[4*g+1][sub], + dm[4*g+2][sub], dm[4*g+3][sub] }; + } + vuc t[MR][2]; + for (int r = 0; r < MR; r++) q5k_codes32(bp[r]->qs, bp[r]->qh, sub, t[r]); + vui rows4[4]; + for (int g = 0; g < 4; g++) + for (int h = 0; h < 2; h++) { + for (int r = 0; r < 4; r++) rows4[r] = (vui)t[4*g + r][h]; + mma_transpose4(rows4, &T->v[ch][16*h + g], 4); + } + } + } + } +} + +extern "C" void q5k_pack_b(const block_q8_K * B, int64_t ldb, + int64_t n, int64_t k, void * packed) { + b4k_t * P = (b4k_t *)packed; + const int64_t nsb = k/QK_K, ns = sl4k(k); + for (int64_t jt = 0; jt < ct4k(n); jt++) + for (int64_t s = 0; s < ns; s++) { + b4k_t * T = &P[jt*ns + s]; + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + for (int64_t b = 0; b < nsl; b++) { + const block_q8_K * yb[NR]; + float dB[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = jt*NR + j; if (jj >= n) jj = n - 1; + yb[j] = &B[jj*ldb + sb0 + b]; + dB[j] = yb[j]->d; + } + T->dB[b][0] = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + T->dB[b][1] = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + for (int sub = 0; sub < 8; sub++) { + const int64_t ch = 8*b + sub; + float TS[NR]; + for (int j = 0; j < NR; j++) + TS[j] = dB[j] * (float)(yb[j]->bsums[2*sub] + yb[j]->bsums[2*sub + 1]); + T->TS[ch][0] = (vfl){ TS[0], TS[1], TS[2], TS[3] }; + T->TS[ch][1] = (vfl){ TS[4], TS[5], TS[6], TS[7] }; + vui rows4[4]; + for (int a = 0; a < 2; a++) { + vuc q[4][2]; + for (int j = 0; j < 4; j++) { + q[j][0] = load16u((const uint8_t *)(yb[4*a + j]->qs) + (32*sub)); + q[j][1] = load16u((const uint8_t *)(yb[4*a + j]->qs) + (32*sub + 16)); + } + for (int h = 0; h < 2; h++) { + for (int j = 0; j < 4; j++) rows4[j] = (vui)q[j][h]; + mma_transpose4(rows4, &T->v[ch][8*h + a], 2); + } + } + } + } + } +} + +// ---- 16x8 microkernel ---- +static void kernel4k_16x8(const a4k_t * PA, const b4k_t * PB, + int64_t nsl, vfl fin[NR][4]) { + for (int64_t b = 0; b < nsl; b++) { + for (int sub = 0; sub < 8; sub++) { + const int64_t ch = 8*b + sub; + const vuc * a = PA->v[ch]; + const vuc * y = PB->v[ch]; + if (ch + 1 < 8*nsl) { +#ifdef PPC_DCBT_STREAM + __asm__ volatile(\"dcbt 0,%0,8\" :: \"r\"(PA->v[ch + 1])); + __asm__ volatile(\"dcbt 0,%0,8\" :: \"r\"(PB->v[ch + 1])); +#else + __builtin_prefetch(PA->v[ch + 1], 0, 3); + __builtin_prefetch((const char *)PA->v[ch + 1] + 256, 0, 3); + __builtin_prefetch(PB->v[ch + 1], 0, 3); +#endif + } + __vector_quad acc[2][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_xxsetaccz(&acc[i][g]); + for (int x = 0; x < 8; x++) { + const vuc y0 = y[2*x], y1 = y[2*x + 1]; + const vuc w0 = a[4*x], w1 = a[4*x+1], w2 = a[4*x+2], w3 = a[4*x+3]; + __builtin_mma_xvi8ger4pp(&acc[0][0], y0, w0); + __builtin_mma_xvi8ger4pp(&acc[0][1], y0, w1); + __builtin_mma_xvi8ger4pp(&acc[0][2], y0, w2); + __builtin_mma_xvi8ger4pp(&acc[0][3], y0, w3); + __builtin_mma_xvi8ger4pp(&acc[1][0], y1, w0); + __builtin_mma_xvi8ger4pp(&acc[1][1], y1, w1); + __builtin_mma_xvi8ger4pp(&acc[1][2], y1, w2); + __builtin_mma_xvi8ger4pp(&acc[1][3], y1, w3); + } + // Stage all accumulators to the stack first (frees VSRs 0-31 + // for the fixup; see q6_k for rationale). + vsi pr[2][4][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_disassemble_acc(pr[i][g], &acc[i][g]); + // fin += P * (dB_j * dsc_g) ; then fin -= TS_j * dm_g + const vfl dB0 = PB->dB[b][0], dB1 = PB->dB[b][1]; + for (int i = 0; i < 2; i++) { + const vfl dB = i ? dB1 : dB0; + const vfl TS = PB->TS[ch][i]; + for (int g = 0; g < 4; g++) { + const vsi * rowsP = pr[i][g]; + const vfl dsc = PA->dsc[ch][g]; + const vfl dm = PA->dm [ch][g]; + const vfl s0 = vec_mul(vec_splat(dB, 0), dsc); + const vfl s1 = vec_mul(vec_splat(dB, 1), dsc); + const vfl s2 = vec_mul(vec_splat(dB, 2), dsc); + const vfl s3 = vec_mul(vec_splat(dB, 3), dsc); + fin[4*i+0][g] = vec_madd(vec_ctf(rowsP[0],0), s0, fin[4*i+0][g]); + fin[4*i+1][g] = vec_madd(vec_ctf(rowsP[1],0), s1, fin[4*i+1][g]); + fin[4*i+2][g] = vec_madd(vec_ctf(rowsP[2],0), s2, fin[4*i+2][g]); + fin[4*i+3][g] = vec_madd(vec_ctf(rowsP[3],0), s3, fin[4*i+3][g]); + fin[4*i+0][g] = vec_nmsub(dm, vec_splat(TS, 0), fin[4*i+0][g]); + fin[4*i+1][g] = vec_nmsub(dm, vec_splat(TS, 1), fin[4*i+1][g]); + fin[4*i+2][g] = vec_nmsub(dm, vec_splat(TS, 2), fin[4*i+2][g]); + fin[4*i+3][g] = vec_nmsub(dm, vec_splat(TS, 3), fin[4*i+3][g]); + } + } + } + } +} + +extern "C" void q5k_gemm_packed(int64_t m, int64_t n, int64_t k, + const void * packedA, const void * packedB, + float * C, int64_t ldc, int ith, int nth) { + const a4k_t * PA = (const a4k_t *)packedA; + const b4k_t * PB = (const b4k_t *)packedB; + const int64_t nsb = k/QK_K, ns = sl4k(k), mt = rt4k(m), njt = ct4k(n); + const int64_t tpt = (mt + nth - 1) / nth; + const int64_t t0 = ith*tpt, t1 = (ith+1)*tpt < mt ? (ith+1)*tpt : mt; + + vfl fin[NR][4]; + for (int64_t it = t0; it < t1; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + for (int64_t jt = 0; jt < njt; jt++) { + for (int j = 0; j < NR; j++) + for (int g = 0; g < 4; g++) fin[j][g] = vec_splats(0.0f); + for (int64_t s = 0; s < ns; s++) { + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + kernel4k_16x8(&PA[it*ns + s], &PB[jt*ns + s], nsl, fin); + } + const int64_t j0 = jt*NR; + const int64_t cols = (n - j0) < NR ? (n - j0) : NR; + for (int64_t cj = 0; cj < cols; cj++) { + float * dst = C + i + (j0 + cj)*ldc; + if (rows == MR) { + for (int g = 0; g < 4; g++) vec_xst(fin[cj][g], 16*g, dst); + } else { + for (int64_t r = 0; r < rows; r++) + dst[r] = fin[cj][r >> 2][r & 3]; + } + } + } + } +} + + +// One-shot driver: packs this thread's row tiles (and, per thread, the +// activations) then runs the packed GEMM. +extern "C" void gemm_q5_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * Av, int64_t lda, const void * Bv, int64_t ldb, + float * C, int64_t ldc, int ith, int nth) { + const block_q5_K * A = (const block_q5_K *)Av; + const block_q8_K * B = (const block_q8_K *)Bv; + int fresh = 0; /* B is packed per-branch below */ + void * PA = ppc_apack_cache_acquire(Av, m, k, 21, q5k_apack_size(m, k), &fresh); + if (PA) { + if (fresh) { q5k_repack_a(A, lda, m, k, PA); + ppc_apack_cache_publish(Av, m, k, 21); } + const int64_t njt = (n + NR - 1) / NR; + if (njt < nth) { + /* n too small to feed every thread by columns (worst + case n == 1 generation: one column, nth-1 idle threads + while scalar row-partitions). Row-partition with the + cached pack instead; the full activation pack is tiny + at these n. Field regression, Q4_K tg32, 2026-07-21. */ + void * PBs = aligned_alloc(64, q5k_bpack_size(n, k)); + if (!PBs) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q5k_pack_b(B, ldb, n, k, PBs); + q5k_gemm_packed(m, n, k, PA, PBs, C, ldc, ith, nth); + free(PBs); + } else { + const int64_t jpt = (njt + nth - 1) / nth; + const int64_t jt0 = (int64_t)ith*jpt; + const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; + if (jt0 < jt1) { + const int64_t j0 = jt0*NR; + const int64_t nc = (n - j0) < (jt1 - jt0)*NR ? (n - j0) : (jt1 - jt0)*NR; + void * PBl = aligned_alloc(64, q5k_bpack_size(nc, k)); + if (!PBl) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q5k_pack_b(B + j0*ldb, ldb, nc, k, PBl); + q5k_gemm_packed(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); + free(PBl); + } + } + } else { + void * PB = aligned_alloc(64, q5k_bpack_size(n, k)); + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q5k_pack_b(B, ldb, n, k, PB); + void * PT = aligned_alloc(64, q5k_apack_size(MR, k)); + if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } + const int64_t mt = (m + MR - 1) / MR; + const int64_t tpt = (mt + nth - 1) / nth; + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + q5k_repack_a(A + i*lda, lda, rows, k, PT); + q5k_gemm_packed(rows, n, k, PT, PB, C + i, ldc, 0, 1); + } + free(PT); + free(PB); + } +} + +#endif // __MMA__ + diff --git a/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp new file mode 100644 index 000000000000..8cee61dc5654 --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/q6k_ppc_mma.cpp @@ -0,0 +1,316 @@ +// q6k_ppc_mma.cpp - POWER10/POWER11 MMA GEMM for block_q6_K x Q8_K. +// Imported from github.com/mavin2009/ppc-mma-kernels (standalone-verified +// against exact double references under qemu -cpu power10; see that +// repo's docs/DESIGN.md). This TU adds a one-shot driver that packs the +// calling thread's row tiles and (per thread) the activations, then runs +// the packed GEMM -- a first integration; a repack.cpp-based load-time +// weight pack is the follow-up. + +#include "ggml-impl.h" +#include "ggml-cpu-impl.h" +#include "ggml-quants.h" +#include "kquants_ppc_mma.h" + +#include +#include +#include +#include +#include + + + + + +static_assert(sizeof(block_q6_K) == QK_K/2 + QK_K/4 + QK_K/16 + sizeof(ggml_half), "bad q6_K"); + + +#if defined(__MMA__) && defined(__powerpc64__) + +typedef vector unsigned char vuc; +typedef vector signed char vsc; +typedef vector unsigned int vui; +typedef vector signed int vsi; +typedef vector float vfl; + +// Unaligned 16-byte load via memcpy: single lxv, well-defined for the +// odd struct offsets several block formats use. +static inline vuc load16u(const void * p) { vuc v; memcpy(&v, p, 16); return v; } + + +#define KC_SB 8 // superblocks per slab (2048 elems) +#define KC_CH16 (KC_SB * 16) // 16-element chunks per slab +#define MR 16 +#define NR 8 + +typedef struct { + vuc v[KC_CH16][16]; // per chunk: 4 depth-steps x 4 rowgroups + vfl dsc[KC_CH16][4]; // d*sc per chunk per rowgroup +} a6k_t; + +typedef struct { + vuc v[KC_CH16][8]; // per chunk: 4 depth-steps x 2 colgroups + vfl dB[KC_SB][2]; + vfl TS[KC_CH16][2]; // dB * 32 * bsums[c] per column +} b6k_t; + +static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { + vui t0 = vec_mergeh(rows[0], rows[1]); + vui t1 = vec_mergel(rows[0], rows[1]); + vui t2 = vec_mergeh(rows[2], rows[3]); + vui t3 = vec_mergel(rows[2], rows[3]); + out[0*stride] = (vuc)vec_xxpermdi(t0, t2, 0); + out[1*stride] = (vuc)vec_xxpermdi(t0, t2, 3); + out[2*stride] = (vuc)vec_xxpermdi(t1, t3, 0); + out[3*stride] = (vuc)vec_xxpermdi(t1, t3, 3); +} + +// 16 unsigned 6-bit codes of chunk c (0..15) of one superblock. +static inline vuc q6k_codes16(const uint8_t * ql, const uint8_t * qh, int c) { + const int h = c >> 3, idx = c & 7, qq = idx >> 1, lo = idx & 1; + vuc nib = load16u((const uint8_t *)(ql) + (64*h + 32*(qq & 1) + 16*lo)); + nib = (qq < 2) ? vec_and(nib, vec_splats((unsigned char)0xF)) + : vec_sr (nib, vec_splats((unsigned char)4)); + vuc hb = load16u((const uint8_t *)(qh) + (32*h + 16*lo)); + hb = vec_sl(vec_and(vec_sr(hb, vec_splats((unsigned char)(2*qq))), + vec_splats((unsigned char)3)), + vec_splats((unsigned char)4)); + return vec_or(nib, hb); +} + +static inline int64_t rt(int64_t m) { return (m + MR - 1) / MR; } +static inline int64_t ct(int64_t n) { return (n + NR - 1) / NR; } +static inline int64_t sl(int64_t k) { return (k/QK_K + KC_SB - 1) / KC_SB; } + +extern "C" size_t q6k_apack_size(int64_t m, int64_t k) { + return (((size_t)(rt(m) * sl(k)) * sizeof(a6k_t)) + 63) & ~(size_t)63; +} +extern "C" size_t q6k_bpack_size(int64_t n, int64_t k) { + return (((size_t)(ct(n) * sl(k)) * sizeof(b6k_t)) + 63) & ~(size_t)63; +} + +extern "C" void q6k_repack_a(const block_q6_K * A, int64_t lda, + int64_t m, int64_t k, void * packed) { + a6k_t * P = (a6k_t *)packed; + const int64_t nsb = k/QK_K, ns = sl(k); + for (int64_t it = 0; it < rt(m); it++) + for (int64_t s = 0; s < ns; s++) { + a6k_t * T = &P[it*ns + s]; + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + for (int64_t b = 0; b < nsl; b++) { + const block_q6_K * bp[MR]; float d[MR]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + bp[r] = &A[rr*lda + sb0 + b]; + d[r] = GGML_FP16_TO_FP32(bp[r]->d); + } + for (int c = 0; c < 16; c++) { + const int64_t ch = 16*b + c; + for (int g = 0; g < 4; g++) + T->dsc[ch][g] = (vfl){ + d[4*g+0]*bp[4*g+0]->scales[c], d[4*g+1]*bp[4*g+1]->scales[c], + d[4*g+2]*bp[4*g+2]->scales[c], d[4*g+3]*bp[4*g+3]->scales[c] }; + vui rows4[4]; + for (int g = 0; g < 4; g++) { + for (int r = 0; r < 4; r++) + rows4[r] = (vui)q6k_codes16(bp[4*g + r]->ql, bp[4*g + r]->qh, c); + mma_transpose4(rows4, &T->v[ch][g], 4); + } + } + } + } +} + +extern "C" void q6k_pack_b(const block_q8_K * B, int64_t ldb, + int64_t n, int64_t k, void * packed) { + b6k_t * P = (b6k_t *)packed; + const int64_t nsb = k/QK_K, ns = sl(k); + for (int64_t jt = 0; jt < ct(n); jt++) + for (int64_t s = 0; s < ns; s++) { + b6k_t * T = &P[jt*ns + s]; + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + for (int64_t b = 0; b < nsl; b++) { + const block_q8_K * yb[NR]; float dB[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = jt*NR + j; if (jj >= n) jj = n - 1; + yb[j] = &B[jj*ldb + sb0 + b]; + dB[j] = yb[j]->d; + } + T->dB[b][0] = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + T->dB[b][1] = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + for (int c = 0; c < 16; c++) { + const int64_t ch = 16*b + c; + float TS[NR]; + for (int j = 0; j < NR; j++) + TS[j] = dB[j] * 32.0f * (float)yb[j]->bsums[c]; + T->TS[ch][0] = (vfl){ TS[0], TS[1], TS[2], TS[3] }; + T->TS[ch][1] = (vfl){ TS[4], TS[5], TS[6], TS[7] }; + vui rows4[4]; + for (int a = 0; a < 2; a++) { + for (int j = 0; j < 4; j++) + rows4[j] = (vui)load16u((const uint8_t *)(yb[4*a + j]->qs) + (16*c)); + mma_transpose4(rows4, &T->v[ch][a], 2); + } + } + } + } +} + +static void kernel6k_16x8(const a6k_t * PA, const b6k_t * PB, + int64_t nsl, vfl fin[NR][4]) { + for (int64_t b = 0; b < nsl; b++) { + const vfl dB0 = PB->dB[b][0], dB1 = PB->dB[b][1]; + for (int c = 0; c < 16; c++) { + const int64_t ch = 16*b + c; + const vuc * a = PA->v[ch]; + const vuc * y = PB->v[ch]; + if (ch + 1 < 16*nsl) { +#ifdef PPC_DCBT_STREAM + __asm__ volatile(\"dcbt 0,%0,8\" :: \"r\"(PA->v[ch + 1])); + __asm__ volatile(\"dcbt 0,%0,8\" :: \"r\"(PB->v[ch + 1])); +#else + __builtin_prefetch(PA->v[ch + 1], 0, 3); + __builtin_prefetch(PB->v[ch + 1], 0, 3); +#endif + } + __vector_quad acc[2][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_xxsetaccz(&acc[i][g]); + for (int x = 0; x < 4; x++) { + const vuc y0 = y[2*x], y1 = y[2*x + 1]; + const vuc w0 = a[4*x], w1 = a[4*x+1], w2 = a[4*x+2], w3 = a[4*x+3]; + __builtin_mma_xvi8ger4pp(&acc[0][0], y0, w0); + __builtin_mma_xvi8ger4pp(&acc[0][1], y0, w1); + __builtin_mma_xvi8ger4pp(&acc[0][2], y0, w2); + __builtin_mma_xvi8ger4pp(&acc[0][3], y0, w3); + __builtin_mma_xvi8ger4pp(&acc[1][0], y1, w0); + __builtin_mma_xvi8ger4pp(&acc[1][1], y1, w1); + __builtin_mma_xvi8ger4pp(&acc[1][2], y1, w2); + __builtin_mma_xvi8ger4pp(&acc[1][3], y1, w3); + } + // Disassemble all accumulators to a stack buffer first: + // frees the acc-aliased VSRs (0-31) before the fixup runs, so + // fin + scale vectors fit the register file without spills. + vsi pr[2][4][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_disassemble_acc(pr[i][g], &acc[i][g]); + // fin += dsc * (dB_j*P - TS_j) (offset correction folded) + for (int i = 0; i < 2; i++) { + const vfl dB = i ? dB1 : dB0; + const vfl TS = PB->TS[ch][i]; + for (int g = 0; g < 4; g++) { + const vsi * rowsP = pr[i][g]; + const vfl dsc = PA->dsc[ch][g]; + vfl t0 = vec_msub(vec_ctf(rowsP[0],0), vec_splat(dB,0), vec_splat(TS,0)); + vfl t1 = vec_msub(vec_ctf(rowsP[1],0), vec_splat(dB,1), vec_splat(TS,1)); + vfl t2 = vec_msub(vec_ctf(rowsP[2],0), vec_splat(dB,2), vec_splat(TS,2)); + vfl t3 = vec_msub(vec_ctf(rowsP[3],0), vec_splat(dB,3), vec_splat(TS,3)); + fin[4*i+0][g] = vec_madd(t0, dsc, fin[4*i+0][g]); + fin[4*i+1][g] = vec_madd(t1, dsc, fin[4*i+1][g]); + fin[4*i+2][g] = vec_madd(t2, dsc, fin[4*i+2][g]); + fin[4*i+3][g] = vec_madd(t3, dsc, fin[4*i+3][g]); + } + } + } + } +} + +extern "C" void q6k_gemm_packed(int64_t m, int64_t n, int64_t k, + const void * packedA, const void * packedB, + float * C, int64_t ldc, int ith, int nth) { + const a6k_t * PA = (const a6k_t *)packedA; + const b6k_t * PB = (const b6k_t *)packedB; + const int64_t nsb = k/QK_K, ns = sl(k), mt = rt(m), njt = ct(n); + const int64_t tpt = (mt + nth - 1) / nth; + const int64_t t0 = ith*tpt, t1 = (ith+1)*tpt < mt ? (ith+1)*tpt : mt; + + vfl fin[NR][4]; + for (int64_t it = t0; it < t1; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + for (int64_t jt = 0; jt < njt; jt++) { + for (int j = 0; j < NR; j++) + for (int g = 0; g < 4; g++) fin[j][g] = vec_splats(0.0f); + for (int64_t s = 0; s < ns; s++) { + const int64_t sb0 = s*KC_SB; + const int64_t nsl = (nsb - sb0) < KC_SB ? (nsb - sb0) : KC_SB; + kernel6k_16x8(&PA[it*ns + s], &PB[jt*ns + s], nsl, fin); + } + const int64_t j0 = jt*NR; + const int64_t cols = (n - j0) < NR ? (n - j0) : NR; + for (int64_t cj = 0; cj < cols; cj++) { + float * dst = C + i + (j0 + cj)*ldc; + if (rows == MR) { + for (int g = 0; g < 4; g++) vec_xst(fin[cj][g], 16*g, dst); + } else { + for (int64_t r = 0; r < rows; r++) + dst[r] = fin[cj][r >> 2][r & 3]; + } + } + } + } +} + + +// One-shot driver: packs this thread's row tiles (and, per thread, the +// activations) then runs the packed GEMM. +extern "C" void gemm_q6_K_q8_K_ppc(int64_t m, int64_t n, int64_t k, + const void * Av, int64_t lda, const void * Bv, int64_t ldb, + float * C, int64_t ldc, int ith, int nth) { + const block_q6_K * A = (const block_q6_K *)Av; + const block_q8_K * B = (const block_q8_K *)Bv; + int fresh = 0; /* B is packed per-branch below */ + void * PA = ppc_apack_cache_acquire(Av, m, k, 22, q6k_apack_size(m, k), &fresh); + if (PA) { + if (fresh) { q6k_repack_a(A, lda, m, k, PA); + ppc_apack_cache_publish(Av, m, k, 22); } + const int64_t njt = (n + NR - 1) / NR; + if (njt < nth) { + /* n too small to feed every thread by columns (worst + case n == 1 generation: one column, nth-1 idle threads + while scalar row-partitions). Row-partition with the + cached pack instead; the full activation pack is tiny + at these n. Field regression, Q4_K tg32, 2026-07-21. */ + void * PBs = aligned_alloc(64, q6k_bpack_size(n, k)); + if (!PBs) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q6k_pack_b(B, ldb, n, k, PBs); + q6k_gemm_packed(m, n, k, PA, PBs, C, ldc, ith, nth); + free(PBs); + } else { + const int64_t jpt = (njt + nth - 1) / nth; + const int64_t jt0 = (int64_t)ith*jpt; + const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; + if (jt0 < jt1) { + const int64_t j0 = jt0*NR; + const int64_t nc = (n - j0) < (jt1 - jt0)*NR ? (n - j0) : (jt1 - jt0)*NR; + void * PBl = aligned_alloc(64, q6k_bpack_size(nc, k)); + if (!PBl) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q6k_pack_b(B + j0*ldb, ldb, nc, k, PBl); + q6k_gemm_packed(m, nc, k, PA, PBl, C + j0*ldc, ldc, 0, 1); + free(PBl); + } + } + } else { + void * PB = aligned_alloc(64, q6k_bpack_size(n, k)); + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } + q6k_pack_b(B, ldb, n, k, PB); + void * PT = aligned_alloc(64, q6k_apack_size(MR, k)); + if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } + const int64_t mt = (m + MR - 1) / MR; + const int64_t tpt = (mt + nth - 1) / nth; + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + q6k_repack_a(A + i*lda, lda, rows, k, PT); + q6k_gemm_packed(rows, n, k, PT, PB, C + i, ldc, 0, 1); + } + free(PT); + free(PB); + } +} + +#endif // __MMA__ + diff --git a/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp b/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp new file mode 100644 index 000000000000..4b5a198bc969 --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.cpp @@ -0,0 +1,472 @@ +// qbit_ppc_mma.cpp - imported from github.com/mavin2009/ppc-mma-kernels +// (v4 production API, standalone-verified under qemu -cpu power10). +// Q1_0 / Q2_0 x Q8_0 on POWER10/POWER11 MMA, with the tensor-keyed +// pack cache and a no-packing GEMV fast path for n == 1. + +#include "ggml-impl.h" +#include "ggml-cpu-impl.h" +#include "ggml-quants.h" +#include "kquants_ppc_mma.h" + +#include +#include +#include +#include +#include + + + + + + + +#if defined(__MMA__) && defined(__powerpc64__) + +typedef vector unsigned char vuc; +typedef vector signed char vsc; +typedef vector unsigned int vui; +typedef vector signed int vsi; +typedef vector float vfl; + +// Unaligned 16-byte load via memcpy: compiles to a single lxv (which is +// alignment-agnostic on POWER) while staying well-defined C++ for any +// source alignment -- several block structs place qs at odd offsets. +static inline vuc load16u(const void * p) { vuc v; memcpy(&v, p, 16); return v; } + + +#define KC_BLKS 16 +#define KC_CHUNKS (KC_BLKS * 4) +#define MR 16 +#define NR 8 +#define QBIT_GEMV_NMAX 2 // n <= this uses the raw-weight GEMV path + +typedef struct { + vuc v[KC_CHUNKS][32]; + vfl dA[KC_BLKS][4]; +} apack_t; + +typedef struct { + vuc v[KC_CHUNKS][16]; + vfl dB[KC_CHUNKS][2]; + vfl E[KC_BLKS][2]; +} bpack_t; + +static inline void mma_transpose4(const vui rows[4], vuc * out, int stride) { + vui t0 = vec_mergeh(rows[0], rows[1]); + vui t1 = vec_mergel(rows[0], rows[1]); + vui t2 = vec_mergeh(rows[2], rows[3]); + vui t3 = vec_mergel(rows[2], rows[3]); + out[0*stride] = (vuc)vec_xxpermdi(t0, t2, 0); + out[1*stride] = (vuc)vec_xxpermdi(t0, t2, 3); + out[2*stride] = (vuc)vec_xxpermdi(t1, t3, 0); + out[3*stride] = (vuc)vec_xxpermdi(t1, t3, 3); +} + +static inline void q1_codes32(vuc raw, int c, vuc v[2]) { + const vuc rep0 = { 0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1 }; + const vuc rep1 = { 2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3 }; + const vuc sh = { 0,1,2,3,4,5,6,7, 0,1,2,3,4,5,6,7 }; + const vuc m1 = vec_splats((unsigned char)1); + const vuc off = vec_splats((unsigned char)(4*c)); + v[0] = vec_and(vec_sr(vec_perm(raw, raw, vec_add(rep0, off)), sh), m1); + v[1] = vec_and(vec_sr(vec_perm(raw, raw, vec_add(rep1, off)), sh), m1); +} + +static inline void q2_codes32(vuc lo, vuc hi, int c, vuc v[2]) { + const vuc rep0 = { 0,0,0,0, 1,1,1,1, 2,2,2,2, 3,3,3,3 }; + const vuc rep1 = { 4,4,4,4, 5,5,5,5, 6,6,6,6, 7,7,7,7 }; + const vuc sh = { 0,2,4,6, 0,2,4,6, 0,2,4,6, 0,2,4,6 }; + const vuc m3 = vec_splats((unsigned char)3); + const vuc off = vec_splats((unsigned char)(8*c)); + v[0] = vec_and(vec_sr(vec_perm(lo, hi, vec_add(rep0, off)), sh), m3); + v[1] = vec_and(vec_sr(vec_perm(lo, hi, vec_add(rep1, off)), sh), m3); +} + +// ---------------- one-time packing API ---------------- + +static inline int64_t n_row_tiles(int64_t m) { return (m + MR - 1) / MR; } +static inline int64_t n_col_tiles(int64_t n) { return (n + NR - 1) / NR; } +static inline int64_t n_slabs(int64_t k) { return (k/128 + KC_BLKS - 1) / KC_BLKS; } + +extern "C" size_t qbit_apack_size(int64_t m, int64_t k) { + return (((size_t)(n_row_tiles(m) * n_slabs(k)) * sizeof(apack_t)) + 63) & ~(size_t)63; +} +extern "C" size_t qbit_bpack_size(int64_t n, int64_t k) { + return (((size_t)(n_col_tiles(n) * n_slabs(k)) * sizeof(bpack_t)) + 63) & ~(size_t)63; +} + +// packed layout: tile-major, slab-minor: P[tile * n_slabs + slab] + +template +static void repack_rows(const BLK * A, int64_t lda, int64_t m, int64_t k, apack_t * P) { + const int64_t kb = k / 128, ns = n_slabs(k); + for (int64_t it = 0; it < n_row_tiles(m); it++) { + for (int64_t s = 0; s < ns; s++) { + apack_t * T = &P[it*ns + s]; + const int64_t blk0 = s*KC_BLKS; + const int64_t nblk = (kb - blk0) < KC_BLKS ? (kb - blk0) : KC_BLKS; + for (int64_t b = 0; b < nblk; b++) { + const BLK * bp[MR]; float d[MR]; + for (int r = 0; r < MR; r++) { + int64_t rr = it*MR + r; if (rr >= m) rr = m - 1; + bp[r] = &A[rr*lda + blk0 + b]; + d[r] = GGML_FP16_TO_FP32(bp[r]->d); + } + for (int g = 0; g < 4; g++) + T->dA[b][g] = (vfl){ d[4*g], d[4*g+1], d[4*g+2], d[4*g+3] }; + for (int c = 0; c < 4; c++) { + vuc t[MR][2]; + for (int r = 0; r < MR; r++) CODES(bp[r], c, t[r]); + vui rows4[4]; + for (int g = 0; g < 4; g++) + for (int h = 0; h < 2; h++) { + for (int r = 0; r < 4; r++) rows4[r] = (vui)t[4*g + r][h]; + mma_transpose4(rows4, &T->v[4*b + c][16*h + g], 4); + } + } + } + } + } +} + +static void q1_blk_codes(const block_q1_0 * bp, int c, vuc v[2]) { + q1_codes32(load16u((const uint8_t *)(bp->qs) + (0)), c, v); +} +static void q2_blk_codes(const block_q2_0 * bp, int c, vuc v[2]) { + q2_codes32(load16u((const uint8_t *)(bp->qs) + (0)), + load16u((const uint8_t *)(bp->qs) + (16)), c, v); +} + +extern "C" void qbit_repack_q1(const block_q1_0 * A, int64_t lda, + int64_t m, int64_t k, void * packed) { + repack_rows(A, lda, m, k, (apack_t *)packed); +} +extern "C" void qbit_repack_q2(const block_q2_0 * A, int64_t lda, + int64_t m, int64_t k, void * packed) { + repack_rows(A, lda, m, k, (apack_t *)packed); +} + +// Pack all of B once; call from one thread (or split by col tile). +extern "C" void qbit_pack_b(const block_q8_0 * B, int64_t ldb, + int64_t n, int64_t k, void * packed) { + bpack_t * P = (bpack_t *)packed; + const int64_t kb = k / 128, ns = n_slabs(k); + for (int64_t jt = 0; jt < n_col_tiles(n); jt++) { + for (int64_t s = 0; s < ns; s++) { + bpack_t * T = &P[jt*ns + s]; + const int64_t blk0 = s*KC_BLKS; + const int64_t nblk = (kb - blk0) < KC_BLKS ? (kb - blk0) : KC_BLKS; + for (int64_t b = 0; b < nblk; b++) { + vfl E0 = vec_splats(0.0f), E1 = vec_splats(0.0f); + for (int c = 0; c < 4; c++) { + const int64_t ch = 4*b + c; + const block_q8_0 * yb[NR]; + float dB[NR], S[NR]; + for (int j = 0; j < NR; j++) { + int64_t jj = jt*NR + j; if (jj >= n) jj = n - 1; + yb[j] = &B[jj*ldb + 4*(blk0 + b) + c]; + dB[j] = GGML_FP16_TO_FP32(yb[j]->d); + } + vui rows4[4]; + for (int a = 0; a < 2; a++) { + vuc q[4][2]; + for (int j = 0; j < 4; j++) { + q[j][0] = load16u((const uint8_t *)(yb[4*a + j]->qs) + (0)); + q[j][1] = load16u((const uint8_t *)(yb[4*a + j]->qs) + (16)); + vsi z = vec_splats(0); + vsi sm = vec_sum4s((vsc)q[j][0], z); + sm = vec_sum4s((vsc)q[j][1], sm); + S[4*a + j] = (float)(sm[0] + sm[1] + sm[2] + sm[3]); + } + for (int h = 0; h < 2; h++) { + for (int j = 0; j < 4; j++) rows4[j] = (vui)q[j][h]; + mma_transpose4(rows4, &T->v[ch][8*h + a], 2); + } + } + vfl dB0 = (vfl){ dB[0], dB[1], dB[2], dB[3] }; + vfl dB1 = (vfl){ dB[4], dB[5], dB[6], dB[7] }; + T->dB[ch][0] = dB0; T->dB[ch][1] = dB1; + E0 = vec_madd(dB0, (vfl){ S[0],S[1],S[2],S[3] }, E0); + E1 = vec_madd(dB1, (vfl){ S[4],S[5],S[6],S[7] }, E1); + } + T->E[b][0] = E0; T->E[b][1] = E1; + } + } + } +} + +// ---------------- GEMM from packed operands ---------------- + +static void kernel_16x8(const apack_t * PA, const bpack_t * PB, + int64_t nblk, float alpha, vfl fin[NR][4]) { + const vfl valpha = vec_splats(alpha); + for (int64_t b = 0; b < nblk; b++) { + for (int c = 0; c < 4; c++) { + const int64_t ch = 4*b + c; + const vuc * a = PA->v[ch]; + const vuc * y = PB->v[ch]; + if (ch + 1 < 4*nblk) { + __builtin_prefetch(PA->v[ch + 1], 0, 3); + __builtin_prefetch((const char *)PA->v[ch + 1] + 256, 0, 3); + __builtin_prefetch(PB->v[ch + 1], 0, 3); + } + __vector_quad acc[2][4]; + for (int i = 0; i < 2; i++) + for (int g = 0; g < 4; g++) + __builtin_mma_xxsetaccz(&acc[i][g]); + for (int x = 0; x < 8; x++) { + const vuc y0 = y[2*x], y1 = y[2*x + 1]; + const vuc w0 = a[4*x], w1 = a[4*x+1], w2 = a[4*x+2], w3 = a[4*x+3]; + __builtin_mma_xvi8ger4pp(&acc[0][0], y0, w0); + __builtin_mma_xvi8ger4pp(&acc[0][1], y0, w1); + __builtin_mma_xvi8ger4pp(&acc[0][2], y0, w2); + __builtin_mma_xvi8ger4pp(&acc[0][3], y0, w3); + __builtin_mma_xvi8ger4pp(&acc[1][0], y1, w0); + __builtin_mma_xvi8ger4pp(&acc[1][1], y1, w1); + __builtin_mma_xvi8ger4pp(&acc[1][2], y1, w2); + __builtin_mma_xvi8ger4pp(&acc[1][3], y1, w3); + } + const vfl dBa0 = vec_mul(PB->dB[ch][0], valpha); + const vfl dBa1 = vec_mul(PB->dB[ch][1], valpha); + for (int i = 0; i < 2; i++) { + const vfl dBa = i ? dBa1 : dBa0; + for (int g = 0; g < 4; g++) { + vsi rowsP[4]; + __builtin_mma_disassemble_acc(rowsP, &acc[i][g]); + const vfl s0 = vec_mul(vec_splat(dBa, 0), PA->dA[b][g]); + const vfl s1 = vec_mul(vec_splat(dBa, 1), PA->dA[b][g]); + const vfl s2 = vec_mul(vec_splat(dBa, 2), PA->dA[b][g]); + const vfl s3 = vec_mul(vec_splat(dBa, 3), PA->dA[b][g]); + fin[4*i + 0][g] = vec_madd(vec_ctf(rowsP[0], 0), s0, fin[4*i + 0][g]); + fin[4*i + 1][g] = vec_madd(vec_ctf(rowsP[1], 0), s1, fin[4*i + 1][g]); + fin[4*i + 2][g] = vec_madd(vec_ctf(rowsP[2], 0), s2, fin[4*i + 2][g]); + fin[4*i + 3][g] = vec_madd(vec_ctf(rowsP[3], 0), s3, fin[4*i + 3][g]); + } + } + } + } + for (int64_t b = 0; b < nblk; b++) + for (int g = 0; g < 4; g++) { + const vfl dA = PA->dA[b][g]; + fin[0][g] = vec_nmsub(dA, vec_splat(PB->E[b][0], 0), fin[0][g]); + fin[1][g] = vec_nmsub(dA, vec_splat(PB->E[b][0], 1), fin[1][g]); + fin[2][g] = vec_nmsub(dA, vec_splat(PB->E[b][0], 2), fin[2][g]); + fin[3][g] = vec_nmsub(dA, vec_splat(PB->E[b][0], 3), fin[3][g]); + fin[4][g] = vec_nmsub(dA, vec_splat(PB->E[b][1], 0), fin[4][g]); + fin[5][g] = vec_nmsub(dA, vec_splat(PB->E[b][1], 1), fin[5][g]); + fin[6][g] = vec_nmsub(dA, vec_splat(PB->E[b][1], 2), fin[6][g]); + fin[7][g] = vec_nmsub(dA, vec_splat(PB->E[b][1], 3), fin[7][g]); + } +} + +// GEMM over pre-packed operands. packedA from qbit_repack_*, +// packedB from qbit_pack_b. Threads split row tiles. +extern "C" void qbit_gemm_packed(int64_t m, int64_t n, int64_t k, float alpha, + const void * packedA, const void * packedB, + float * C, int64_t ldc, int ith, int nth) { + const apack_t * PA = (const apack_t *)packedA; + const bpack_t * PB = (const bpack_t *)packedB; + const int64_t kb = k/128, ns = n_slabs(k), mt = n_row_tiles(m), njt = n_col_tiles(n); + const int64_t tpt = (mt + nth - 1) / nth; + const int64_t t0 = ith*tpt, t1 = (ith+1)*tpt < mt ? (ith+1)*tpt : mt; + + vfl fin[NR][4]; + for (int64_t it = t0; it < t1; it++) { + const int64_t i = it*MR; + const int64_t rows = (m - i) < MR ? (m - i) : MR; + for (int64_t jt = 0; jt < njt; jt++) { + for (int j = 0; j < NR; j++) + for (int g = 0; g < 4; g++) fin[j][g] = vec_splats(0.0f); + for (int64_t s = 0; s < ns; s++) { + const int64_t blk0 = s*KC_BLKS; + const int64_t nblk = (kb - blk0) < KC_BLKS ? (kb - blk0) : KC_BLKS; + kernel_16x8(&PA[it*ns + s], &PB[jt*ns + s], nblk, alpha, fin); + } + const int64_t j0 = jt*NR; + const int64_t cols = (n - j0) < NR ? (n - j0) : NR; + for (int64_t cj = 0; cj < cols; cj++) { + float * dst = C + i + (j0 + cj)*ldc; + if (rows == MR) { + for (int g = 0; g < 4; g++) vec_xst(fin[cj][g], 16*g, dst); + } else { + for (int64_t r = 0; r < rows; r++) + dst[r] = fin[cj][r >> 2][r & 3]; + } + } + } + } +} + +// ---------------- GEMV path (n <= QBIT_GEMV_NMAX), raw weights ---------------- + +typedef struct { float dB; float S; } gemv_bmeta_t; // per chunk + +static void gemv_prep_b(const block_q8_0 * y, int64_t nch, gemv_bmeta_t * M) { + for (int64_t c = 0; c < nch; c++) { + M[c].dB = GGML_FP16_TO_FP32(y[c].d); + vsi z = vec_splats(0); + vsi s = vec_sum4s((vsc)load16u((const uint8_t *)(y[c].qs) + (0)), z); + s = vec_sum4s((vsc)load16u((const uint8_t *)(y[c].qs) + (16)), s); + M[c].S = (float)(s[0] + s[1] + s[2] + s[3]); + } +} + +static inline int hsum(vsi s) { return s[0] + s[1] + s[2] + s[3]; } + +static float gemv_row_q1(const block_q1_0 * a, const block_q8_0 * y, + const gemv_bmeta_t * M, int64_t kb) { + const vuc rep0 = { 0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1 }; + const vuc rep1 = { 2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3 }; + const vuc bitsel = { 1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128 }; + float sumf = 0.0f; + for (int64_t b = 0; b < kb; b++) { + const float dA = GGML_FP16_TO_FP32(a[b].d); + vuc raw = load16u((const uint8_t *)(a[b].qs) + (0)); + for (int c = 0; c < 4; c++) { + const vuc off = vec_splats((unsigned char)(4*c)); + vuc e0 = vec_perm(raw, raw, vec_add(rep0, off)); + vuc e1 = vec_perm(raw, raw, vec_add(rep1, off)); + vuc m0 = (vuc)vec_cmpeq(vec_and(e0, bitsel), bitsel); + vuc m1 = (vuc)vec_cmpeq(vec_and(e1, bitsel), bitsel); + const int8_t * q = y[4*b + c].qs; + vuc y0 = load16u((const uint8_t *)(q) + (0)); + vuc y1 = load16u((const uint8_t *)(q) + (16)); + vsi z = vec_splats(0); + vsi p = vec_sum4s((vsc)vec_and(y0, m0), z); + p = vec_sum4s((vsc)vec_and(y1, m1), p); + const gemv_bmeta_t * mm = &M[4*b + c]; + sumf += dA * mm->dB * (2.0f*(float)hsum(p) - mm->S); + } + } + return sumf; +} + +static float gemv_row_q2(const block_q2_0 * a, const block_q8_0 * y, + const gemv_bmeta_t * M, int64_t kb) { + const vuc rep0 = { 0,0,0,0, 1,1,1,1, 2,2,2,2, 3,3,3,3 }; + const vuc rep1 = { 4,4,4,4, 5,5,5,5, 6,6,6,6, 7,7,7,7 }; + const vuc sel0 = { 1,4,16,64, 1,4,16,64, 1,4,16,64, 1,4,16,64 }; + const vuc sel1 = { 2,8,32,128, 2,8,32,128, 2,8,32,128, 2,8,32,128 }; + float sumf = 0.0f; + for (int64_t b = 0; b < kb; b++) { + const float dA = GGML_FP16_TO_FP32(a[b].d); + vuc lo = load16u((const uint8_t *)(a[b].qs) + (0)); + vuc hi = load16u((const uint8_t *)(a[b].qs) + (16)); + for (int c = 0; c < 4; c++) { + const vuc off = vec_splats((unsigned char)(8*c)); + vuc e0 = vec_perm(lo, hi, vec_add(rep0, off)); + vuc e1 = vec_perm(lo, hi, vec_add(rep1, off)); + vuc ma0 = (vuc)vec_cmpeq(vec_and(e0, sel0), sel0); + vuc mb0 = (vuc)vec_cmpeq(vec_and(e0, sel1), sel1); + vuc ma1 = (vuc)vec_cmpeq(vec_and(e1, sel0), sel0); + vuc mb1 = (vuc)vec_cmpeq(vec_and(e1, sel1), sel1); + const int8_t * q = y[4*b + c].qs; + vuc y0 = load16u((const uint8_t *)(q) + (0)); + vuc y1 = load16u((const uint8_t *)(q) + (16)); + vsi z = vec_splats(0); + vsi p0 = vec_sum4s((vsc)vec_and(y0, ma0), z); + p0 = vec_sum4s((vsc)vec_and(y1, ma1), p0); + vsi p1 = vec_sum4s((vsc)vec_and(y0, mb0), z); + p1 = vec_sum4s((vsc)vec_and(y1, mb1), p1); + const gemv_bmeta_t * mm = &M[4*b + c]; + sumf += dA * mm->dB * ((float)hsum(p0) + 2.0f*(float)hsum(p1) - mm->S); + } + } + return sumf; +} + +extern "C" void qbit_gemv_q1(int64_t m, int64_t n, int64_t k, + const block_q1_0 * A, int64_t lda, const block_q8_0 * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth) { + const int64_t kb = k/128; + gemv_bmeta_t * M = (gemv_bmeta_t *)malloc(sizeof(gemv_bmeta_t)*4*kb); + const int64_t rpt = (m + nth - 1)/nth, i0 = ith*rpt, i1 = (ith+1)*rpt < m ? (ith+1)*rpt : m; + for (int64_t j = 0; j < n; j++) { + gemv_prep_b(B + j*ldb, 4*kb, M); + for (int64_t i = i0; i < i1; i++) + C[i + j*ldc] = gemv_row_q1(A + i*lda, B + j*ldb, M, kb); + } + free(M); +} + +extern "C" void qbit_gemv_q2(int64_t m, int64_t n, int64_t k, + const block_q2_0 * A, int64_t lda, const block_q8_0 * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth) { + const int64_t kb = k/128; + gemv_bmeta_t * M = (gemv_bmeta_t *)malloc(sizeof(gemv_bmeta_t)*4*kb); + const int64_t rpt = (m + nth - 1)/nth, i0 = ith*rpt, i1 = (ith+1)*rpt < m ? (ith+1)*rpt : m; + for (int64_t j = 0; j < n; j++) { + gemv_prep_b(B + j*ldb, 4*kb, M); + for (int64_t i = i0; i < i1; i++) + C[i + j*ldc] = gemv_row_q2(A + i*lda, B + j*ldb, M, kb); + } + free(M); +} + + +// dispatch entries (names preserved from the original v3 integration): +// n == 1 takes the GEMV path -- no packing, no cache, mask-select and +// vsum4s only; larger n uses the packed GEMM through the pack cache. +#define QBIT_DRIVER(NAME, BLKA, REPACK, GEMV, ALPHA, VARIANT) \ +extern "C" void NAME(int64_t m, int64_t n, int64_t k, \ + const void * Av, int64_t lda, const void * Bv, int64_t ldb, \ + float * C, int64_t ldc, int ith, int nth) { \ + const BLKA * A = (const BLKA *)Av; \ + const block_q8_0 * B = (const block_q8_0 *)Bv; \ + if (n == 1) { GEMV(m, n, k, A, lda, B, ldb, C, ldc, ith, nth); return; } \ + int fresh = 0; \ + void * PA = ppc_apack_cache_acquire(Av, m, k, VARIANT, \ + qbit_apack_size(m, k), &fresh); \ + if (PA) { \ + if (fresh) { REPACK(A, lda, m, k, PA); \ + ppc_apack_cache_publish(Av, m, k, VARIANT); } \ + const int64_t njt = (n + NR - 1) / NR; \ + if (njt < nth) { \ + /* n too small to feed every thread by columns (worst \ + case n == 1 generation: one column, nth-1 idle threads \ + while scalar row-partitions). Row-partition with the \ + cached pack instead; the full activation pack is tiny \ + at these n. Field regression, Q4_K tg32, 2026-07-21. */ \ + void * PBs = aligned_alloc(64, qbit_bpack_size(n, k)); \ + if (!PBs) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + qbit_pack_b(B, ldb, n, k, PBs); \ + qbit_gemm_packed(m, n, k, ALPHA, PA, PBs, C, ldc, ith, nth); \ + free(PBs); \ + } else { \ + const int64_t jpt = (njt + nth - 1) / nth; \ + const int64_t jt0 = (int64_t)ith*jpt; \ + const int64_t jt1 = (ith+1)*jpt < njt ? (ith+1)*jpt : njt; \ + if (jt0 < jt1) { \ + const int64_t j0 = jt0*NR; \ + const int64_t nc = (n - j0) < (jt1 - jt0)*NR ? (n - j0) \ + : (jt1 - jt0)*NR; \ + void * PBl = aligned_alloc(64, qbit_bpack_size(nc, k)); \ + if (!PBl) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + qbit_pack_b(B + j0*ldb, ldb, nc, k, PBl); \ + qbit_gemm_packed(m, nc, k, ALPHA, PA, PBl, C + j0*ldc, ldc, 0, 1); \ + free(PBl); \ + } \ + } \ + } else { \ + void * PB = aligned_alloc(64, qbit_bpack_size(n, k)); \ + if (!PB) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + qbit_pack_b(B, ldb, n, k, PB); \ + void * PT = aligned_alloc(64, qbit_apack_size(MR, k)); \ + if (!PT) { GGML_ABORT("ppc-mma: pack alloc failed"); } \ + const int64_t mt = (m + MR - 1) / MR; \ + const int64_t tpt = (mt + nth - 1) / nth; \ + for (int64_t it = ith*tpt; it < (ith+1)*tpt && it < mt; it++) { \ + const int64_t i = it*MR; \ + const int64_t rows = (m - i) < MR ? (m - i) : MR; \ + REPACK(A + i*lda, lda, rows, k, PT); \ + qbit_gemm_packed(rows, n, k, ALPHA, PT, PB, C + i, ldc, 0, 1); \ + } \ + free(PT); \ + free(PB); \ + } \ +} +QBIT_DRIVER(gemm_q1_0_q8_0_ppc_v3, block_q1_0, qbit_repack_q1, qbit_gemv_q1, 2.0f, 30) +QBIT_DRIVER(gemm_q2_0_q8_0_ppc_v3, block_q2_0, qbit_repack_q2, qbit_gemv_q2, 1.0f, 31) + +#endif // __MMA__ + diff --git a/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.h b/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.h new file mode 100644 index 000000000000..349e75520251 --- /dev/null +++ b/ggml/src/ggml-cpu/llamafile/qbit_ppc_mma.h @@ -0,0 +1,18 @@ +#pragma once +// POWER10/POWER11 MMA GEMM for PrismML Q1_0/Q2_0 weights x Q8_0 activations. +// k is in ELEMENTS (multiple of 128); lda in weight blocks, ldb in q8 blocks; +// C column-major float. A/B are block_q1_0|block_q2_0 / block_q8_0 arrays. +// From github.com/mavin2009/ppc-mma-kernels. +#include +#ifdef __cplusplus +extern "C" { +#endif +void gemm_q1_0_q8_0_ppc_v3(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +void gemm_q2_0_q8_0_ppc_v3(int64_t m, int64_t n, int64_t k, + const void * A, int64_t lda, const void * B, int64_t ldb, + float * C, int64_t ldc, int ith, int nth); +#ifdef __cplusplus +} +#endif diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp index e13828e3be6f..790cdbf94a4d 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp +++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp @@ -49,6 +49,8 @@ #endif #include "sgemm.h" +#include "qbit_ppc_mma.h" +#include "kquants_ppc_mma.h" #include "ggml-impl.h" #include "ggml-cpu-impl.h" #include "ggml-quants.h" @@ -3625,7 +3627,11 @@ class tinyBLAS_PPC { } else if constexpr(RM == 8 && RN == 8) { KERNEL_8x8(ii, jj); } else { - static_assert(false, "RN/RM values not supported"); + // dependent form: GCC < 13 evaluates a non-dependent + // static_assert(false) eagerly even in a discarded + // constexpr branch (pre-P2593); RM != RM defers it to + // instantiation, which never occurs for supported tiles. + static_assert(RM != RM, "RN/RM values not supported"); } } @@ -3950,12 +3956,341 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 return false; if (m < 8 && m != 4) return false; - tinyBLAS_Q0_PPC tb{ - k, (const block_q8_0 *)A, lda, - (const block_q8_0 *)B, ldb, - (float *)C, ldc, - params->ith, params->nth}; - tb.matmul(m, n); + // routed through the pack-cached MMA kernels (patch 0011); replaces + // tinyBLAS_Q0_PPC, whose per-call weight repack, scalar comparray + // fixup and absent GEMV path made it the slow producer here. + gemm_q8_0_q8_0_ppc(m, n, k * QK8_0, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_Q4_K: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) + gemm_q4_K_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_Q5_K: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) + gemm_q5_K_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_Q6_K: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) + gemm_q6_K_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_Q2_K: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) + gemm_q2_K_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_Q3_K: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) + gemm_q3_K_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_IQ4_XS: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) + gemm_iq4_xs_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_Q4_1: { + if (Btype != GGML_TYPE_Q8_1) + return false; +#if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) + gemm_q4_1_q8_1_ppc(m, n, k * QK4_1, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_Q5_1: { + if (Btype != GGML_TYPE_Q8_1) + return false; +#if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) + gemm_q5_1_q8_1_ppc(m, n, k * QK5_1, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_TQ2_0: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) + gemm_tq2_0_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_TQ1_0: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + // 0015 guard deliberately absent for this format: its ggml + // vec_dot is compute-starved on Power (2-3%% of the memory + // wall, VALIDATION-POWER10.md #9), so the cached packed path + // wins generation despite the int8 expansion (patch 0017). + gemm_tq1_0_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_IQ2_XXS: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + if (n > 1 && n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) + gemm_iq2_xxs_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_IQ3_XXS: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + if (n > 1 && n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) + gemm_iq3_xxs_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_IQ3_S: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + if (n > 1 && n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) + gemm_iq3_s_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_IQ1_S: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + if (n > 1 && n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) + gemm_iq1_s_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_MXFP4: { + if (Btype != GGML_TYPE_Q8_0) + return false; +#if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) + gemm_mxfp4_q8_0_ppc(m, n, k * QK_MXFP4, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_IQ2_XS: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) + gemm_iq2_xs_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_IQ2_S: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) + gemm_iq2_s_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_IQ1_M: { + if (Btype != GGML_TYPE_Q8_K) + return false; +#if defined(__MMA__) + // 0015 guard deliberately absent for this format: its ggml + // vec_dot is compute-starved on Power (2-3%% of the memory + // wall, VALIDATION-POWER10.md #9), so the cached packed path + // wins generation despite the int8 expansion (patch 0017). + gemm_iq1_m_q8_K_ppc(m, n, k * QK_K, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_NVFP4: { + if (Btype != GGML_TYPE_Q8_0) + return false; +#if defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) + gemm_nvfp4_q8_0_ppc(m, n, k * QK_NVFP4, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_Q1_0: { + if (Btype != GGML_TYPE_Q8_0) + return false; +#if defined(__MMA__) + gemm_q1_0_q8_0_ppc_v3(m, n, k * QK1_0, + A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; +#else + return false; +#endif + } + + case GGML_TYPE_Q2_0: { + if (Btype != GGML_TYPE_Q8_0) + return false; +#if defined(__MMA__) + gemm_q2_0_q8_0_ppc_v3(m, n, k * QK2_0, + A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); return true; #else return false; @@ -3987,12 +4322,11 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 return false; if (m < 8 && m != 4) return false; - tinyBLAS_Q0_PPC tb{ - k, (const block_q4_0 *)A, lda, - (const block_q8_0 *)B, ldb, - (float *)C, ldc, - params->ith, params->nth}; - tb.matmul(m, n); + // routed through the pack-cached MMA kernels (patch 0011); replaces + // tinyBLAS_Q0_PPC, whose per-call weight repack, scalar comparray + // fixup and absent GEMV path made it the slow producer here. + gemm_q4_0_q8_0_ppc(m, n, k * QK4_0, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); return true; #else return false; @@ -4010,6 +4344,14 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 params->ith, params->nth}; tb.matmul(m, n); return true; +#elif defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) + gemm_q5_0_q8_0_ppc(m, n, k * QK5_0, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; #else return false; #endif @@ -4026,6 +4368,14 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 params->ith, params->nth}; tb.matmul(m, n); return true; +#elif defined(__MMA__) + if (n < 8) + return false; // packed path reads int8-expanded weights; + // below one column tile vec_dot wins the + // bandwidth race on silicon (patch 0015) + gemm_iq4_nl_q8_0_ppc(m, n, k * QK4_NL, A, lda, B, ldb, + (float *)C, ldc, params->ith, params->nth); + return true; #else return false; #endif