From 30821e4c683646b8f41bba0dff256caec0d0ff37 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 20 Aug 2026 06:10:22 +0000 Subject: [PATCH] Improve SVE2 optimizations of class ResizerByteArea2x2. Rewrite the Area 2x2 row update to load full SVE vectors, pair same-channel neighbors with svtbl, and horizontally add with svaddlb instead of four small tbl gathers per output vector. Also vectorize the area result store. Record the change in release notes for 7.2.165. Co-authored-by: igor.ermolaev --- docs/2026.html | 1 + src/Simd/SimdSve2ResizerArea.cpp | 132 +++++++++++++++++++++++++------ 2 files changed, 107 insertions(+), 26 deletions(-) diff --git a/docs/2026.html b/docs/2026.html index c5514ae468..baeb3766b8 100644 --- a/docs/2026.html +++ b/docs/2026.html @@ -92,6 +92,7 @@
Improving
  • SVE2 optimizations of function ReduceGray3x3.
  • SVE2 optimizations of function ReduceGray4x4.
  • SVE2 optimizations of function ReduceGray5x5.
    • +
  • SVE2 optimizations of class ResizerByteArea2x2.
  • SVE2 optimizations of function SobelDx.
  • SVE2 optimizations of function SobelDxAbs.
  • SVE2 optimizations of function SobelDxAbsSum.
    • diff --git a/src/Simd/SimdSve2ResizerArea.cpp b/src/Simd/SimdSve2ResizerArea.cpp index 2e1a5e9276..ad26767f13 100644 --- a/src/Simd/SimdSve2ResizerArea.cpp +++ b/src/Simd/SimdSve2ResizerArea.cpp @@ -126,18 +126,74 @@ namespace Simd { } - template SIMD_INLINE svuint8_t ResizerByteArea2x2Index(size_t count) + SIMD_INLINE svuint8_t ResizerByteArea2x2ShuffleRc2() { - uint8_t index[SIMD_SVE2_VECTOR_SIZE_MAX] = { 0 }; - for (size_t i = 0; i < count; ++i) - index[i] = uint8_t(2 * N * (i / N) + i % N); + const svbool_t all = svptrue_b8(); + svuint8_t i = svindex_u8(0, 1); + svuint8_t base = svand_n_u8_x(all, i, 0xFC); + svuint8_t even = svlsl_n_u8_x(all, svand_n_u8_x(all, i, 1), 1); + svuint8_t odd = svlsr_n_u8_x(all, svand_n_u8_x(all, i, 2), 1); + return svadd_u8_x(all, base, svadd_u8_x(all, even, odd)); + } + + SIMD_INLINE svuint8_t ResizerByteArea2x2ShuffleRc4() + { + const svbool_t all = svptrue_b8(); + svuint8_t i = svindex_u8(0, 1); + svuint8_t base = svand_n_u8_x(all, i, 0xF8); + svuint8_t p = svand_n_u8_x(all, i, 7); + svuint8_t lo = svlsl_n_u8_x(all, svand_n_u8_x(all, p, 1), 2); + svuint8_t hi = svlsr_n_u8_x(all, p, 1); + return svadd_u8_x(all, base, svadd_u8_x(all, lo, hi)); + } + + SIMD_INLINE svuint8_t ResizerByteArea2x2ShuffleBgr() + { + uint8_t index[SIMD_SVE2_VECTOR_SIZE_MAX]; + const size_t A = svcntb(); + const size_t srcStep = AlignLoAny(A, size_t(6)); + for (size_t i = 0; i < A; ++i) + { + if (i < srcStep) + index[i] = uint8_t((i / 6) * 6 + ((i % 6) % 2) * 3 + (i % 6) / 2); + else + index[i] = 0; + } return svld1_u8(svptrue_b8(), index); } - SIMD_INLINE svuint32_t ResizerByteArea2x2Load(const uint8_t* src, const svuint8_t& index, size_t count) + template SIMD_INLINE svuint8_t ResizerByteArea2x2Shuffle() + { + if (N == 2) + return ResizerByteArea2x2ShuffleRc2(); + else if (N == 3) + return ResizerByteArea2x2ShuffleBgr(); + else if (N == 4) + return ResizerByteArea2x2ShuffleRc4(); + else + return svindex_u8(0, 1); + } + + template SIMD_INLINE svuint8_t ResizerByteArea2x2ShuffleSrc(const svuint8_t& src, const svuint8_t& index) + { + if (N == 1) + return src; + else + return svtbl_u8(src, index); + } + + template SIMD_INLINE size_t ResizerByteArea2x2SrcStep() + { + const size_t A = svcntb(); + return N == 3 ? AlignLoAny(A, size_t(6)) : A; + } + + SIMD_INLINE svuint16_t ResizerByteArea2x2PairSum(const svuint8_t& s0, const svuint8_t& s1) { - svuint8_t bytes = svld1_u8(svwhilelt_b8((size_t)0, 2 * count), src); - return svunpklo_u32(svunpklo_u16(svtbl_u8(bytes, index))); + const svbool_t mask = svptrue_b16(); + svuint16_t sum0 = svaddlb_u16(s0, svext_u8(s0, s0, 1)); + svuint16_t sum1 = svaddlb_u16(s1, svext_u8(s1, s1, 1)); + return svadd_u16_x(mask, sum0, sum1); } template SIMD_INLINE void ResizerByteArea2x2Update(const svbool_t& mask, int32_t* dst, const svint32_t& value) @@ -150,25 +206,33 @@ namespace Simd svst1_s32(mask, dst, svadd_s32_x(mask, svld1_s32(mask, dst), value)); } - template SIMD_INLINE void ResizerByteArea2x2RowUpdate(const uint8_t* src0, const uint8_t* src1, size_t size, int32_t val, int32_t* dst) + template SIMD_INLINE void ResizerByteArea2x2StoreSum(const svuint16_t& sum, const svint32_t& val, int32_t* dst, size_t dstN) + { + const size_t F = svcntw(); + svbool_t maskLo = svwhilelt_b32((size_t)0, dstN); + svbool_t maskHi = svwhilelt_b32(F, dstN); + ResizerByteArea2x2Update(maskLo, dst + 0, svmul_s32_x(maskLo, svreinterpret_s32_u32(svunpklo_u32(sum)), val)); + ResizerByteArea2x2Update(maskHi, dst + F, svmul_s32_x(maskHi, svreinterpret_s32_u32(svunpkhi_u32(sum)), val)); + } + + template SIMD_INLINE void ResizerByteArea2x2RowUpdate(const uint8_t* src0, const uint8_t* src1, size_t size, int32_t val, int32_t* dst, const svuint8_t& index) { if (update == UpdateAdd && val == 0) return; - size_t size2N = AlignLoAny(size, 2 * N); - size_t F = svcntw(), step = AlignLoAny(F, N), dstSize = size2N / 2; - svuint8_t index = ResizerByteArea2x2Index(step); + const size_t size2N = AlignLoAny(size, 2 * N); + const size_t dstSize = size2N / 2; + const size_t srcStep = ResizerByteArea2x2SrcStep(); + const size_t dstStep = srcStep / 2; svint32_t _val = svdup_n_s32(val); - size_t j = 0; - for (; j < dstSize; j += step) + size_t i = 0, j = 0; + for (; j < dstSize; i += srcStep, j += dstStep) { - size_t count = dstSize - j < step ? dstSize - j : step; - svbool_t mask = svwhilelt_b32((size_t)0, count); - const uint8_t* s0 = src0 + 2 * j; - const uint8_t* s1 = src1 + 2 * j; - svuint32_t sum = svadd_u32_x(mask, ResizerByteArea2x2Load(s0, index, count), ResizerByteArea2x2Load(s0 + N, index, count)); - sum = svadd_u32_x(mask, sum, ResizerByteArea2x2Load(s1, index, count)); - sum = svadd_u32_x(mask, sum, ResizerByteArea2x2Load(s1 + N, index, count)); - ResizerByteArea2x2Update(mask, dst + j, svmul_s32_x(mask, svreinterpret_s32_u32(sum), _val)); + size_t srcN = size2N - i < srcStep ? size2N - i : srcStep; + size_t dstN = dstSize - j < dstStep ? dstSize - j : dstStep; + svbool_t srcMask = svwhilelt_b8((size_t)0, srcN); + svuint8_t s0 = ResizerByteArea2x2ShuffleSrc(svld1_u8(srcMask, src0 + i), index); + svuint8_t s1 = ResizerByteArea2x2ShuffleSrc(svld1_u8(srcMask, src1 + i), index); + ResizerByteArea2x2StoreSum(ResizerByteArea2x2PairSum(s0, s1), _val, dst + j, dstN); } if (size2N < size) Base::ResizerByteArea2x2RowUpdate(src0 + size2N, src1 + size2N, val, dst + dstSize); @@ -176,16 +240,32 @@ namespace Simd template SIMD_INLINE void ResizerByteArea2x2RowSum(const uint8_t* src, size_t stride, size_t count, size_t size, int32_t curr, int32_t zero, int32_t next, bool tail, int32_t* dst) { + svuint8_t index = ResizerByteArea2x2Shuffle(); size_t c = 0; if (count) { - ResizerByteArea2x2RowUpdate(src, src + stride, size, curr, dst), src += 2 * stride, c += 2; + ResizerByteArea2x2RowUpdate(src, src + stride, size, curr, dst, index), src += 2 * stride, c += 2; for (; c < count; c += 2, src += 2 * stride) - ResizerByteArea2x2RowUpdate(src, src + stride, size, zero, dst); - ResizerByteArea2x2RowUpdate(src, tail ? src : src + stride, size, zero - next, dst); + ResizerByteArea2x2RowUpdate(src, src + stride, size, zero, dst, index); + ResizerByteArea2x2RowUpdate(src, tail ? src : src + stride, size, zero - next, dst, index); } else - ResizerByteArea2x2RowUpdate(src, tail ? src : src + stride, size, curr - next, dst); + ResizerByteArea2x2RowUpdate(src, tail ? src : src + stride, size, curr - next, dst, index); + } + + template SIMD_INLINE void ResizerByteAreaResult(const int32_t* src, size_t count, int32_t curr, int32_t zero, int32_t next, uint8_t* dst) + { + svbool_t mask = svwhilelt_b32((size_t)0, N); + svint32_t _zero = svdup_n_s32(zero); + svint32_t sum = svmul_s32_x(mask, svld1_s32(mask, src), svdup_n_s32(curr)); + for (size_t i = 0; i < count; ++i) + { + src += N; + sum = svmla_s32_x(mask, sum, svld1_s32(mask, src), _zero); + } + sum = svmla_s32_x(mask, sum, svld1_s32(mask, src), svdup_n_s32(-next)); + sum = svasr_n_s32_x(mask, svadd_n_s32_x(mask, sum, Base::AREA_ROUND), Base::AREA_SHIFT); + svst1b_u32(mask, dst, svreinterpret_u32_s32(sum)); } template void ResizerByteArea2x2::Run(const uint8_t* src, size_t srcStride, uint8_t* dst, size_t dstStride) @@ -202,7 +282,7 @@ namespace Simd for (size_t dx = 0; dx < dstW; dx++, dst += N) { size_t xn = ix[dx + 1] - ix[dx]; - Base::ResizerByteAreaResult(buf, xn, ax[dx], ax0, ax[dx + 1], dst), buf += xn * N; + ResizerByteAreaResult(buf, xn, ax[dx], ax0, ax[dx + 1], dst), buf += xn * N; } } }