diff --git a/docs/2026.html b/docs/2026.html index baeb3766b8..eff69f1b79 100644 --- a/docs/2026.html +++ b/docs/2026.html @@ -92,6 +92,7 @@
Improving
  • SVE2 optimizations of function ReduceGray3x3.
  • SVE2 optimizations of function ReduceGray4x4.
  • SVE2 optimizations of function ReduceGray5x5.
  • +
  • SVE2 optimizations of class ResizerByteArea1x1.
  • SVE2 optimizations of class ResizerByteArea2x2.
  • SVE2 optimizations of function SobelDx.
  • SVE2 optimizations of function SobelDxAbs.
  • diff --git a/src/Simd/SimdSve2ResizerArea.cpp b/src/Simd/SimdSve2ResizerArea.cpp index ad26767f13..73c6dcd1ad 100644 --- a/src/Simd/SimdSve2ResizerArea.cpp +++ b/src/Simd/SimdSve2ResizerArea.cpp @@ -36,41 +36,101 @@ namespace Simd { } - template SIMD_INLINE void ResizerByteArea1x1Update(const svbool_t& mask, int32_t* dst, const svint32_t& value) + template SIMD_INLINE void ResizerByteAreaUpdate(const svbool_t& mask, int32_t* dst, const svint32_t& value) { svst1_s32(mask, dst, value); } - template<> SIMD_INLINE void ResizerByteArea1x1Update(const svbool_t& mask, int32_t* dst, const svint32_t& value) + template<> SIMD_INLINE void ResizerByteAreaUpdate(const svbool_t& mask, int32_t* dst, const svint32_t& value) { svst1_s32(mask, dst, svadd_s32_x(mask, svld1_s32(mask, dst), value)); } + SIMD_INLINE void ResizerByteAreaUnpackU8(const svuint8_t& src, svint32_t& d0, svint32_t& d1, svint32_t& d2, svint32_t& d3) + { + svuint16_t lo = svunpklo_u16(src); + svuint16_t hi = svunpkhi_u16(src); + d0 = svreinterpret_s32_u32(svunpklo_u32(lo)); + d1 = svreinterpret_s32_u32(svunpkhi_u32(lo)); + d2 = svreinterpret_s32_u32(svunpklo_u32(hi)); + d3 = svreinterpret_s32_u32(svunpkhi_u32(hi)); + } + + template SIMD_INLINE void ResizerByteArea1x1Store(int32_t* dst, const svint32_t& v0, const svint32_t& v1, + const svint32_t& v2, const svint32_t& v3, size_t count) + { + const size_t F = svcntw(); + ResizerByteAreaUpdate(svwhilelt_b32((size_t)0, count), dst + 0 * F, v0); + ResizerByteAreaUpdate(svwhilelt_b32(F, count), dst + 1 * F, v1); + ResizerByteAreaUpdate(svwhilelt_b32(2 * F, count), dst + 2 * F, v2); + ResizerByteAreaUpdate(svwhilelt_b32(3 * F, count), dst + 3 * F, v3); + } + + template SIMD_INLINE void ResizerByteArea1x1Block(const uint8_t* src0, int32_t* dst, const svint32_t& a0, + const svbool_t& mask8, const svbool_t& mask32, size_t count) + { + svint32_t d0, d1, d2, d3; + ResizerByteAreaUnpackU8(svld1_u8(mask8, src0), d0, d1, d2, d3); + ResizerByteArea1x1Store(dst, + svmul_s32_x(mask32, d0, a0), + svmul_s32_x(mask32, d1, a0), + svmul_s32_x(mask32, d2, a0), + svmul_s32_x(mask32, d3, a0), + count); + } + + template SIMD_INLINE void ResizerByteArea1x1Block(const uint8_t* src0, const uint8_t* src1, int32_t* dst, + const svint32_t& a0, const svint32_t& a1, const svbool_t& mask8, const svbool_t& mask32, size_t count) + { + svint32_t s00, s01, s02, s03, s10, s11, s12, s13; + ResizerByteAreaUnpackU8(svld1_u8(mask8, src0), s00, s01, s02, s03); + ResizerByteAreaUnpackU8(svld1_u8(mask8, src1), s10, s11, s12, s13); + ResizerByteArea1x1Store(dst, + svmla_s32_x(mask32, svmul_s32_x(mask32, s00, a0), s10, a1), + svmla_s32_x(mask32, svmul_s32_x(mask32, s01, a0), s11, a1), + svmla_s32_x(mask32, svmul_s32_x(mask32, s02, a0), s12, a1), + svmla_s32_x(mask32, svmul_s32_x(mask32, s03, a0), s13, a1), + count); + } + template SIMD_INLINE void ResizerByteArea1x1RowUpdate(const uint8_t* src0, size_t size, int32_t a0, int32_t* dst) { + if (update == UpdateAdd && a0 == 0) + return; + const size_t A = svcntb(); + const svbool_t mask8 = svptrue_b8(); + const svbool_t mask32 = svptrue_b32(); svint32_t _a0 = svdup_n_s32(a0); - size_t F = svcntw(); - for (size_t i = 0; i < size; i += F) + size_t i = 0; + for (; i + 2 * A <= size; i += 2 * A, dst += 2 * A) { - svbool_t mask = svwhilelt_b32(i, size); - svint32_t sum = svmul_s32_x(mask, svld1ub_s32(mask, src0 + i), _a0); - ResizerByteArea1x1Update(mask, dst + i, sum); + ResizerByteArea1x1Block(src0 + i, dst, _a0, mask8, mask32, A); + ResizerByteArea1x1Block(src0 + i + A, dst + A, _a0, mask8, mask32, A); } + for (; i + A <= size; i += A, dst += A) + ResizerByteArea1x1Block(src0 + i, dst, _a0, mask8, mask32, A); + if (i < size) + ResizerByteArea1x1Block(src0 + i, dst, _a0, svwhilelt_b8((size_t)0, size - i), mask32, size - i); } template SIMD_INLINE void ResizerByteArea1x1RowUpdate(const uint8_t* src0, size_t stride, size_t size, int32_t a0, int32_t a1, int32_t* dst) { + const size_t A = svcntb(); + const svbool_t mask8 = svptrue_b8(); + const svbool_t mask32 = svptrue_b32(); svint32_t _a0 = svdup_n_s32(a0); svint32_t _a1 = svdup_n_s32(a1); const uint8_t* src1 = src0 + stride; - size_t F = svcntw(); - for (size_t i = 0; i < size; i += F) + size_t i = 0; + for (; i + 2 * A <= size; i += 2 * A, dst += 2 * A) { - svbool_t mask = svwhilelt_b32(i, size); - svint32_t sum = svmul_s32_x(mask, svld1ub_s32(mask, src0 + i), _a0); - sum = svmla_s32_x(mask, sum, svld1ub_s32(mask, src1 + i), _a1); - ResizerByteArea1x1Update(mask, dst + i, sum); + ResizerByteArea1x1Block(src0 + i, src1 + i, dst, _a0, _a1, mask8, mask32, A); + ResizerByteArea1x1Block(src0 + i + A, src1 + i + A, dst + A, _a0, _a1, mask8, mask32, A); } + for (; i + A <= size; i += A, dst += A) + ResizerByteArea1x1Block(src0 + i, src1 + i, dst, _a0, _a1, mask8, mask32, A); + if (i < size) + ResizerByteArea1x1Block(src0 + i, src1 + i, dst, _a0, _a1, svwhilelt_b8((size_t)0, size - i), mask32, size - i); } SIMD_INLINE void ResizerByteArea1x1RowSum(const uint8_t* src, size_t stride, size_t count, size_t size, int32_t curr, int32_t zero, int32_t next, int32_t* dst) @@ -88,6 +148,21 @@ namespace Simd ResizerByteArea1x1RowUpdate(src, size, curr - next, dst); } + template SIMD_INLINE void ResizerByteAreaResult(const int32_t* src, size_t count, int32_t curr, int32_t zero, int32_t next, uint8_t* dst) + { + svbool_t mask = svwhilelt_b32((size_t)0, N); + svint32_t _zero = svdup_n_s32(zero); + svint32_t sum = svmul_s32_x(mask, svld1_s32(mask, src), svdup_n_s32(curr)); + for (size_t i = 0; i < count; ++i) + { + src += N; + sum = svmla_s32_x(mask, sum, svld1_s32(mask, src), _zero); + } + sum = svmla_s32_x(mask, sum, svld1_s32(mask, src), svdup_n_s32(-next)); + sum = svasr_n_s32_x(mask, svadd_n_s32_x(mask, sum, Base::AREA_ROUND), Base::AREA_SHIFT); + svst1b_u32(mask, dst, svreinterpret_u32_s32(sum)); + } + template void ResizerByteArea1x1::Run(const uint8_t* src, size_t srcStride, uint8_t* dst, size_t dstStride) { size_t dstW = _param.dstW, rowSize = _param.srcW * N, rowRest = dstStride - dstW * N; @@ -101,7 +176,7 @@ namespace Simd for (size_t dx = 0; dx < dstW; dx++, dst += N) { size_t xn = ix[dx + 1] - ix[dx]; - Base::ResizerByteAreaResult(buf, xn, ax[dx], ax0, ax[dx + 1], dst), buf += xn * N; + ResizerByteAreaResult(buf, xn, ax[dx], ax0, ax[dx + 1], dst), buf += xn * N; } } } @@ -126,146 +201,164 @@ namespace Simd { } - SIMD_INLINE svuint8_t ResizerByteArea2x2ShuffleRc2() - { - const svbool_t all = svptrue_b8(); - svuint8_t i = svindex_u8(0, 1); - svuint8_t base = svand_n_u8_x(all, i, 0xFC); - svuint8_t even = svlsl_n_u8_x(all, svand_n_u8_x(all, i, 1), 1); - svuint8_t odd = svlsr_n_u8_x(all, svand_n_u8_x(all, i, 2), 1); - return svadd_u8_x(all, base, svadd_u8_x(all, even, odd)); - } - - SIMD_INLINE svuint8_t ResizerByteArea2x2ShuffleRc4() - { - const svbool_t all = svptrue_b8(); - svuint8_t i = svindex_u8(0, 1); - svuint8_t base = svand_n_u8_x(all, i, 0xF8); - svuint8_t p = svand_n_u8_x(all, i, 7); - svuint8_t lo = svlsl_n_u8_x(all, svand_n_u8_x(all, p, 1), 2); - svuint8_t hi = svlsr_n_u8_x(all, p, 1); - return svadd_u8_x(all, base, svadd_u8_x(all, lo, hi)); - } - - SIMD_INLINE svuint8_t ResizerByteArea2x2ShuffleBgr() + SIMD_INLINE bool InitResizerByteArea2x2Index(uint8_t index[4][SIMD_SVE2_VECTOR_SIZE_MAX]) { - uint8_t index[SIMD_SVE2_VECTOR_SIZE_MAX]; const size_t A = svcntb(); - const size_t srcStep = AlignLoAny(A, size_t(6)); + assert(A <= SIMD_SVE2_VECTOR_SIZE_MAX); for (size_t i = 0; i < A; ++i) { + index[0][i] = uint8_t((i & 0xFC) + ((i & 1) << 1) + ((i & 2) >> 1)); + size_t p = i & 7; + index[1][i] = uint8_t((i & 0xF8) + ((p & 1) << 2) + (p >> 1)); + } + const size_t srcStep = AlignLoAny(2 * A, size_t(6)); + for (size_t i = 0; i < 2 * A; ++i) + { + uint8_t value = 0; if (i < srcStep) - index[i] = uint8_t((i / 6) * 6 + ((i % 6) % 2) * 3 + (i % 6) / 2); + value = uint8_t((i / 6) * 6 + ((i % 6) % 2) * 3 + (i % 6) / 2); + if (i < A) + index[2][i] = value; else - index[i] = 0; + index[3][i - A] = value; } - return svld1_u8(svptrue_b8(), index); + return true; } - template SIMD_INLINE svuint8_t ResizerByteArea2x2Shuffle() - { - if (N == 2) - return ResizerByteArea2x2ShuffleRc2(); - else if (N == 3) - return ResizerByteArea2x2ShuffleBgr(); - else if (N == 4) - return ResizerByteArea2x2ShuffleRc4(); - else - return svindex_u8(0, 1); - } + SIMD_ALIGNED(SIMD_ALIGN) uint8_t RESIZER_BYTE_AREA_2X2_INDEX[4][SIMD_SVE2_VECTOR_SIZE_MAX]; + const bool RESIZER_BYTE_AREA_2X2_INDEX_INITED = InitResizerByteArea2x2Index(RESIZER_BYTE_AREA_2X2_INDEX); - template SIMD_INLINE svuint8_t ResizerByteArea2x2ShuffleSrc(const svuint8_t& src, const svuint8_t& index) + SIMD_INLINE svuint8_t ResizerByteArea2x2ShuffleRc2() { - if (N == 1) - return src; - else - return svtbl_u8(src, index); + return svld1_u8(svptrue_b8(), RESIZER_BYTE_AREA_2X2_INDEX[0]); } - template SIMD_INLINE size_t ResizerByteArea2x2SrcStep() + SIMD_INLINE svuint8_t ResizerByteArea2x2ShuffleRc4() { - const size_t A = svcntb(); - return N == 3 ? AlignLoAny(A, size_t(6)) : A; + return svld1_u8(svptrue_b8(), RESIZER_BYTE_AREA_2X2_INDEX[1]); } SIMD_INLINE svuint16_t ResizerByteArea2x2PairSum(const svuint8_t& s0, const svuint8_t& s1) { const svbool_t mask = svptrue_b16(); - svuint16_t sum0 = svaddlb_u16(s0, svext_u8(s0, s0, 1)); - svuint16_t sum1 = svaddlb_u16(s1, svext_u8(s1, s1, 1)); - return svadd_u16_x(mask, sum0, sum1); + svuint16_t sum = svadalp_u16_x(mask, svdup_n_u16(0), s0); + return svadalp_u16_x(mask, sum, s1); } - template SIMD_INLINE void ResizerByteArea2x2Update(const svbool_t& mask, int32_t* dst, const svint32_t& value) + template SIMD_INLINE void ResizerByteArea2x2StoreSum(const svuint16_t& sum, const svint32_t& val, int32_t* dst, size_t dstN) { - svst1_s32(mask, dst, value); + const size_t F = svcntw(); + svbool_t maskLo = svwhilelt_b32((size_t)0, dstN); + svbool_t maskHi = svwhilelt_b32(F, dstN); + ResizerByteAreaUpdate(maskLo, dst + 0, svmul_s32_x(maskLo, svreinterpret_s32_u32(svunpklo_u32(sum)), val)); + ResizerByteAreaUpdate(maskHi, dst + F, svmul_s32_x(maskHi, svreinterpret_s32_u32(svunpkhi_u32(sum)), val)); } - template<> SIMD_INLINE void ResizerByteArea2x2Update(const svbool_t& mask, int32_t* dst, const svint32_t& value) + template SIMD_INLINE svuint8_t ResizerByteArea2x2LoadColor(const uint8_t* src, const svuint8_t& index, const svbool_t& mask) { - svst1_s32(mask, dst, svadd_s32_x(mask, svld1_s32(mask, dst), value)); + svuint8_t value = svld1_u8(mask, src); + if (N == 1) + return value; + else + return svtbl_u8(value, index); } - template SIMD_INLINE void ResizerByteArea2x2StoreSum(const svuint16_t& sum, const svint32_t& val, int32_t* dst, size_t dstN) + template SIMD_INLINE void ResizerByteArea2x2RowUpdateColor(const uint8_t* src0, const uint8_t* src1, + size_t size2N, size_t dstSize, int32_t* dst, const svint32_t& val, const svuint8_t& index) { - const size_t F = svcntw(); - svbool_t maskLo = svwhilelt_b32((size_t)0, dstN); - svbool_t maskHi = svwhilelt_b32(F, dstN); - ResizerByteArea2x2Update(maskLo, dst + 0, svmul_s32_x(maskLo, svreinterpret_s32_u32(svunpklo_u32(sum)), val)); - ResizerByteArea2x2Update(maskHi, dst + F, svmul_s32_x(maskHi, svreinterpret_s32_u32(svunpkhi_u32(sum)), val)); + const size_t A = svcntb(); + const size_t HA = A / 2; + const svbool_t mask8 = svptrue_b8(); + size_t i = 0, j = 0; + for (; i + 2 * A <= size2N; i += 2 * A, j += A) + { + svuint8_t s00 = ResizerByteArea2x2LoadColor(src0 + i, index, mask8); + svuint8_t s10 = ResizerByteArea2x2LoadColor(src1 + i, index, mask8); + svuint8_t s01 = ResizerByteArea2x2LoadColor(src0 + i + A, index, mask8); + svuint8_t s11 = ResizerByteArea2x2LoadColor(src1 + i + A, index, mask8); + ResizerByteArea2x2StoreSum(ResizerByteArea2x2PairSum(s00, s10), val, dst + j, HA); + ResizerByteArea2x2StoreSum(ResizerByteArea2x2PairSum(s01, s11), val, dst + j + HA, HA); + } + for (; i + A <= size2N; i += A, j += HA) + { + svuint8_t s0 = ResizerByteArea2x2LoadColor(src0 + i, index, mask8); + svuint8_t s1 = ResizerByteArea2x2LoadColor(src1 + i, index, mask8); + ResizerByteArea2x2StoreSum(ResizerByteArea2x2PairSum(s0, s1), val, dst + j, HA); + } + if (i < size2N) + { + size_t srcN = size2N - i; + svbool_t srcMask = svwhilelt_b8((size_t)0, srcN); + svuint8_t s0 = ResizerByteArea2x2LoadColor(src0 + i, index, srcMask); + svuint8_t s1 = ResizerByteArea2x2LoadColor(src1 + i, index, srcMask); + ResizerByteArea2x2StoreSum(ResizerByteArea2x2PairSum(s0, s1), val, dst + j, dstSize - j); + } + } + + template SIMD_INLINE void ResizerByteArea2x2RowUpdateBgr(const uint8_t* src0, const uint8_t* src1, + size_t size2N, size_t dstSize, int32_t* dst, const svint32_t& val) + { + const size_t A = svcntb(); + const size_t HA = A / 2; + const size_t srcStep = AlignLoAny(2 * A, size_t(6)); + const size_t dstStep = srcStep / 2; + const svbool_t mask8 = svptrue_b8(); + svuint8_t index0 = svld1_u8(mask8, RESIZER_BYTE_AREA_2X2_INDEX[2]); + svuint8_t index1 = svld1_u8(mask8, RESIZER_BYTE_AREA_2X2_INDEX[3]); + size_t i = 0, j = 0; + for (; i + srcStep <= size2N; i += srcStep, j += dstStep) + { + svbool_t mask1 = svwhilelt_b8((size_t)0, srcStep - A); + svuint8x2_t t0 = svcreate2_u8(svld1_u8(mask8, src0 + i), svld1_u8(mask1, src0 + i + A)); + svuint8x2_t t1 = svcreate2_u8(svld1_u8(mask8, src1 + i), svld1_u8(mask1, src1 + i + A)); + ResizerByteArea2x2StoreSum(ResizerByteArea2x2PairSum(svtbl2_u8(t0, index0), svtbl2_u8(t1, index0)), val, dst + j, HA); + ResizerByteArea2x2StoreSum(ResizerByteArea2x2PairSum(svtbl2_u8(t0, index1), svtbl2_u8(t1, index1)), val, dst + j + HA, dstStep - HA); + } + if (i < size2N) + { + size_t srcN = size2N - i; + size_t dstN = dstSize - j; + svbool_t mask0 = svwhilelt_b8((size_t)0, srcN); + svbool_t mask1 = svwhilelt_b8((size_t)0, srcN > A ? srcN - A : 0); + svuint8x2_t t0 = svcreate2_u8(svld1_u8(mask0, src0 + i), svld1_u8(mask1, src0 + i + A)); + svuint8x2_t t1 = svcreate2_u8(svld1_u8(mask0, src1 + i), svld1_u8(mask1, src1 + i + A)); + size_t dst0 = dstN < HA ? dstN : HA; + ResizerByteArea2x2StoreSum(ResizerByteArea2x2PairSum(svtbl2_u8(t0, index0), svtbl2_u8(t1, index0)), val, dst + j, dst0); + ResizerByteArea2x2StoreSum(ResizerByteArea2x2PairSum(svtbl2_u8(t0, index1), svtbl2_u8(t1, index1)), val, dst + j + HA, dstN - dst0); + } } - template SIMD_INLINE void ResizerByteArea2x2RowUpdate(const uint8_t* src0, const uint8_t* src1, size_t size, int32_t val, int32_t* dst, const svuint8_t& index) + template SIMD_INLINE void ResizerByteArea2x2RowUpdate(const uint8_t* src0, const uint8_t* src1, size_t size, int32_t val, int32_t* dst) { if (update == UpdateAdd && val == 0) return; const size_t size2N = AlignLoAny(size, 2 * N); const size_t dstSize = size2N / 2; - const size_t srcStep = ResizerByteArea2x2SrcStep(); - const size_t dstStep = srcStep / 2; svint32_t _val = svdup_n_s32(val); - size_t i = 0, j = 0; - for (; j < dstSize; i += srcStep, j += dstStep) - { - size_t srcN = size2N - i < srcStep ? size2N - i : srcStep; - size_t dstN = dstSize - j < dstStep ? dstSize - j : dstStep; - svbool_t srcMask = svwhilelt_b8((size_t)0, srcN); - svuint8_t s0 = ResizerByteArea2x2ShuffleSrc(svld1_u8(srcMask, src0 + i), index); - svuint8_t s1 = ResizerByteArea2x2ShuffleSrc(svld1_u8(srcMask, src1 + i), index); - ResizerByteArea2x2StoreSum(ResizerByteArea2x2PairSum(s0, s1), _val, dst + j, dstN); - } + if (N == 3) + ResizerByteArea2x2RowUpdateBgr(src0, src1, size2N, dstSize, dst, _val); + else if (N == 4) + ResizerByteArea2x2RowUpdateColor(src0, src1, size2N, dstSize, dst, _val, ResizerByteArea2x2ShuffleRc4()); + else if (N == 2) + ResizerByteArea2x2RowUpdateColor(src0, src1, size2N, dstSize, dst, _val, ResizerByteArea2x2ShuffleRc2()); + else + ResizerByteArea2x2RowUpdateColor(src0, src1, size2N, dstSize, dst, _val, svindex_u8(0, 1)); if (size2N < size) Base::ResizerByteArea2x2RowUpdate(src0 + size2N, src1 + size2N, val, dst + dstSize); } template SIMD_INLINE void ResizerByteArea2x2RowSum(const uint8_t* src, size_t stride, size_t count, size_t size, int32_t curr, int32_t zero, int32_t next, bool tail, int32_t* dst) { - svuint8_t index = ResizerByteArea2x2Shuffle(); size_t c = 0; if (count) { - ResizerByteArea2x2RowUpdate(src, src + stride, size, curr, dst, index), src += 2 * stride, c += 2; + ResizerByteArea2x2RowUpdate(src, src + stride, size, curr, dst), src += 2 * stride, c += 2; for (; c < count; c += 2, src += 2 * stride) - ResizerByteArea2x2RowUpdate(src, src + stride, size, zero, dst, index); - ResizerByteArea2x2RowUpdate(src, tail ? src : src + stride, size, zero - next, dst, index); + ResizerByteArea2x2RowUpdate(src, src + stride, size, zero, dst); + ResizerByteArea2x2RowUpdate(src, tail ? src : src + stride, size, zero - next, dst); } else - ResizerByteArea2x2RowUpdate(src, tail ? src : src + stride, size, curr - next, dst, index); - } - - template SIMD_INLINE void ResizerByteAreaResult(const int32_t* src, size_t count, int32_t curr, int32_t zero, int32_t next, uint8_t* dst) - { - svbool_t mask = svwhilelt_b32((size_t)0, N); - svint32_t _zero = svdup_n_s32(zero); - svint32_t sum = svmul_s32_x(mask, svld1_s32(mask, src), svdup_n_s32(curr)); - for (size_t i = 0; i < count; ++i) - { - src += N; - sum = svmla_s32_x(mask, sum, svld1_s32(mask, src), _zero); - } - sum = svmla_s32_x(mask, sum, svld1_s32(mask, src), svdup_n_s32(-next)); - sum = svasr_n_s32_x(mask, svadd_n_s32_x(mask, sum, Base::AREA_ROUND), Base::AREA_SHIFT); - svst1b_u32(mask, dst, svreinterpret_u32_s32(sum)); + ResizerByteArea2x2RowUpdate(src, tail ? src : src + stride, size, curr - next, dst); } template void ResizerByteArea2x2::Run(const uint8_t* src, size_t srcStride, uint8_t* dst, size_t dstStride) @@ -302,4 +395,3 @@ namespace Simd } #endif } -