From 69bcb240b5feaf6be1ccbee21c1c6cbd32788e4f Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Thu, 20 Aug 2026 13:11:07 +0000
Subject: [PATCH] Improve SVE2 optimizations of SimdGetColSums.

Replace unpack-and-add with SVE2 widening even/odd adds (svaddwb/svaddwt), keep that layout in the 16-bit buffer, and zip back to natural order only when converting to 32-bit. Unroll the inner loop across four vectors and apply the same approach to GetAbsDxColSums.

Co-authored-by: igor.ermolaev <igor.ermolaev@altezza.org>
---
 docs/2026.html               |   2 +
 src/Simd/SimdSve2ColSums.cpp | 310 +++++++++++++++++++++++++++--------
 2 files changed, 247 insertions(+), 65 deletions(-)

diff --git a/docs/2026.html b/docs/2026.html
index de0fbf7d99..5461f8d872 100644
--- a/docs/2026.html
+++ b/docs/2026.html
@@ -105,6 +105,8 @@ <h5>Improving</h5>
  <li>SVE2 optimizations of function SynetConvert32fTo8u.</li>
  <li>SVE2 optimizations of function SynetConvert8uTo32f.</li>
  <li>SVE2 optimizations of function WinogradKernel3x3Block2x2SetInput.</li>
+ <li>SVE2 optimizations of function GetColSums.</li>
+ <li>SVE2 optimizations of function GetAbsDxColSums.</li>
 </ul>
 <h5>Renaming</h5>
 <ul>
diff --git a/src/Simd/SimdSve2ColSums.cpp b/src/Simd/SimdSve2ColSums.cpp
index 8877ff443a..146a7b87e2 100644
--- a/src/Simd/SimdSve2ColSums.cpp
+++ b/src/Simd/SimdSve2ColSums.cpp
@@ -53,77 +53,168 @@ namespace Simd
             };
         }
 
-        SIMD_INLINE void GetAbsDxColSums(const uint8_t* src, const svbool_t& mask8, const svbool_t& lo16, const svbool_t& hi16, size_t half, uint16_t* sums)
+        SIMD_INLINE void AddColSum(const uint8_t* src, const svbool_t& mask, svuint16_t& even, svuint16_t& odd)
         {
-            svuint8_t diff = svabd_u8_x(mask8, svld1_u8(mask8, src), svld1_u8(mask8, src + 1));
-            svst1_u16(lo16, sums, svadd_u16_x(lo16, svld1_u16(lo16, sums), svunpklo_u16(diff)));
-            svst1_u16(hi16, sums + half, svadd_u16_x(hi16, svld1_u16(hi16, sums + half), svunpkhi_u16(diff)));
+            svuint8_t val = svld1_u8(mask, src);
+            even = svaddwb_u16(even, val);
+            odd = svaddwt_u16(odd, val);
         }
 
-        SIMD_INLINE void AddSums16To32(const uint16_t* src, uint32_t* dst, size_t col, size_t width, size_t half)
+        SIMD_INLINE void AddColSum4(const uint8_t* src, size_t A, const svbool_t& mask,
+            svuint16_t& e0, svuint16_t& o0, svuint16_t& e1, svuint16_t& o1,
+            svuint16_t& e2, svuint16_t& o2, svuint16_t& e3, svuint16_t& o3)
         {
-            svbool_t mask16 = svwhilelt_b16(col, width);
-            svbool_t lo32 = svwhilelt_b32(col, width);
-            svbool_t hi32 = svwhilelt_b32(col + half, width);
-            svuint16_t sums16 = svld1_u16(mask16, src + col);
-            svst1_u32(lo32, dst + col, svadd_u32_x(lo32, svld1_u32(lo32, dst + col), svunpklo_u32(sums16)));
-            svst1_u32(hi32, dst + col + half, svadd_u32_x(hi32, svld1_u32(hi32, dst + col + half), svunpkhi_u32(sums16)));
+            AddColSum(src + 0 * A, mask, e0, o0);
+            AddColSum(src + 1 * A, mask, e1, o1);
+            AddColSum(src + 2 * A, mask, e2, o2);
+            AddColSum(src + 3 * A, mask, e3, o3);
         }
 
-        //-------------------------------------------------------------------------------------------------
+        SIMD_INLINE void AddAbsDxColSum(const uint8_t* src, const svbool_t& mask, svuint16_t& even, svuint16_t& odd)
+        {
+            svuint8_t diff = svabd_u8_x(mask, svld1_u8(mask, src), svld1_u8(mask, src + 1));
+            even = svaddwb_u16(even, diff);
+            odd = svaddwt_u16(odd, diff);
+        }
+
+        SIMD_INLINE void AddAbsDxColSum4(const uint8_t* src, size_t A, const svbool_t& mask,
+            svuint16_t& e0, svuint16_t& o0, svuint16_t& e1, svuint16_t& o1,
+            svuint16_t& e2, svuint16_t& o2, svuint16_t& e3, svuint16_t& o3)
+        {
+            AddAbsDxColSum(src + 0 * A, mask, e0, o0);
+            AddAbsDxColSum(src + 1 * A, mask, e1, o1);
+            AddAbsDxColSum(src + 2 * A, mask, e2, o2);
+            AddAbsDxColSum(src + 3 * A, mask, e3, o3);
+        }
+
+        SIMD_INLINE void LoadEvenOdd(const uint16_t* dst, size_t half, const svbool_t& mask16, svuint16_t& even, svuint16_t& odd)
+        {
+            even = svld1_u16(mask16, dst);
+            odd = svld1_u16(mask16, dst + half);
+        }
 
-        SIMD_INLINE void AddColSum16(const svuint8_t& src, const svbool_t& lo16, const svbool_t& hi16, svuint16_t& lo, svuint16_t& hi)
+        SIMD_INLINE void StoreEvenOdd(uint16_t* dst, size_t half, const svbool_t& mask16, const svuint16_t& even, const svuint16_t& odd)
         {
-            lo = svadd_u16_x(lo16, lo, svunpklo_u16(src));
-            hi = svadd_u16_x(hi16, hi, svunpkhi_u16(src));
+            svst1_u16(mask16, dst, even);
+            svst1_u16(mask16, dst + half, odd);
         }
 
-        SIMD_INLINE void GetColSum16x1(const uint8_t* src, const svbool_t& mask8, const svbool_t& lo16, const svbool_t& hi16, size_t half, uint16_t* dst)
+        SIMD_INLINE void AddEvenOdd16To32(const uint16_t* src, uint32_t* dst, size_t half, size_t quarter)
         {
-            svuint16_t lo = svld1_u16(lo16, dst);
-            svuint16_t hi = svld1_u16(hi16, dst + half);
-            AddColSum16(svld1_u8(mask8, src), lo16, hi16, lo, hi);
-            svst1_u16(lo16, dst, lo);
-            svst1_u16(hi16, dst + half, hi);
+            const svbool_t mask16 = svptrue_b16();
+            const svbool_t mask32 = svptrue_b32();
+            svuint16_t even = svld1_u16(mask16, src);
+            svuint16_t odd = svld1_u16(mask16, src + half);
+            svuint16_t natLo = svzip1_u16(even, odd);
+            svuint16_t natHi = svzip2_u16(even, odd);
+            svst1_u32(mask32, dst + 0 * quarter, svadd_u32_x(mask32, svld1_u32(mask32, dst + 0 * quarter), svunpklo_u32(natLo)));
+            svst1_u32(mask32, dst + 1 * quarter, svadd_u32_x(mask32, svld1_u32(mask32, dst + 1 * quarter), svunpkhi_u32(natLo)));
+            svst1_u32(mask32, dst + 2 * quarter, svadd_u32_x(mask32, svld1_u32(mask32, dst + 2 * quarter), svunpklo_u32(natHi)));
+            svst1_u32(mask32, dst + 3 * quarter, svadd_u32_x(mask32, svld1_u32(mask32, dst + 3 * quarter), svunpkhi_u32(natHi)));
         }
 
-        SIMD_INLINE void GetColSum16x4(const uint8_t* src, size_t stride, const svbool_t& mask8, const svbool_t& lo16, const svbool_t& hi16, size_t half, uint16_t* dst)
+        SIMD_INLINE void GetColSum16x1(const uint8_t* src, const svbool_t& mask, size_t half, uint16_t* dst)
         {
-            svuint16_t lo = svld1_u16(lo16, dst);
-            svuint16_t hi = svld1_u16(hi16, dst + half);
-            AddColSum16(svld1_u8(mask8, src + 0 * stride), lo16, hi16, lo, hi);
-            AddColSum16(svld1_u8(mask8, src + 1 * stride), lo16, hi16, lo, hi);
-            AddColSum16(svld1_u8(mask8, src + 2 * stride), lo16, hi16, lo, hi);
-            AddColSum16(svld1_u8(mask8, src + 3 * stride), lo16, hi16, lo, hi);
-            svst1_u16(lo16, dst, lo);
-            svst1_u16(hi16, dst + half, hi);
+            const svbool_t mask16 = svptrue_b16();
+            svuint16_t even, odd;
+            LoadEvenOdd(dst, half, mask16, even, odd);
+            AddColSum(src, mask, even, odd);
+            StoreEvenOdd(dst, half, mask16, even, odd);
         }
 
-        SIMD_INLINE void GetColSum16x8(const uint8_t* src, size_t stride, const svbool_t& mask8, const svbool_t& lo16, const svbool_t& hi16, size_t half, uint16_t* dst)
+        SIMD_INLINE void GetColSum16x4(const uint8_t* src, size_t stride, const svbool_t& mask, size_t half, uint16_t* dst)
         {
-            svuint16_t lo = svld1_u16(lo16, dst);
-            svuint16_t hi = svld1_u16(hi16, dst + half);
-            AddColSum16(svld1_u8(mask8, src + 0 * stride), lo16, hi16, lo, hi);
-            AddColSum16(svld1_u8(mask8, src + 1 * stride), lo16, hi16, lo, hi);
-            AddColSum16(svld1_u8(mask8, src + 2 * stride), lo16, hi16, lo, hi);
-            AddColSum16(svld1_u8(mask8, src + 3 * stride), lo16, hi16, lo, hi);
-            AddColSum16(svld1_u8(mask8, src + 4 * stride), lo16, hi16, lo, hi);
-            AddColSum16(svld1_u8(mask8, src + 5 * stride), lo16, hi16, lo, hi);
-            AddColSum16(svld1_u8(mask8, src + 6 * stride), lo16, hi16, lo, hi);
-            AddColSum16(svld1_u8(mask8, src + 7 * stride), lo16, hi16, lo, hi);
-            svst1_u16(lo16, dst, lo);
-            svst1_u16(hi16, dst + half, hi);
+            const svbool_t mask16 = svptrue_b16();
+            svuint16_t even, odd;
+            LoadEvenOdd(dst, half, mask16, even, odd);
+            AddColSum(src + 0 * stride, mask, even, odd);
+            AddColSum(src + 1 * stride, mask, even, odd);
+            AddColSum(src + 2 * stride, mask, even, odd);
+            AddColSum(src + 3 * stride, mask, even, odd);
+            StoreEvenOdd(dst, half, mask16, even, odd);
+        }
+
+        SIMD_INLINE void GetColSum16x8(const uint8_t* src, size_t stride, const svbool_t& mask, size_t half, uint16_t* dst)
+        {
+            const svbool_t mask16 = svptrue_b16();
+            svuint16_t even, odd;
+            LoadEvenOdd(dst, half, mask16, even, odd);
+            AddColSum(src + 0 * stride, mask, even, odd);
+            AddColSum(src + 1 * stride, mask, even, odd);
+            AddColSum(src + 2 * stride, mask, even, odd);
+            AddColSum(src + 3 * stride, mask, even, odd);
+            AddColSum(src + 4 * stride, mask, even, odd);
+            AddColSum(src + 5 * stride, mask, even, odd);
+            AddColSum(src + 6 * stride, mask, even, odd);
+            AddColSum(src + 7 * stride, mask, even, odd);
+            StoreEvenOdd(dst, half, mask16, even, odd);
+        }
+
+        SIMD_INLINE void GetColSum16x8x4(const uint8_t* src, size_t stride, size_t A, const svbool_t& mask, size_t half, uint16_t* dst)
+        {
+            const svbool_t mask16 = svptrue_b16();
+            svuint16_t e0, o0, e1, o1, e2, o2, e3, o3;
+            LoadEvenOdd(dst + 0 * A, half, mask16, e0, o0);
+            LoadEvenOdd(dst + 1 * A, half, mask16, e1, o1);
+            LoadEvenOdd(dst + 2 * A, half, mask16, e2, o2);
+            LoadEvenOdd(dst + 3 * A, half, mask16, e3, o3);
+            AddColSum4(src + 0 * stride, A, mask, e0, o0, e1, o1, e2, o2, e3, o3);
+            AddColSum4(src + 1 * stride, A, mask, e0, o0, e1, o1, e2, o2, e3, o3);
+            AddColSum4(src + 2 * stride, A, mask, e0, o0, e1, o1, e2, o2, e3, o3);
+            AddColSum4(src + 3 * stride, A, mask, e0, o0, e1, o1, e2, o2, e3, o3);
+            AddColSum4(src + 4 * stride, A, mask, e0, o0, e1, o1, e2, o2, e3, o3);
+            AddColSum4(src + 5 * stride, A, mask, e0, o0, e1, o1, e2, o2, e3, o3);
+            AddColSum4(src + 6 * stride, A, mask, e0, o0, e1, o1, e2, o2, e3, o3);
+            AddColSum4(src + 7 * stride, A, mask, e0, o0, e1, o1, e2, o2, e3, o3);
+            StoreEvenOdd(dst + 0 * A, half, mask16, e0, o0);
+            StoreEvenOdd(dst + 1 * A, half, mask16, e1, o1);
+            StoreEvenOdd(dst + 2 * A, half, mask16, e2, o2);
+            StoreEvenOdd(dst + 3 * A, half, mask16, e3, o3);
+        }
+
+        SIMD_INLINE void GetColSum16x4x4(const uint8_t* src, size_t stride, size_t A, const svbool_t& mask, size_t half, uint16_t* dst)
+        {
+            const svbool_t mask16 = svptrue_b16();
+            svuint16_t e0, o0, e1, o1, e2, o2, e3, o3;
+            LoadEvenOdd(dst + 0 * A, half, mask16, e0, o0);
+            LoadEvenOdd(dst + 1 * A, half, mask16, e1, o1);
+            LoadEvenOdd(dst + 2 * A, half, mask16, e2, o2);
+            LoadEvenOdd(dst + 3 * A, half, mask16, e3, o3);
+            AddColSum4(src + 0 * stride, A, mask, e0, o0, e1, o1, e2, o2, e3, o3);
+            AddColSum4(src + 1 * stride, A, mask, e0, o0, e1, o1, e2, o2, e3, o3);
+            AddColSum4(src + 2 * stride, A, mask, e0, o0, e1, o1, e2, o2, e3, o3);
+            AddColSum4(src + 3 * stride, A, mask, e0, o0, e1, o1, e2, o2, e3, o3);
+            StoreEvenOdd(dst + 0 * A, half, mask16, e0, o0);
+            StoreEvenOdd(dst + 1 * A, half, mask16, e1, o1);
+            StoreEvenOdd(dst + 2 * A, half, mask16, e2, o2);
+            StoreEvenOdd(dst + 3 * A, half, mask16, e3, o3);
+        }
+
+        SIMD_INLINE void GetColSum16x1x4(const uint8_t* src, size_t A, const svbool_t& mask, size_t half, uint16_t* dst)
+        {
+            const svbool_t mask16 = svptrue_b16();
+            svuint16_t e0, o0, e1, o1, e2, o2, e3, o3;
+            LoadEvenOdd(dst + 0 * A, half, mask16, e0, o0);
+            LoadEvenOdd(dst + 1 * A, half, mask16, e1, o1);
+            LoadEvenOdd(dst + 2 * A, half, mask16, e2, o2);
+            LoadEvenOdd(dst + 3 * A, half, mask16, e3, o3);
+            AddColSum4(src, A, mask, e0, o0, e1, o1, e2, o2, e3, o3);
+            StoreEvenOdd(dst + 0 * A, half, mask16, e0, o0);
+            StoreEvenOdd(dst + 1 * A, half, mask16, e1, o1);
+            StoreEvenOdd(dst + 2 * A, half, mask16, e2, o2);
+            StoreEvenOdd(dst + 3 * A, half, mask16, e3, o3);
         }
 
         void GetColSums(const uint8_t* src, size_t stride, size_t width, size_t height, uint32_t* sums)
         {
             const size_t A = svcntb(), HA = svcnth(), F = svcntw();
+            const size_t A4 = A * 4;
             const size_t alignedLoWidth = AlignLo(width, A);
             const size_t alignedHiWidth = AlignHi(width, A);
+            const size_t alignedLoWidth4 = AlignLo(width, A4);
             const size_t stepSize = 256;
             const size_t stepCount = DivHi(height, stepSize);
             const svbool_t body8 = svptrue_b8();
-            const svbool_t body16 = svptrue_b16();
 
             Buffer buffer(alignedHiWidth);
             memset(buffer.sums32, 0, sizeof(uint32_t) * alignedHiWidth);
@@ -140,37 +231,102 @@ namespace Simd
                 for (; row < rowEnd8; row += 8)
                 {
                     size_t col = 0;
+                    for (; col < alignedLoWidth4; col += A4)
+                        GetColSum16x8x4(rowSrc + col, stride, A, body8, HA, buffer.sums16 + col);
                     for (; col < alignedLoWidth; col += A)
-                        GetColSum16x8(rowSrc + col, stride, body8, body16, body16, HA, buffer.sums16 + col);
+                        GetColSum16x8(rowSrc + col, stride, body8, HA, buffer.sums16 + col);
                     if (col < width)
-                        GetColSum16x8(rowSrc + col, stride, svwhilelt_b8(col, width), svwhilelt_b16(col, width), svwhilelt_b16(col + HA, width), HA, buffer.sums16 + col);
+                        GetColSum16x8(rowSrc + col, stride, svwhilelt_b8(col, width), HA, buffer.sums16 + col);
                     rowSrc += 8 * stride;
                 }
                 for (; row < rowEnd4; row += 4)
                 {
                     size_t col = 0;
+                    for (; col < alignedLoWidth4; col += A4)
+                        GetColSum16x4x4(rowSrc + col, stride, A, body8, HA, buffer.sums16 + col);
                     for (; col < alignedLoWidth; col += A)
-                        GetColSum16x4(rowSrc + col, stride, body8, body16, body16, HA, buffer.sums16 + col);
+                        GetColSum16x4(rowSrc + col, stride, body8, HA, buffer.sums16 + col);
                     if (col < width)
-                        GetColSum16x4(rowSrc + col, stride, svwhilelt_b8(col, width), svwhilelt_b16(col, width), svwhilelt_b16(col + HA, width), HA, buffer.sums16 + col);
+                        GetColSum16x4(rowSrc + col, stride, svwhilelt_b8(col, width), HA, buffer.sums16 + col);
                     rowSrc += 4 * stride;
                 }
                 for (; row < rowEnd; ++row)
                 {
                     size_t col = 0;
+                    for (; col < alignedLoWidth4; col += A4)
+                        GetColSum16x1x4(rowSrc + col, A, body8, HA, buffer.sums16 + col);
                     for (; col < alignedLoWidth; col += A)
-                        GetColSum16x1(rowSrc + col, body8, body16, body16, HA, buffer.sums16 + col);
+                        GetColSum16x1(rowSrc + col, body8, HA, buffer.sums16 + col);
                     if (col < width)
-                        GetColSum16x1(rowSrc + col, svwhilelt_b8(col, width), svwhilelt_b16(col, width), svwhilelt_b16(col + HA, width), HA, buffer.sums16 + col);
+                        GetColSum16x1(rowSrc + col, svwhilelt_b8(col, width), HA, buffer.sums16 + col);
                     rowSrc += stride;
                 }
 
-                for (size_t col = 0; col < alignedHiWidth; col += HA)
-                    AddSums16To32(buffer.sums16, buffer.sums32, col, alignedHiWidth, F);
+                for (size_t col = 0; col < alignedHiWidth; col += A)
+                    AddEvenOdd16To32(buffer.sums16 + col, buffer.sums32 + col, HA, F);
             }
             memcpy(sums, buffer.sums32, sizeof(uint32_t) * width);
         }
 
+        SIMD_INLINE void GetAbsDxColSum16x1(const uint8_t* src, const svbool_t& mask, size_t half, uint16_t* dst)
+        {
+            const svbool_t mask16 = svptrue_b16();
+            svuint16_t even, odd;
+            LoadEvenOdd(dst, half, mask16, even, odd);
+            AddAbsDxColSum(src, mask, even, odd);
+            StoreEvenOdd(dst, half, mask16, even, odd);
+        }
+
+        SIMD_INLINE void GetAbsDxColSum16x4(const uint8_t* src, size_t stride, const svbool_t& mask, size_t half, uint16_t* dst)
+        {
+            const svbool_t mask16 = svptrue_b16();
+            svuint16_t even, odd;
+            LoadEvenOdd(dst, half, mask16, even, odd);
+            AddAbsDxColSum(src + 0 * stride, mask, even, odd);
+            AddAbsDxColSum(src + 1 * stride, mask, even, odd);
+            AddAbsDxColSum(src + 2 * stride, mask, even, odd);
+            AddAbsDxColSum(src + 3 * stride, mask, even, odd);
+            StoreEvenOdd(dst, half, mask16, even, odd);
+        }
+
+        SIMD_INLINE void GetAbsDxColSum16x8(const uint8_t* src, size_t stride, const svbool_t& mask, size_t half, uint16_t* dst)
+        {
+            const svbool_t mask16 = svptrue_b16();
+            svuint16_t even, odd;
+            LoadEvenOdd(dst, half, mask16, even, odd);
+            AddAbsDxColSum(src + 0 * stride, mask, even, odd);
+            AddAbsDxColSum(src + 1 * stride, mask, even, odd);
+            AddAbsDxColSum(src + 2 * stride, mask, even, odd);
+            AddAbsDxColSum(src + 3 * stride, mask, even, odd);
+            AddAbsDxColSum(src + 4 * stride, mask, even, odd);
+            AddAbsDxColSum(src + 5 * stride, mask, even, odd);
+            AddAbsDxColSum(src + 6 * stride, mask, even, odd);
+            AddAbsDxColSum(src + 7 * stride, mask, even, odd);
+            StoreEvenOdd(dst, half, mask16, even, odd);
+        }
+
+        SIMD_INLINE void GetAbsDxColSum16x8x4(const uint8_t* src, size_t stride, size_t A, const svbool_t& mask, size_t half, uint16_t* dst)
+        {
+            const svbool_t mask16 = svptrue_b16();
+            svuint16_t e0, o0, e1, o1, e2, o2, e3, o3;
+            LoadEvenOdd(dst + 0 * A, half, mask16, e0, o0);
+            LoadEvenOdd(dst + 1 * A, half, mask16, e1, o1);
+            LoadEvenOdd(dst + 2 * A, half, mask16, e2, o2);
+            LoadEvenOdd(dst + 3 * A, half, mask16, e3, o3);
+            AddAbsDxColSum4(src + 0 * stride, A, mask, e0, o0, e1, o1, e2, o2, e3, o3);
+            AddAbsDxColSum4(src + 1 * stride, A, mask, e0, o0, e1, o1, e2, o2, e3, o3);
+            AddAbsDxColSum4(src + 2 * stride, A, mask, e0, o0, e1, o1, e2, o2, e3, o3);
+            AddAbsDxColSum4(src + 3 * stride, A, mask, e0, o0, e1, o1, e2, o2, e3, o3);
+            AddAbsDxColSum4(src + 4 * stride, A, mask, e0, o0, e1, o1, e2, o2, e3, o3);
+            AddAbsDxColSum4(src + 5 * stride, A, mask, e0, o0, e1, o1, e2, o2, e3, o3);
+            AddAbsDxColSum4(src + 6 * stride, A, mask, e0, o0, e1, o1, e2, o2, e3, o3);
+            AddAbsDxColSum4(src + 7 * stride, A, mask, e0, o0, e1, o1, e2, o2, e3, o3);
+            StoreEvenOdd(dst + 0 * A, half, mask16, e0, o0);
+            StoreEvenOdd(dst + 1 * A, half, mask16, e1, o1);
+            StoreEvenOdd(dst + 2 * A, half, mask16, e2, o2);
+            StoreEvenOdd(dst + 3 * A, half, mask16, e3, o3);
+        }
+
         void GetAbsDxColSums(const uint8_t* src, size_t stride, size_t width, size_t height, uint32_t* sums)
         {
             if (width < 2)
@@ -181,35 +337,59 @@ namespace Simd
             }
 
             width--;
-            const size_t A = svlen(svuint8_t()), HA = svlen(svuint16_t());
-            const size_t widthA = AlignLo(width, A);
-            const size_t widthB = AlignHi(width, A);
+            const size_t A = svcntb(), HA = svcnth(), F = svcntw();
+            const size_t A4 = A * 4;
+            const size_t alignedLoWidth = AlignLo(width, A);
+            const size_t alignedHiWidth = AlignHi(width, A);
+            const size_t alignedLoWidth4 = AlignLo(width, A4);
             const svbool_t body8 = svptrue_b8();
-            const svbool_t body16 = svptrue_b16();
             const size_t stepSize = SCHAR_MAX + 1;
             const size_t stepCount = (height + SCHAR_MAX) / stepSize;
 
-            Buffer buffer(widthB);
-            memset(buffer.sums32, 0, sizeof(uint32_t) * widthB);
+            Buffer buffer(alignedHiWidth);
+            memset(buffer.sums32, 0, sizeof(uint32_t) * alignedHiWidth);
             for (size_t step = 0; step < stepCount; ++step)
             {
                 const size_t rowStart = step * stepSize;
                 const size_t rowEnd = Min(rowStart + stepSize, height);
+                const size_t rowEnd4 = rowStart + AlignLo(rowEnd - rowStart, 4);
+                const size_t rowEnd8 = rowStart + AlignLo(rowEnd - rowStart, 8);
 
-                memset(buffer.sums16, 0, sizeof(uint16_t) * widthB);
+                memset(buffer.sums16, 0, sizeof(uint16_t) * alignedHiWidth);
                 const uint8_t* rowSrc = src + rowStart * stride;
-                for (size_t row = rowStart; row < rowEnd; ++row)
+                size_t row = rowStart;
+                for (; row < rowEnd8; row += 8)
+                {
+                    size_t col = 0;
+                    for (; col < alignedLoWidth4; col += A4)
+                        GetAbsDxColSum16x8x4(rowSrc + col, stride, A, body8, HA, buffer.sums16 + col);
+                    for (; col < alignedLoWidth; col += A)
+                        GetAbsDxColSum16x8(rowSrc + col, stride, body8, HA, buffer.sums16 + col);
+                    if (col < width)
+                        GetAbsDxColSum16x8(rowSrc + col, stride, svwhilelt_b8(col, width), HA, buffer.sums16 + col);
+                    rowSrc += 8 * stride;
+                }
+                for (; row < rowEnd4; row += 4)
+                {
+                    size_t col = 0;
+                    for (; col < alignedLoWidth; col += A)
+                        GetAbsDxColSum16x4(rowSrc + col, stride, body8, HA, buffer.sums16 + col);
+                    if (col < width)
+                        GetAbsDxColSum16x4(rowSrc + col, stride, svwhilelt_b8(col, width), HA, buffer.sums16 + col);
+                    rowSrc += 4 * stride;
+                }
+                for (; row < rowEnd; ++row)
                 {
                     size_t col = 0;
-                    for (; col < widthA; col += A)
-                        GetAbsDxColSums(rowSrc + col, body8, body16, body16, HA, buffer.sums16 + col);
+                    for (; col < alignedLoWidth; col += A)
+                        GetAbsDxColSum16x1(rowSrc + col, body8, HA, buffer.sums16 + col);
                     if (col < width)
-                        GetAbsDxColSums(rowSrc + col, svwhilelt_b8(col, width), svwhilelt_b16(col, width), svwhilelt_b16(col + HA, width), HA, buffer.sums16 + col);
+                        GetAbsDxColSum16x1(rowSrc + col, svwhilelt_b8(col, width), HA, buffer.sums16 + col);
                     rowSrc += stride;
                 }
 
-                for (size_t col = 0; col < width; col += svcntw() * 2)
-                    AddSums16To32(buffer.sums16, buffer.sums32, col, width, svcntw());
+                for (size_t col = 0; col < alignedHiWidth; col += A)
+                    AddEvenOdd16To32(buffer.sums16 + col, buffer.sums32 + col, HA, F);
             }
             memcpy(sums, buffer.sums32, sizeof(uint32_t) * width);
             sums[width] = 0;