[gaynor-anthropic]

Restructures the double-fast extDict loop the way the noDict loop already is: the ip+1 long-table lookup is issued early, candidate checks use cmov instead of branching on index validity, and repcodes that cannot be valid in the block are disabled up front. This path dominates level-3 compression when a small-window stream is flushed every few KB, since nearly every block is then compressed in extDict mode.

The first three commits are preparatory refactors: sharing the cmov/branch candidate-check helpers between fast and double-fast (with an aarch64 csel so GCC keeps them branchless), and force-inlined ZSTD_count variants so GCC does not stop inlining under distro hardening flags.

Results

Measured against dev, change in throughput.

Benchmark	Sapphire Rapids (GCC 13)	Neoverse-V2 (GCC 15)
streaming, 32 KB window, 4 KB flushes, silesia.tar	+24%	+22%
streaming, 32 KB window, small flushed messages	+7% to +15%	+10% to +11%
fullbench -b5 (extDict), 1 MB silesia chunks	+12% to +40%	not measured
one-shot zstd -b3, silesia.tar / enwik8	+1% / +2%	up to +3%
one-shot zstd -b4	0%	+5% to +6%
one-shot zstd -b1, -b2	0%	0%

One-shot output is byte-identical. Streaming output changes slightly with the parse once the window slides: -0.15% bytes on silesia in 4 KB flushes with a 32 KB window, +0.04% on the message stream.

[terrelln]

What does the performance look like on recent clang versions?

[gaynor-anthropic]

Looks to be more or less a no-op on x86-64, but still a sizable win on arm64.

Benchmark	Sapphire Rapids (clang 23.1)	Neoverse-V2 (clang 23.1)
streaming, 32 KB window, 4 KB flushes, silesia.tar	+1%	+25%
streaming, 32 KB window, small flushed messages	+2% to +3%	+13% to +15%
fullbench -b5 (extDict), 1 MB silesia chunks	+1% to +5%	+9% to +32%
one-shot zstd -b3, silesia.tar / enwik8	+2% / +1%	+2% / +3%
one-shot zstd -b4	0% / −1%	0% / −1%
one-shot zstd -b1, -b2	0%	0%

[gaynor-anthropic]

Today in computers are bad, it looks like this may be super specific to Clang+x86, and the following patch would get wins there:

Subject: [PATCH] dfast: keep extDict segment-base selects as conditional moves

The pipelined double-fast extDict loop addresses each table candidate
through dictBase or base, chosen by comparing the candidate's index
with prefixStartIndex. The candidates met within one block lie on both
sides of that index in no learnable order (previous-segment positions
and never-written slots below it, current-segment positions above), so
the loop makes that choice with a select rather than a branch. LLVM's
x86 cmov-conversion pass turns these selects into branches, which in a
clang x86-64 build mispredict about 360 more times per 4.5 KB flushed
message (930 vs. 570); marking the comparisons __builtin_unpredictable
keeps the conditional moves.

Compression speed versus unpatched dev, clang 23, Sapphire Rapids
(x86-64), level 3, one core; streaming rows use a 32 KB window:

    workload                                 unmarked  marked
    4.5-8.9 KB flushed messages,
      2,048 live streams interleaved         +2-3%     +10-13%
      one stream                             +2%       +19-23%
    silesia.tar, 4 KB flushes, 16 streams    +1%       +26%
    fullbench -b5 -l3, first MB of each
      silesia file                           0-6%      +12-59%

GCC (x86-64, aarch64) and clang on aarch64 generate identical object
code marked or unmarked, so their numbers do not change.

UNPREDICTABLE() sits next to LIKELY()/UNLIKELY() in compiler.h and is
the plain condition where __builtin_unpredictable does not exist.
---
 lib/common/compiler.h           | 11 +++++++++++
 lib/compress/zstd_double_fast.c | 14 +++++++++-----
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/lib/common/compiler.h b/lib/common/compiler.h
index 5e70570e..6b933997 100644
--- a/lib/common/compiler.h
+++ b/lib/common/compiler.h
@@ -191,6 +191,17 @@
 #define UNLIKELY(x) (x)
 #endif
 
+/* Tell the compiler that a condition follows no pattern a branch predictor
+ * could learn, so that a value chosen on it is better computed with a
+ * conditional move than reached through a branch.
+ * As with LIKELY/UNLIKELY, only use it where it measurably matters.
+ */
+#if __has_builtin(__builtin_unpredictable)
+#  define UNPREDICTABLE(x) (__builtin_unpredictable(x))
+#else
+#  define UNPREDICTABLE(x) (x)
+#endif
+
 #if __has_builtin(__builtin_unreachable) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5)))
 #  define ZSTD_UNREACHABLE do { assert(0), __builtin_unreachable(); } while (0)
 #else
diff --git a/lib/compress/zstd_double_fast.c b/lib/compress/zstd_double_fast.c
index 77ecb035..23d3731d 100644
--- a/lib/compress/zstd_double_fast.c
+++ b/lib/compress/zstd_double_fast.c
@@ -716,16 +716,20 @@ size_t ZSTD_compressBlock_doubleFast_extDict_generic(
 
         hl0 = ZSTD_hashPtr(ip, hBitsL, 8);
         idxl0 = hashLong[hl0];
-        matchl0 = (idxl0 < prefixStartIndex ? dictBase : base) + idxl0;
+        /* Table candidates met within one block lie on both sides of
+         * prefixStartIndex in no learnable order: the comparisons choosing a
+         * segment base in this loop are marked UNPREDICTABLE so that the choice
+         * is made with a conditional move rather than a branch. */
+        matchl0 = (UNPREDICTABLE(idxl0 < prefixStartIndex) ? dictBase : base) + idxl0;
 
         /* Inner Loop: one iteration per search / position */
         do {
             const size_t hs0 = ZSTD_hashPtr(ip, hBitsS, mls);
             const U32 repIndex = (U32)(ip-base) + 1 - offset_1;
-            const BYTE* const repMatch = (repIndex < prefixStartIndex ? dictBase : base) + repIndex;
+            const BYTE* const repMatch = (UNPREDICTABLE(repIndex < prefixStartIndex) ? dictBase : base) + repIndex;
             idxs0 = hashSmall[hs0];
             curr = (U32)(ip-base);
-            matchs0 = (idxs0 < prefixStartIndex ? dictBase : base) + idxs0;
+            matchs0 = (UNPREDICTABLE(idxs0 < prefixStartIndex) ? dictBase : base) + idxs0;
 
             hashLong[hl0] = hashSmall[hs0] = curr;   /* update hash tables */
 
@@ -757,7 +761,7 @@ size_t ZSTD_compressBlock_doubleFast_extDict_generic(
             }
 
             idxl1 = hashLong[hl1];
-            matchl1 = (idxl1 < prefixStartIndex ? dictBase : base) + idxl1;
+            matchl1 = (UNPREDICTABLE(idxl1 < prefixStartIndex) ? dictBase : base) + idxl1;
 
             /* check short match */
             if (matchShortFound(ip, matchs0, idxs0, dictStartIndex)) {
@@ -860,7 +864,7 @@ _match_stored:
             while (ip <= ilimit) {
                 U32 const current2 = (U32)(ip-base);
                 U32 const repIndex2 = current2 - offset_2;
-                const BYTE* const repMatch2 = (repIndex2 < prefixStartIndex ? dictBase : base) + repIndex2;
+                const BYTE* const repMatch2 = (UNPREDICTABLE(repIndex2 < prefixStartIndex) ? dictBase : base) + repIndex2;
                 if ( ((ZSTD_index_overlap_check(prefixStartIndex, repIndex2)) & (offset_2 > 0))
                   && (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
                     const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
-- 
2.55.0

Happy to add that to this PR, or do it as a follow up.

[gaynor-anthropic]

I'm looking at the regression-test failure

[terrelln]

@gaynor-anthropic I've started trying to reproduce the claimed win today, and so far haven't been successful. So far I've been testing with gcc-11 on an AMD Turin machine, and clang-21 on an M4.

In order to reduce the number of variables, could you please share a benchmark harness that measures the performance of the streaming use case?

So far I've been using fullbench -b5 with compression level 3, and using the zstd CLI writing to /dev/null (which uses extDict almost exclusively) at level 3 with various window sizes to change the probability a match is found in the extDict (smaller window should increase probability a match is in extDict rather than prefix).

I think the direction of the patch generally makes sense, especially the ZSTD_selectAddr change, so I want to make sure that we can run exactly the same benchmark, so I can reproduce the win.

[gaynor-anthropic]

Absolute minimal reproducer:

head -c 1048576 silesia.tar > sil1M && ./tests/fullbench -b5 -l3 sil1M

For a stand-alone harness:

/* flushbench: streaming compression throughput with a flush every `chunk` bytes.
 * usage: flushbench FILE [level=3] [windowLog=15] [chunk=4096] [streams=1] [rounds=7]
 * The file is cut into `streams` equal segments, one CCtx each, fed round-robin. */
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <zstd.h>

#define CHECK(f) do { size_t const r_ = (f); if (ZSTD_isError(r_)) { \
    fprintf(stderr, "%s: %s\n", #f, ZSTD_getErrorName(r_)); exit(1); } } while (0)

static double now(void) {
    struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t);
    return (double)t.tv_sec + 1e-9 * (double)t.tv_nsec;
}

int main(int argc, char** argv) {
    if (argc < 2) { fprintf(stderr, "usage: %s FILE [level] [windowLog] [chunk] [streams] [rounds]\n", argv[0]); return 1; }
    int    const level   = argc > 2 ? atoi(argv[2]) : 3;
    int    const wlog    = argc > 3 ? atoi(argv[3]) : 15;
    size_t const chunk   = argc > 4 ? (size_t)atol(argv[4]) : 4096;
    size_t const streams = argc > 5 ? (size_t)atol(argv[5]) : 1;
    int    const rounds  = argc > 6 ? atoi(argv[6]) : 7;

    FILE* const f = fopen(argv[1], "rb");
    if (!f) { perror(argv[1]); return 1; }
    fseek(f, 0, SEEK_END); size_t const size = (size_t)ftell(f); fseek(f, 0, SEEK_SET);
    char* const src = malloc(size);
    if (!src || fread(src, 1, size, f) != size) { fprintf(stderr, "read error\n"); return 1; }
    fclose(f);

    size_t const seg = size / streams;          /* tail remainder is ignored */
    size_t const cap = ZSTD_compressBound(chunk);
    char* const dst = malloc(cap);
    ZSTD_CCtx** const cctx = malloc(streams * sizeof(*cctx));
    for (size_t s = 0; s < streams; s++) {
        cctx[s] = ZSTD_createCCtx();
        CHECK(ZSTD_CCtx_setParameter(cctx[s], ZSTD_c_compressionLevel, level));
        CHECK(ZSTD_CCtx_setParameter(cctx[s], ZSTD_c_windowLog, wlog));
    }

    double best = 1e30; size_t csize = 0;
    for (int r = 0; r < rounds; r++) {
        for (size_t s = 0; s < streams; s++) CHECK(ZSTD_CCtx_reset(cctx[s], ZSTD_reset_session_only));
        csize = 0;
        double const t0 = now();
        for (size_t off = 0; off < seg; off += chunk) {
            size_t const n = seg - off < chunk ? seg - off : chunk;
            for (size_t s = 0; s < streams; s++) {
                ZSTD_inBuffer  in  = { src + s * seg + off, n, 0 };
                ZSTD_outBuffer out = { dst, cap, 0 };
                size_t const rem = ZSTD_compressStream2(cctx[s], &out, &in, ZSTD_e_flush);
                CHECK(rem);
                if (rem || in.pos != n) { fprintf(stderr, "flush incomplete\n"); return 1; }
                csize += out.pos;
            }
        }
        double const dt = now() - t0;
        if (dt < best) best = dt;
    }
    printf("level %d wlog %d chunk %zu streams %zu: %.1f MB/s, %zu -> %zu bytes\n",
           level, wlog, chunk, streams, (double)(seg * streams) / best / 1e6, seg * streams, csize);
    return 0;
}

which can be run like ./flushbench silesia.tar 3 15 4096.

[terrelln]

Thanks for sharing that @gaynor-anthropic, that will definitely help in measuring this PR! I'm going on vacation for October, so @Cyan4973 will have to take over the review

[gaynor-anthropic]

Cheers, hopefully it's somewhere fun!