Spill cold string values to an append-only SSD log

src/Makefile

@@ -605,6 +605,9 @@ ENGINE_SERVER_OBJ = \
     crdt/merge.o \
     crdt/peer.o \
     rdb_forkless.o \
+    tier/vlog.o \
+    tier/evict.o \
+    tier/compact.o \
     rdma.o \
     release.o \
     replication.o \

src/aof.c

@@ -32,6 +32,7 @@
 #include "listpack.h"
 #include "sds.h"
 #include "server.h"
+#include "tier/tier.h"
 #include "ordered_index.h"
 #include "bio.h"
 #include "rio.h"
@@ -1925,6 +1926,13 @@ int rioWriteBulkObject(rio *r, robj *obj) {
     }
     if (obj->encoding == OBJ_ENCODING_INT) {
         return rioWriteBulkLongLong(r, (long)objectGetVal(obj));
+    } else if (obj->encoding == OBJ_ENCODING_TIERED) {
+        sds raw = tierReadRef(objectGetVal(obj));
+        int n;
+        if (!raw) return 0;
+        n = rioWriteBulkString(r, raw, sdslen(raw));
+        sdsfree(raw);
+        return n;
     } else if (sdsEncodedObject(obj)) {
         return rioWriteBulkString(r, objectGetVal(obj), sdslen(objectGetVal(obj)));
     } else {

src/config.c

@@ -3483,6 +3483,8 @@ standardConfig static_configs[] = {
     createBoolConfig("syslog-enabled", NULL, IMMUTABLE_CONFIG, server.syslog_enabled, 0, NULL, NULL),
     createBoolConfig("cluster-enabled", NULL, IMMUTABLE_CONFIG, server.cluster_enabled, 0, NULL, NULL),
     createBoolConfig("crdt-enabled", NULL, MODIFIABLE_CONFIG, server.crdt_enabled, 0, NULL, NULL),
+    createBoolConfig("tiering-enabled", NULL, MODIFIABLE_CONFIG, server.tiering_enabled, 0, NULL, NULL),
+    createBoolConfig("tiering-debug-exit-before-rename", NULL, MODIFIABLE_CONFIG, server.tiering_debug_exit_before_rename, 0, NULL, NULL),
     createBoolConfig("appendonly", NULL, MODIFIABLE_CONFIG | DENY_LOADING_CONFIG, server.aof_enabled, 0, NULL, updateAppendOnly),
     createBoolConfig("cluster-allow-reads-when-down", NULL, MODIFIABLE_CONFIG, server.cluster_allow_reads_when_down, 0, NULL, NULL),
     createBoolConfig("cluster-allow-pubsubshard-when-down", NULL, MODIFIABLE_CONFIG, server.cluster_allow_pubsubshard_when_down, 1, NULL, NULL),
@@ -3506,6 +3508,7 @@ standardConfig static_configs[] = {
     /* String Configs */
     createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.acl_filename, "", NULL, NULL),
     createStringConfig("crdt-peers", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, server.crdt_peers, "", NULL, NULL),
+    createStringConfig("tiering-path", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tiering_path, "tier.vlog", NULL, NULL),
     createStringConfig("unixsocket", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.unixsocket, NULL, NULL, NULL),
     createStringConfig("unixsocketgroup", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.unix_ctx_config.group, NULL, NULL, NULL),
     createStringConfig("pidfile", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.pidfile, NULL, NULL, NULL),
@@ -3571,6 +3574,10 @@ standardConfig static_configs[] = {
     createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.config_databases, 16, INTEGER_CONFIG, NULL, NULL),
     createIntConfig("crdt-region-id", NULL, MODIFIABLE_CONFIG, 1, 65535, server.crdt_region_id, 1, INTEGER_CONFIG, NULL, NULL),
     createLongLongConfig("crdt-tombstone-gc-ms", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.crdt_tombstone_gc_ms, 60000, INTEGER_CONFIG, NULL, NULL),
+    createIntConfig("tiering-min-value", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.tiering_min_value, 128, INTEGER_CONFIG, NULL, NULL),
+    createIntConfig("tiering-idle-seconds", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tiering_idle_seconds, 60, INTEGER_CONFIG, NULL, NULL),
+    createIntConfig("tiering-debug-crash-after", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tiering_debug_crash_after, 0, INTEGER_CONFIG, NULL, NULL),
+    createLongLongConfig("tiering-max-ram", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.tiering_max_ram, 0, MEMORY_CONFIG, NULL, NULL),
     createIntConfig("cluster-databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.config_databases_cluster, 1, INTEGER_CONFIG, NULL, NULL),
     createIntConfig("port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.port, 6379, INTEGER_CONFIG, NULL, updatePort),                                               /* TCP port. */
     createIntConfig("io-threads", NULL, DEBUG_CONFIG | MODIFIABLE_CONFIG, 1, IO_THREADS_MAX_NUM, server.io_threads_num, 1, INTEGER_CONFIG, NULL, updateIOThreads), /* Single threaded by default */

src/db.c

@@ -30,6 +30,7 @@
 #include "server.h"
 #include "crdt/peer.h"
 #include "rdb_forkless.h"
+#include "tier/tier.h"
 #include "listpack.h"
 #include "hotkeys.h"
 #include "ordered_index.h"
@@ -127,6 +128,8 @@ robj *lookupKey(serverDb *db, robj *key, int flags) {
             val->lru = lrulfu_touch(val->lru);
         }
 
+        if (server.tiering_enabled && objectGetType(val) == OBJ_STRING && objectGetEncoding(val) == OBJ_ENCODING_TIERED)
+            val = tierPromote(db, key, val);
         if (!(flags & (LOOKUP_NOSTATS | LOOKUP_WRITE))) server.stat_keyspace_hits++;
         /* TODO: Use separate hits stats for WRITE */
     } else {

src/object.c

@@ -30,6 +30,7 @@
 
 #include "hashtable.h"
 #include "server.h"
+#include "tier/tier.h"
 #include "ordered_index.h"
 #include "serverassert.h"
 #include "functions.h"
@@ -619,6 +620,14 @@ robj *dupStringObject(const robj *o) {
         d->encoding = OBJ_ENCODING_INT;
         d->val_ptr = o->val_ptr;
         return d;
+    case OBJ_ENCODING_TIERED: {
+        sds raw = tierReadRef(objectGetVal(o));
+        robj *copy;
+        if (!raw) serverPanic("Tiered string unreadable");
+        copy = createRawStringObject(raw, sdslen(raw));
+        sdsfree(raw);
+        return copy;
+    }
     default: serverPanic("Wrong encoding."); break;
     }
 }
@@ -700,6 +709,9 @@ robj *createModuleObject(moduleType *mt, void *value) {
 void freeStringObject(robj *o) {
     if (objectGetEncoding(o) == OBJ_ENCODING_RAW) {
         sdsfree(objectGetVal(o));
+    } else if (objectGetEncoding(o) == OBJ_ENCODING_TIERED) {
+        tierMarkDead(objectGetVal(o));
+        zfree(objectGetVal(o));
     }
 }
 
@@ -1073,6 +1085,13 @@ robj *getDecodedObject(robj *o) {
         incrRefCount(o);
         return o;
     }
+    if (objectGetType(o) == OBJ_STRING && objectGetEncoding(o) == OBJ_ENCODING_TIERED) {
+        sds raw = tierReadRef(objectGetVal(o));
+        if (!raw) serverPanic("Tiered string unreadable");
+        dec = createStringObject(raw, sdslen(raw));
+        sdsfree(raw);
+        return dec;
+    }
     if (objectGetType(o) == OBJ_STRING && objectGetEncoding(o) == OBJ_ENCODING_INT) {
         char buf[32];
 

src/rdb.c

@@ -51,6 +51,7 @@
 #include "compression_stream.h"
 #include "forkless.h"
 #include "rdb_forkless.h"
+#include "tier/tier.h"
 
 #include <math.h>
 #include <fcntl.h>
@@ -596,6 +597,14 @@ ssize_t rdbSaveStringObject(rio *rdb, robj *obj) {
             return n;
         }
     }
+    if (obj->encoding == OBJ_ENCODING_TIERED) {
+        sds raw = tierReadRef(objectGetVal(obj));
+        ssize_t n;
+        if (!raw) return -1;
+        n = rdbSaveRawString(rdb, (unsigned char *)raw, sdslen(raw));
+        sdsfree(raw);
+        return n;
+    }
     if (obj->encoding == OBJ_ENCODING_INT) {
         return rdbSaveLongLongAsStringObject(rdb, (long)objectGetVal(obj));
     } else {

src/server.c

@@ -61,6 +61,7 @@
 #include "forkless.h"
 #include "crdt/peer.h"
 #include "rdb_forkless.h"
+#include "tier/tier.h"
 
 #include "eval.h"
 #include "bgiteration.h"
@@ -1696,6 +1697,7 @@ long long serverCron(struct aeEventLoop *eventLoop, long long id, void *clientDa
     databasesCron();
     crdtCron();
     snapshotForklessCron();
+    tierCron();
 
     /* Start a scheduled AOF rewrite if this was requested by the user while
      * a BGSAVE was in progress. We don't start the rewrite if there is an
@@ -6741,6 +6743,11 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) {
                 "mem_overhead_db_hashtable_rehashing:%zu\r\n", mh->overhead_db_hashtable_rehashing,
                 "active_defrag_running:%d\r\n", server.active_defrag_cpu_percent,
                 "lazyfree_pending_objects:%zu\r\n", lazyfreeGetPendingObjectsCount(),
+                "tiering_enabled:%d\r\n", server.tiering_enabled,
+                "tiering_spills:%lld\r\n", tierSpills(),
+                "tiering_promotions:%lld\r\n", tierPromotions(),
+                "tiering_bytes:%lld\r\n", tierBytesOnDisk(),
+                "tiering_dead_bytes:%lld\r\n", tierDeadBytes(),
                 "lazyfreed_objects:%zu\r\n", lazyfreeGetFreedObjectsCount()));
         freeMemoryOverheadData(mh);
     }

src/server.h

@@ -843,6 +843,7 @@ typedef struct ValkeyModuleType moduleType;
 #define OBJ_ENCODING_LISTPACK 11  /* Encoded as a listpack */
 #define OBJ_ENCODING_LISTPACK2 12 /* Encoded as a listpack with metadata tag */
 #define OBJ_ENCODING_PATH_HASH 13 /* Path hash backed by a radix tree */
+#define OBJ_ENCODING_TIERED 14    /* String payload lives in the tiering value log */
 
 #define OBJ_REFCOUNT_BITS 29
 #define OBJ_SHARED_REFCOUNT ((1 << OBJ_REFCOUNT_BITS) - 1) /* Global object never destroyed. */
@@ -2223,6 +2224,13 @@ struct valkeyServer {
     long long crdt_tombstone_gc_ms;
     int snapshot_mode;                    /* SNAPSHOT_MODE_FORK or SNAPSHOT_MODE_FORKLESS */
     int snapshot_forkless_slice_keys;     /* Max keys written per cron tick in forkless mode */
+    int tiering_enabled;
+    char *tiering_path;
+    int tiering_min_value;
+    long long tiering_max_ram;
+    int tiering_idle_seconds;
+    int tiering_debug_crash_after;
+    int tiering_debug_exit_before_rename;
     int key_load_delay;                   /* Delay in microseconds between keys while
                                            * loading aof or rdb. (for testings). negative
                                            * value means fractions of microseconds (on average). */

src/tier/compact.c

@@ -0,0 +1,32 @@
+/* Gather live tier pointers and rewrite the value log.
+ *
+ * Copyright (c) 2026, Cache Me Outside contributors.
+ * Licensed under the BSD 3-Clause License, the same terms as the server.
+ */
+#include "tier/tier.h"
+
+int tierCompactLive(void) {
+    tierRef **live = NULL;
+    int n = 0, cap = 0, dbid;
+    int rc;
+    for (dbid = 0; dbid < server.dbnum; dbid++) {
+        serverDb *db = server.db[dbid];
+        kvstoreIterator *it;
+        void *next;
+        if (!db) continue;
+        it = kvstoreIteratorInit(db->keys, 0);
+        while (kvstoreIteratorNext(it, &next)) {
+            robj *o = next;
+            if (objectGetType(o) != OBJ_STRING || objectGetEncoding(o) != OBJ_ENCODING_TIERED) continue;
+            if (n == cap) {
+                cap = cap ? cap * 2 : 16;
+                live = zrealloc(live, sizeof(*live) * (size_t)cap);
+            }
+            live[n++] = objectGetVal(o);
+        }
+        kvstoreIteratorRelease(it);
+    }
+    rc = tierCompact(live, n);
+    zfree(live);
+    return rc;
+}

src/tier/evict.c

@@ -0,0 +1,85 @@
+/* Move cold large strings to the value log and bring them back on read.
+ *
+ * Copyright (c) 2026, Cache Me Outside contributors.
+ * Licensed under the BSD 3-Clause License, the same terms as the server.
+ */
+#include "tier/tier.h"
+
+/* Replace the string body with a log pointer and free the body now.
+ * dbReplaceValue() can leave the old allocation charged: a refcount above 1
+ * only decrements, and the swap path hands the sds to a throwaway object that
+ * lazy-free may not reclaim before the next INFO. The key stays the same
+ * hashtable entry, so this only rewrites that entry's value pointer. */
+static int tierSpillObject(serverDb *db, robj *key, robj *o) {
+    sds raw;
+    tierRef *ref;
+    UNUSED(db);
+    UNUSED(key);
+    if (!o || objectGetType(o) != OBJ_STRING) return 0;
+    if (objectGetEncoding(o) != OBJ_ENCODING_RAW || o->hasembval) return 0;
+    /* Another owner still needs the bytes. Try again on a later tick. */
+    if (objectGetRefcount(o) != 1) return 0;
+    raw = objectGetVal(o);
+    if (sdslen(raw) < (size_t)server.tiering_min_value) return 0;
+    if (server.tiering_idle_seconds > 0 && objectGetLRUIdleSecs(o) < (uint32_t)server.tiering_idle_seconds) return 0;
+    ref = tierAppend(raw, sdslen(raw));
+    if (!ref) return 0;
+    objectSetEncoding(o, OBJ_ENCODING_TIERED);
+    objectSetVal(o, ref);
+    sdsfree(raw);
+    return 1;
+}
+
+robj *tierPromote(serverDb *db, robj *key, robj *o) {
+    tierRef *ref;
+    sds raw;
+    robj *neu;
+    if (!o || objectGetEncoding(o) != OBJ_ENCODING_TIERED) return o;
+    ref = objectGetVal(o);
+    raw = tierReadRef(ref);
+    if (!raw) return o;
+    neu = createObject(OBJ_STRING, raw);
+    dbReplaceValue(db, key, &neu);
+    tierNotePromotion();
+    return neu;
+}
+
+static int tierVisit(serverDb *db, int budget) {
+    kvstoreIterator *it;
+    void *next;
+    int spilled = 0;
+    if (!db || kvstoreSize(db->keys) == 0 || budget <= 0) return 0;
+    it = kvstoreIteratorInit(db->keys, HASHTABLE_ITER_SAFE);
+    while (budget > 0 && kvstoreIteratorNext(it, &next)) {
+        robj *o = next;
+        sds keystr;
+        robj key;
+        budget--;
+        if (objectGetType(o) != OBJ_STRING) continue;
+        if (objectGetEncoding(o) == OBJ_ENCODING_TIERED) continue;
+        keystr = objectGetKey(o);
+        initStaticStringObject(key, keystr);
+        if (server.tiering_max_ram > 0 && (long long)zmalloc_used_memory() <= server.tiering_max_ram &&
+            server.tiering_idle_seconds > 0 && objectGetLRUIdleSecs(o) < (uint32_t)server.tiering_idle_seconds)
+            continue;
+        if (tierSpillObject(db, &key, o)) spilled++;
+        if (server.tiering_max_ram > 0 && (long long)zmalloc_used_memory() <= server.tiering_max_ram && spilled > 0 &&
+            server.tiering_idle_seconds == 0)
+            break;
+    }
+    kvstoreIteratorRelease(it);
+    return spilled;
+}
+
+void tierCron(void) {
+    int dbid;
+    if (!server.tiering_enabled) return;
+    for (dbid = 0; dbid < server.dbnum; dbid++) {
+        if (server.db[dbid]) tierVisit(server.db[dbid], 64);
+    }
+    if (tierDeadBytes() > 4096 && tierDeadBytes() * 2 > tierBytesOnDisk()) {
+        /* Collected by compact.c */
+        extern int tierCompactLive(void);
+        tierCompactLive();
+    }
+}

src/tier/tier.h

@@ -0,0 +1,29 @@
+#ifndef CMO_TIER_H
+#define CMO_TIER_H
+
+#include "server.h"
+
+/* String values live in an append-only log. The object keeps only this
+ * pointer. OBJ_ENCODING_TIERED is defined in server.h. A command never sees
+ * that encoding: lookup promotes the value back to a normal string first. */
+
+typedef struct tierRef {
+    uint64_t offset;
+    uint32_t len;
+    uint64_t crc;
+} tierRef;
+
+void tierCron(void);
+robj *tierPromote(serverDb *db, robj *key, robj *o);
+tierRef *tierAppend(const char *buf, size_t len);
+sds tierReadRef(const tierRef *ref);
+void tierMarkDead(const tierRef *ref);
+void tierFreeRef(tierRef *ref);
+void tierNotePromotion(void);
+int tierCompact(tierRef **live, int nlive);
+long long tierBytesOnDisk(void);
+long long tierSpills(void);
+long long tierPromotions(void);
+long long tierDeadBytes(void);
+
+#endif