cherry-pick: lwIP adapter (#452) and Redis rate limiting (#453) to main

.github/workflows/interop.yml

@@ -80,3 +80,25 @@ jobs:
 
     - name: Test requests through 400 address changes
       run: make -j"$(nproc)" http3-migration-e2e CWIST_WEBRTC=0 PYTHON="$RUNNER_TEMP/h3-migration/bin/python"
+
+  redis-rate-limit:
+    name: Shared Redis rate limit (two processes)
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - name: Install build dependencies and isolated Redis
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y cmake libcurl4-openssl-dev libnghttp2-dev libbrotli-dev libzstd-dev redis-server
+
+    - name: Test shared window, failures and bounded Redis calls
+      run: |
+        make -j"$(nproc)" SANITIZE=address,undefined WERROR=1 CWIST_HTTP3=0 CWIST_WEBRTC=0 redis-rate-limit-e2e test_redis_timeout test_redis_frame
+      env:
+        ASAN_OPTIONS: detect_leaks=1:abort_on_error=1:halt_on_error=1
+        UBSAN_OPTIONS: halt_on_error=1:abort_on_error=1

.gitignore

@@ -10,6 +10,7 @@ bindings/rust/Cargo.lock
 # Test binaries
 test_sstring
 test_baremetal_port
+test_lwip_adapter
 test_http
 test_http_client
 test_siphash
@@ -118,6 +119,9 @@ test_middleware_ctx
 test_builtin_mw_chain
 test_proto_desc
 test_rate_limit
+/test_rate_limit_redis
+/test_redis_timeout
+/test_redis_frame
 test_redis
 test_scheduler
 test_secure_headers

.gitmodules

@@ -32,3 +32,6 @@
 [submodule "lib/mbedtls"]
 	path = lib/mbedtls
 	url = https://github.com/Mbed-TLS/mbedtls
+[submodule "lib/lwip"]
+	path = lib/lwip
+	url = https://github.com/lwip-tcpip/lwip

Makefile

@@ -386,6 +386,7 @@ SRCS = src/core/sstring/sstring.c \
        src/core/utils/json_heal.c \
        src/core/utils/zod.c \
        src/sys/app/middleware.c \
+       src/sys/app/rate_limit_redis.c \
        src/sys/app/otel.c \
        src/sys/app/config.c \
        src/sys/app/logger.c \
@@ -536,6 +537,13 @@ wasip2-smoke: $(WASIP2_ARCHIVE)
 clean-wasip2:
 	rm -rf .wasip2-build libcwist_wasip2_*.a wasip2_smoke.wasm wasm_abi_layout.wasm
 
+# --- lwIP (vendored for the adapter tests, issue #447 Phase 3) ---------------------
+# The core and IPv4 sources; tests/lwip/lwipopts.h turns off what the adapter
+# tests do not use, so the extra files compile to nothing.
+LWIP_DIR = lib/lwip
+LWIP_CORE_SRCS = $(wildcard $(LWIP_DIR)/src/core/*.c $(LWIP_DIR)/src/core/ipv4/*.c)
+LWIP_TEST_INCLUDES = -Itests/lwip -I$(LWIP_DIR)/src/include
+
 # --- Bare metal (arm-none-eabi / Cortex-M, issue #447) -------------------------
 # The WASI core (WASM_SRCS + WASIP2_EXTRA_SRCS) minus sqlite/db, the test
 # client and the WASI compat layer, built for a single-threaded firmware on
@@ -584,9 +592,22 @@ $(BAREMETAL_HELLO): $(BAREMETAL_ARCHIVE) tests/baremetal/hello.c tests/baremetal
 	    tests/baremetal/hello.c tests/baremetal/board.c $(BAREMETAL_ARCHIVE) -lm
 	$(BAREMETAL_CROSS)size $@
 
-baremetal-smoke: $(BAREMETAL_HELLO)
+# The lwIP adapter on Cortex-M: CWIST + src/sys/port/lwip.c + the vendored
+# lwIP core (tests/lwip/lwipopts.h), client and server over loopback.
+BAREMETAL_LWIP_SRCS = tests/baremetal/hello_lwip.c tests/baremetal/board.c src/sys/port/lwip.c \
+	tests/lwip/loop_client.c $(LWIP_CORE_SRCS)
+$(BAREMETAL_BUILD_DIR)/hello-lwip.elf: $(BAREMETAL_ARCHIVE) $(BAREMETAL_LWIP_SRCS) tests/baremetal/stm32h743.ld \
+	    tests/lwip/lwipopts.h tests/lwip/arch/cc.h tests/lwip/loop_client.h include/cwist/sys/port/lwip.h
+	$(BAREMETAL_CROSS)gcc $(BAREMETAL_CFLAGS) $(LWIP_TEST_INCLUDES) -DBOARD_QEMU_USER -nostartfiles \
+	    --specs=nosys.specs -T tests/baremetal/stm32h743.ld -Wl,--gc-sections \
+	    -Wl,--no-warn-rwx-segments -Wl,-Map=$(@:.elf=.map) -o $@ \
+	    $(BAREMETAL_LWIP_SRCS) $(BAREMETAL_ARCHIVE) -lm
+	$(BAREMETAL_CROSS)size $@
+
+baremetal-smoke: $(BAREMETAL_HELLO) $(BAREMETAL_BUILD_DIR)/hello-lwip.elf
 	$(QEMU_ARM) -cpu cortex-m7 $(BAREMETAL_BUILD_DIR)/hello.elf
 	$(QEMU_ARM) -cpu cortex-m7 $(BAREMETAL_BUILD_DIR)/hello-rtc.elf
+	$(QEMU_ARM) -cpu cortex-m7 $(BAREMETAL_BUILD_DIR)/hello-lwip.elf
 
 clean-baremetal:
 	rm -rf $(BAREMETAL_BUILD_DIR) $(BAREMETAL_ARCHIVE)
@@ -1264,6 +1285,7 @@ TEST_TARGETS = test_worker_affinity \
                test_seq_auth \
                test_sha256 \
                test_baremetal_port \
+               test_lwip_adapter \
                test_error \
                test_middleware_jwt \
                test_inline_exports \
@@ -1319,6 +1341,9 @@ TEST_TARGETS = test_worker_affinity \
                test_metrics \
                test_access_log \
                test_rate_limit \
+               test_rate_limit_redis \
+               test_redis_timeout \
+               test_redis_frame \
                test_cache \
                test_bdr \
                test_bdr_cache \
@@ -1580,6 +1605,14 @@ test_sha256: tests/test_sha256.c include/cwist/core/crypto/sha256.h
 	$(CC) $(CFLAGS) -o $@ tests/test_sha256.c
 	./$@
 
+# lwIP adapter (src/sys/port/lwip.c) on the host: the vendored lwIP core in
+# NO_SYS mode with tests/lwip/lwipopts.h, client and server over loopback.
+test_lwip_adapter: $(LIB_NAME) tests/test_lwip_adapter.c src/sys/port/lwip.c include/cwist/sys/port/lwip.h \
+	    tests/lwip/loop_client.c tests/lwip/loop_client.h tests/lwip/lwipopts.h tests/lwip/arch/cc.h
+	$(CC) $(CFLAGS) $(LWIP_TEST_INCLUDES) -o $@ tests/test_lwip_adapter.c src/sys/port/lwip.c \
+	    tests/lwip/loop_client.c $(LWIP_CORE_SRCS) $(LIB_NAME) $(LIBS)
+	./$@
+
 # Bare-metal port layer on the host: siphash.c and baremetal_time.c built as
 # CWIST_BAREMETAL (no libcwist.a), with baremetal_time.c's libc names renamed
 # so they do not collide with the host's.
@@ -1879,6 +1912,25 @@ test_rate_limit: $(LIB_NAME) tests/test_rate_limit.c
 	$(CC) $(CFLAGS) -o test_rate_limit tests/test_rate_limit.c $(LIB_NAME) $(LIBS)
 	./test_rate_limit
 
+test_rate_limit_redis: $(LIB_NAME) tests/test_rate_limit_redis.c
+	$(CC) $(CFLAGS) -o $@ tests/test_rate_limit_redis.c $(LIB_NAME) $(LIBS)
+	./$@
+
+test_redis_timeout: $(LIB_NAME) tests/test_redis_timeout.c
+	$(CC) $(CFLAGS) -o $@ tests/test_redis_timeout.c $(LIB_NAME) $(LIBS)
+	./$@
+
+src/net/redis/cwist_redis.o: src/net/redis/redis_frame_internal.h
+
+test_redis_frame: tests/test_redis_frame.c src/net/redis/redis_frame_internal.h
+	$(CC) $(CFLAGS) -o $@ tests/test_redis_frame.c
+	./$@
+
+.PHONY: redis-rate-limit-e2e
+redis-rate-limit-e2e: $(LIB_NAME) tests/test_rate_limit_redis.c
+	$(CC) $(CFLAGS) -o test_rate_limit_redis tests/test_rate_limit_redis.c $(LIB_NAME) $(LIBS)
+	$(PYTHON) scripts/ci/redis_rate_limit_e2e.py ./test_rate_limit_redis
+
 test_cache: $(LIB_NAME) tests/test_cache.c
 	$(CC) $(CFLAGS) -o test_cache tests/test_cache.c $(LIB_NAME) $(LIBS)
 	./test_cache

docs/api/middleware.md

@@ -53,6 +53,84 @@ Limits the number of requests per minute from a single IP.
 cwist_app_use(app, cwist_mw_rate_limit_ip(60));
 ```
 
+### Redis-backed shared rate limiter
+
+The local limiter above keeps its buckets in one process. To share one
+limit across independent server processes, give each process its own
+**dedicated pool on DB 0, with the default user and no AUTH**, and use the
+same prefix and limit. Replacement connections start fresh: AUTH, SELECT,
+HELLO and other connection state are not replayed. Legacy calls that share
+this pool may also receive fresh connections after a timed call discards one.
+Use one prefix per rate policy: different limits with the same prefix
+share one counter, not independent buckets.
+
+```c
+cwist_redis_pool_t *pool = cwist_redis_pool_create("127.0.0.1", 6379, 4);
+cwist_rate_limit_redis_cfg_t cfg = {
+    .pool = pool,
+    .requests_per_minute = 60,
+    .key_prefix = "my-service:rate:",
+    .timeout_ms = 50,
+    .fail_closed = true,
+};
+cwist_middleware_func limit = cwist_mw_rate_limit_ip_redis(&cfg);
+/* Inside main: fail startup if registration did not succeed. */
+if (!pool || !limit) return 1;
+cwist_app_use(app, limit);
+/* One process per instance: do not fork a live pool. */
+cwist_app_listen_ex(app, 8080, 1, -1);
+cwist_redis_pool_destroy(pool);
+```
+
+Each key is `prefix + peer-IP`. One Lua EVAL increments the counter and
+sets a 60-second expiry on the first request. It repairs missing expiry
+without extending a running window. The window starts at the first hit,
+not a process-local clock boundary. Allowed and rejected attempts count.
+Over-limit replies are 429 with a conservative `Retry-After: 60`.
+
+With `fail_closed = false` (default), backend errors allow the request.
+With `true`, errors return 503, not 429. This includes timeout, pool
+exhaustion, malformed replies, and a missing socket peer. A failed command
+is never retried: it might already have incremented the counter. There is
+no local fallback counter.
+
+The request-time budget covers pool wait, cached-peer reconnect, write,
+and reply I/O. Zero selects 50 ms; the maximum is 60000 ms. This is a
+blocking middleware, not an async reactor call. Scheduling delays are not
+a real-time guarantee. Initial pool creation and DNS are outside this
+budget. The first backend failure per config and process emits a warning
+with the error code and policy to stderr. Keys and credentials are not logged.
+The Redis command budget does not bound application logging.
+Failed initial pool creation or invalid config returns NULL.
+
+Create pools after fork, separately in each serving process. Inherited
+pools are rejected by the timed API. Keep the pool alive until every call
+has stopped, then destroy it. Do not register a pre-fork pool for automatic
+`CWIST_WORKERS` children; this API does not add a worker-start hook. Eight
+distinct immutable configs fit per process; equal configs reuse a slot.
+Prefixes are copied and limited to 128 bytes.
+
+Only the socket peer is used. Forwarded headers cannot choose the key.
+A request without a connected TCP peer fd (for example, a QUIC request)
+follows the missing-peer failure policy; this API does not extract QUIC
+path metadata. IPv6 keys use the full address (/128), not a /64 prefix;
+clients that rotate IPv6 addresses obtain separate buckets.
+Behind a reverse proxy, this limits the proxy address unless a trusted
+upstream preserves client source addresses. Redis RESP2 is plaintext: use
+loopback, a private network, or a separate TLS tunnel. The default user's
+ACL must allow EVAL, INCR, PTTL, and PEXPIRE. A server requiring AUTH makes
+every call fail; the configured failure policy then applies to every request.
+Eviction, failover, or external deletion
+can reset counters; this is not a global consistency guarantee across
+Redis data loss.
+
+Run `make redis-rate-limit-e2e` with redis-server installed. The runner
+starts its own loopback Redis and never silently skips. Two C processes
+create separate pools and invoke the middleware with real TCP peer fds.
+Their 40 attempts share seven admissions. The test also checks TTL,
+expiry, namespaces, and failure policies. The normal unit target skips
+only the live-Redis part when no test port is supplied.
+
 ### CORS Middleware
 Enables Cross-Origin Resource Sharing (CORS) support.
 - Adds `Access-Control-Allow-Origin: *` to all responses.

docs/api/v4.0-api-baseline.txt

@@ -4,7 +4,7 @@
 # (parameter names dropped) and struct/union/enum layouts in member order.
 # `make api-check` (CI) fails when the headers no longer match this file.
 
-== functions (783) ==
+== functions (789) ==
 cwist_accept_socket
 cwist_alloc
 cwist_alloc_array
@@ -502,6 +502,10 @@ cwist_logger_log
 cwist_logger_set_colors
 cwist_logger_set_level
 cwist_logger_warn
+cwist_lwip_active_connections
+cwist_lwip_close
+cwist_lwip_config_init
+cwist_lwip_listen
 cwist_make_socket_ipv4
 cwist_malloc
 cwist_malloc_shim
@@ -540,6 +544,7 @@ cwist_mw_metrics
 cwist_mw_otel_get_trace_id
 cwist_mw_otel_trace
 cwist_mw_rate_limit_ip
+cwist_mw_rate_limit_ip_redis
 cwist_mw_rate_limit_reset
 cwist_mw_request_id
 cwist_mw_session
@@ -647,6 +652,7 @@ cwist_redis_del
 cwist_redis_expire
 cwist_redis_get
 cwist_redis_pool_command_argv
+cwist_redis_pool_command_argv_reply_timeout
 cwist_redis_pool_create
 cwist_redis_pool_del
 cwist_redis_pool_destroy
@@ -789,7 +795,7 @@ cwist_zod_parse
 cwist_zod_print_errors
 cwist_zod_validate
 
-== function signatures (784) ==
+== function signatures (790) ==
 cwist_accept_socket: cwist_error_t (int, struct sockaddr *, void (*)(int client_fd, void *ctx), void *)
 cwist_alloc: void * (size_t)
 cwist_alloc_array: void * (size_t, size_t)
@@ -1288,6 +1294,10 @@ cwist_logger_log: void (cwist_logger *, cwist_log_level_t, const char *, ...)
 cwist_logger_set_colors: void (cwist_logger *, bool)
 cwist_logger_set_level: void (cwist_logger *, cwist_log_level_t)
 cwist_logger_warn: void (cwist_logger *, const char *, ...)
+cwist_lwip_active_connections: size_t (const cwist_lwip_server *)
+cwist_lwip_close: void (cwist_lwip_server *)
+cwist_lwip_config_init: void (cwist_lwip_config *)
+cwist_lwip_listen: cwist_lwip_server * (cwist_app *, const cwist_lwip_config *)
 cwist_make_socket_ipv4: int (struct sockaddr_in *, const char *, uint16_t, uint16_t)
 cwist_malloc: void * (size_t)
 cwist_malloc_shim: void * (size_t)
@@ -1326,6 +1336,7 @@ cwist_mw_metrics: cwist_middleware_func (void)
 cwist_mw_otel_get_trace_id: const char * (const cwist_http_request *)
 cwist_mw_otel_trace: cwist_middleware_func (const cwist_otel_cfg_t *)
 cwist_mw_rate_limit_ip: cwist_middleware_func (int)
+cwist_mw_rate_limit_ip_redis: cwist_middleware_func (const cwist_rate_limit_redis_cfg_t *)
 cwist_mw_rate_limit_reset: void (void)
 cwist_mw_request_id: cwist_middleware_func (const char *)
 cwist_mw_session: cwist_middleware_func (cwist_app *)
@@ -1433,6 +1444,7 @@ cwist_redis_del: cwist_error_t (cwist_redis_t *, const char *)
 cwist_redis_expire: cwist_error_t (cwist_redis_t *, const char *, int)
 cwist_redis_get: cwist_error_t (cwist_redis_t *, const char *, char **)
 cwist_redis_pool_command_argv: cwist_error_t (cwist_redis_pool_t *, size_t, const void *const *, const size_t *, char **, size_t *)
+cwist_redis_pool_command_argv_reply_timeout: cwist_error_t (cwist_redis_pool_t *, size_t, const void *const *, const size_t *, unsigned timeout_ms, cwist_redis_reply_t **)
 cwist_redis_pool_create: cwist_redis_pool_t * (const char *, int, size_t)
 cwist_redis_pool_del: cwist_error_t (cwist_redis_pool_t *, const char *)
 cwist_redis_pool_destroy: void (cwist_redis_pool_t *)
@@ -1599,7 +1611,7 @@ cwist_wasm_component_dispose: void (void *)
 cwist_wasm_dispatch_memory: const char * (cwist_app *, const char *, size_t, size_t *)
 cwist_wasm_free: void (void *)
 
-== types (struct/union/enum names) (153) ==
+== types (struct/union/enum names) (155) ==
 __natsConnection
 __prim_cwist_error_t
 bdr_blob_t
@@ -1688,6 +1700,8 @@ cwist_json_builder
 cwist_jwt_chunk
 cwist_jwt_claims
 cwist_logger
+cwist_lwip_config
+cwist_lwip_server
 cwist_metric
 cwist_metric_id
 cwist_metric_type
@@ -1754,7 +1768,7 @@ cwist_zod_result
 quic_conn_fc
 quic_stream_fc
 
-== typedefs (207) ==
+== typedefs (210) ==
 __prim_cwist_error_t
 bdr_blob_t
 bdr_entry_t
@@ -1868,6 +1882,8 @@ cwist_jwt_claims
 cwist_log_format_t
 cwist_log_level_t
 cwist_logger
+cwist_lwip_config
+cwist_lwip_server
 cwist_macro_log_level_t
 cwist_metric_id_t
 cwist_metric_t
@@ -1903,6 +1919,7 @@ cwist_pb_writer
 cwist_query_bucket
 cwist_query_map
 cwist_query_map_iter_func
+cwist_rate_limit_redis_cfg_t
 cwist_rdbms_provider_t
 cwist_rdbms_runtime
 cwist_reactor_cb_t
@@ -2196,7 +2213,7 @@ CWIST_WS_FRAME_TEXT
 
 == globals (extern) (0) ==
 
-== layouts (83) ==
+== layouts (84) ==
 enum cwist_async_action_t [0] CWIST_ASYNC_CLOSE = 0
 enum cwist_async_action_t [1] CWIST_ASYNC_REARM
 enum cwist_async_action_t [2] CWIST_ASYNC_DETACH
@@ -2703,6 +2720,11 @@ struct cwist_query_map [1] size_t size
 struct cwist_query_map [2] uint8_t seed[16]
 struct cwist_query_map [3] void *arena
 struct cwist_query_map [4] size_t count
+struct cwist_rate_limit_redis_cfg_t [0] cwist_redis_pool_t *pool
+struct cwist_rate_limit_redis_cfg_t [1] int requests_per_minute
+struct cwist_rate_limit_redis_cfg_t [2] const char *key_prefix
+struct cwist_rate_limit_redis_cfg_t [3] unsigned timeout_ms
+struct cwist_rate_limit_redis_cfg_t [4] bool fail_closed
 struct cwist_rdbms_runtime [0] cwist_rdbms_provider_t provider
 struct cwist_rdbms_runtime [1] int port
 struct cwist_rdbms_runtime [2] char *host