feat: preserve unknown envelope keys, drop forbidden keys on decode (1.8.0)

Envelope and Meta gain an extras component so unknown top-level and meta keys
survive decode/re-encode, including the retry, DLQ and redrive paths. The five
forbidden keys are dropped on decode with a warning instead of being captured.
Payload minLength now counts Unicode code points. Adds conformance runners for
the roundtrip, data_shape, forbidden_keys and payload_schema_unicode sections
and a dependabot configuration.

Author: muhammetsafak

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "maven"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

CHANGELOG.md

@@ -9,6 +9,39 @@ The envelope wire format is versioned separately by `meta.schema_version`
 
 ## [Unreleased]
 
+## [1.8.0] - 2026-10-01
+
+MINOR release: additive public API (`extras`, `withAttempts`, `withDeadLetter`, a new
+`decode` overload). Note that `Envelope` and `Meta` gain a record component, so a Java 21
+record *deconstruction pattern* listing the old component count (`case Envelope(var a, …)`)
+must add the `extras` slot; `new Envelope(...)` / `new Meta(...)` calls with the previous
+argument counts keep compiling.
+
+### Fixed
+- **Unknown keys are preserved (message-envelope.md §4/§8, GR-5).** `Envelope` and `Meta`
+  gain an `extras` component (`Map<String,Object>`, insertion-ordered, unmodifiable, never
+  `null`): `EnvelopeCodec.decode` collects unknown top-level and `meta` keys into it, and
+  `encode` writes the known fields in their canonical order first, then the extras. Every
+  re-emit path carries them: retry (`Envelope.withAttempts`), DLQ (`DeadLetters.annotate`)
+  and redrive (`Redrive.reset`); the outbox relay already publishes the stored bytes verbatim.
+  Previously a decode → re-encode hop silently dropped them.
+- **Forbidden keys (§10, K-15).** `timestamp`, `meta.max_retries`, `meta.attempts`,
+  `meta.source` and `meta.ts` are never captured into extras and never encoded. Decode does
+  not reject them: it drops them and emits a warning naming the JSON pointer (e.g.
+  `/meta/attempts`) — via `System.Logger` at `WARNING`, or to the callback of the new
+  `EnvelopeCodec.decode(String, Consumer<String>)` overload.
+- **`minLength` counts Unicode code points** (ADR-0024), not UTF-16 code units — an emoji is
+  one character, matching every other SDK's payload validator.
+
+### Added
+- Additive API: `Envelope(..., extras)` / `Meta(..., extras)` canonical constructors (the
+  previous 6-arg / 5-arg constructors remain and yield empty extras),
+  `Envelope.withAttempts(int)`, `Envelope.withDeadLetter(DeadLetter)`,
+  `EnvelopeCodec.decode(String, Consumer<String>)`.
+- Conformance runners for the `roundtrip`, `data_shape`, `forbidden_keys` and
+  `payload_schema_unicode` manifest sections (none may be skipped).
+- `.github/dependabot.yml` (Maven + GitHub Actions, weekly).
+
 ## [1.7.0] - 2026-06-21
 
 ### Added

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.babelqueue</groupId>
     <artifactId>babelqueue-core</artifactId>
-    <version>1.7.0</version>
+    <version>1.8.0</version>
     <packaging>jar</packaging>
 
     <name>BabelQueue Core</name>

src/main/java/com/babelqueue/DeadLetters.java

@@ -37,12 +37,7 @@ public static Envelope annotate(
             attempts,
             EnvelopeCodec.SOURCE_LANG);
 
-        return new Envelope(
-            envelope.job(),
-            envelope.traceId(),
-            envelope.data(),
-            envelope.meta(),
-            envelope.attempts(),
-            deadLetter);
+        // withDeadLetter keeps every other component, including unknown top-level/meta keys.
+        return envelope.withDeadLetter(deadLetter);
     }
 }

src/main/java/com/babelqueue/Envelope.java

@@ -1,5 +1,7 @@
 package com.babelqueue;
 
+import java.util.Collections;
+import java.util.LinkedHashMap;
 import java.util.Map;
 
 /**
@@ -12,18 +14,57 @@
  * {@link EnvelopeCodec#decode}. The record is immutable; {@link DeadLetters}
  * returns a copy with a {@link DeadLetter} attached.
  *
+ * <p>Top-level keys this SDK does not know (a newer producer's additions) are kept in
+ * {@link #extras()} — in their original order — so a decode → re-emit chain (retry, DLQ,
+ * redrive) forwards them unchanged (message-envelope.md §8: consumers MUST tolerate unknown
+ * keys). Forbidden keys (§10) are never captured there and never encoded.
+ *
  * @param job        the message URN (never a class name)
  * @param traceId    correlation id, preserved across every hop
  * @param data       the pure-JSON payload
  * @param meta       the immutable metadata block
  * @param attempts   the top-level transport retry counter
  * @param deadLetter the dead-letter block, or {@code null} until dead-lettered
+ * @param extras     unknown top-level keys, insertion-ordered and unmodifiable (never {@code null})
  */
 public record Envelope(
     String job,
     String traceId,
     Map<String, Object> data,
     Meta meta,
     int attempts,
-    DeadLetter deadLetter
-) {}
+    DeadLetter deadLetter,
+    Map<String, Object> extras
+) {
+
+    /** Normalises {@code extras} to an unmodifiable, insertion-ordered copy ({@code null} → empty). */
+    public Envelope {
+        extras = extras == null || extras.isEmpty()
+            ? Map.of()
+            : Collections.unmodifiableMap(new LinkedHashMap<>(extras));
+    }
+
+    /** An envelope with no unknown top-level keys (the pre-1.8.0 shape). */
+    public Envelope(
+        String job,
+        String traceId,
+        Map<String, Object> data,
+        Meta meta,
+        int attempts,
+        DeadLetter deadLetter) {
+        this(job, traceId, data, meta, attempts, deadLetter, null);
+    }
+
+    /**
+     * A copy with {@code attempts} replaced — the retry/release path. Everything else,
+     * including the {@code meta} and top-level {@link #extras()}, is carried over unchanged.
+     */
+    public Envelope withAttempts(int attempts) {
+        return new Envelope(job, traceId, data, meta, attempts, deadLetter, extras);
+    }
+
+    /** A copy with the {@code dead_letter} block replaced ({@code null} removes it); extras are kept. */
+    public Envelope withDeadLetter(DeadLetter deadLetter) {
+        return new Envelope(job, traceId, data, meta, attempts, deadLetter, extras);
+    }
+}

src/main/java/com/babelqueue/EnvelopeCodec.java

@@ -2,7 +2,10 @@
 
 import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
 import java.util.UUID;
+import java.util.function.Consumer;
 
 /**
  * Builds, encodes and decodes the canonical BabelQueue envelope — the single Java
@@ -21,6 +24,23 @@ public final class EnvelopeCodec {
     /** The value stamped into {@code meta.lang} for envelopes produced here. */
     public static final String SOURCE_LANG = "java";
 
+    /** Top-level keys the codec maps to {@link Envelope} components ({@code urn} is the inbound alias of {@code job}). */
+    private static final Set<String> KNOWN_TOP_LEVEL =
+        Set.of("job", "urn", "trace_id", "data", "meta", "attempts", "dead_letter");
+
+    /** {@code meta} keys the codec maps to {@link Meta} components. */
+    private static final Set<String> KNOWN_META =
+        Set.of("id", "queue", "lang", "schema_version", "created_at");
+
+    /** Forbidden top-level keys (message-envelope.md §10) — never captured, never encoded. */
+    private static final Set<String> FORBIDDEN_TOP_LEVEL = Set.of("timestamp");
+
+    /** Forbidden {@code meta} keys (message-envelope.md §10) — never captured, never encoded. */
+    private static final Set<String> FORBIDDEN_META =
+        Set.of("max_retries", "attempts", "source", "ts");
+
+    private static final System.Logger LOGGER = System.getLogger(EnvelopeCodec.class.getName());
+
     private EnvelopeCodec() {}
 
     /** Build the canonical envelope for a {@code (urn, data)} pair on the default queue. */
@@ -79,7 +99,10 @@ public static Envelope fromMessage(PolyglotMessage message, String queue) {
 
     /**
      * Encode the envelope as compact UTF-8 JSON. Slashes and non-ASCII are left
-     * unescaped, matching the other SDK cores; the field order is canonical.
+     * unescaped, matching the other SDK cores; the field order is canonical. Known
+     * fields are written first in their canonical order, followed by the unknown keys
+     * kept in {@link Envelope#extras()} / {@link Meta#extras()} in their original order.
+     * Forbidden keys (§10) are never written, even if a caller put them into extras.
      */
     public static String encode(Envelope envelope) {
         Map<String, Object> root = new LinkedHashMap<>();
@@ -95,6 +118,7 @@ public static String encode(Envelope envelope) {
             metaMap.put("lang", meta.lang());
             metaMap.put("schema_version", meta.schemaVersion());
             metaMap.put("created_at", meta.createdAt());
+            appendExtras(metaMap, meta.extras(), KNOWN_META, FORBIDDEN_META);
         }
         root.put("meta", metaMap);
         root.put("attempts", envelope.attempts());
@@ -112,16 +136,46 @@ public static String encode(Envelope envelope) {
             root.put("dead_letter", dlMap);
         }
 
+        appendExtras(root, envelope.extras(), KNOWN_TOP_LEVEL, FORBIDDEN_TOP_LEVEL);
         return Json.write(root);
     }
 
+    private static void appendExtras(
+        Map<String, Object> target, Map<String, Object> extras, Set<String> known, Set<String> forbidden) {
+        if (extras == null) {
+            return;
+        }
+        for (Map.Entry<String, Object> entry : extras.entrySet()) {
+            String key = entry.getKey();
+            if (key == null || known.contains(key) || forbidden.contains(key) || target.containsKey(key)) {
+                continue;
+            }
+            target.put(key, entry.getValue());
+        }
+    }
+
     /**
      * Parse a raw JSON body into an {@link Envelope}. Malformed or non-object input
      * yields an empty envelope (so {@link #accepts} returns {@code false}); the
      * {@code urn} inbound alias is resolved into {@code job}. Does not validate the
      * contents — call {@link #accepts} first.
+     *
+     * <p>Unknown top-level and {@code meta} keys are preserved in the envelope's
+     * extras. Forbidden keys (message-envelope.md §10) are dropped and a warning is
+     * logged via {@link System.Logger} at {@code WARNING} level; use
+     * {@link #decode(String, Consumer)} to receive the warnings directly.
      */
     public static Envelope decode(String raw) {
+        return decode(raw, warning -> LOGGER.log(System.Logger.Level.WARNING, warning));
+    }
+
+    /**
+     * Same as {@link #decode(String)}, but each warning — one per forbidden key found
+     * and dropped, naming its JSON pointer (e.g. {@code /meta/attempts}) — is passed to
+     * {@code onWarning} instead of the logger. Forbidden keys do not reject the message.
+     */
+    public static Envelope decode(String raw, Consumer<String> onWarning) {
+        Objects.requireNonNull(onWarning, "onWarning");
         Object parsed;
         try {
             parsed = Json.parse(raw);
@@ -140,13 +194,29 @@ public static Envelope decode(String raw) {
             }
         }
 
+        Map<String, Object> extras = new LinkedHashMap<>();
+        for (Map.Entry<?, ?> entry : map.entrySet()) {
+            String key = String.valueOf(entry.getKey());
+            if (FORBIDDEN_TOP_LEVEL.contains(key)) {
+                onWarning.accept(forbiddenWarning("/" + key));
+            } else if (!KNOWN_TOP_LEVEL.contains(key)) {
+                extras.put(key, entry.getValue());
+            }
+        }
+
         return new Envelope(
             job,
             asString(map.get("trace_id")),
             asMap(map.get("data")),
-            parseMeta(map.get("meta")),
+            parseMeta(map.get("meta"), onWarning),
             asInt(map.get("attempts"), 0),
-            parseDeadLetter(map.get("dead_letter")));
+            parseDeadLetter(map.get("dead_letter")),
+            extras);
+    }
+
+    private static String forbiddenWarning(String pointer) {
+        return "BabelQueue: dropped forbidden envelope key " + pointer
+            + " (message-envelope.md §10); it will not be re-emitted";
     }
 
     /** The message URN — the canonical {@code job}, with the {@code urn} alias resolved by {@link #decode}. */
@@ -176,17 +246,27 @@ private static Envelope empty() {
         return new Envelope(null, null, null, null, 0, null);
     }
 
-    private static Meta parseMeta(Object value) {
+    private static Meta parseMeta(Object value, Consumer<String> onWarning) {
         Map<String, Object> map = asMap(value);
         if (map == null) {
             return null;
         }
+        Map<String, Object> extras = new LinkedHashMap<>();
+        for (Map.Entry<String, Object> entry : map.entrySet()) {
+            String key = entry.getKey();
+            if (FORBIDDEN_META.contains(key)) {
+                onWarning.accept(forbiddenWarning("/meta/" + key));
+            } else if (!KNOWN_META.contains(key)) {
+                extras.put(key, entry.getValue());
+            }
+        }
         return new Meta(
             asString(map.get("id")),
             asString(map.get("queue")),
             asString(map.get("lang")),
             asInt(map.get("schema_version"), 0),
-            asLong(map.get("created_at"), 0L));
+            asLong(map.get("created_at"), 0L),
+            extras);
     }
 
     private static DeadLetter parseDeadLetter(Object value) {

src/main/java/com/babelqueue/Meta.java

@@ -1,18 +1,42 @@
 package com.babelqueue;
 
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
 /**
  * The immutable per-message metadata block of a {@link Envelope}.
  *
+ * <p>{@code meta} keys this SDK does not know are kept in {@link #extras()} (insertion
+ * order preserved) and re-emitted after the known keys, so consumers stay
+ * forward-compatible (message-envelope.md §4: consumers MUST ignore — and a re-emitting
+ * hop MUST NOT drop — unknown {@code meta} keys). Forbidden keys (§10) are never kept.
+ *
  * @param id            a unique identifier for this specific message
  * @param queue         the logical queue the message was produced for
  * @param lang          the source SDK language (e.g. {@code "java"})
  * @param schemaVersion the wire envelope schema version
  * @param createdAt     creation time in Unix milliseconds, UTC
+ * @param extras        unknown {@code meta} keys, insertion-ordered and unmodifiable (never {@code null})
  */
 public record Meta(
     String id,
     String queue,
     String lang,
     int schemaVersion,
-    long createdAt
-) {}
+    long createdAt,
+    Map<String, Object> extras
+) {
+
+    /** Normalises {@code extras} to an unmodifiable, insertion-ordered copy ({@code null} → empty). */
+    public Meta {
+        extras = extras == null || extras.isEmpty()
+            ? Map.of()
+            : Collections.unmodifiableMap(new LinkedHashMap<>(extras));
+    }
+
+    /** A meta block with no unknown keys (the pre-1.8.0 shape). */
+    public Meta(String id, String queue, String lang, int schemaVersion, long createdAt) {
+        this(id, queue, lang, schemaVersion, createdAt, null);
+    }
+}

src/main/java/com/babelqueue/Redrive.java

@@ -132,10 +132,10 @@ public record Result(int redriven, int skipped, List<Item> items) {
     /**
      * Returns a copy of {@code env} reset for reprocessing: the {@code dead_letter} block is
      * removed and {@code attempts} reset to 0; {@code job}, {@code trace_id}, {@code data} and
-     * {@code meta} are preserved verbatim.
+     * {@code meta} are preserved verbatim, as are unknown top-level keys ({@link Envelope#extras()}).
      */
     public static Envelope reset(Envelope env) {
-        return new Envelope(env.job(), env.traceId(), env.data(), env.meta(), 0, null);
+        return new Envelope(env.job(), env.traceId(), env.data(), env.meta(), 0, null, env.extras());
     }
 
     /**

src/main/java/com/babelqueue/schema/PayloadValidator.java

@@ -104,7 +104,9 @@ private static String checkString(Map<?, ?> schema, Object value, String path) {
         if (!(value instanceof String str)) {
             return violation(path, "not_a_string");
         }
-        if (schema.get("minLength") instanceof Number min && str.length() < min.intValue()) {
+        // JSON Schema counts string length in Unicode code points, not UTF-16 units.
+        if (schema.get("minLength") instanceof Number min
+            && str.codePointCount(0, str.length()) < min.intValue()) {
             return violation(path, "below_min_length");
         }
         return null;