Fix encoder correctness gaps (float precision, control-char escaping, datetime) and add toml-test encoder harness

bin/toml-encoder

@@ -0,0 +1,115 @@
+#!/usr/bin/env php
+<?php
+
+/**
+ * toml-test encoder harness.
+ *
+ * Reads toml-test "tagged" JSON from STDIN and writes TOML to STDOUT, mirroring
+ * the decoder harness in bin/toml-decoder. Used to exercise the encoder against
+ * the official toml-test corpus (encoder mode) and in CI round-trip checks.
+ */
+
+require __DIR__ . '/../vendor/autoload.php';
+
+use PhpCollective\Toml\Toml;
+use PhpCollective\Toml\TomlVersion;
+use PhpCollective\Toml\Value\LocalDate;
+use PhpCollective\Toml\Value\LocalDateTime;
+use PhpCollective\Toml\Value\LocalTime;
+
+$input = stream_get_contents(STDIN);
+$version = match (getenv('TOML_VERSION')) {
+    '1.0' => TomlVersion::V10,
+    default => TomlVersion::V11,
+};
+
+try {
+    $decoded = json_decode((string)$input, false, 512, JSON_THROW_ON_ERROR);
+    $data = convertNode($decoded);
+    if (!is_array($data)) {
+        throw new RuntimeException('Top-level TOML value must be a table');
+    }
+
+    echo Toml::encode($data, new PhpCollective\Toml\Encoder\EncoderOptions(version: $version));
+} catch (Throwable $e) {
+    fwrite(STDERR, $e->getMessage() . "\n");
+    exit(1);
+}
+
+/**
+ * Converts a toml-test tagged JSON node into the PHP value the encoder expects.
+ */
+function convertNode(mixed $node): mixed
+{
+    if (is_array($node)) {
+        return array_map('convertNode', $node);
+    }
+
+    if (!$node instanceof stdClass) {
+        throw new RuntimeException('Unexpected JSON scalar at structural position');
+    }
+
+    // The current toml-test format represents arrays as plain JSON arrays (handled
+    // above); older tagged output uses {"type": "array", "value": [...]}. Support both.
+    if (isset($node->type) && $node->type === 'array' && isset($node->value) && is_array($node->value)) {
+        return array_map('convertNode', $node->value);
+    }
+
+    if (isTaggedLeaf($node)) {
+        return convertLeaf($node->type, $node->value);
+    }
+
+    $table = [];
+    foreach (get_object_vars($node) as $key => $child) {
+        $table[$key] = convertNode($child);
+    }
+
+    return $table;
+}
+
+/**
+ * A tagged leaf is an object with exactly the string keys `type` and `value`.
+ */
+function isTaggedLeaf(stdClass $node): bool
+{
+    $vars = get_object_vars($node);
+    if (count($vars) !== 2 || !array_key_exists('type', $vars) || !array_key_exists('value', $vars)) {
+        return false;
+    }
+
+    return is_string($vars['type']) && is_string($vars['value']);
+}
+
+function convertLeaf(string $type, string $value): mixed
+{
+    return match ($type) {
+        'string' => $value,
+        'integer' => toIntegerValue($value),
+        'bool' => $value === 'true',
+        'float' => toFloatValue($value),
+        'datetime' => new DateTimeImmutable($value),
+        'datetime-local' => new LocalDateTime($value),
+        'date-local' => new LocalDate($value),
+        'time-local' => new LocalTime($value),
+        default => throw new RuntimeException("Unknown tagged type: {$type}"),
+    };
+}
+
+function toIntegerValue(string $value): int
+{
+    if (preg_match('/^[+-]?\d+$/', $value) !== 1) {
+        throw new RuntimeException("Invalid integer literal: {$value}");
+    }
+
+    return (int)$value;
+}
+
+function toFloatValue(string $value): float
+{
+    return match ($value) {
+        'inf', '+inf' => INF,
+        '-inf' => -INF,
+        'nan', '+nan', '-nan' => NAN,
+        default => (float)$value,
+    };
+}

docs/reference/limitations.md

@@ -139,6 +139,17 @@ Toml::encode(['obj' => new MyClass()]);
 Toml::encode(['obj' => (array)$myObject]);
 ```
 
+## Empty Tables
+
+PHP arrays cannot distinguish an empty table from an empty array, so `encode()` emits an empty PHP array as an empty TOML array (`key = []`), never as an empty table (`[key]`).
+
+```php
+Toml::encode(['settings' => []]);
+// Output: settings = []   (not an empty [settings] table)
+```
+
+**Workaround:** Add at least one key, or build an `encodeDocument()` AST when an explicit empty table header is required. A pure decode/encode round trip of an empty table is therefore not byte-preserving in normalized mode.
+
 ## Recursive Structures
 
 Circular references throw `EncodeException`:

docs/reference/support-matrix.md

@@ -68,15 +68,16 @@ The default API behavior is TOML 1.1-compatible. Strict TOML 1.0 parsing/decodin
 
 | Feature | Status | Notes |
 |---------|--------|-------|
-| Strings | Supported | Encoded as basic strings |
+| Strings | Supported | Encoded as basic strings; control characters are escaped (`\uXXXX` / shorthand) so output stays valid TOML |
 | Integers | Supported | |
-| Floats | Supported | |
+| Floats | Supported | Round-tripped at full `double` precision (shortest exact representation) |
 | Booleans | Supported | |
 | Arrays | Supported | |
 | Nested tables | Supported | |
 | Array of tables | Supported | |
 | Quoted keys when needed | Supported | |
-| `DateTimeInterface` | Partial | Encoded as offset datetime with microseconds and offset |
+| `DateTimeInterface` | Partial | Encoded as offset datetime; zero fractional seconds are omitted and a `+00:00` offset is emitted as `Z` |
+| Empty tables | Not Yet | An empty PHP array encodes as `[]`; PHP cannot distinguish it from an empty table |
 | `PhpCollective\Toml\Value\LocalDate` | Supported | Encoded as local date literal |
 | `PhpCollective\Toml\Value\LocalTime` | Supported | Encoded as local time literal; strict TOML 1.0 mode normalizes missing seconds |
 | `PhpCollective\Toml\Value\LocalDateTime` | Supported | Encoded as local datetime literal; strict TOML 1.0 mode normalizes missing seconds |
@@ -135,6 +136,10 @@ These results were measured against the library's `bin/toml-decoder` adapter for
 
 Strict TOML 1.0 mode closes the previously documented invalid-case gaps for syntax that TOML 1.1 relaxes: multiline inline-table layout, inline-table trailing commas, `\xHH` byte escapes, and optional seconds in local times/datetimes.
 
+A leading UTF-8 BOM (`U+FEFF`) at the very start of a document is accepted and skipped, matching the `toml-test` corpus and common parsers.
+
+A matching `bin/toml-encoder` adapter implements the `toml-test` encoder protocol (tagged JSON to TOML). It is exercised by `TomlTestEncoderTest`, which encodes each fixture and decodes the result back to confirm valid, semantically equivalent output. Both adapters are skipped automatically when the `toml-test` corpus is not present locally.
+
 ## Recommended Use
 
 This library is well suited for:

src/Encoder/Encoder.php

@@ -140,7 +140,14 @@ private function encodeValue(mixed $value): string
             if (is_nan($value)) {
                 return 'nan';
             }
-            $str = (string)$value;
+
+            // (string) casts obey the `precision` ini (default 14) and silently drop
+            // significant digits, so floats no longer round-trip. json_encode() honors
+            // `serialize_precision=-1` and emits the shortest exact representation.
+            $str = json_encode($value, JSON_PRESERVE_ZERO_FRACTION);
+            if ($str === false) {
+                $str = (string)$value;
+            }
             if (!str_contains($str, '.') && !str_contains($str, 'e') && !str_contains($str, 'E')) {
                 $str .= '.0';
             }
@@ -157,7 +164,7 @@ private function encodeValue(mixed $value): string
         }
 
         if ($value instanceof DateTimeInterface) {
-            return $value->format('Y-m-d\TH:i:s.uP');
+            return $this->formatOffsetDateTime($value);
         }
 
         if ($value instanceof ValueLocalDateTime) {
@@ -302,7 +309,7 @@ private function encodeAstValue(Value $value): string
             $value instanceof IntegerValue => $this->encodeAstIntegerValue($value),
             $value instanceof FloatValue => $this->isReusableFloat($value) ? $value->raw : $this->encodeValue($value->value),
             $value instanceof BoolValue => $this->isReusableBool($value) ? $value->raw : ($value->value ? 'true' : 'false'),
-            $value instanceof OffsetDateTime => $this->isReusableOffsetDateTime($value) ? $value->raw : $value->value->format('Y-m-d\TH:i:s.uP'),
+            $value instanceof OffsetDateTime => $this->isReusableOffsetDateTime($value) ? $value->raw : $this->formatOffsetDateTime($value->value),
             $value instanceof LocalDateTime => $this->isReusableLocalDateTime($value) ? $value->raw : $this->normalizeLocalDateTimeLiteral($value->value),
             $value instanceof LocalDate => $this->isReusableLocalDate($value) ? $value->raw : $value->value,
             $value instanceof LocalTime => $this->isReusableLocalTime($value) ? $value->raw : $this->normalizeLocalTimeLiteral($value->value),
@@ -628,6 +635,25 @@ private function normalizeLocalTimeLiteral(string $value): string
             : $value;
     }
 
+    /**
+     * Formats an offset datetime, omitting the fractional-seconds part when it is
+     * zero (and trimming trailing zeros otherwise) so a value without sub-second
+     * precision does not gain a spurious `.000000`.
+     */
+    private function formatOffsetDateTime(DateTimeInterface $value): string
+    {
+        $literal = $value->format('Y-m-d\TH:i:s');
+        $fraction = rtrim($value->format('u'), '0');
+        if ($fraction !== '') {
+            $literal .= '.' . $fraction;
+        }
+
+        // Emit the idiomatic `Z` for UTC rather than `+00:00`.
+        $offset = $value->format('P');
+
+        return $literal . ($offset === '+00:00' ? 'Z' : $offset);
+    }
+
     private function encodeMultilineBasicString(string $value): string
     {
         $escaped = str_replace(
@@ -636,7 +662,9 @@ private function encodeMultilineBasicString(string $value): string
             $value,
         );
 
-        return "\"\"\"\n{$escaped}\"\"\"";
+        // Literal newlines stay raw in multiline strings; other control characters
+        // (the regex deliberately excludes \n) must still be escaped.
+        return "\"\"\"\n" . $this->escapeControlChars($escaped) . '"""';
     }
 
     /**
@@ -706,7 +734,7 @@ private function assertAstValueSupportsVersion(Value $value): void
         }
 
         if ($value instanceof OffsetDateTime) {
-            $literal = $value->raw !== '' ? $value->raw : $value->value->format('Y-m-d\TH:i:s.uP');
+            $literal = $value->raw !== '' ? $value->raw : $this->formatOffsetDateTime($value->value);
             if (!TemporalValidator::isValidOffsetDateTime($literal, TomlVersion::V10)) {
                 throw new EncodeException('Source-aware TOML 1.0 output cannot preserve offset datetimes without seconds');
             }
@@ -1264,7 +1292,22 @@ private function encodeString(string $value): string
             $value,
         );
 
-        return '"' . $escaped . '"';
+        return '"' . $this->escapeControlChars($escaped) . '"';
+    }
+
+    /**
+     * Escapes any remaining control characters (U+0000-U+001F and U+007F) that have
+     * no shorthand escape as `\uXXXX`. TOML basic strings forbid raw control bytes,
+     * so emitting them would produce invalid output. Operates byte-wise; UTF-8
+     * continuation bytes are always >= 0x80 and never match this range.
+     */
+    private function escapeControlChars(string $value): string
+    {
+        return preg_replace_callback(
+            '/[\x00-\x08\x0B\x0E-\x1F\x7F]/',
+            static fn (array $match): string => sprintf('\u%04X', ord($match[0])),
+            $value,
+        ) ?? $value;
     }
 
     /**

src/Lexer/Lexer.php

@@ -39,6 +39,12 @@ public function tokenize(): Generator
             return;
         }
 
+        // Skip a single optional leading UTF-8 BOM (U+FEFF, bytes EF BB BF). The
+        // toml-test conformance suite treats a leading BOM as valid input.
+        if ($this->pos === 0 && str_starts_with($this->input, "\u{FEFF}")) {
+            $this->pos = 3;
+        }
+
         while ($this->pos < $this->length) {
             $char = $this->input[$this->pos];
 

tests/Conformance/TomlTestDecoderTest.php

@@ -22,13 +22,54 @@ public function testTomlDecoderMatchesTaggedJsonFixtures(string $fixtureBase): v
 
         $output = $this->runDecoder(file_get_contents($tomlPath) ?: '');
 
+        // toml-test stores float values in varied textual forms (`300`, `1000.0`,
+        // `3.0e14`); the reference runner compares them numerically, so we do too.
         $this->assertEquals(
-            $this->decodeJsonFile($jsonPath),
-            $this->decodeJson($output),
+            self::normalizeFloatLeaves($this->decodeJsonFile($jsonPath)),
+            self::normalizeFloatLeaves($this->decodeJson($output)),
             basename($tomlPath),
         );
     }
 
+    /**
+     * Recursively replaces every `{"type": "float", "value": "..."}` leaf's textual
+     * value with a numeric one so comparisons are value-based rather than string-based.
+     *
+     * @param array<array-key, mixed> $data
+     *
+     * @return array<array-key, mixed>
+     */
+    public static function normalizeFloatLeaves(array $data): array
+    {
+        if (
+            ($data['type'] ?? null) === 'float'
+            && array_key_exists('value', $data)
+            && is_string($data['value'])
+        ) {
+            $data['value'] = self::canonicalFloat($data['value']);
+
+            return $data;
+        }
+
+        foreach ($data as $key => $value) {
+            if (is_array($value)) {
+                $data[$key] = self::normalizeFloatLeaves($value);
+            }
+        }
+
+        return $data;
+    }
+
+    private static function canonicalFloat(string $value): string|float
+    {
+        return match (strtolower($value)) {
+            'nan', '+nan', '-nan' => 'nan',
+            'inf', '+inf' => INF,
+            '-inf' => -INF,
+            default => (float)$value,
+        };
+    }
+
     public function testNumericLikeBareKeyWithLeadingZeroParsesAsKey(): void
     {
         $output = $this->runDecoder("-01 = true\n");