diff --git a/bin/toml-encoder b/bin/toml-encoder new file mode 100755 index 0000000..0ca1fa9 --- /dev/null +++ b/bin/toml-encoder @@ -0,0 +1,115 @@ +#!/usr/bin/env php + TomlVersion::V10, + default => TomlVersion::V11, +}; + +try { + $decoded = json_decode((string)$input, false, 512, JSON_THROW_ON_ERROR); + $data = convertNode($decoded); + if (!is_array($data)) { + throw new RuntimeException('Top-level TOML value must be a table'); + } + + echo Toml::encode($data, new PhpCollective\Toml\Encoder\EncoderOptions(version: $version)); +} catch (Throwable $e) { + fwrite(STDERR, $e->getMessage() . "\n"); + exit(1); +} + +/** + * Converts a toml-test tagged JSON node into the PHP value the encoder expects. + */ +function convertNode(mixed $node): mixed +{ + if (is_array($node)) { + return array_map('convertNode', $node); + } + + if (!$node instanceof stdClass) { + throw new RuntimeException('Unexpected JSON scalar at structural position'); + } + + // The current toml-test format represents arrays as plain JSON arrays (handled + // above); older tagged output uses {"type": "array", "value": [...]}. Support both. + if (isset($node->type) && $node->type === 'array' && isset($node->value) && is_array($node->value)) { + return array_map('convertNode', $node->value); + } + + if (isTaggedLeaf($node)) { + return convertLeaf($node->type, $node->value); + } + + $table = []; + foreach (get_object_vars($node) as $key => $child) { + $table[$key] = convertNode($child); + } + + return $table; +} + +/** + * A tagged leaf is an object with exactly the string keys `type` and `value`. + */ +function isTaggedLeaf(stdClass $node): bool +{ + $vars = get_object_vars($node); + if (count($vars) !== 2 || !array_key_exists('type', $vars) || !array_key_exists('value', $vars)) { + return false; + } + + return is_string($vars['type']) && is_string($vars['value']); +} + +function convertLeaf(string $type, string $value): mixed +{ + return match ($type) { + 'string' => $value, + 'integer' => toIntegerValue($value), + 'bool' => $value === 'true', + 'float' => toFloatValue($value), + 'datetime' => new DateTimeImmutable($value), + 'datetime-local' => new LocalDateTime($value), + 'date-local' => new LocalDate($value), + 'time-local' => new LocalTime($value), + default => throw new RuntimeException("Unknown tagged type: {$type}"), + }; +} + +function toIntegerValue(string $value): int +{ + if (preg_match('/^[+-]?\d+$/', $value) !== 1) { + throw new RuntimeException("Invalid integer literal: {$value}"); + } + + return (int)$value; +} + +function toFloatValue(string $value): float +{ + return match ($value) { + 'inf', '+inf' => INF, + '-inf' => -INF, + 'nan', '+nan', '-nan' => NAN, + default => (float)$value, + }; +} diff --git a/docs/reference/limitations.md b/docs/reference/limitations.md index 4e2842a..029e69c 100644 --- a/docs/reference/limitations.md +++ b/docs/reference/limitations.md @@ -139,6 +139,17 @@ Toml::encode(['obj' => new MyClass()]); Toml::encode(['obj' => (array)$myObject]); ``` +## Empty Tables + +PHP arrays cannot distinguish an empty table from an empty array, so `encode()` emits an empty PHP array as an empty TOML array (`key = []`), never as an empty table (`[key]`). + +```php +Toml::encode(['settings' => []]); +// Output: settings = [] (not an empty [settings] table) +``` + +**Workaround:** Add at least one key, or build an `encodeDocument()` AST when an explicit empty table header is required. A pure decode/encode round trip of an empty table is therefore not byte-preserving in normalized mode. + ## Recursive Structures Circular references throw `EncodeException`: diff --git a/docs/reference/support-matrix.md b/docs/reference/support-matrix.md index 25c7233..7d8aa07 100644 --- a/docs/reference/support-matrix.md +++ b/docs/reference/support-matrix.md @@ -68,15 +68,16 @@ The default API behavior is TOML 1.1-compatible. Strict TOML 1.0 parsing/decodin | Feature | Status | Notes | |---------|--------|-------| -| Strings | Supported | Encoded as basic strings | +| Strings | Supported | Encoded as basic strings; control characters are escaped (`\uXXXX` / shorthand) so output stays valid TOML | | Integers | Supported | | -| Floats | Supported | | +| Floats | Supported | Round-tripped at full `double` precision (shortest exact representation) | | Booleans | Supported | | | Arrays | Supported | | | Nested tables | Supported | | | Array of tables | Supported | | | Quoted keys when needed | Supported | | -| `DateTimeInterface` | Partial | Encoded as offset datetime with microseconds and offset | +| `DateTimeInterface` | Partial | Encoded as offset datetime; zero fractional seconds are omitted and a `+00:00` offset is emitted as `Z` | +| Empty tables | Not Yet | An empty PHP array encodes as `[]`; PHP cannot distinguish it from an empty table | | `PhpCollective\Toml\Value\LocalDate` | Supported | Encoded as local date literal | | `PhpCollective\Toml\Value\LocalTime` | Supported | Encoded as local time literal; strict TOML 1.0 mode normalizes missing seconds | | `PhpCollective\Toml\Value\LocalDateTime` | Supported | Encoded as local datetime literal; strict TOML 1.0 mode normalizes missing seconds | @@ -135,6 +136,10 @@ These results were measured against the library's `bin/toml-decoder` adapter for Strict TOML 1.0 mode closes the previously documented invalid-case gaps for syntax that TOML 1.1 relaxes: multiline inline-table layout, inline-table trailing commas, `\xHH` byte escapes, and optional seconds in local times/datetimes. +A leading UTF-8 BOM (`U+FEFF`) at the very start of a document is accepted and skipped, matching the `toml-test` corpus and common parsers. + +A matching `bin/toml-encoder` adapter implements the `toml-test` encoder protocol (tagged JSON to TOML). It is exercised by `TomlTestEncoderTest`, which encodes each fixture and decodes the result back to confirm valid, semantically equivalent output. Both adapters are skipped automatically when the `toml-test` corpus is not present locally. + ## Recommended Use This library is well suited for: diff --git a/src/Encoder/Encoder.php b/src/Encoder/Encoder.php index 80bf421..ee07e9c 100644 --- a/src/Encoder/Encoder.php +++ b/src/Encoder/Encoder.php @@ -140,7 +140,14 @@ private function encodeValue(mixed $value): string if (is_nan($value)) { return 'nan'; } - $str = (string)$value; + + // (string) casts obey the `precision` ini (default 14) and silently drop + // significant digits, so floats no longer round-trip. json_encode() honors + // `serialize_precision=-1` and emits the shortest exact representation. + $str = json_encode($value, JSON_PRESERVE_ZERO_FRACTION); + if ($str === false) { + $str = (string)$value; + } if (!str_contains($str, '.') && !str_contains($str, 'e') && !str_contains($str, 'E')) { $str .= '.0'; } @@ -157,7 +164,7 @@ private function encodeValue(mixed $value): string } if ($value instanceof DateTimeInterface) { - return $value->format('Y-m-d\TH:i:s.uP'); + return $this->formatOffsetDateTime($value); } if ($value instanceof ValueLocalDateTime) { @@ -302,7 +309,7 @@ private function encodeAstValue(Value $value): string $value instanceof IntegerValue => $this->encodeAstIntegerValue($value), $value instanceof FloatValue => $this->isReusableFloat($value) ? $value->raw : $this->encodeValue($value->value), $value instanceof BoolValue => $this->isReusableBool($value) ? $value->raw : ($value->value ? 'true' : 'false'), - $value instanceof OffsetDateTime => $this->isReusableOffsetDateTime($value) ? $value->raw : $value->value->format('Y-m-d\TH:i:s.uP'), + $value instanceof OffsetDateTime => $this->isReusableOffsetDateTime($value) ? $value->raw : $this->formatOffsetDateTime($value->value), $value instanceof LocalDateTime => $this->isReusableLocalDateTime($value) ? $value->raw : $this->normalizeLocalDateTimeLiteral($value->value), $value instanceof LocalDate => $this->isReusableLocalDate($value) ? $value->raw : $value->value, $value instanceof LocalTime => $this->isReusableLocalTime($value) ? $value->raw : $this->normalizeLocalTimeLiteral($value->value), @@ -628,6 +635,25 @@ private function normalizeLocalTimeLiteral(string $value): string : $value; } + /** + * Formats an offset datetime, omitting the fractional-seconds part when it is + * zero (and trimming trailing zeros otherwise) so a value without sub-second + * precision does not gain a spurious `.000000`. + */ + private function formatOffsetDateTime(DateTimeInterface $value): string + { + $literal = $value->format('Y-m-d\TH:i:s'); + $fraction = rtrim($value->format('u'), '0'); + if ($fraction !== '') { + $literal .= '.' . $fraction; + } + + // Emit the idiomatic `Z` for UTC rather than `+00:00`. + $offset = $value->format('P'); + + return $literal . ($offset === '+00:00' ? 'Z' : $offset); + } + private function encodeMultilineBasicString(string $value): string { $escaped = str_replace( @@ -636,7 +662,9 @@ private function encodeMultilineBasicString(string $value): string $value, ); - return "\"\"\"\n{$escaped}\"\"\""; + // Literal newlines stay raw in multiline strings; other control characters + // (the regex deliberately excludes \n) must still be escaped. + return "\"\"\"\n" . $this->escapeControlChars($escaped) . '"""'; } /** @@ -706,7 +734,7 @@ private function assertAstValueSupportsVersion(Value $value): void } if ($value instanceof OffsetDateTime) { - $literal = $value->raw !== '' ? $value->raw : $value->value->format('Y-m-d\TH:i:s.uP'); + $literal = $value->raw !== '' ? $value->raw : $this->formatOffsetDateTime($value->value); if (!TemporalValidator::isValidOffsetDateTime($literal, TomlVersion::V10)) { throw new EncodeException('Source-aware TOML 1.0 output cannot preserve offset datetimes without seconds'); } @@ -1264,7 +1292,22 @@ private function encodeString(string $value): string $value, ); - return '"' . $escaped . '"'; + return '"' . $this->escapeControlChars($escaped) . '"'; + } + + /** + * Escapes any remaining control characters (U+0000-U+001F and U+007F) that have + * no shorthand escape as `\uXXXX`. TOML basic strings forbid raw control bytes, + * so emitting them would produce invalid output. Operates byte-wise; UTF-8 + * continuation bytes are always >= 0x80 and never match this range. + */ + private function escapeControlChars(string $value): string + { + return preg_replace_callback( + '/[\x00-\x08\x0B\x0E-\x1F\x7F]/', + static fn (array $match): string => sprintf('\u%04X', ord($match[0])), + $value, + ) ?? $value; } /** diff --git a/src/Lexer/Lexer.php b/src/Lexer/Lexer.php index 99f3765..075aad9 100644 --- a/src/Lexer/Lexer.php +++ b/src/Lexer/Lexer.php @@ -39,6 +39,12 @@ public function tokenize(): Generator return; } + // Skip a single optional leading UTF-8 BOM (U+FEFF, bytes EF BB BF). The + // toml-test conformance suite treats a leading BOM as valid input. + if ($this->pos === 0 && str_starts_with($this->input, "\u{FEFF}")) { + $this->pos = 3; + } + while ($this->pos < $this->length) { $char = $this->input[$this->pos]; diff --git a/tests/Conformance/TomlTestDecoderTest.php b/tests/Conformance/TomlTestDecoderTest.php index 2258e12..c5d3ac5 100644 --- a/tests/Conformance/TomlTestDecoderTest.php +++ b/tests/Conformance/TomlTestDecoderTest.php @@ -22,13 +22,54 @@ public function testTomlDecoderMatchesTaggedJsonFixtures(string $fixtureBase): v $output = $this->runDecoder(file_get_contents($tomlPath) ?: ''); + // toml-test stores float values in varied textual forms (`300`, `1000.0`, + // `3.0e14`); the reference runner compares them numerically, so we do too. $this->assertEquals( - $this->decodeJsonFile($jsonPath), - $this->decodeJson($output), + self::normalizeFloatLeaves($this->decodeJsonFile($jsonPath)), + self::normalizeFloatLeaves($this->decodeJson($output)), basename($tomlPath), ); } + /** + * Recursively replaces every `{"type": "float", "value": "..."}` leaf's textual + * value with a numeric one so comparisons are value-based rather than string-based. + * + * @param array $data + * + * @return array + */ + public static function normalizeFloatLeaves(array $data): array + { + if ( + ($data['type'] ?? null) === 'float' + && array_key_exists('value', $data) + && is_string($data['value']) + ) { + $data['value'] = self::canonicalFloat($data['value']); + + return $data; + } + + foreach ($data as $key => $value) { + if (is_array($value)) { + $data[$key] = self::normalizeFloatLeaves($value); + } + } + + return $data; + } + + private static function canonicalFloat(string $value): string|float + { + return match (strtolower($value)) { + 'nan', '+nan', '-nan' => 'nan', + 'inf', '+inf' => INF, + '-inf' => -INF, + default => (float)$value, + }; + } + public function testNumericLikeBareKeyWithLeadingZeroParsesAsKey(): void { $output = $this->runDecoder("-01 = true\n"); diff --git a/tests/Conformance/TomlTestEncoderTest.php b/tests/Conformance/TomlTestEncoderTest.php new file mode 100644 index 0000000..7995bf1 --- /dev/null +++ b/tests/Conformance/TomlTestEncoderTest.php @@ -0,0 +1,112 @@ +markTestSkipped('toml-test corpus is not available in this environment.'); + } + + $jsonPath = $fixtureBase . '.json'; + $expected = file_get_contents($jsonPath); + $this->assertNotFalse($expected); + + $toml = $this->runScript(__DIR__ . '/../../bin/toml-encoder', $expected); + $actual = $this->runScript(__DIR__ . '/../../bin/toml-decoder', $toml); + + // Floats are compared numerically (see TomlTestDecoderTest::normalizeFloatLeaves). + $this->assertEquals( + TomlTestDecoderTest::normalizeFloatLeaves($this->decodeJson($expected)), + TomlTestDecoderTest::normalizeFloatLeaves($this->decodeJson($actual)), + basename($fixtureBase), + ); + } + + /** + * @return iterable + */ + public static function encoderFixtureProvider(): iterable + { + $fixtures = [ + 'valid/string/escape-esc', + 'valid/string/escapes', + 'valid/string/hex-escape', + 'valid/string/unicode-escape', + 'valid/float/exponent', + 'valid/float/zero', + 'valid/float/long', + 'valid/datetime/datetime', + 'valid/datetime/milliseconds', + 'valid/datetime/timezone', + 'valid/array/array', + 'valid/key/alphanum', + ]; + + foreach ($fixtures as $fixture) { + yield $fixture => ['/tmp/toml-test/tests/' . $fixture]; + } + } + + private function runScript(string $script, string $input): string + { + $descriptorSpec = [ + 0 => ['pipe', 'r'], + 1 => ['pipe', 'w'], + 2 => ['pipe', 'w'], + ]; + + $process = proc_open([PHP_BINARY, $script], $descriptorSpec, $pipes); + $this->assertIsResource($process); + + fwrite($pipes[0], $input); + fclose($pipes[0]); + + $stdout = stream_get_contents($pipes[1]); + fclose($pipes[1]); + $stderr = stream_get_contents($pipes[2]); + fclose($pipes[2]); + + $exitCode = proc_close($process); + + $this->assertSame(0, $exitCode, trim((string)$stderr)); + $this->assertNotFalse($stdout); + + return $stdout; + } + + /** + * @return array + */ + private function decodeJson(string $json): array + { + try { + $decoded = json_decode($json, true, 512, JSON_THROW_ON_ERROR); + } catch (JsonException $e) { + $this->fail($e->getMessage()); + } + + $this->assertIsArray($decoded); + + return $decoded; + } +} diff --git a/tests/Encoder/EncoderTest.php b/tests/Encoder/EncoderTest.php index c9dbbed..dba9b0c 100644 --- a/tests/Encoder/EncoderTest.php +++ b/tests/Encoder/EncoderTest.php @@ -16,6 +16,7 @@ use PhpCollective\Toml\Value\LocalDateTime; use PhpCollective\Toml\Value\LocalTime; use PhpCollective\Toml\Value\TomlInteger; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; final class EncoderTest extends TestCase @@ -700,4 +701,90 @@ public function testDiffFriendlyPresetProducesExpectedOutput(): void // Large array becomes multiline with trailing commas $this->assertStringContainsString("large = [\n 1,\n 2,\n 3,\n 4,\n]", $result); } + + /** + * @return iterable + */ + public static function precisionFloatProvider(): iterable + { + yield 'pi' => [3.141592653589793]; + yield 'long-decimal' => [123456789.123456789]; + yield 'max-double' => [1.7976931348623157e308]; + yield 'min-normal' => [2.2250738585072014e-308]; + yield 'tenth' => [0.1]; + } + + #[DataProvider('precisionFloatProvider')] + public function testEncodeFloatRoundTripsWithFullPrecision(float $value): void + { + // (string) casts obey precision=14 and drop digits; the encoder must emit a + // representation that decodes back to the exact same double. + $encoder = new Encoder(new EncoderOptions()); + + $toml = $encoder->encode(['x' => $value]); + + $this->assertSame($value, Toml::decode($toml)['x']); + } + + public function testEncodeEscapesControlCharactersInBasicString(): void + { + $encoder = new Encoder(new EncoderOptions()); + + $result = $encoder->encode([ + 'esc' => "\x1B", + 'nul' => "\x00", + 'del' => "\x7F", + 'unit' => "\x1F", + ]); + + $this->assertStringContainsString('esc = "\u001B"', $result); + $this->assertStringContainsString('nul = "\u0000"', $result); + $this->assertStringContainsString('del = "\u007F"', $result); + $this->assertStringContainsString('unit = "\u001F"', $result); + } + + public function testEncodeControlCharactersRoundTrip(): void + { + $encoder = new Encoder(new EncoderOptions()); + $value = "a\x00b\x1Bc\x7Fd"; + + $toml = $encoder->encode(['x' => $value]); + + $this->assertSame($value, Toml::decode($toml)['x']); + } + + public function testEncodeOffsetDateTimeOmitsZeroMicroseconds(): void + { + // A value without sub-second precision must not gain a spurious `.000000`. + $encoder = new Encoder(new EncoderOptions()); + + $result = $encoder->encode([ + 'x' => new DateTimeImmutable('1979-05-27T07:32:00-08:00'), + ]); + + $this->assertStringContainsString('x = 1979-05-27T07:32:00-08:00', $result); + $this->assertStringNotContainsString('.000000', $result); + } + + public function testEncodeOffsetDateTimeKeepsNonZeroFraction(): void + { + $encoder = new Encoder(new EncoderOptions()); + + $result = $encoder->encode([ + 'x' => new DateTimeImmutable('1979-05-27T07:32:00.500000-08:00'), + ]); + + $this->assertStringContainsString('x = 1979-05-27T07:32:00.5-08:00', $result); + } + + public function testEncodeOffsetDateTimeUsesZuluForUtc(): void + { + $encoder = new Encoder(new EncoderOptions()); + + $result = $encoder->encode([ + 'x' => new DateTimeImmutable('1987-07-05T17:45:00+00:00'), + ]); + + $this->assertStringContainsString('x = 1987-07-05T17:45:00Z', $result); + } } diff --git a/tests/Parser/ParserTest.php b/tests/Parser/ParserTest.php index d8fcf73..29be887 100644 --- a/tests/Parser/ParserTest.php +++ b/tests/Parser/ParserTest.php @@ -104,4 +104,36 @@ public function testArrayTrailingCommaAllowed(): void $this->assertInstanceOf(KeyValue::class, $kv); $this->assertSame([1, 2, 3], $kv->value->getValue()); } + + public function testParseSkipsLeadingUtf8Bom(): void + { + $parser = new Parser(); + $doc = $parser->parse("\u{FEFF}a = 1"); + + $this->assertEmpty($parser->getErrors()); + $kv = $doc->items[0]; + $this->assertInstanceOf(KeyValue::class, $kv); + $this->assertSame(['a'], $kv->key->parts); + $this->assertSame(1, $kv->value->getValue()); + } + + public function testParseSkipsLeadingBomBeforeComment(): void + { + $parser = new Parser(); + $doc = $parser->parse("\u{FEFF}# comment\nb = 2"); + + $this->assertEmpty($parser->getErrors()); + $kv = $doc->items[0]; + $this->assertInstanceOf(KeyValue::class, $kv); + $this->assertSame(['b'], $kv->key->parts); + } + + public function testBomOnlyStrippedAtStart(): void + { + // A BOM that is not the very first byte is a normal (invalid) character. + $parser = new Parser(); + $parser->parse("a = 1\n\u{FEFF}b = 2"); + + $this->assertNotEmpty($parser->getErrors()); + } }