ext/json: replace packed charmap bitmap with a byte-indexed lookup table

.github/actions/verify-generated-files/action.yml

@@ -11,6 +11,7 @@ runs:
         scripts/gdb/debug_gdb_scripts_gen.php
         Zend/zend_vm_gen.php
         ext/tokenizer/tokenizer_data_gen.php
+        ext/json/gen_json_escape_table.php
         build/gen_stub.php -f --generate-optimizer-info --verify
         ext/phar/makestub.php
         .github/scripts/test-directory-unchanged.sh .

ext/json/Makefile.frag

@@ -3,3 +3,8 @@ $(srcdir)/json_scanner.c $(srcdir)/php_json_scanner_defs.h: $(srcdir)/json_scann
 
 $(srcdir)/json_parser.tab.c $(srcdir)/json_parser.tab.h: $(srcdir)/json_parser.y
 	@$(YACC) $(YFLAGS) --defines -l $(srcdir)/json_parser.y -o $(srcdir)/json_parser.tab.c
+
+$(srcdir)/php_json_escape_table.h: $(srcdir)/gen_json_escape_table.php
+	@if test ! -z "$(PHP)"; then \
+		$(PHP) $(srcdir)/gen_json_escape_table.php; \
+	fi;

ext/json/gen_json_escape_table.php

@@ -0,0 +1,50 @@
+#!/usr/bin/env php
+<?php
+/**
+ * Generates php_json_escape_table.h. This file is not necessary to build
+ * PHP -- it's only necessary to regenerate php_json_escape_table.h if the
+ * set of bytes php_json_escape_string() must escape ever changes.
+ *
+ * Usage: php gen_json_escape_table.php
+ */
+
+function is_dirty(int $b): bool {
+    // Control characters (< 0x20), non-ASCII bytes (>= 0x80, which need
+    // UTF-8 decoding), and the ASCII specials " \ / < > & '.
+    return $b < 0x20 || $b >= 0x80
+        || $b === 0x22 // "
+        || $b === 0x5c // backslash
+        || $b === 0x2f // /
+        || $b === 0x3c // <
+        || $b === 0x3e // >
+        || $b === 0x26 // &
+        || $b === 0x27; // '
+}
+
+$result = <<<'HEADER'
+/* This file was generated by ext/json/gen_json_escape_table.php.
+ *
+ *                     DO NOT EDIT THIS FILE!
+ *
+ * Classifies which bytes php_json_escape_string() must escape: control
+ * characters (< 0x20), non-ASCII bytes (>= 0x80, which need UTF-8
+ * decoding), and the ASCII specials " \ / < > & '.
+ */
+
+static const bool php_json_escape_dirty_table[256] = {
+
+HEADER;
+
+for ($row = 0; $row < 16; $row++) {
+    $values = [];
+    for ($col = 0; $col < 16; $col++) {
+        $b = $row * 16 + $col;
+        $values[] = is_dirty($b) ? '1' : '0';
+    }
+    $result .= "\t" . implode(', ', $values) . ",\n";
+}
+
+$result .= "};\n";
+
+file_put_contents(__DIR__ . '/php_json_escape_table.h', $result);
+echo "Generated php_json_escape_table.h\n";

ext/json/json_encoder.c

@@ -22,6 +22,7 @@
 #include "zend_smart_str.h"
 #include "php_json.h"
 #include "php_json_encoder.h"
+#include "php_json_escape_table.h"
 #include "zend_portability.h"
 #include <zend_exceptions.h>
 #include "zend_enum.h"
@@ -384,12 +385,8 @@ zend_result php_json_escape_string(
 	pos = 0;
 
 	do {
-		static const uint32_t charmap[8] = {
-			0xffffffff, 0x500080c4, 0x10000000, 0x00000000,
-			0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff};
-
 		unsigned int us = (unsigned char)s[pos];
-		if (EXPECTED(!ZEND_BIT_TEST(charmap, us))) {
+		if (EXPECTED(!php_json_escape_dirty_table[us])) {
 			pos++;
 			len--;
 			if (len == 0) {

ext/json/php_json_escape_table.h

@@ -0,0 +1,27 @@
+/* This file was generated by ext/json/gen_json_escape_table.php.
+ *
+ *                     DO NOT EDIT THIS FILE!
+ *
+ * Classifies which bytes php_json_escape_string() must escape: control
+ * characters (< 0x20), non-ASCII bytes (>= 0x80, which need UTF-8
+ * decoding), and the ASCII specials " \ / < > & '.
+ */
+
+static const bool php_json_escape_dirty_table[256] = {
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+	0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+};

ext/json/tests/json_encode_byte_classification.phpt

@@ -0,0 +1,68 @@
+--TEST--
+json_encode() escapes every byte value 0x00-0xFF correctly
+--FILE--
+<?php
+
+/* Independent reference implementation of PHP's JSON string-escaping
+ * rules, deliberately not sharing any logic with ext/json's C
+ * implementation, so this test verifies behavior rather than trivially
+ * agreeing with whatever the implementation currently does. */
+function expected_escape(int $b, int $options): string {
+    switch ($b) {
+        case 0x22: return ($options & JSON_HEX_QUOT) ? "\\u0022" : '\"';
+        case 0x5c: return '\\\\';
+        case 0x2f: return ($options & JSON_UNESCAPED_SLASHES) ? '/' : '\/';
+        case 0x08: return '\b';
+        case 0x0c: return '\f';
+        case 0x0a: return '\n';
+        case 0x0d: return '\r';
+        case 0x09: return '\t';
+        case 0x3c: return ($options & JSON_HEX_TAG) ? "\\u003C" : '<';
+        case 0x3e: return ($options & JSON_HEX_TAG) ? "\\u003E" : '>';
+        case 0x26: return ($options & JSON_HEX_AMP) ? "\\u0026" : '&';
+        case 0x27: return ($options & JSON_HEX_APOS) ? "\\u0027" : "'";
+    }
+    if ($b < 0x20) {
+        return sprintf('\u%04x', $b);
+    }
+    return chr($b);
+}
+
+function check_ascii_range(int $options): void {
+    for ($b = 0x00; $b < 0x80; $b++) {
+        $expected = '"' . expected_escape($b, $options) . '"';
+        $actual = json_encode(chr($b), $options);
+        if ($actual !== $expected) {
+            printf("MISMATCH (options=%d) at byte 0x%02x: expected %s got %s\n",
+                $options, $b, var_export($expected, true), var_export($actual, true));
+        }
+    }
+}
+
+function check_lone_high_bytes(): void {
+    /* A single byte >= 0x80 is never valid UTF-8 on its own -- every one
+     * of these must be rejected as invalid UTF-8, not silently passed
+     * through unescaped. */
+    for ($b = 0x80; $b <= 0xff; $b++) {
+        $actual = json_encode(chr($b));
+        if ($actual !== false) {
+            printf("MISMATCH at byte 0x%02x: expected false (invalid UTF-8) got %s\n",
+                $b, var_export($actual, true));
+        }
+        if (json_last_error() !== JSON_ERROR_UTF8) {
+            printf("MISMATCH at byte 0x%02x: expected JSON_ERROR_UTF8, got error code %d\n",
+                $b, json_last_error());
+        }
+    }
+}
+
+check_ascii_range(0);
+check_ascii_range(JSON_UNESCAPED_SLASHES);
+check_ascii_range(JSON_HEX_QUOT | JSON_HEX_TAG | JSON_HEX_AMP | JSON_HEX_APOS);
+check_ascii_range(JSON_HEX_QUOT | JSON_HEX_TAG | JSON_HEX_AMP | JSON_HEX_APOS | JSON_UNESCAPED_SLASHES);
+check_lone_high_bytes();
+
+echo "Done\n";
+?>
+--EXPECT--
+Done