From da3e292a3a4aa381adca70d65e3ffd0ac996f2b4 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 9 Sep 2026 21:08:56 +0300 Subject: [PATCH 01/66] Give ParparVM real primitive class objects; Integer.TYPE was null javac lowers a primitive class literal to a read of the boxed type's own TYPE field, so `TYPE = int.class` inside Integer's initializer compiles to `getstatic TYPE; putstatic TYPE` -- it reads the field it is initializing and leaves it null. Integer, Long, Byte, Character and Double all declared TYPE that way and all had a null one; Short, Boolean and Float had no TYPE at all; and Void.TYPE was java.lang.Void, the wrapper, rather than void. Nothing threw. Measured on a translated binary before this change: TYPE Integer null=true TYPE Double null=true TYPE Void name=java.lang.Void map size 2 of 6 m.get(Integer.TYPE) -> "JAVA_DOUBLE" A Map keyed on them collapses onto the single null key, so every lookup answers with whatever was stored last. The translator's own Util.ctypeMap/sigTypeMap are exactly that shape, keyed on all nine, which is how this surfaced: it would have typed every primitive alike and emitted syntactically valid C with every primitive type wrong. The JDK declares a native for this for the same reason, and so does this: - nine scalar `struct clazz` objects in cn1_globals.m, with designated rather than positional initializers so a future field added to struct clazz cannot silently shift every value the way it would in the generated ones beside them - java_lang_Class_getPrimitiveClass, taking an int code rather than the JDK's String name -- this runs inside the wrapper class initializers, which are among the earliest code in the process, and decoding a Java String here would drag String.getBytes and the charset machinery into Integer's own clinit - isAssignableFrom and isInstance now test primitiveType before calling instanceofFunction, which indexes tables by classId; a primitive class carries a sentinel classId that no table has a row for __codenameOneParentClsReference has to be set to class__java_lang_Class as the generated clazz objects do. CN1_CLASS_OF reads it to find the vtable when a clazz is used as an ordinary object, which is what happens the moment one becomes a Map key -- leaving it zero segfaults on the first hashCode(), well away from anything that names it. PrimitiveTypeIntegrationTest compares a translated run against a real JVM rather than a hard-coded expectation, because the failure was self-consistent and silent: only an independent reference catches it. Confirmed to fail when TYPE = int.class is put back. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.h | 28 +++ vm/ByteCodeTranslator/src/cn1_globals.m | 48 ++++ vm/ByteCodeTranslator/src/nativeMethods.m | 44 ++++ vm/JavaAPI/src/java/lang/Boolean.java | 6 + vm/JavaAPI/src/java/lang/Byte.java | 2 +- vm/JavaAPI/src/java/lang/Character.java | 2 +- vm/JavaAPI/src/java/lang/Class.java | 30 +++ vm/JavaAPI/src/java/lang/Double.java | 2 +- vm/JavaAPI/src/java/lang/Float.java | 6 + vm/JavaAPI/src/java/lang/Integer.java | 2 +- vm/JavaAPI/src/java/lang/Long.java | 2 +- vm/JavaAPI/src/java/lang/Short.java | 6 + vm/JavaAPI/src/java/lang/Void.java | 2 +- .../PrimitiveTypeIntegrationTest.java | 215 ++++++++++++++++++ .../tools/translator/PrimitiveTypeApp.java | 75 ++++++ 15 files changed, 464 insertions(+), 6 deletions(-) create mode 100644 vm/tests/src/test/java/com/codename1/tools/translator/PrimitiveTypeIntegrationTest.java create mode 100644 vm/tests/src/test/resources/com/codename1/tools/translator/PrimitiveTypeApp.java diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index b12ce93136d..6f0517b7073 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -2827,6 +2827,34 @@ extern struct clazz class_array1__JAVA_DOUBLE; extern struct clazz class_array2__JAVA_DOUBLE; extern struct clazz class_array3__JAVA_DOUBLE; +/** + * The nine scalar primitive class objects -- int.class, Integer.TYPE and friends. + * + * javac lowers a primitive class literal to a read of the boxed type's own TYPE + * field, so `TYPE = int.class` inside Integer's initializer compiles to + * `getstatic TYPE; putstatic TYPE` -- it reads the field it is initializing and + * leaves it null. Every wrapper that declared TYPE that way had a null one, and + * a Map keyed on them collapsed to a single entry, so a lookup for int answered + * with whatever type was stored last. Nothing threw. The wrappers now go through + * java_lang_Class_getPrimitiveClass, which hands back one of these. + * + * classId is CN1_PRIMITIVE_CLASS_ID for all nine: these never take part in an + * instanceof, and instanceofFunction indexes tables by classId, so the callers + * that could reach one (isAssignableFrom, isInstance) test primitiveType first + * rather than indexing with a value no table has a row for. + */ +#define CN1_PRIMITIVE_CLASS_ID (-1) + +extern struct clazz cn1_primitive_class_int; +extern struct clazz cn1_primitive_class_long; +extern struct clazz cn1_primitive_class_short; +extern struct clazz cn1_primitive_class_byte; +extern struct clazz cn1_primitive_class_char; +extern struct clazz cn1_primitive_class_float; +extern struct clazz cn1_primitive_class_double; +extern struct clazz cn1_primitive_class_boolean; +extern struct clazz cn1_primitive_class_void; + extern JAVA_OBJECT newString(CODENAME_ONE_THREAD_STATE, int length, JAVA_CHAR data[]); /** * Like newStringFromCString but DECODES, in the PLATFORM's encoding, instead of diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index c4a93106b1a..8825ffac59f 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -956,6 +956,54 @@ static void init_gc_thresholds() { //#define DEBUG_GC_OBJECTS_IN_HEAP +/** + * Scalar primitive class objects. See the comment on CN1_PRIMITIVE_CLASS_ID in + * cn1_globals.h for why these exist and why their classId is a sentinel. + * + * baseClass is 0 because int.class.getSuperclass() is null, which + * java_lang_Class_getSuperclass already returns for a null baseClass. isArray is + * false and arrayType is 0: these are the scalar types, not the array classes, + * which already exist as class_arrayN__JAVA_*. + * + * Designated initializers, unlike the positional generated ones beside them, so + * that a future field added to struct clazz cannot silently shift every value. + */ +/* + * __codenameOneParentClsReference is the class OF this object. Every generated + * clazz sets it to class__java_lang_Class, and CN1_CLASS_OF reads it to find the + * vtable when a clazz is used as an ordinary object -- which is what happens the + * moment one becomes a Map key. Leaving it zero segfaults on the first + * hashCode(), well away from anything that names it. + * + * The comment sits outside the macro on purpose: backslash-newline splicing + * happens before comments are removed, so an unbackslashed comment line inside + * the macro would silently end the definition. + */ +#define CN1_DEFINE_PRIMITIVE_CLASS(cname, jname) \ +struct clazz cn1_primitive_class_##cname = { \ + .__codenameOneParentClsReference = &class__java_lang_Class, \ + .classId = CN1_PRIMITIVE_CLASS_ID, \ + .clsName = jname, \ + .isArray = JAVA_FALSE, \ + .dimensions = 0, \ + .arrayType = 0, \ + .primitiveType = JAVA_TRUE, \ + .baseClass = 0, \ + .baseInterfaces = EMPTY_INTERFACES, \ + .baseInterfaceCount = 0, \ + .initialized = JAVA_TRUE \ +} + +CN1_DEFINE_PRIMITIVE_CLASS(int, "int"); +CN1_DEFINE_PRIMITIVE_CLASS(long, "long"); +CN1_DEFINE_PRIMITIVE_CLASS(short, "short"); +CN1_DEFINE_PRIMITIVE_CLASS(byte, "byte"); +CN1_DEFINE_PRIMITIVE_CLASS(char, "char"); +CN1_DEFINE_PRIMITIVE_CLASS(float, "float"); +CN1_DEFINE_PRIMITIVE_CLASS(double, "double"); +CN1_DEFINE_PRIMITIVE_CLASS(boolean, "boolean"); +CN1_DEFINE_PRIMITIVE_CLASS(void, "void"); + struct clazz class_array1__JAVA_BOOLEAN = { DEBUG_GC_INIT 0, 0, 0, 0, 0, 0, 0, cn1_array_1_id_JAVA_BOOLEAN, "boolean[]", JAVA_TRUE, 1, &class__java_lang_Boolean, JAVA_TRUE, &class__java_lang_Object, EMPTY_INTERFACES, 0, 0, 0 }; diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 4e88a1d8ebc..245bca39bb9 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -1973,6 +1973,41 @@ JAVA_OBJECT java_lang_Class_getName___R_java_lang_String(CODENAME_ONE_THREAD_STA return newStringFromCString(threadStateData, clz->clsName); } +/** + * Backs Integer.TYPE and the eight other wrapper TYPE fields. The JDK needs a + * native here for the same reason we do: `TYPE = int.class` cannot initialize the + * field, because javac lowers a primitive class literal to a read of that very + * field (getstatic TYPE; putstatic TYPE), leaving it null. + * + * Takes an int code rather than the JDK's String name deliberately. This runs + * inside the wrapper class initializers, which are among the earliest code in the + * process, and decoding a Java String here would drag in String.getBytes and the + * charset machinery during Integer's own clinit. An int argument allocates + * nothing and initializes nothing. + * + * The codes are an implementation detail shared only with java/lang/Class.java; + * they are matched by CN1_PRIM_* there. + */ +JAVA_OBJECT java_lang_Class_getPrimitiveClass___int_R_java_lang_Class(CODENAME_ONE_THREAD_STATE, JAVA_INT typeCode) { + switch(typeCode) { + case 0: return (JAVA_OBJECT)&cn1_primitive_class_int; + case 1: return (JAVA_OBJECT)&cn1_primitive_class_long; + case 2: return (JAVA_OBJECT)&cn1_primitive_class_short; + case 3: return (JAVA_OBJECT)&cn1_primitive_class_byte; + case 4: return (JAVA_OBJECT)&cn1_primitive_class_char; + case 5: return (JAVA_OBJECT)&cn1_primitive_class_float; + case 6: return (JAVA_OBJECT)&cn1_primitive_class_double; + case 7: return (JAVA_OBJECT)&cn1_primitive_class_boolean; + case 8: return (JAVA_OBJECT)&cn1_primitive_class_void; + } + // Only java/lang/Class.java calls this, always with one of its own constants, + // so this is unreachable short of the two files disagreeing. Returning null + // would restore exactly the silent null TYPE this code exists to remove. + fprintf(stderr, "getPrimitiveClass: unknown primitive type code %d\n", (int)typeCode); + exit(1); + return JAVA_NULL; +} + JAVA_BOOLEAN java_lang_Class_isArray___R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT cls) { struct clazz* clz = (struct clazz*)cls; return clz->isArray; @@ -1988,6 +2023,12 @@ JAVA_BOOLEAN java_lang_Class_isArray___R_boolean(CODENAME_ONE_THREAD_STATE, JAVA JAVA_BOOLEAN java_lang_Class_isAssignableFrom___java_lang_Class_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT cls, JAVA_OBJECT cls2) { struct clazz* clz1 = (struct clazz*)cls; struct clazz* clz2 = (struct clazz*)cls2; + // A primitive class carries CN1_PRIMITIVE_CLASS_ID, which indexes no row of + // the instanceof tables, so it must never reach instanceofFunction. The JDK + // rule is also simply identity: int is assignable only from int. + if(clz1->primitiveType || clz2->primitiveType) { + return clz1 == clz2 ? JAVA_TRUE : JAVA_FALSE; + } // A.isAssignableFrom(B): target is A, the class under test is B. return instanceofFunction(clz1->classId, clz2->classId); } @@ -1995,6 +2036,9 @@ JAVA_BOOLEAN java_lang_Class_isAssignableFrom___java_lang_Class_R_boolean(CODENA JAVA_BOOLEAN java_lang_Class_isInstance___java_lang_Object_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT cls, JAVA_OBJECT obj) { if(obj == JAVA_NULL) { return JAVA_FALSE; } struct clazz* clz1 = (struct clazz*)cls; + // No object is ever an instance of a primitive class, and its sentinel + // classId indexes no instanceof table row -- see isAssignableFrom above. + if(((struct clazz*)cls)->primitiveType) { return JAVA_FALSE; } struct clazz* clz2 = (struct clazz*)CN1_CLASS_OF(obj); // tag-aware: a tagged Integer has no header // A.isInstance(o): target is A, the class under test is o's class. These were // reversed, so isInstance searched the TARGET's supertype table for the diff --git a/vm/JavaAPI/src/java/lang/Boolean.java b/vm/JavaAPI/src/java/lang/Boolean.java index 043fee9956d..2f56aa21520 100644 --- a/vm/JavaAPI/src/java/lang/Boolean.java +++ b/vm/JavaAPI/src/java/lang/Boolean.java @@ -27,6 +27,12 @@ * Since: JDK1.0, CLDC 1.0 */ public final class Boolean implements Comparable { + + /** + * The class object for the primitive type this class wraps. + */ + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_BOOLEAN); + /** * The Boolean object corresponding to the primitive value false. */ diff --git a/vm/JavaAPI/src/java/lang/Byte.java b/vm/JavaAPI/src/java/lang/Byte.java index 9a7fa9d99e9..b7b5ff186f0 100644 --- a/vm/JavaAPI/src/java/lang/Byte.java +++ b/vm/JavaAPI/src/java/lang/Byte.java @@ -28,7 +28,7 @@ */ public final class Byte extends Number implements Comparable { - public static final Class TYPE = byte.class; + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_BYTE); public static final int SIZE = 8; /** diff --git a/vm/JavaAPI/src/java/lang/Character.java b/vm/JavaAPI/src/java/lang/Character.java index 93ce6f67946..448f24d6a51 100644 --- a/vm/JavaAPI/src/java/lang/Character.java +++ b/vm/JavaAPI/src/java/lang/Character.java @@ -155,7 +155,7 @@ public final class Character implements Comparable{ //public static final int MAX_RADIX = 36; //public static final char MIN_VALUE = '\0'; //public static final char MAX_VALUE = '\uFFFF'; - public static final Class TYPE = char.class; + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_CHAR); public static final byte UNASSIGNED = 0; public static final byte UPPERCASE_LETTER = 1; public static final byte LOWERCASE_LETTER = 2; diff --git a/vm/JavaAPI/src/java/lang/Class.java b/vm/JavaAPI/src/java/lang/Class.java index 00b5f6466ec..4e2f0be4ce3 100644 --- a/vm/JavaAPI/src/java/lang/Class.java +++ b/vm/JavaAPI/src/java/lang/Class.java @@ -139,6 +139,36 @@ public java.io.InputStream getResourceAsStream(java.lang.String name){ return null; } + /** + * Type codes for {@link #getPrimitiveClass(int)}. Shared only with + * nativeMethods.m, which switches on the same values. + */ + static final int CN1_PRIM_INT = 0; + static final int CN1_PRIM_LONG = 1; + static final int CN1_PRIM_SHORT = 2; + static final int CN1_PRIM_BYTE = 3; + static final int CN1_PRIM_CHAR = 4; + static final int CN1_PRIM_FLOAT = 5; + static final int CN1_PRIM_DOUBLE = 6; + static final int CN1_PRIM_BOOLEAN = 7; + static final int CN1_PRIM_VOID = 8; + + /** + * Returns the class object for a primitive type, e.g. the one + * {@code int.class} and {@link Integer#TYPE} denote. + * + * The wrapper classes cannot initialize their {@code TYPE} fields with a + * primitive class literal: javac lowers {@code int.class} to a read of + * {@code Integer.TYPE} itself, so {@code TYPE = int.class} compiles to + * {@code getstatic TYPE; putstatic TYPE} and leaves the field null. The JDK + * declares an equivalent native for the same reason. + * + * Takes an int code rather than a name so that it allocates nothing and + * decodes nothing: it runs inside the wrapper class initializers, which are + * among the earliest code in the process. + */ + static native Class getPrimitiveClass(int typeCode); + /** * Determines if this Class object represents an array class. */ diff --git a/vm/JavaAPI/src/java/lang/Double.java b/vm/JavaAPI/src/java/lang/Double.java index 22f161feb5d..a58612b9c38 100644 --- a/vm/JavaAPI/src/java/lang/Double.java +++ b/vm/JavaAPI/src/java/lang/Double.java @@ -29,7 +29,7 @@ */ public final class Double extends Number implements Comparable { - public static final Class TYPE = double.class; + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_DOUBLE); /** * The largest positive finite value of type double. It is equal to the value returned by Double.longBitsToDouble(0x7fefffffffffffffL) * See Also:Constant Field Values diff --git a/vm/JavaAPI/src/java/lang/Float.java b/vm/JavaAPI/src/java/lang/Float.java index 5b257d00f64..a0a32ba6971 100644 --- a/vm/JavaAPI/src/java/lang/Float.java +++ b/vm/JavaAPI/src/java/lang/Float.java @@ -28,6 +28,12 @@ * Since: JDK1.0, CLDC 1.1 */ public final class Float extends Number implements Comparable { + + /** + * The class object for the primitive type this class wraps. + */ + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_FLOAT); + /** * The largest positive value of type float. It is equal to the value returned by Float.intBitsToFloat(0x7f7fffff). * See Also:Constant Field Values diff --git a/vm/JavaAPI/src/java/lang/Integer.java b/vm/JavaAPI/src/java/lang/Integer.java index 0bcf391a733..f9842015071 100644 --- a/vm/JavaAPI/src/java/lang/Integer.java +++ b/vm/JavaAPI/src/java/lang/Integer.java @@ -29,7 +29,7 @@ */ public final class Integer extends Number implements Comparable { - public static final Class TYPE = int.class; + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_INT); private static final char[] DIGITS = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', diff --git a/vm/JavaAPI/src/java/lang/Long.java b/vm/JavaAPI/src/java/lang/Long.java index 0e938265153..fce50a48abd 100644 --- a/vm/JavaAPI/src/java/lang/Long.java +++ b/vm/JavaAPI/src/java/lang/Long.java @@ -29,7 +29,7 @@ */ public final class Long extends Number implements Comparable { - public static Class TYPE = long.class; + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_LONG); /** * The largest value of type long. diff --git a/vm/JavaAPI/src/java/lang/Short.java b/vm/JavaAPI/src/java/lang/Short.java index 233a1698268..f0800e1f219 100644 --- a/vm/JavaAPI/src/java/lang/Short.java +++ b/vm/JavaAPI/src/java/lang/Short.java @@ -27,6 +27,12 @@ * Since: JDK1.1, CLDC 1.0 */ public final class Short extends Number implements Comparable { + + /** + * The class object for the primitive type this class wraps. + */ + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_SHORT); + /** * The maximum value a Short can have. * See Also:Constant Field Values diff --git a/vm/JavaAPI/src/java/lang/Void.java b/vm/JavaAPI/src/java/lang/Void.java index c1391f982e0..96dbd87a71e 100644 --- a/vm/JavaAPI/src/java/lang/Void.java +++ b/vm/JavaAPI/src/java/lang/Void.java @@ -27,5 +27,5 @@ * @author Shai Almog */ public final class Void { - public static final Class TYPE = Void.class; + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_VOID); } diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/PrimitiveTypeIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/PrimitiveTypeIntegrationTest.java new file mode 100644 index 00000000000..c73f2e967d3 --- /dev/null +++ b/vm/tests/src/test/java/com/codename1/tools/translator/PrimitiveTypeIntegrationTest.java @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import org.junit.jupiter.api.Test; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Pins the nine primitive class objects -- {@code Integer.TYPE} and friends -- against + * the JVM. + * + *

These were all null on ParparVM until the primitive {@code struct clazz} objects + * existed. javac lowers a primitive class literal to a read of the boxed type's own + * {@code TYPE} field, so {@code TYPE = int.class} compiled to + * {@code getstatic TYPE; putstatic TYPE} and left the field null. Nothing threw: a + * {@code Map} keyed on them collapsed to a single entry and answered every lookup with + * whatever had been stored last, which is exactly how the translator's own + * primitive-to-C-type maps in {@code Util} would have typed every primitive alike.

+ * + *

Comparing against a real JVM rather than a hard-coded expectation is deliberate -- + * the failure mode here was self-consistent and silent, so only an independent + * reference catches it.

+ */ +class PrimitiveTypeIntegrationTest { + + @Test + void primitiveClassObjectsMatchTheJvm() throws Exception { + Parser.cleanup(); + + Path sourceDir = Files.createTempDirectory("primitive-type-sources"); + Path classesDir = Files.createTempDirectory("primitive-type-classes"); + Path javaApiDir = Files.createTempDirectory("primitive-type-java-api"); + + Path source = sourceDir.resolve("PrimitiveTypeApp.java"); + Files.write(source, loadAppSource().getBytes(StandardCharsets.UTF_8)); + + CompilerHelper.CompilerConfig config = selectCompiler(); + if (config == null) { + fail("No compatible compiler available for the primitive type integration test"); + } + assertTrue(CompilerHelper.isJavaApiCompatible(config), + "JDK " + config.jdkVersion + " must target matching bytecode level for JavaAPI"); + + CompilerHelper.compileJavaAPI(javaApiDir, config); + + List compileArgs = new ArrayList<>(); + compileArgs.add("-source"); + compileArgs.add(config.targetVersion); + compileArgs.add("-target"); + compileArgs.add(config.targetVersion); + if (CompilerHelper.useClasspath(config)) { + compileArgs.add("-classpath"); + compileArgs.add(javaApiDir.toString()); + } else { + compileArgs.add("-bootclasspath"); + compileArgs.add(javaApiDir.toString()); + compileArgs.add("-Xlint:-options"); + } + compileArgs.add("-d"); + compileArgs.add(classesDir.toString()); + compileArgs.add(source.toString()); + + assertEquals(0, CompilerHelper.compile(config.jdkHome, compileArgs), + "PrimitiveTypeApp should compile against the JavaAPI"); + + Map expected = parseCases(runJavaMain(config, classesDir, javaApiDir)); + assertFalse(expected.isEmpty(), "JVM run should emit cases"); + + CompilerHelper.copyDirectory(javaApiDir, classesDir); + + Path outputDir = Files.createTempDirectory("primitive-type-output"); + CleanTargetIntegrationTest.runTranslator(classesDir, outputDir, "PrimitiveTypeApp"); + + Path distDir = outputDir.resolve("dist"); + Path cmakeLists = distDir.resolve("CMakeLists.txt"); + assertTrue(Files.exists(cmakeLists), "Translator should emit a CMake project"); + CleanTargetIntegrationTest.replaceLibraryWithExecutableTarget(cmakeLists, "PrimitiveTypeApp-src"); + + Path buildDir = distDir.resolve("build"); + Files.createDirectories(buildDir); + CleanTargetIntegrationTest.runCommand(Arrays.asList( + "cmake", + "-S", distDir.toString(), + "-B", buildDir.toString(), + "-DCMAKE_C_COMPILER=clang", + "-DCMAKE_OBJC_COMPILER=clang" + ), distDir); + CleanTargetIntegrationTest.runCommand(Arrays.asList("cmake", "--build", buildDir.toString()), distDir); + + Path executable = buildDir.resolve("PrimitiveTypeApp"); + String parparOutput = CleanTargetIntegrationTest.runCommand( + Arrays.asList(executable.toString()), buildDir); + assertTrue(parparOutput.contains("DONE"), + "ParparVM run should complete. Output: " + parparOutput); + + Map actual = parseCases(parparOutput); + assertEquals(expected.keySet(), actual.keySet(), "ParparVM should emit the same cases"); + + List differences = new ArrayList<>(); + for (Map.Entry entry : expected.entrySet()) { + if (!entry.getValue().equals(actual.get(entry.getKey()))) { + differences.add(entry.getKey() + + "\n jvm : " + entry.getValue() + + "\n parparvm: " + actual.get(entry.getKey())); + } + } + assertTrue(differences.isEmpty(), + "Primitive class objects diverged from the JVM:\n" + String.join("\n", differences)); + + // Stated explicitly so a regression names the original symptom rather than + // showing up only as a generic diff. + assertEquals("int", actual.get("name.int"), "Integer.TYPE must be the int class"); + assertEquals("void", actual.get("name.void"), "Void.TYPE must be void, not java.lang.Void"); + assertEquals("9", actual.get("distinctIdentities"), + "the nine primitive class objects must be distinct"); + assertEquals("9", actual.get("mapSize"), + "a Map keyed on the nine must hold nine entries, not collapse onto null"); + assertEquals("0", actual.get("lookupFailures"), + "each primitive class must look up its own value"); + } + + private Map parseCases(String output) { + Map cases = new LinkedHashMap<>(); + for (String line : output.split("\\R")) { + if (!line.startsWith("CASE|")) { + continue; + } + String body = line.substring("CASE|".length()); + int separator = body.indexOf('|'); + assertTrue(separator > 0, "Malformed case line: " + line); + cases.put(body.substring(0, separator), body.substring(separator + 1)); + } + return cases; + } + + private String loadAppSource() throws Exception { + java.io.InputStream in = PrimitiveTypeIntegrationTest.class + .getResourceAsStream("/com/codename1/tools/translator/PrimitiveTypeApp.java"); + assertNotNull(in, "PrimitiveTypeApp.java test resource should exist"); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) { + return reader.lines().collect(Collectors.joining("\n")) + "\n"; + } + } + + private String runJavaMain(CompilerHelper.CompilerConfig config, Path classesDir, Path javaApiDir) + throws Exception { + String javaExe = config.jdkHome.resolve("bin").resolve(CompilerHelper.executableName("java")).toString(); + ProcessBuilder pb = new ProcessBuilder( + javaExe, + "-cp", + classesDir + System.getProperty("path.separator") + javaApiDir, + "PrimitiveTypeApp" + ); + pb.redirectErrorStream(true); + + Process process = pb.start(); + String output; + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8))) { + output = reader.lines().collect(Collectors.joining("\n")); + } + assertEquals(0, process.waitFor(), "JVM run should exit cleanly. Output: " + output); + return output; + } + + private CompilerHelper.CompilerConfig selectCompiler() { + String[] preferredTargets = {"11", "17", "21", "25", "1.8"}; + for (String target : preferredTargets) { + List configs = CompilerHelper.getAvailableCompilers(target); + for (CompilerHelper.CompilerConfig config : configs) { + if (CompilerHelper.isJavaApiCompatible(config)) { + return config; + } + } + } + return null; + } +} diff --git a/vm/tests/src/test/resources/com/codename1/tools/translator/PrimitiveTypeApp.java b/vm/tests/src/test/resources/com/codename1/tools/translator/PrimitiveTypeApp.java new file mode 100644 index 00000000000..fd3753208dc --- /dev/null +++ b/vm/tests/src/test/resources/com/codename1/tools/translator/PrimitiveTypeApp.java @@ -0,0 +1,75 @@ +import java.util.HashMap; +import java.util.Map; + +/** + * Emits the identity and behaviour of the nine primitive class objects so the JVM + * and ParparVM runs can be compared line for line. + * + * These were all null on ParparVM before the primitive class objects existed: + * javac lowers a primitive class literal to a read of the boxed type's own TYPE + * field, so `TYPE = int.class` compiled to `getstatic TYPE; putstatic TYPE`. + * Nothing threw -- a Map keyed on them simply collapsed to one entry and answered + * every lookup with whatever was stored last. + */ +public class PrimitiveTypeApp { + public static void main(String[] args) { + Class[] types = { + Integer.TYPE, Long.TYPE, Short.TYPE, Byte.TYPE, Character.TYPE, + Float.TYPE, Double.TYPE, Boolean.TYPE, Void.TYPE + }; + String[] names = { + "int", "long", "short", "byte", "char", + "float", "double", "boolean", "void" + }; + + for (int i = 0; i < types.length; i++) { + System.out.println("CASE|name." + names[i] + "|" + + (types[i] == null ? "" : types[i].getName())); + } + + // Distinct identities. Any two collapsing is the failure that made the + // translator's own primitive-to-C-type maps answer wrongly. + int distinct = 0; + for (int i = 0; i < types.length; i++) { + boolean unique = true; + for (int j = 0; j < i; j++) { + if (types[i] == types[j]) { + unique = false; + } + } + if (unique) { + distinct++; + } + } + System.out.println("CASE|distinctIdentities|" + distinct); + + // The shape the translator's Util actually uses. + Map byType = new HashMap(); + for (int i = 0; i < types.length; i++) { + byType.put(types[i], names[i]); + } + System.out.println("CASE|mapSize|" + byType.size()); + + int lookupFailures = 0; + for (int i = 0; i < types.length; i++) { + if (!names[i].equals(byType.get(types[i]))) { + lookupFailures++; + } + } + System.out.println("CASE|lookupFailures|" + lookupFailures); + + // isPrimitive, and the two natives that index instanceof tables by classId + // and so must special-case a primitive class rather than look it up. + System.out.println("CASE|isPrimitive.int|" + Integer.TYPE.isPrimitive()); + System.out.println("CASE|isPrimitive.boxed|" + Integer.class.isPrimitive()); + System.out.println("CASE|isArray.int|" + Integer.TYPE.isArray()); + System.out.println("CASE|assignable.self|" + Integer.TYPE.isAssignableFrom(Integer.TYPE)); + System.out.println("CASE|assignable.cross|" + Integer.TYPE.isAssignableFrom(Long.TYPE)); + System.out.println("CASE|assignable.boxed|" + Integer.TYPE.isAssignableFrom(Integer.class)); + System.out.println("CASE|isInstance.boxed|" + Integer.TYPE.isInstance(Integer.valueOf(1))); + System.out.println("CASE|isInstance.string|" + Integer.TYPE.isInstance("x")); + System.out.println("CASE|boxedNotPrimitive|" + (Integer.TYPE == Integer.class)); + + System.out.println("DONE"); + } +} From 87be20fc9f1676b08b06e0512b90e8ee0e33adbb Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 9 Sep 2026 22:00:26 +0300 Subject: [PATCH 02/66] Make ParparVM self-hosting: the translator now translates itself ByteCodeTranslator's own bytecode, plus ASM's, translates to C and compiles into a native binary that performs real translations. vm/selfhost/build-selfhost.sh builds it and verify-selfhost.sh compares its output against the JVM-hosted translator's. The point is validation. The translator is a ~37k-line program that exercises collections, strings, file I/O, exceptions and the GC at scale, so running both builds over the same input and diffing the emitted C is an end-to-end conformance test of the whole VM -- one whose corpus grows on its own as the translator does. It has already earned that: three defects fell out of it, each invisible to every existing test because each was self-consistent on HotSpot. - C label names came from identity hash codes. ASM's Label.toString() is "L" + System.identityHashCode(this), so the emitted C was irreproducible; and on ParparVM, whose identity hash is the object pointer narrowed to int and so often negative, it emitted label_L-180306432001, which C reads as a subtraction. Every method with a try/catch failed to compile. Labels are now numbered per method in bytecode order. - C local-variable declarations were emitted in HashSet iteration order, so the same input produced different C. debugVarEntries had already had to learn this for the debug side-table; the declarations had the same defect and now share its comparator. - Class.getResourceAsStream returned a hard-coded null on every ParparVM target. It now consults resources linked into the executable through a weakly-defined cn1FindResource -- which the generated resource table overrides on targets that embed them -- and then a search path from CN1_RESOURCE_PATH. JavaAPI grows only where ASM's bytecode forces it, because ASM is a jar we cannot edit: Integer.rotateLeft, Double.doubleToRawLongBits, Float.floatToRawIntBits, the three-argument Class.forName, and TypeNotPresentException. Everything the translator's own source needed was removed from the translator instead: - String.split/replaceAll are gone from it entirely (Util.splitLiteral, splitWhitespace, collapseWhitespace, rewriteLocalObjectRefs). Declaring them in JavaAPI would have collided with BytecodeComplianceMojo, which rewrites those calls onto com.codename1.util.regex precisely because JavaAPI lacks them. UtilStringHelperTest holds each replacement against the JDK original over ~6000 fuzzed inputs; it caught one real divergence, that String.split returns { s } when the pattern never matches instead of dropping the trailing empty. - The 24 two-argument System.getProperty calls go through Util.getProperty, which uses the one-argument form JavaAPI does have and falls back to getenv. That also makes the knobs work in a native build, which no -D can. - The ~110 java.nio.file calls are plain java.io again. Parser and ConcatenatingFileOutputStream already used those constructors under the zero-findings SpotBugs gate, so both idioms already coexisted. - java.util.zip is confined to ArchiveClassScanner and DebugSymbolCompressor, and NativeSignatureVerifier's command-line half moved to NativeSignatureVerifierCli. JavaAPI cannot gain java.util.zip: it is mirrored by Ports/CLDC11, where the package does not belong. Splitting out the CLI also removes the second main() that made ByteCodeClass refuse the translation with "Multiple main classes". vm/selfhost/stubs holds no-op replacements used only by the native build, for the JavaScript target and those two zip users. Gate D (the native translator against itself) passes. Gate A (JVM against native) is at 245 of 247 files byte-identical, and binaries built from the two trees produce identical output. The remaining two files are java_util_HashMap.c/.h, where the native pass culls seven more methods and emits them as stubs; both trees compile, link and run correctly, but the two runtimes should not disagree. Not nondeterminism (gate D passes on both) and not identity-hash order (tested by re-running the JVM under -XX:hashCode=2, byte-identical output). Written up in vm/selfhost/README.md. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/check-native-signatures.sh | 4 +- .../tools/translator/ArchiveClassScanner.java | 91 ++++++ .../tools/translator/ByteCodeClass.java | 2 +- .../tools/translator/ByteCodeTranslator.java | 163 +++++----- .../tools/translator/BytecodeMethod.java | 58 +++- .../translator/DebugSymbolCompressor.java | 60 ++++ .../translator/JavascriptNativeRegistry.java | 1 + .../translator/NativeSignatureVerifier.java | 145 ++------- .../NativeSignatureVerifierCli.java | 116 +++++++ .../codename1/tools/translator/Parser.java | 32 +- .../com/codename1/tools/translator/Util.java | 291 ++++++++++++++++++ .../translator/bytecodes/CustomInvoke.java | 4 +- .../translator/bytecodes/CustomJump.java | 4 +- .../tools/translator/bytecodes/Invoke.java | 4 +- .../tools/translator/bytecodes/Jump.java | 4 +- .../bytecodes/LabelInstruction.java | 59 +++- .../bytecodes/SwitchInstruction.java | 8 +- .../tools/translator/bytecodes/TryCatch.java | 10 +- .../src/javascript/parparvm_runtime.js | 23 +- vm/ByteCodeTranslator/src/nativeMethods.m | 50 ++- vm/JavaAPI/src/java/lang/Class.java | 80 ++++- vm/JavaAPI/src/java/lang/Double.java | 8 + vm/JavaAPI/src/java/lang/Float.java | 14 + vm/JavaAPI/src/java/lang/Integer.java | 12 + .../java/lang/TypeNotPresentException.java | 44 +++ vm/selfhost/README.md | 93 ++++++ vm/selfhost/build-selfhost.sh | 108 +++++++ .../tools/translator/ArchiveClassScanner.java | 45 +++ .../translator/DebugSymbolCompressor.java | 43 +++ .../translator/JavascriptBundleWriter.java | 47 +++ .../translator/JavascriptMethodGenerator.java | 45 +++ .../translator/JavascriptNativeRegistry.java | 47 +++ .../translator/JavascriptReachability.java | 46 +++ .../JavascriptSuspensionAnalysis.java | 45 +++ vm/selfhost/verify-selfhost.sh | 90 ++++++ .../translator/UtilStringHelperTest.java | 170 ++++++++++ 36 files changed, 1809 insertions(+), 257 deletions(-) create mode 100644 vm/ByteCodeTranslator/src/com/codename1/tools/translator/ArchiveClassScanner.java create mode 100644 vm/ByteCodeTranslator/src/com/codename1/tools/translator/DebugSymbolCompressor.java create mode 100644 vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifierCli.java create mode 100644 vm/JavaAPI/src/java/lang/TypeNotPresentException.java create mode 100644 vm/selfhost/README.md create mode 100755 vm/selfhost/build-selfhost.sh create mode 100644 vm/selfhost/stubs/com/codename1/tools/translator/ArchiveClassScanner.java create mode 100644 vm/selfhost/stubs/com/codename1/tools/translator/DebugSymbolCompressor.java create mode 100644 vm/selfhost/stubs/com/codename1/tools/translator/JavascriptBundleWriter.java create mode 100644 vm/selfhost/stubs/com/codename1/tools/translator/JavascriptMethodGenerator.java create mode 100644 vm/selfhost/stubs/com/codename1/tools/translator/JavascriptNativeRegistry.java create mode 100644 vm/selfhost/stubs/com/codename1/tools/translator/JavascriptReachability.java create mode 100644 vm/selfhost/stubs/com/codename1/tools/translator/JavascriptSuspensionAnalysis.java create mode 100755 vm/selfhost/verify-selfhost.sh create mode 100644 vm/tests/src/test/java/com/codename1/tools/translator/UtilStringHelperTest.java diff --git a/scripts/check-native-signatures.sh b/scripts/check-native-signatures.sh index beb98e60838..cd467807da4 100755 --- a/scripts/check-native-signatures.sh +++ b/scripts/check-native-signatures.sh @@ -36,7 +36,7 @@ for arg in "$@"; do esac done -if [[ ! -f "$TRANSLATOR/com/codename1/tools/translator/NativeSignatureVerifier.class" ]]; then +if [[ ! -f "$TRANSLATOR/com/codename1/tools/translator/NativeSignatureVerifierCli.class" ]]; then echo "check-native-signatures: building the translator" >&2 (cd "$REPO_ROOT/vm" && mvn -q -B -pl ByteCodeTranslator -am package -DskipTests) fi @@ -95,7 +95,7 @@ for entry in "${PORTS[@]}"; do echo "== $name" if ! java -cp "$TRANSLATOR:$(cat "$ASM_CP_FILE")" \ - com.codename1.tools.translator.NativeSignatureVerifier "${args[@]}"; then + com.codename1.tools.translator.NativeSignatureVerifierCli "${args[@]}"; then status=1 fi checked=$((checked + 1)) diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ArchiveClassScanner.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ArchiveClassScanner.java new file mode 100644 index 00000000000..1a7120a5667 --- /dev/null +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ArchiveClassScanner.java @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Enumeration; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +/** + * Collects the native methods declared by every class inside a jar or zip. + * + * Split out of {@link NativeSignatureVerifier} because it is that class's only use + * of {@code java.util.zip}, and it is reachable only from the offline command-line + * entry point that scripts/check-native-signatures.sh drives -- never from a + * translation. Isolating it is what lets the rest of the verifier compile against + * ParparVM's JavaAPI, which has no java.util.zip and cannot gain one: JavaAPI is + * mirrored by Ports/CLDC11, where the package does not belong. + * + * The translator itself never reads an archive. Every caller extracts a jar into a + * directory of class files before invoking it. + */ +final class ArchiveClassScanner { + private ArchiveClassScanner() { + } + + /** + * Entries are visited in sorted order so that two runs over the same archive + * report findings in the same order. + */ + static void collect(File archive, List into) throws IOException { + ZipFile zip = new ZipFile(archive); + try { + List names = new ArrayList(); + for (Enumeration e = zip.entries(); e.hasMoreElements();) { + ZipEntry entry = e.nextElement(); + if (!entry.isDirectory() && entry.getName().endsWith(".class") + && !entry.getName().endsWith("module-info.class")) { + names.add(entry.getName()); + } + } + Collections.sort(names); + for (String name : names) { + InputStream in = zip.getInputStream(zip.getEntry(name)); + try { + NativeSignatureVerifier.collectFromClassBytes(readAll(in), into); + } finally { + in.close(); + } + } + } finally { + zip.close(); + } + } + + private static byte[] readAll(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int read; + while ((read = in.read(buffer)) > 0) { + out.write(buffer, 0, read); + } + return out.toByteArray(); + } +} diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java index 5701065e0b8..8c80e1903e3 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java @@ -1027,7 +1027,7 @@ public String generateCCode(List allClasses) { buildInstanceFieldList(fullFieldList); String nullCheck = ""; - if (System.getProperty("fieldNullChecks", "false").equals("true")) { + if (Util.getProperty("fieldNullChecks", "false").equals("true")) { nullCheck = "if(__cn1T == JAVA_NULL){throwException(getThreadLocalData(), __NEW_INSTANCE_java_lang_NullPointerException(getThreadLocalData()));}\n"; } for(ByteCodeField fld : fullFieldList) { diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java index 585249e0c2e..64760b89ae2 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java @@ -25,13 +25,14 @@ import java.io.DataInputStream; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -147,9 +148,9 @@ private static void sortByName(File[] files) { } void execute(File sourceDir, File outputDir) throws Exception { - File[] directoryList = sourceDir.listFiles(pathname -> + File[] directoryList = Util.listFiles(sourceDir, pathname -> !pathname.isHidden() && !pathname.getName().startsWith(".") && pathname.isDirectory()); - File[] fileList = sourceDir.listFiles(pathname -> + File[] fileList = Util.listFiles(sourceDir, pathname -> !pathname.isHidden() && !pathname.getName().startsWith(".") && !pathname.isDirectory()); // listFiles() returns whatever order the filesystem hands back, which can // differ between two builds of the same input (the app classes are @@ -171,7 +172,7 @@ void execute(File sourceDir, File outputDir) throws Exception { } else { if(!f.isDirectory() && !isBuildMetadata(f)) { // copy the file to the dest dir - copy(Files.newInputStream(f.toPath()), Files.newOutputStream(new File(outputDir, f.getName()).toPath())); + copy(new FileInputStream(f), new FileOutputStream(new File(outputDir, f.getName()))); } } } @@ -198,7 +199,7 @@ private void copyDir(File source, File destDir) throws IOException { if(f.isDirectory()) { copyDir(f, destFile); } else { - copy(Files.newInputStream(f.toPath()), Files.newOutputStream(new File(destFile, f.getName()).toPath())); + copy(new FileInputStream(f), new FileOutputStream(new File(destFile, f.getName()))); } } } @@ -212,7 +213,7 @@ private void copyDir(File source, File destDir) throws IOException { * engine compiled in. Set by the platform builders from their class scan. */ static boolean isBundledSqliteEnabled() { - return "true".equals(System.getProperty("cn1.sqlite", "false")); + return "true".equals(Util.getProperty("cn1.sqlite", "false")); } /** @@ -220,7 +221,7 @@ static boolean isBundledSqliteEnabled() { * system libsqlite3, which has no cipher support, with the bundled engine. */ static boolean isBundledSqliteCipherEnabled() { - return "true".equals(System.getProperty("cn1.sqlcipher", "false")); + return "true".equals(Util.getProperty("cn1.sqlcipher", "false")); } /** @@ -253,7 +254,7 @@ static boolean isBundledSqliteCipherEnabled() { * shipping target. */ public static boolean isCheckedCastsEnabled() { - return "true".equalsIgnoreCase(System.getProperty("cn1.checkedCasts", "false")); + return "true".equalsIgnoreCase(Util.getProperty("cn1.checkedCasts", "false")); } /// Writes the bundled SQLite engine into a source root, or takes it back out. @@ -276,12 +277,12 @@ private static void emitBundledSqlite(File srcRoot) throws IOException { File sqliteCipherMarker = new File(srcRoot, "cn1_sqlite3_cipher.h"); if (isBundledSqliteEnabled()) { copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_sqlite3.c"), - Files.newOutputStream(sqliteUnity.toPath())); + new FileOutputStream(sqliteUnity)); replaceInFile(sqliteUnity, "//#define CN1_INCLUDE_SQLITE", "#define CN1_INCLUDE_SQLITE"); copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_sqlite3.h"), - Files.newOutputStream(sqliteHeader.toPath())); + new FileOutputStream(sqliteHeader)); copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_sqlite3_amalgamation.h"), - Files.newOutputStream(sqliteAmalgamation.toPath())); + new FileOutputStream(sqliteAmalgamation)); } else { deleteIfPresent(sqliteUnity); deleteIfPresent(sqliteHeader); @@ -294,7 +295,7 @@ private static void emitBundledSqlite(File srcRoot) throws IOException { // flags. Emitted only for an application that configures encryption, so everyone else // compiles the engine as plain SQLite and links no keying code at all. copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_sqlite3_cipher.h"), - Files.newOutputStream(sqliteCipherMarker.toPath())); + new FileOutputStream(sqliteCipherMarker)); } else { // Left behind, this would put the ciphers back into an engine emitted without them -- // __has_include does not care which run wrote the file. @@ -341,7 +342,7 @@ public static void main(String[] args) throws Exception { final String appType = args[7]; final String addFrameworks = args[8]; // we accept 3 argument output types, input directory and output directory - if (System.getProperty("saveUnitTests", "false").equals("true")) { + if (Util.getProperty("saveUnitTests", "false").equals("true")) { System.out.println("Generating Unit Tests"); ByteCodeClass.setSaveUnitTests(true); } @@ -358,7 +359,7 @@ public static void main(String[] args) throws Exception { // Unrecognized output type falls back to the plain copy-through default handler recognizedOutputType = false; } - String[] sourceDirectories = args[1].split(";"); + String[] sourceDirectories = Util.splitLiteral(args[1], ';'); File[] sources = new File[sourceDirectories.length]; for(int iter = 0 ; iter < sourceDirectories.length ; iter++) { sources[iter] = new File(sourceDirectories[iter]); @@ -436,31 +437,31 @@ private static void handleCleanOutput(ByteCodeTranslator b, File[] sources, File b.execute(sources, srcRoot); File cn1Globals = new File(srcRoot, "cn1_globals.h"); - copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_globals.h"), Files.newOutputStream(cn1Globals.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_globals.h"), new FileOutputStream(cn1Globals)); File cn1Intrinsics = new File(srcRoot, "cn1_intrinsics.h"); - copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_intrinsics.h"), Files.newOutputStream(cn1Intrinsics.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_intrinsics.h"), new FileOutputStream(cn1Intrinsics)); // Virtual threads: the switch is a few instructions of assembly per // architecture, so the .S travels with the runtime rather than being // generated. A project that gets the C and not the .S links against a // missing symbol, which is at least loud. emitVirtualThreadRuntime(srcRoot); - if (System.getProperty("INCLUDE_NPE_CHECKS", "false").equals("true")) { + if (Util.getProperty("INCLUDE_NPE_CHECKS", "false").equals("true")) { replaceInFile(cn1Globals, "//#define CN1_INCLUDE_NPE_CHECKS", "#define CN1_INCLUDE_NPE_CHECKS"); } - if ("true".equalsIgnoreCase(System.getProperty("cn1.onDeviceDebug", "false"))) { + if ("true".equalsIgnoreCase(Util.getProperty("cn1.onDeviceDebug", "false"))) { replaceInFile(cn1Globals, "//#define CN1_ON_DEVICE_DEBUG", "#define CN1_ON_DEVICE_DEBUG"); } File cn1GlobalsC = new File(srcRoot, "cn1_globals.c"); - copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_globals.m"), Files.newOutputStream(cn1GlobalsC.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_globals.m"), new FileOutputStream(cn1GlobalsC)); File nativeMethodsC = new File(srcRoot, "nativeMethods.c"); - copy(ByteCodeTranslator.class.getResourceAsStream("/nativeMethods.m"), Files.newOutputStream(nativeMethodsC.toPath())); - if (System.getProperty("USE_RPMALLOC", "false").equals("true")) { + copy(ByteCodeTranslator.class.getResourceAsStream("/nativeMethods.m"), new FileOutputStream(nativeMethodsC)); + if (Util.getProperty("USE_RPMALLOC", "false").equals("true")) { File malloc = new File(srcRoot, "malloc.c"); - copy(ByteCodeTranslator.class.getResourceAsStream("/malloc.c"), Files.newOutputStream(malloc.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/malloc.c"), new FileOutputStream(malloc)); File rpmalloc = new File(srcRoot, "rpmalloc.c"); - copy(ByteCodeTranslator.class.getResourceAsStream("/rpmalloc.c"), Files.newOutputStream(rpmalloc.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/rpmalloc.c"), new FileOutputStream(rpmalloc)); File rpmalloch = new File(srcRoot, "rpmalloc.h"); - copy(ByteCodeTranslator.class.getResourceAsStream("/rpmalloc.h"), Files.newOutputStream(rpmalloch.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/rpmalloc.h"), new FileOutputStream(rpmalloch)); } // The bundled SQLite engine is emitted only for applications that actually use // com.codename1.db, so everyone else pays nothing for it. cn1_sqlite3.c is gated on @@ -470,31 +471,31 @@ private static void handleCleanOutput(ByteCodeTranslator b, File[] sources, File // the engine is present and as stubs when it is not, so an application that references // com.codename1.db links regardless of how the translator was invoked. File sqliteBindings = new File(srcRoot, "cn1_db_sqlite_impl.h"); - copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_db_sqlite_impl.h"), Files.newOutputStream(sqliteBindings.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_db_sqlite_impl.h"), new FileOutputStream(sqliteBindings)); emitBundledSqlite(srcRoot); File xmlvm = new File(srcRoot, "xmlvm.h"); - copy(ByteCodeTranslator.class.getResourceAsStream("/xmlvm.h"), Files.newOutputStream(xmlvm.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/xmlvm.h"), new FileOutputStream(xmlvm)); // Win32 POSIX compatibility shim. Always emitted; both files are gated on // _WIN32 internally, so they compile to nothing on iOS/macOS/Linux and // provide pthreads/usleep/gettimeofday on Windows (clang-cl / MSVC ABI). File cn1WinCompatH = new File(srcRoot, "cn1_win_compat.h"); - copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_win_compat.h"), Files.newOutputStream(cn1WinCompatH.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_win_compat.h"), new FileOutputStream(cn1WinCompatH)); File cn1WinCompatC = new File(srcRoot, "cn1_win_compat.c"); - copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_win_compat.c"), Files.newOutputStream(cn1WinCompatC.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_win_compat.c"), new FileOutputStream(cn1WinCompatC)); Parser.writeOutput(srcRoot); File javaIoFileHeader = new File(srcRoot, "java_io_File.h"); if (javaIoFileHeader.exists()) { File javaIoFileC = new File(srcRoot, "java_io_File_runtime.c"); - copy(ByteCodeTranslator.class.getResourceAsStream("/java_io_File.m"), Files.newOutputStream(javaIoFileC.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/java_io_File.m"), new FileOutputStream(javaIoFileC)); } File classMethodIndexM = new File(srcRoot, "cn1_class_method_index.m"); if (classMethodIndexM.exists()) { File classMethodIndexC = new File(srcRoot, "cn1_class_method_index.c"); - copy(Files.newInputStream(classMethodIndexM.toPath()), Files.newOutputStream(classMethodIndexC.toPath())); + copy(new FileInputStream(classMethodIndexM), new FileOutputStream(classMethodIndexC)); if(!classMethodIndexM.delete()) { System.err.println("Deletion of " + classMethodIndexM.getAbsolutePath() + " failed"); } @@ -563,14 +564,14 @@ private static void embedWindowsResources(File[] sources, File srcRoot) throws I int id = 1; for (java.util.Map.Entry e : resources.entrySet()) { File staged = new File(resDir, "res" + id); - copy(Files.newInputStream(e.getValue().toPath()), Files.newOutputStream(staged.toPath())); + copy(new FileInputStream(e.getValue()), new FileOutputStream(staged)); // RC filenames are resolved relative to the .rc (srcRoot); llvm-rc and // rc.exe both accept forward slashes. rc.append(id).append(" RCDATA \"cn1_resources/res").append(id).append("\"\n"); table.append(" {\"").append(escapeCString(e.getKey())).append("\", ").append(id).append("},\n"); id++; } - Files.write(new File(srcRoot, "cn1_resources.rc").toPath(), + Util.writeBytes(new File(srcRoot, "cn1_resources.rc"), rc.toString().getBytes(StandardCharsets.UTF_8)); } @@ -583,7 +584,7 @@ private static void embedWindowsResources(File[] sources, File srcRoot) throws I table.append(" }\n"); table.append(" return 0;\n"); table.append("}\n"); - Files.write(new File(srcRoot, "cn1_resources_table.c").toPath(), + Util.writeBytes(new File(srcRoot, "cn1_resources_table.c"), table.toString().getBytes(StandardCharsets.UTF_8)); } @@ -632,7 +633,7 @@ private static void embedLinuxResources(File[] sources, File srcRoot) throws IOE int id = 1; for (java.util.Map.Entry e : resources.entrySet()) { File staged = new File(resDir, "res" + id); - copy(Files.newInputStream(e.getValue().toPath()), Files.newOutputStream(staged.toPath())); + copy(new FileInputStream(e.getValue()), new FileOutputStream(staged)); // Absolute path so .incbin resolves regardless of the assembler's // working directory (the build runs out of a separate build dir). String incPath = escapeCString(staged.getAbsolutePath().replace('\\', '/')); @@ -646,7 +647,7 @@ private static void embedLinuxResources(File[] sources, File srcRoot) throws IOE .append("[]; extern const unsigned char cn1res_").append(id).append("_end[];\n"); id++; } - Files.write(new File(srcRoot, "cn1_resources_data.S").toPath(), + Util.writeBytes(new File(srcRoot, "cn1_resources_data.S"), asm.toString().getBytes(StandardCharsets.UTF_8)); } @@ -671,7 +672,7 @@ private static void embedLinuxResources(File[] sources, File srcRoot) throws IOE table.append(" if (lenOut) { *lenOut = 0; }\n"); table.append(" return 0;\n"); table.append("}\n"); - Files.write(new File(srcRoot, "cn1_resources_table.c").toPath(), + Util.writeBytes(new File(srcRoot, "cn1_resources_table.c"), table.toString().getBytes(StandardCharsets.UTF_8)); } @@ -700,7 +701,7 @@ private static void collectResources(File root, File dir, java.util.LinkedHashMa || ext.equals("mm") || ext.equals("rc")) { continue; } - String rel = root.toPath().relativize(f.toPath()).toString().replace('\\', '/'); + String rel = Util.relativePath(root, f); String key = "/" + rel; if (!out.containsKey(key)) { out.put(key, f); @@ -761,7 +762,7 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File launchImageLaunchimage.mkdirs(); //cleanDir(launchImageLaunchimage); - copy(ByteCodeTranslator.class.getResourceAsStream("/LaunchImages.json"), Files.newOutputStream(new File(launchImageLaunchimage, "Contents.json").toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/LaunchImages.json"), new FileOutputStream(new File(launchImageLaunchimage, "Contents.json"))); } File appIconAppiconset = new File(imagesXcassets, "AppIcon.appiconset"); @@ -772,7 +773,7 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File // wants the 16..512 @1x/@2x "mac" idiom ladder. copy(ByteCodeTranslator.class.getResourceAsStream( platform.hasIosDeviceIdioms() ? "/Icons.json" : "/Icons-macos.json"), - Files.newOutputStream(new File(appIconAppiconset, "Contents.json").toPath())); + new FileOutputStream(new File(appIconAppiconset, "Contents.json"))); File xcproj = new File(root, appName + ".xcodeproj"); @@ -785,34 +786,34 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File b.execute(sources, srcRoot); File cn1Globals = new File(srcRoot, "cn1_globals.h"); - copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_globals.h"), Files.newOutputStream(cn1Globals.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_globals.h"), new FileOutputStream(cn1Globals)); File cn1Intrinsics = new File(srcRoot, "cn1_intrinsics.h"); - copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_intrinsics.h"), Files.newOutputStream(cn1Intrinsics.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_intrinsics.h"), new FileOutputStream(cn1Intrinsics)); // Virtual threads: the switch is a few instructions of assembly per // architecture, so the .S travels with the runtime rather than being // generated. A project that gets the C and not the .S links against a // missing symbol, which is at least loud. emitVirtualThreadRuntime(srcRoot); - if (System.getProperty("INCLUDE_NPE_CHECKS", "false").equals("true")) { + if (Util.getProperty("INCLUDE_NPE_CHECKS", "false").equals("true")) { replaceInFile(cn1Globals, "//#define CN1_INCLUDE_NPE_CHECKS", "#define CN1_INCLUDE_NPE_CHECKS"); } - if ("true".equalsIgnoreCase(System.getProperty("cn1.onDeviceDebug", "false"))) { + if ("true".equalsIgnoreCase(Util.getProperty("cn1.onDeviceDebug", "false"))) { replaceInFile(cn1Globals, "//#define CN1_ON_DEVICE_DEBUG", "#define CN1_ON_DEVICE_DEBUG"); } File cn1GlobalsM = new File(srcRoot, "cn1_globals.m"); - copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_globals.m"), Files.newOutputStream(cn1GlobalsM.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_globals.m"), new FileOutputStream(cn1GlobalsM)); File nativeMethods = new File(srcRoot, "nativeMethods.m"); - copy(ByteCodeTranslator.class.getResourceAsStream("/nativeMethods.m"), Files.newOutputStream(nativeMethods.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/nativeMethods.m"), new FileOutputStream(nativeMethods)); File javaIoFileM = new File(srcRoot, "java_io_File.m"); - copy(ByteCodeTranslator.class.getResourceAsStream("/java_io_File.m"), Files.newOutputStream(javaIoFileM.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/java_io_File.m"), new FileOutputStream(javaIoFileM)); - if (System.getProperty("USE_RPMALLOC", "false").equals("true")) { + if (Util.getProperty("USE_RPMALLOC", "false").equals("true")) { File malloc = new File(srcRoot, "malloc.c"); - copy(ByteCodeTranslator.class.getResourceAsStream("/malloc.c"), Files.newOutputStream(malloc.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/malloc.c"), new FileOutputStream(malloc)); File rpmalloc = new File(srcRoot, "rpmalloc.c"); - copy(ByteCodeTranslator.class.getResourceAsStream("/rpmalloc.c"), Files.newOutputStream(rpmalloc.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/rpmalloc.c"), new FileOutputStream(rpmalloc)); File rpmalloch = new File(srcRoot, "rpmalloc.h"); - copy(ByteCodeTranslator.class.getResourceAsStream("/rpmalloc.h"), Files.newOutputStream(rpmalloch.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/rpmalloc.h"), new FileOutputStream(rpmalloch)); } // The bundled SQLite engine is emitted only for applications that actually use // com.codename1.db, so everyone else pays nothing for it. cn1_sqlite3.c is gated on @@ -822,29 +823,29 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File // the engine is present and as stubs when it is not, so an application that references // com.codename1.db links regardless of how the translator was invoked. File sqliteBindings = new File(srcRoot, "cn1_db_sqlite_impl.h"); - copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_db_sqlite_impl.h"), Files.newOutputStream(sqliteBindings.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_db_sqlite_impl.h"), new FileOutputStream(sqliteBindings)); emitBundledSqlite(srcRoot); Parser.writeOutput(srcRoot); File templateInfoPlist = new File(srcRoot, appName + "-Info.plist"); - copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template/template-Info.plist"), Files.newOutputStream(templateInfoPlist.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template/template-Info.plist"), new FileOutputStream(templateInfoPlist)); File templatePch = new File(srcRoot, appName + "-Prefix.pch"); - copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template/template-Prefix.pch"), Files.newOutputStream(templatePch.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template/template-Prefix.pch"), new FileOutputStream(templatePch)); File xmlvm = new File(srcRoot, "xmlvm.h"); - copy(ByteCodeTranslator.class.getResourceAsStream("/xmlvm.h"), Files.newOutputStream(xmlvm.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/xmlvm.h"), new FileOutputStream(xmlvm)); File projectWorkspaceData = new File(projectXCworkspace, "contents.xcworkspacedata"); - copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template.xcodeproj/project.xcworkspace/contents.xcworkspacedata"), Files.newOutputStream(projectWorkspaceData.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template.xcodeproj/project.xcworkspace/contents.xcworkspacedata"), new FileOutputStream(projectWorkspaceData)); replaceInFile(projectWorkspaceData, "KitchenSink", appName); File projectPbx = new File(xcproj, "project.pbxproj"); - copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template.xcodeproj/project.pbxproj"), Files.newOutputStream(projectPbx.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template.xcodeproj/project.pbxproj"), new FileOutputStream(projectPbx)); - String[] sourceFiles = srcRoot.list((pathname, string) -> + String[] sourceFiles = Util.list(srcRoot, (pathname, string) -> string.endsWith(".bundle") || string.endsWith(".xcdatamodeld") || !pathname.isHidden() && !string.startsWith(".") && !"Images.xcassets".equals(string)); StringBuilder fileOneEntry = new StringBuilder(); @@ -861,7 +862,7 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File List includeFrameworks = new ArrayList<>(); Set optionalFrameworks = new HashSet<>(); - for (String optionalFramework : System.getProperty("optional.frameworks", "").split(";")) { + for (String optionalFramework : Util.splitLiteral(Util.getProperty("optional.frameworks", ""), ';')) { optionalFramework = optionalFramework.trim(); if (!optionalFramework.isEmpty()) { optionalFrameworks.add(optionalFramework); @@ -918,7 +919,7 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File includeFrameworks.add("libz.dylib"); includeFrameworks.add("AVKit.framework"); if(!addFrameworks.equalsIgnoreCase("none")) { - includeFrameworks.addAll(Arrays.asList(addFrameworks.split(";"))); + includeFrameworks.addAll(Arrays.asList(Util.splitLiteral(addFrameworks, ';'))); } int currentValue = 0xF63EAAA; @@ -1064,7 +1065,7 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File "***FRAMEWORKS2***", frameworks2.toString(), "***RESOURCES***", resources.toString()); } - String bundleVersion = System.getProperty("bundleVersionNumber", appVersion); + String bundleVersion = Util.getProperty("bundleVersionNumber", appVersion); replaceInFile(templateInfoPlist, "com.codename1pkg", appPackageName, "${PRODUCT_NAME}", appDisplayName, "VERSION_VALUE", appVersion, "VERSION_BUNDLE_VALUE", bundleVersion); } @@ -1082,7 +1083,7 @@ private static void writeCmakeProject(File projectRoot, File srcRoot, String app boolean windows = "windows".equalsIgnoreCase(appType); boolean linux = "linux".equalsIgnoreCase(appType); boolean executable = windows || linux; - try (Writer writer = new OutputStreamWriter(Files.newOutputStream(cmakeLists.toPath()), StandardCharsets.UTF_8)) { + try (Writer writer = new OutputStreamWriter(new FileOutputStream(cmakeLists), "UTF-8")) { writer.append("cmake_minimum_required(VERSION 3.10)\n"); // The native Windows port mixes the translated C runtime with a C++ // layer for the COM APIs that have no C binding (DirectWrite), so the @@ -1472,12 +1473,12 @@ private static String getFileType(String s) { // to be mutated. Also, expire the temporary byte[] buffer so it can // be collected. // - private static StringBuilder readFileAsStringBuilder(File sourceFile) throws IOException + private static String readFileAsString(File sourceFile) throws IOException { - try(DataInputStream dis = new DataInputStream(Files.newInputStream(sourceFile.toPath()))) { + try(DataInputStream dis = new DataInputStream(new FileInputStream(sourceFile))) { byte[] data = new byte[(int) sourceFile.length()]; dis.readFully(data); - return new StringBuilder(new String(data, StandardCharsets.UTF_8)); + return new String(data, StandardCharsets.UTF_8); } } // @@ -1488,21 +1489,33 @@ private static StringBuilder readFileAsStringBuilder(File sourceFile) throws IOE // process for large projects. // private static void replaceInFile(File sourceFile, String... values) throws IOException { - StringBuilder str = readFileAsStringBuilder(sourceFile); + String str = readFileAsString(sourceFile); int totchanges = 0; - // perform the mutations on stringbuilder, which ought to implement - // these operations efficiently. + // One pass per target, appending the untouched runs into a fresh builder + // rather than mutating in place. StringBuilder.indexOf/replace do not exist + // in ParparVM's JavaAPI, which the translator has to compile against in + // order to translate itself; this keeps the same single-buffer-per-pass + // memory shape the in-place version had, so the OutOfMemoryError this + // method was written to avoid stays avoided. for (int iter = 0; iter < values.length; iter += 2) { String target = values[iter]; String replacement = values[iter + 1]; - int index = 0; - while ((index = str.indexOf(target, index)) >= 0) { - int targetSize = target.length(); - str.replace(index, index + targetSize, replacement); - index += replacement.length(); + int index = str.indexOf(target); + if (index < 0) { + continue; + } + StringBuilder rewritten = new StringBuilder(str.length()); + int from = 0; + while (index >= 0) { + rewritten.append(str, from, index); + rewritten.append(replacement); + from = index + target.length(); + index = str.indexOf(target, from); totchanges++; } + rewritten.append(str, from, str.length()); + str = rewritten.toString(); } // @@ -1511,8 +1524,8 @@ private static void replaceInFile(File sourceFile, String... values) throws IOEx if(verbose) { System.out.println("Rewrite " + sourceFile + " with " + totchanges + " changes"); } - try(Writer fios = new OutputStreamWriter(Files.newOutputStream(sourceFile.toPath()), StandardCharsets.UTF_8)) { - fios.write(str.toString()); + try(Writer fios = new OutputStreamWriter(new FileOutputStream(sourceFile), "UTF-8")) { + fios.write(str); } } @@ -1569,7 +1582,7 @@ private static void emitVirtualThreadRuntime(File srcRoot) throws IOException { // cause, where the link error later names only a symbol. throw new IOException("virtual-thread runtime resource missing: " + name); } - copy(in, Files.newOutputStream(new File(srcRoot, name).toPath())); + copy(in, new FileOutputStream(new File(srcRoot, name))); } } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java index 35609a140c8..babf4d9fd49 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java @@ -104,12 +104,12 @@ public static void setDependencyGraph(MethodDependencyGraph dependencyGraph) { private int maxLocals; private static boolean acceptStaticOnEquals; private static final boolean FORCE_VOLATILE_LOCALS = - "true".equalsIgnoreCase(System.getProperty("CN1_FORCE_VOLATILE_LOCALS", "false")); + "true".equalsIgnoreCase(Util.getProperty("CN1_FORCE_VOLATILE_LOCALS", "false")); // Frameless codegen gate (-Dcn1.frameless, default on). When off the // eligibility predicate always returns false, so every method emits the // legacy frame code byte-for-byte identical to before. private static final boolean FRAMELESS_ENABLED = - "true".equalsIgnoreCase(System.getProperty("cn1.frameless", "true")); + "true".equalsIgnoreCase(Util.getProperty("cn1.frameless", "true")); // PHASE 3b: extend frameless codegen to OBJECT-BEARING methods (-Dcn1.frameless.objects, // default off). Such a method keeps its object operand stack + object locals in a // method-local C array on the native stack; the C runtime (built with @@ -117,14 +117,14 @@ public static void setDependencyGraph(MethodDependencyGraph dependencyGraph) { // stopped thread's native stack. With this OFF, only primitive-only methods are // frameless (identical to the prior phase). Requires the conservative-GC runtime. private static final boolean FRAMELESS_OBJECTS_ENABLED = - "true".equalsIgnoreCase(System.getProperty("cn1.frameless.objects", "true")); + "true".equalsIgnoreCase(Util.getProperty("cn1.frameless.objects", "true")); // PHASE 3b: extend object-frameless to INSTANCE methods (receiver `this` becomes a // conservatively-scanned C parameter). Now DEFAULT ON: the intermittent multi-threaded // failure that previously gated this off was a pre-existing Thread.start/join visibility // race (alive set on the worker thread async after start() returned), fixed in // java_lang_Thread_start__ (993331107); with it fixed, MtStress is 50/50 deterministic. private static final boolean FRAMELESS_INSTANCE_ENABLED = - "true".equalsIgnoreCase(System.getProperty("cn1.frameless.instance", "true")); + "true".equalsIgnoreCase(Util.getProperty("cn1.frameless.instance", "true")); private int methodOffset; private boolean forceVirtual; private boolean virtualOverriden; @@ -162,7 +162,7 @@ public static void setDependencyGraph(MethodDependencyGraph dependencyGraph) { optimizerOn = op == null || op.equalsIgnoreCase("on"); //optimizerOn = false; - onDeviceDebug = "true".equalsIgnoreCase(System.getProperty("cn1.onDeviceDebug", "false")); + onDeviceDebug = "true".equalsIgnoreCase(Util.getProperty("cn1.onDeviceDebug", "false")); } public static boolean isOnDeviceDebug() { @@ -1395,6 +1395,18 @@ public List debugVarEntries() { return rows; } + /** + * Every local, in a deterministic order, for emitting the C declarations. + * + * Unlike {@link #debugVarEntries} this drops nothing: a local whose slot lies + * outside the frame still needs its declaration, it just has no debug row. + */ + private List declarationOrderedLocals() { + List ordered = new ArrayList(localVariables); + Collections.sort(ordered, DEBUG_VAR_ORDER); + return ordered; + } + /** Slot first, then storage qualifier, so a reused slot's rows stay adjacent. */ private static final Comparator DEBUG_VAR_ORDER = new Comparator() { @Override @@ -1629,29 +1641,29 @@ private void fixUpBarebone() { CustomJump cj = (CustomJump)i; String cmp = cj.getCustomCompareCode(); if (cmp != null) { - cj.setCustomCompareCode(cmp.replaceAll("locals\\[(\\d+)\\]\\.data\\.o", "olocals_$1_")); + cj.setCustomCompareCode(Util.rewriteLocalObjectRefs(cmp)); } } else if (i instanceof CustomIntruction) { CustomIntruction ci = (CustomIntruction)i; String code = ci.getCode(); if (code != null) { - ci.setCode(code.replaceAll("locals\\[(\\d+)\\]\\.data\\.o", "olocals_$1_")); + ci.setCode(Util.rewriteLocalObjectRefs(code)); } String complexCode = ci.getComplexCode(); if (complexCode != null) { - ci.setComplexCode(complexCode.replaceAll("locals\\[(\\d+)\\]\\.data\\.o", "olocals_$1_")); + ci.setComplexCode(Util.rewriteLocalObjectRefs(complexCode)); } } else if (i instanceof CustomInvoke) { CustomInvoke ci = (CustomInvoke)i; String target = ci.getTargetObjectLiteral(); if (target != null) { - ci.setTargetObjectLiteral(target.replaceAll("locals\\[(\\d+)\\]\\.data\\.o", "olocals_$1_")); + ci.setTargetObjectLiteral(Util.rewriteLocalObjectRefs(target)); } String[] args = ci.getLiteralArgs(); if (args != null) { for (int j=0; j added = new HashSet(); - for (LocalVariable lv : localVariables) { + // Sorted, not in localVariables iteration order: that is a HashSet, so the + // order of these declarations varied between builds of the same input. + // debugVarEntries already had to learn this for the debug side-table; the + // C declarations had the same defect and it stayed invisible because + // HotSpot's identity hash is stable within a run. Translating the + // translator with itself is what surfaced it -- a different runtime, a + // different order, and the same input produced different C. + for (LocalVariable lv : declarationOrderedLocals()) { String variableName = lv.getQualifier() + "locals_"+lv.getIndex()+"_"; if (!added.contains(variableName) && (barebone || lv.getQualifier() != 'o')) { added.add(variableName); @@ -2174,7 +2193,7 @@ public void appendVirtualMethodC(String cls, StringBuilder b, String offset, boo b.append(cls); b.append("(threadStateData);\n "); } - if (System.getProperty("INCLUDE_NPE_CHECKS", "false").equals("true")) { + if (Util.getProperty("INCLUDE_NPE_CHECKS", "false").equals("true")) { b.append("\n if(__cn1ThisObject == JAVA_NULL) THROW_NULL_POINTER_EXCEPTION();\n "); } if(!returnType.isVoid()) { @@ -2359,7 +2378,7 @@ public NativeSignatureVerifier.Signature getNativeSignature() { } return new NativeSignatureVerifier.Signature(symbol.toString(), clsName, methodName, overloadPrefix, cReturnType.toString().trim(), params, - prototype.toString().trim().replaceAll("\\s+", " ")); + Util.collapseWhitespace(prototype.toString().trim())); } public boolean isAbstract() { @@ -2448,6 +2467,10 @@ public void addDebugInfo(int line) { } public void addLabel(Label l) { + // Named here, in bytecode order, so the generated C label is a function of the + // method alone. See LabelInstruction.assignLabelName. + com.codename1.tools.translator.bytecodes.LabelInstruction.assignLabelName(l, nextLabelIndex); + nextLabelIndex++; addInstruction(new com.codename1.tools.translator.bytecodes.LabelInstruction(l)); } @@ -2455,6 +2478,9 @@ public void addInvoke(int opcode, String owner, String name, String desc, boolea addInstruction(new Invoke(opcode, owner, name, desc, itf)); } + /** Per-method label counter; see addLabel. */ + private int nextLabelIndex; + public void setMaxes(int maxStack, int maxLocals) { this.maxLocals = maxLocals; this.maxStack = maxStack; @@ -2755,7 +2781,7 @@ public void setEliminated(boolean eliminated) { private int varCounter = 0; // Master off-switch: -DCN1_DISABLE_BCE=true reverts to fully-checked array access. private static final boolean DISABLE_BCE = - "true".equalsIgnoreCase(System.getProperty("CN1_DISABLE_BCE", "false")); + "true".equalsIgnoreCase(Util.getProperty("CN1_DISABLE_BCE", "false")); /** * Prove-safe array-bounds-check elimination. Conservative and fail-closed: @@ -2931,7 +2957,7 @@ private static boolean bceForeignEntry(java.util.List r, java.util. // the whole struct to registers. // ------------------------------------------------------------------ private static final boolean DISABLE_SCALAR_REPLACE = - "true".equalsIgnoreCase(System.getProperty("CN1_DISABLE_SCALAR_REPLACE", "false")); + "true".equalsIgnoreCase(Util.getProperty("CN1_DISABLE_SCALAR_REPLACE", "false")); private static String srMangle(String s) { return s.replace('.', '_').replace('/', '_').replace('$', '_'); @@ -3223,7 +3249,7 @@ private void scalarReplaceStackAllocations() { // can't dispatch to an escaping override. // ------------------------------------------------------------------ private static final boolean DISABLE_SB_STACK_ALLOC = - "true".equalsIgnoreCase(System.getProperty("CN1_DISABLE_SB_STACK_ALLOC", "false")); + "true".equalsIgnoreCase(Util.getProperty("CN1_DISABLE_SB_STACK_ALLOC", "false")); private static final String SB_OWNER = "java/lang/StringBuilder"; /** Slots consumed by the argument list of a method descriptor (no receiver). */ diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/DebugSymbolCompressor.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/DebugSymbolCompressor.java new file mode 100644 index 00000000000..1f9b06dbab7 --- /dev/null +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/DebugSymbolCompressor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.zip.GZIPOutputStream; + +/** + * Compresses the on-device-debug symbol table. + * + * This is the translator's only use of {@code java.util.zip}, and it exists as its + * own class so that it is the only thing that has to be replaced when the + * translator is compiled against ParparVM's JavaAPI in order to translate itself. + * JavaAPI has no java.util.zip and cannot gain one: it is mirrored by + * Ports/CLDC11, where the package does not belong. + * + * Nothing else needs the package. The translator reads directories of class files, + * never archives -- every caller extracts a jar before invoking it -- and + * {@code NativeSignatureVerifier}'s archive scan lives behind its own command-line + * entry point. + * + * Symbol tables are large and highly repetitive, so compressing keeps a debug + * binary's footprint modest. + */ +final class DebugSymbolCompressor { + private DebugSymbolCompressor() { + } + + static byte[] gzip(ByteArrayOutputStream raw) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(raw.size() / 3 + 64); + GZIPOutputStream gz = new GZIPOutputStream(out); + try { + raw.writeTo(gz); + } finally { + gz.close(); + } + return out.toByteArray(); + } +} diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptNativeRegistry.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptNativeRegistry.java index 04be9ad4685..d1faac4c9f0 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptNativeRegistry.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptNativeRegistry.java @@ -40,6 +40,7 @@ enum NativeCategory { "cn1_java_lang_Class_forNameImpl_java_lang_String_R_java_lang_Class", "cn1_java_lang_Class_getComponentType_R_java_lang_Class", "cn1_java_lang_Class_getNameImpl_R_java_lang_String", + "cn1_java_lang_Class_getPrimitiveClass_int_R_java_lang_Class", "cn1_java_lang_Class_getName_R_java_lang_String", "cn1_java_lang_Class_getSuperclass_R_java_lang_Class", "cn1_java_lang_Class_hashCode_R_int", diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifier.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifier.java index 3db5a800842..4ae3e021028 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifier.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifier.java @@ -27,7 +27,6 @@ import org.objectweb.asm.MethodVisitor; import org.objectweb.asm.Opcodes; -import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; @@ -48,8 +47,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; /** * Checks that every {@code native} method in a translated project has a C @@ -400,24 +397,35 @@ public int size() { } } - /** Reads {@link #IGNORE_FILE}: one symbol or {@code prefix*} per line. */ + /** + * Reads {@link #IGNORE_FILE}: one symbol or {@code prefix*} per line. + * + * Splits the file itself rather than using a BufferedReader, which ParparVM's + * JavaAPI does not declare -- this runs during translation, so it has to compile + * when the translator is built against that JavaAPI to translate itself. + */ static void readIgnoreFile(File file, Set into) throws IOException { - BufferedReader reader = new BufferedReader( - new InputStreamReader(new FileInputStream(file), UTF8)); - try { - String line; - while ((line = reader.readLine()) != null) { - int hash = line.indexOf('#'); - if (hash >= 0) { - line = line.substring(0, hash); - } - line = line.trim(); - if (line.length() > 0) { - into.add(line); - } + String text = new String(readAll(file), UTF8); + int start = 0; + while (start <= text.length()) { + int end = text.indexOf('\n', start); + String line = end < 0 ? text.substring(start) : text.substring(start, end); + // Accept CRLF as readLine did. + if (line.endsWith("\r")) { + line = line.substring(0, line.length() - 1); + } + int hash = line.indexOf('#'); + if (hash >= 0) { + line = line.substring(0, hash); + } + line = line.trim(); + if (line.length() > 0) { + into.add(line); + } + if (end < 0) { + break; } - } finally { - reader.close(); + start = end + 1; } } @@ -817,7 +825,7 @@ private static List splitTopLevel(String text) { private static String normalizeParameter(String declaration) { String text = declaration.replace("*", " * ").trim(); List tokens = new ArrayList( - Arrays.asList(text.split("\\s+"))); + Arrays.asList(Util.splitWhitespace(text))); // "CODENAME_ONE_THREAD_STATE" is a macro that expands to a full declaration // and carries no separate name to strip. if (tokens.size() > 1 && !"CODENAME_ONE_THREAD_STATE".equals(tokens.get(0))) { @@ -1172,7 +1180,7 @@ public static List collectFromClasses(File root) throws IOException { if (root.isDirectory()) { collectClassesFromDirectory(root, found); } else if (root.getName().endsWith(".jar") || root.getName().endsWith(".zip")) { - collectClassesFromArchive(root, found); + ArchiveClassScanner.collect(root, found); } else if (root.getName().endsWith(".class")) { collectFromClassBytes(readAll(root), found); } @@ -1195,32 +1203,7 @@ private static void collectClassesFromDirectory(File dir, List into) } } - private static void collectClassesFromArchive(File archive, List into) throws IOException { - ZipFile zip = new ZipFile(archive); - try { - List names = new ArrayList(); - for (Enumeration e = zip.entries(); e.hasMoreElements();) { - ZipEntry entry = e.nextElement(); - if (!entry.isDirectory() && entry.getName().endsWith(".class") - && !entry.getName().endsWith("module-info.class")) { - names.add(entry.getName()); - } - } - Collections.sort(names); - for (String name : names) { - InputStream in = zip.getInputStream(zip.getEntry(name)); - try { - collectFromClassBytes(readAll(in), into); - } finally { - in.close(); - } - } - } finally { - zip.close(); - } - } - - private static void collectFromClassBytes(byte[] bytes, final List into) { + static void collectFromClassBytes(byte[] bytes, final List into) { final String[] owner = new String[1]; new ClassReader(bytes).accept(new ClassVisitor(Opcodes.ASM9) { @Override @@ -1293,74 +1276,6 @@ private static boolean isIdentifier(String s) { return true; } - public static void main(String[] args) throws IOException { - List classRoots = new ArrayList(); - List nativeRoots = new ArrayList(); - boolean orphans = true; - for (int iter = 0; iter < args.length; iter++) { - if ("--classes".equals(args[iter]) && iter + 1 < args.length) { - classRoots.add(new File(args[++iter])); - } else if ("--natives".equals(args[iter]) && iter + 1 < args.length) { - nativeRoots.add(new File(args[++iter])); - } else if ("--no-orphans".equals(args[iter])) { - orphans = false; - } else { - System.err.println("unrecognised argument: " + args[iter]); - usage(); - System.exit(2); - } - } - if (classRoots.isEmpty() || nativeRoots.isEmpty()) { - usage(); - System.exit(2); - } - - List required = new ArrayList(); - for (File root : classRoots) { - if (!root.exists()) { - System.err.println("NativeSignatureVerifier: no such path: " + root); - System.exit(2); - } - required.addAll(collectFromClasses(root)); - } - List sources = new ArrayList(); - for (File root : nativeRoots) { - if (!root.exists()) { - System.err.println("NativeSignatureVerifier: no such path: " + root); - System.exit(2); - } - sources.addAll(root.isDirectory() - ? listNativeSourcesRecursive(root) : Collections.singletonList(root)); - } - - SourceIndex index = new SourceIndex(sources); - List problems = verify(required, index); - if (!orphans) { - List filtered = new ArrayList(); - for (Problem problem : problems) { - if (problem.kind != Kind.ORPHAN) { - filtered.add(problem); - } - } - problems = filtered; - } - - if (problems.isEmpty()) { - System.out.println("NativeSignatureVerifier: " + required.size() - + " native method(s) all resolve against " + index.size() - + " C definition(s) in " + sources.size() + " file(s)."); - return; - } - int fatal = report(problems, Mode.STRICT, - required.size() + " native methods, " + sources.size() + " native sources", true); - System.exit(fatal > 0 ? 1 : 0); - } - - private static void usage() { - System.err.println("usage: NativeSignatureVerifier --classes DIR_OR_JAR [--classes ...]" - + " --natives DIR [--natives ...] [--no-orphans]"); - } - private NativeSignatureVerifier() { } } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifierCli.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifierCli.java new file mode 100644 index 00000000000..83f082c0d93 --- /dev/null +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifierCli.java @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Command-line entry point for {@link NativeSignatureVerifier}, driven by + * scripts/check-native-signatures.sh. + * + * Split out of the verifier for two reasons, both about the self-hosted translator + * build. It is the half that scans jars, so it is the half that needs + * java.util.zip -- which JavaAPI cannot gain, being mirrored by Ports/CLDC11. And a + * second class carrying a {@code main} makes ByteCodeClass.addMethod refuse the + * translation outright with "Multiple main classes", since the clean target does + * not set a preferred main class the way the JavaScript target does. + * + * A translation never comes through here: the verifier's in-process entry points + * are what Parser calls. + */ +public final class NativeSignatureVerifierCli { + private NativeSignatureVerifierCli() { + } + + public static void main(String[] args) throws IOException { + List classRoots = new ArrayList(); + List nativeRoots = new ArrayList(); + boolean orphans = true; + for (int iter = 0; iter < args.length; iter++) { + if ("--classes".equals(args[iter]) && iter + 1 < args.length) { + classRoots.add(new File(args[++iter])); + } else if ("--natives".equals(args[iter]) && iter + 1 < args.length) { + nativeRoots.add(new File(args[++iter])); + } else if ("--no-orphans".equals(args[iter])) { + orphans = false; + } else { + System.err.println("unrecognised argument: " + args[iter]); + usage(); + System.exit(2); + } + } + if (classRoots.isEmpty() || nativeRoots.isEmpty()) { + usage(); + System.exit(2); + } + + List required = new ArrayList(); + for (File root : classRoots) { + if (!root.exists()) { + System.err.println("NativeSignatureVerifier: no such path: " + root); + System.exit(2); + } + required.addAll(NativeSignatureVerifier.collectFromClasses(root)); + } + List sources = new ArrayList(); + for (File root : nativeRoots) { + if (!root.exists()) { + System.err.println("NativeSignatureVerifier: no such path: " + root); + System.exit(2); + } + sources.addAll(root.isDirectory() + ? NativeSignatureVerifier.listNativeSourcesRecursive(root) : Collections.singletonList(root)); + } + + NativeSignatureVerifier.SourceIndex index = new NativeSignatureVerifier.SourceIndex(sources); + List problems = NativeSignatureVerifier.verify(required, index); + if (!orphans) { + List filtered = new ArrayList(); + for (NativeSignatureVerifier.Problem problem : problems) { + if (problem.kind != NativeSignatureVerifier.Kind.ORPHAN) { + filtered.add(problem); + } + } + problems = filtered; + } + + if (problems.isEmpty()) { + System.out.println("NativeSignatureVerifier: " + required.size() + + " native method(s) all resolve against " + index.size() + + " C definition(s) in " + sources.size() + " file(s)."); + return; + } + int fatal = NativeSignatureVerifier.report(problems, NativeSignatureVerifier.Mode.STRICT, + required.size() + " native methods, " + sources.size() + " native sources", true); + System.exit(fatal > 0 ? 1 : 0); + } + + private static void usage() { + System.err.println("usage: NativeSignatureVerifier --classes DIR_OR_JAR [--classes ...]" + + " --natives DIR [--natives ...] [--no-orphans]"); + } +} diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java index d462fe03d28..555c9702d91 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java @@ -25,7 +25,6 @@ import java.io.*; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.util.*; import org.objectweb.asm.AnnotationVisitor; @@ -162,7 +161,7 @@ public static void parse(File sourceFile) throws Exception { } BytecodeMethod.setDependencyGraph(dependencyGraph); ClassReader r; - try (InputStream in = Files.newInputStream(sourceFile.toPath())) { + try (InputStream in = new FileInputStream(sourceFile)) { r = new ClassReader(in); } Parser p = new Parser(); @@ -273,7 +272,7 @@ public static int jdwpAccessFlagsOf(ByteCodeField bf) { */ private static void writeSymbolSidecar(File outputDirectory) throws IOException { java.io.ByteArrayOutputStream raw = new java.io.ByteArrayOutputStream(1 << 20); - try (Writer w = new OutputStreamWriter(raw, StandardCharsets.UTF_8)) { + try (Writer w = new OutputStreamWriter(raw, "UTF-8")) { w.write("version\t1\n"); for (ByteCodeClass bc : classes) { String src = bc.getSourceFile(); @@ -355,14 +354,10 @@ private static void writeSymbolSidecar(File outputDirectory) throws IOException // gzip the payload — symbol tables are large and highly repetitive, // so this keeps the debug binary's footprint modest. - java.io.ByteArrayOutputStream gzOut = new java.io.ByteArrayOutputStream(raw.size() / 3 + 64); - try (java.util.zip.GZIPOutputStream gz = new java.util.zip.GZIPOutputStream(gzOut)) { - raw.writeTo(gz); - } - byte[] gz = gzOut.toByteArray(); + byte[] gz = DebugSymbolCompressor.gzip(raw); File f = new File(outputDirectory, "cn1_debug_symbols.c"); - try (Writer w = new OutputStreamWriter(Files.newOutputStream(f.toPath()), StandardCharsets.UTF_8)) { + try (Writer w = new OutputStreamWriter(new FileOutputStream(f), "UTF-8")) { w.write("/* Auto-generated by the Codename One iOS translator. Do not edit.\n"); w.write(" * On-device-debug symbol table (gzip-compressed), streamed to the\n"); w.write(" * desktop debug proxy over CMD_GET_SYMBOLS. */\n"); @@ -378,8 +373,8 @@ private static void writeSymbolSidecar(File outputDirectory) throws IOException } w.write("0x"); int b = gz[i] & 0xff; - w.write(Character.forDigit(b >> 4, 16)); - w.write(Character.forDigit(b & 0xf, 16)); + w.write(Util.hexDigit(b >> 4)); + w.write(Util.hexDigit(b & 0xf)); w.write(','); w.write((i & 15) == 15 ? '\n' : ' '); } @@ -864,7 +859,7 @@ public static void writeOutput(File outputDirectory) throws Exception { generateClassAndMethodIndexHeader(outputDirectory); - boolean concatenate = "true".equals(System.getProperty("concatenateFiles", "false")); + boolean concatenate = "true".equals(Util.getProperty("concatenateFiles", "false")); ConcatenatingFileOutputStream cos = concatenate ? new ConcatenatingFileOutputStream(outputDirectory) : null; for(ByteCodeClass bc : classes) { @@ -903,7 +898,7 @@ public static void writeOutput(File outputDirectory) throws Exception { } private static void readNativeFiles(File outputDirectory) throws IOException { - File[] mFiles = outputDirectory.listFiles(file -> + File[] mFiles = Util.listFiles(outputDirectory, file -> file.getName().endsWith(".m") || file.getName().endsWith("." + ByteCodeTranslator.output.extension())); if(mFiles == null) { return; @@ -1111,7 +1106,14 @@ private static int cullClasses(boolean found, int depth) { // 2nd pass to mark classes as eliminated so that we can propagate down to each // method of the class to mark it eliminated so that virtual methods // aren't included later on when writing virtual methods - Set removedClasses = new HashSet<>(classes); + // LinkedHashSet, not HashSet: ByteCodeClass overrides neither equals nor + // hashCode, so a HashSet here iterates in identity-hash order. Elimination + // is greedy and monotone -- isMethodUsed treats an already-eliminated + // caller as no caller -- so with a cycle in the call graph the ORDER + // decides which member of the cycle survives. Two runtimes hash + // identities differently and culled different methods from the same + // input; translating the translator with itself is what exposed it. + Set removedClasses = new LinkedHashSet<>(classes); tmp.forEach(removedClasses::remove); int nfound = 0; for (ByteCodeClass cls : removedClasses) { @@ -1154,7 +1156,7 @@ private static void writeFile(ByteCodeClass cls, File outputDir, ConcatenatingFi // it back to one file per class. writeBufferInstead != null && ByteCodeTranslator.output.isApple() ? writeBufferInstead : - Files.newOutputStream(new File(outputDir, cls.getClsName() + "." + ByteCodeTranslator.output.extension()).toPath()); + new FileOutputStream(new File(outputDir, cls.getClsName() + "." + ByteCodeTranslator.output.extension())); if (outMain instanceof ConcatenatingFileOutputStream) { ((ConcatenatingFileOutputStream)outMain).beginNextFile(cls.getClsName()); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Util.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Util.java index a02f3520ae8..c8da001f38e 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Util.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Util.java @@ -24,6 +24,10 @@ import com.codename1.tools.translator.bytecodes.Instruction; import com.codename1.tools.translator.bytecodes.TryCatch; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -422,4 +426,291 @@ public static char[] getStackOutputTypes(Instruction instr) { } } + + /** + * Writes {@code data} to {@code target}, replacing it. + * + * Stands in for {@code Files.write(Path, byte[])}. ParparVM's JavaAPI has no + * java.nio.file, and the translator has to compile against it to be able to + * translate itself, so the whole translator stays on java.io. + */ + public static void writeBytes(File target, byte[] data) throws IOException { + OutputStream out = new FileOutputStream(target); + try { + out.write(data); + } finally { + out.close(); + } + } + + /** + * The path of {@code f} relative to {@code root}, with '/' separators. + * + * Stands in for {@code root.toPath().relativize(f.toPath())} for the one case + * that needs it: {@code f} is always found by walking {@code root}, so it is + * always underneath it and no ".." segment can arise. + */ + public static String relativePath(File root, File f) { + String rootPath = root.getAbsolutePath(); + String filePath = f.getAbsolutePath(); + if (filePath.startsWith(rootPath)) { + filePath = filePath.substring(rootPath.length()); + } + filePath = filePath.replace('\\', '/'); + while (filePath.startsWith("/")) { + filePath = filePath.substring(1); + } + return filePath; + } + + /** + * Java's {@code \s}: the six characters the regex engine treats as whitespace. + * Deliberately not Character.isWhitespace, which differs -- it excludes the + * vertical tab and accepts many Unicode separators. + * + * 0x0B rather than an escape because a raw control byte in a source file is + * what check-control-characters.py exists to reject. + */ + private static boolean isRegexWhitespace(char c) { + return c == ' ' || c == '\t' || c == '\n' || c == 0x0B || c == '\f' || c == '\r'; + } + + /** + * Equivalent of {@code s.split(String.valueOf(separator))} for a separator that + * is not a regex metacharacter, including the trailing-empty-string removal + * String.split does at the default limit of zero. + * + * The translator has to compile against ParparVM's JavaAPI in order to translate + * itself, and String.split is not declared there. It is one of the methods + * BytecodeComplianceMojo rewrites onto com.codename1.util.regex precisely + * because JavaAPI lacks it, so adding it there would leave two regex engines and + * a rewrite rule whose premise had become false. The few call sites here lose + * the regex instead. + */ + public static String[] splitLiteral(String s, char separator) { + // String.split returns { s } when the pattern never matches, WITHOUT the + // trailing-empty removal below -- so "".split(";") is { "" }, not { }. Missing + // this is the one way a hand-written splitter and the regex part company on + // an input a caller can actually produce (an unset build hint). + if (s.indexOf(separator) < 0) { + return new String[] { s }; + } + List parts = new ArrayList(); + int start = 0; + for (int i = 0; i < s.length(); i++) { + if (s.charAt(i) == separator) { + parts.add(s.substring(start, i)); + start = i + 1; + } + } + parts.add(s.substring(start)); + int end = parts.size(); + while (end > 0 && parts.get(end - 1).isEmpty()) { + end--; + } + return parts.subList(0, end).toArray(new String[end]); + } + + /** + * Equivalent of {@code s.split("\\s+")}, including the leading empty string + * String.split produces when the input starts with whitespace, and the removal + * of trailing empty strings. See {@link #splitLiteral} for why this is not a + * regex. + */ + public static String[] splitWhitespace(String s) { + // See splitLiteral: no match means { s }, trailing-empty removal skipped. + boolean matched = false; + for (int j = 0; j < s.length(); j++) { + if (isRegexWhitespace(s.charAt(j))) { + matched = true; + break; + } + } + if (!matched) { + return new String[] { s }; + } + List parts = new ArrayList(); + int i = 0; + int start = 0; + while (i < s.length()) { + if (isRegexWhitespace(s.charAt(i))) { + parts.add(s.substring(start, i)); + while (i < s.length() && isRegexWhitespace(s.charAt(i))) { + i++; + } + start = i; + } else { + i++; + } + } + parts.add(s.substring(start)); + int end = parts.size(); + while (end > 0 && parts.get(end - 1).isEmpty()) { + end--; + } + return parts.subList(0, end).toArray(new String[end]); + } + + /** + * Equivalent of {@code s.replaceAll("\\s+", " ")}. See {@link #splitLiteral} + * for why this is not a regex. + */ + public static String collapseWhitespace(String s) { + StringBuilder b = new StringBuilder(s.length()); + int i = 0; + while (i < s.length()) { + char c = s.charAt(i); + if (isRegexWhitespace(c)) { + b.append(' '); + while (i < s.length() && isRegexWhitespace(s.charAt(i))) { + i++; + } + } else { + b.append(c); + i++; + } + } + return b.toString(); + } + + /** + * Equivalent of + * {@code s.replaceAll("locals\\[(\\d+)\\]\\.data\\.o", "olocals_$1_")}: rewrites + * an indexed object local into the scalar-replaced name the barebone path emits. + * + * Besides removing the regex (see {@link #splitLiteral}), this drops a Pattern + * compile that used to happen once per barebone method in every build. + */ + public static String rewriteLocalObjectRefs(String s) { + final String prefix = "locals["; + final String suffix = "].data.o"; + int at = s.indexOf(prefix); + if (at < 0) { + return s; + } + StringBuilder b = new StringBuilder(s.length()); + int from = 0; + while (at >= 0) { + int digits = at + prefix.length(); + int end = digits; + while (end < s.length() && s.charAt(end) >= '0' && s.charAt(end) <= '9') { + end++; + } + if (end > digits && s.startsWith(suffix, end)) { + b.append(s, from, at); + b.append("olocals_").append(s, digits, end).append('_'); + from = end + suffix.length(); + } else { + // \d+ needs at least one digit and "].data.o" must follow it, so this + // occurrence is not a match; copy it through and keep scanning after it. + b.append(s, from, digits); + from = digits; + } + at = s.indexOf(prefix, from); + } + b.append(s, from, s.length()); + return b.toString(); + } + + /** + * {@code System.getProperty(key, defaultValue)}, falling back to the + * environment. + * + * ParparVM's JavaAPI declares only the one-argument form, and it returns null + * unconditionally -- a native binary has no -D to read. The translator has to + * compile against that JavaAPI in order to translate itself, so the two-argument + * form is provided here instead of being added to JavaAPI, and every knob gains + * an environment spelling that works in a translated build. cn1.sqlite is read + * from CN1_SQLITE, INCLUDE_NPE_CHECKS from INCLUDE_NPE_CHECKS. + * + * NativeSignatureVerifier.mode() already reached for getenv for exactly this + * reason; this generalizes it rather than adding a second convention. + */ + public static String getProperty(String key, String defaultValue) { + String value = System.getProperty(key); + if (value == null) { + value = System.getenv(environmentName(key)); + } + return value == null ? defaultValue : value; + } + + /** + * "cn1.sqlite" -> "CN1_SQLITE". Folded by hand: String.toUpperCase is locale + * sensitive and CN1 has no java.util.Locale to ask for the root locale, so on a + * Turkish device the 'i' of "cn1.sqlite" would not fold to 'I' and the variable + * would never be found. + */ + private static String environmentName(String key) { + StringBuilder b = new StringBuilder(key.length()); + for (int i = 0; i < key.length(); i++) { + char c = key.charAt(i); + if (c >= 'a' && c <= 'z') { + b.append((char) (c - 'a' + 'A')); + } else if ((c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) { + b.append(c); + } else { + b.append('_'); + } + } + return b.toString(); + } + + /** + * Stands in for {@code java.io.FileFilter}, which ParparVM's JavaAPI does not + * declare. Kept as a functional interface so the call sites keep their lambdas. + */ + public interface FileMatcher { + boolean accept(File file); + } + + /** + * Stands in for {@code java.io.FilenameFilter}. + */ + public interface FileNameMatcher { + boolean accept(File dir, String name); + } + + /** + * {@code dir.listFiles(filter)}, including its null return when {@code dir} is + * not a directory -- callers test for it. + */ + public static File[] listFiles(File dir, FileMatcher matcher) { + File[] all = dir.listFiles(); + if (all == null) { + return null; + } + List kept = new ArrayList(all.length); + for (int i = 0; i < all.length; i++) { + if (matcher.accept(all[i])) { + kept.add(all[i]); + } + } + return kept.toArray(new File[kept.size()]); + } + + /** + * {@code dir.list(filter)}, including its null return when {@code dir} is not a + * directory. + */ + public static String[] list(File dir, FileNameMatcher matcher) { + String[] all = dir.list(); + if (all == null) { + return null; + } + List kept = new ArrayList(all.length); + for (int i = 0; i < all.length; i++) { + if (matcher.accept(dir, all[i])) { + kept.add(all[i]); + } + } + return kept.toArray(new String[kept.size()]); + } + + /** + * {@code Character.forDigit(digit, 16)} for a digit already known to be in + * range. JavaAPI has no forDigit. + */ + public static char hexDigit(int digit) { + return (char) (digit < 10 ? '0' + digit : 'a' - 10 + digit); + } } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomInvoke.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomInvoke.java index 2ec078764bd..67bf006a8c1 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomInvoke.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomInvoke.java @@ -343,7 +343,7 @@ public boolean appendExpression(StringBuilder b) { ArrayList args = new ArrayList<>(); String returnVal = BytecodeMethod.appendMethodSignatureSuffixFromDesc(desc, bld, args); if (isVirtualCall) { - BytecodeMethod.addVirtualMethodsInvoked(bld.substring("virtual_".length())); + BytecodeMethod.addVirtualMethodsInvoked(bld.toString().substring("virtual_".length())); } else { // keep in sync with Invoke: direct/devirtualized calls of the mapped // String/StringBuilder natives get the inlined fast path @@ -595,7 +595,7 @@ public void appendInstruction(StringBuilder b) { ArrayList args = new ArrayList<>(); String returnVal = BytecodeMethod.appendMethodSignatureSuffixFromDesc(desc, bld, args); if (isVirtualCall) { - BytecodeMethod.addVirtualMethodsInvoked(bld.substring("virtual_".length())); + BytecodeMethod.addVirtualMethodsInvoked(bld.toString().substring("virtual_".length())); } else { // keep in sync with Invoke: direct/devirtualized calls of the mapped // String/StringBuilder natives get the inlined fast path diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomJump.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomJump.java index b93718c8429..07ab36aa1e9 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomJump.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomJump.java @@ -68,13 +68,13 @@ public void appendInstruction(StringBuilder b, List instructions) { if(TryCatch.isTryCatchInMethod()) { b.append("JUMP_TO(label_"); - b.append(label.toString()); + b.append(LabelInstruction.labelName(label)); b.append(", "); b.append(LabelInstruction.getLabelCatchDepth(label, instructions)); b.append(");\n"); } else { b.append("goto label_"); - b.append(label.toString()); + b.append(LabelInstruction.labelName(label)); b.append(";\n"); } if(customSuffix != null) { diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Invoke.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Invoke.java index 4f7adfa789c..ba0a5c67f6e 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Invoke.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Invoke.java @@ -364,7 +364,7 @@ public void appendInstruction(StringBuilder b) { ArrayList args = new ArrayList<>(); String returnVal = BytecodeMethod.appendMethodSignatureSuffixFromDesc(desc, bld, args); if (isVirtualCall) { - BytecodeMethod.addVirtualMethodsInvoked(bld.substring("virtual_".length())); + BytecodeMethod.addVirtualMethodsInvoked(bld.toString().substring("virtual_".length())); } else { // direct/devirtualized calls of the hottest String/StringBuilder // natives get the call-site-inlined fast path (cn1_intrinsics.h) @@ -501,7 +501,7 @@ public void appendInstruction(StringBuilder b) { // Master off-switch: -DCN1_DISABLE_INLINE=true disables trivial-method inlining. private static final boolean DISABLE_INLINE = - "true".equalsIgnoreCase(System.getProperty("CN1_DISABLE_INLINE", "false")); + "true".equalsIgnoreCase(Util.getProperty("CN1_DISABLE_INLINE", "false")); /** * If this invoke is a direct (provably monomorphic) instance call to a trivial diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Jump.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Jump.java index cc0594d406d..718b0c21c24 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Jump.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Jump.java @@ -112,13 +112,13 @@ public void appendInstruction(StringBuilder b, List instructions) { if(TryCatch.isTryCatchInMethod()) { b.append("JUMP_TO(label_"); - b.append(label.toString()); + b.append(LabelInstruction.labelName(label)); b.append(", "); b.append(LabelInstruction.getLabelCatchDepth(label, instructions)); b.append(");\n"); } else { b.append("goto label_"); - b.append(label.toString()); + b.append(LabelInstruction.labelName(label)); b.append(";\n"); } } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/LabelInstruction.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/LabelInstruction.java index 0c3d85e1ceb..f7c228030c7 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/LabelInstruction.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/LabelInstruction.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.HashMap; +import java.util.IdentityHashMap; import java.util.Hashtable; import java.util.List; import java.util.Map; @@ -56,6 +57,55 @@ static class Pair { // a lot of strings. private static Map usedLabels = new Hashtable(); + /** + * Stable names for the C labels generated from ASM labels. + * + * These used to be {@code Label.toString()}, which ASM defines as + * {@code "L" + System.identityHashCode(this)}. That made the emitted C depend on + * identity hash codes, with two consequences. It is not reproducible -- nothing + * promises an identity hash is stable. And it is not even VALID on a runtime + * whose identity hash can be negative: ParparVM's is the object pointer narrowed + * to int, so a self-hosted translator emitted {@code label_L-180306432001}, which + * C reads as a subtraction, and every method with a try/catch failed to compile. + * + * Numbering is per method and assigned in bytecode order as the labels are + * visited, so a method's C depends only on that method. A global counter would + * work too, but it would make every method downstream of any change renumber, + * which turns one real difference into thousands when two outputs are compared. + * + * C labels are function-scoped, so the same name in two methods is not a clash. + * + * An IdentityHashMap because Label overrides neither equals nor hashCode, and two + * distinct labels must never share a name. + */ + private static final Map labelNames = new IdentityHashMap(); + + /** + * Names {@code l} as the {@code index}th label of its method. Called from + * BytecodeMethod.addLabel while the method is being parsed. + */ + public static void assignLabelName(Label l, int index) { + if (!labelNames.containsKey(l)) { + labelNames.put(l, "L" + index); + } + } + + /** + * The C label name for {@code l}. + * + * Every label reaching emission has been through addLabel, so the fallback is + * unreachable; it is spelled with a distinct prefix so that if it ever does fire + * it cannot collide with a real per-method name. + */ + public static String labelName(Label l) { + String name = labelNames.get(l); + if (name == null) { + name = "Lx" + labelNames.size(); + labelNames.put(l, name); + } + return name; + } + // cleanup between passes, free the garbage! public static void cleanup() { @@ -63,6 +113,7 @@ public static void cleanup() tryEndLabels.clear(); labelCatchDepth.clear(); usedLabels.clear(); + labelNames.clear(); } public LabelInstruction(org.objectweb.asm.Label parent) { super(-1); @@ -160,7 +211,7 @@ public void appendInstruction(StringBuilder b) { return; } b.append("\nlabel_"); - b.append(parent); + b.append(labelName(parent)); b.append(":\n"); Integer tryCount = tryEndLabels.get(parent); if(tryCount != null) { @@ -181,19 +232,19 @@ public void appendInstruction(StringBuilder b) { for(int iter = strs.size() - 1; iter >= 0 ; iter--) { Pair s = strs.get(iter); b.append(" tryBlockOffset"); - b.append(parent); + b.append(labelName(parent)); b.append(s.cls); b.append(s.counter); b.append(" = threadStateData->tryBlockOffset;\n"); b.append(" BEGIN_TRY("); b.append(s.cls); b.append(", catch_"); - b.append(parent); + b.append(labelName(parent)); b.append(s.cls); b.append(s.counter); //b.append("); NSLog(@\"Begin try on: %s %d off: %i\\n\", __FILE__, __LINE__, getThreadLocalData()->tryBlockOffset);"); b.append(");\n restoreTo"); - b.append(parent); + b.append(labelName(parent)); b.append(s.cls); b.append(s.counter); b.append(" = threadStateData->threadObjectStackOffset;\n"); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/SwitchInstruction.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/SwitchInstruction.java index 711b8d20867..7858d24321d 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/SwitchInstruction.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/SwitchInstruction.java @@ -55,13 +55,13 @@ public void appendInstruction(StringBuilder b, List instructions) { b.append(keys[iter]); if(TryCatch.isTryCatchInMethod()) { b.append(": JUMP_TO(label_"); - b.append(labels[iter].toString()); + b.append(LabelInstruction.labelName(labels[iter])); b.append(", "); b.append(LabelInstruction.getLabelCatchDepth(labels[iter], instructions)); b.append(");\n"); } else { b.append(": goto label_"); - b.append(labels[iter].toString()); + b.append(LabelInstruction.labelName(labels[iter])); b.append(";\n"); } } @@ -69,13 +69,13 @@ public void appendInstruction(StringBuilder b, List instructions) { if(dflt != null) { if(TryCatch.isTryCatchInMethod()) { b.append(" default: JUMP_TO(label_"); - b.append(dflt.toString()); + b.append(LabelInstruction.labelName(dflt)); b.append(", "); b.append(LabelInstruction.getLabelCatchDepth(dflt, instructions)); b.append(");\n"); } else { b.append(" default: goto label_"); - b.append(dflt.toString()); + b.append(LabelInstruction.labelName(dflt)); b.append(";\n"); } } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TryCatch.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TryCatch.java index a097c80b5b1..d36cf7ad31a 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TryCatch.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TryCatch.java @@ -109,21 +109,21 @@ public void appendInstruction(StringBuilder b, List instructions) { // threadObjectStackOffset from trash and later callee frames were // allocated on top of this frame's locals. clang happened to spill. b.append(" volatile int restoreTo"); - b.append(start); + b.append(LabelInstruction.labelName(start)); b.append(cid); b.append(counter); b.append(";\n volatile int tryBlockOffset"); - b.append(start); + b.append(LabelInstruction.labelName(start)); b.append(cid); b.append(counter); b.append(";\n DEFINE_CATCH_BLOCK(catch_"); - b.append(start); + b.append(LabelInstruction.labelName(start)); b.append(cid); b.append(counter); b.append(", label_"); - b.append(handler); + b.append(LabelInstruction.labelName(handler)); b.append(", restoreTo"); - b.append(start); + b.append(LabelInstruction.labelName(start)); b.append(cid); b.append(counter); b.append(");\n"); diff --git a/vm/ByteCodeTranslator/src/javascript/parparvm_runtime.js b/vm/ByteCodeTranslator/src/javascript/parparvm_runtime.js index 3e8027247da..69330508f58 100644 --- a/vm/ByteCodeTranslator/src/javascript/parparvm_runtime.js +++ b/vm/ByteCodeTranslator/src/javascript/parparvm_runtime.js @@ -78,7 +78,11 @@ const PRIMITIVE_INFO = { JAVA_BYTE: { javaName: "byte", descriptor: "B" }, JAVA_SHORT: { javaName: "short", descriptor: "S" }, JAVA_INT: { javaName: "int", descriptor: "I" }, - JAVA_LONG: { javaName: "long", descriptor: "J" } + JAVA_LONG: { javaName: "long", descriptor: "J" }, + // void is a primitive class too -- Void.TYPE is one, and unlike the other + // eight it is not reachable through a primitive class literal, so nothing + // needed it here until getPrimitiveClass did. + JAVA_VOID: { javaName: "void", descriptor: "V" } }; const jsObjectWrappers = typeof WeakMap === "function" ? new WeakMap() : null; const externalIdentityMap = typeof WeakMap === "function" ? new WeakMap() : null; @@ -5755,6 +5759,23 @@ bindNative(["cn1_java_lang_Class_getComponentType_R_java_lang_Class"], function( } return classObjectForName(def.componentClass); }); +// Backs the wrapper classes' TYPE fields. The JavaAPI cannot initialize them with +// a primitive class literal: javac lowers `int.class` to a read of Integer.TYPE +// itself, so `TYPE = int.class` compiles to `getstatic TYPE; putstatic TYPE` and +// leaves the field null. The codes match Class.CN1_PRIM_* in the JavaAPI and the +// switch in nativeMethods.m. +// +// _primClass covers the same ground for a primitive class literal appearing in +// ordinary code; this is the path taken by the wrapper clinits themselves. +bindNative(["cn1_java_lang_Class_getPrimitiveClass_int_R_java_lang_Class"], function(typeCode) { + const names = ["JAVA_INT", "JAVA_LONG", "JAVA_SHORT", "JAVA_BYTE", "JAVA_CHAR", + "JAVA_FLOAT", "JAVA_DOUBLE", "JAVA_BOOLEAN", "JAVA_VOID"]; + const name = names[typeCode | 0]; + if (!name) { + throw new Error("getPrimitiveClass: unknown primitive type code " + typeCode); + } + return classObjectForName(name); +}); bindNative(["cn1_java_lang_Class_isPrimitive_R_boolean"], function(__cn1ThisObject) { return __cn1ThisObject.__classDef && __cn1ThisObject.__classDef.isPrimitive ? 1 : 0; }); bindNative(["cn1_java_lang_reflect_Array_newInstanceImpl_java_lang_Class_int_R_java_lang_Object"], function(componentClass, length) { if (!componentClass || !componentClass.__classDef) { diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 245bca39bb9..da508632c4e 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -1463,16 +1463,6 @@ JAVA_LONG java_lang_Double_doubleToLongBits___double_R_long(CODENAME_ONE_THREAD_ return u.l; } -JAVA_LONG java_lang_Double_doubleToRawLongBits___double_R_long(CODENAME_ONE_THREAD_STATE, JAVA_DOUBLE n1) { - union { - JAVA_DOUBLE d; - JAVA_LONG l; - } u; - - u.d = n1; - return u.l; -} - JAVA_FLOAT java_lang_Float_intBitsToFloat___int_R_float(CODENAME_ONE_THREAD_STATE, JAVA_INT n1) { union { @@ -2008,6 +1998,46 @@ JAVA_OBJECT java_lang_Class_getPrimitiveClass___int_R_java_lang_Class(CODENAME_O return JAVA_NULL; } +/** + * Resources linked into the executable, backing Class.getResourceAsStream. + * + * cn1FindResource has a weak definition here that finds nothing. A target that + * embeds resources emits a strong one (the generated cn1_resources_table.c) and + * overrides it; everywhere else this one stands and getResourceAsStream falls + * through to the filesystem. That keeps every existing target unchanged -- + * getResourceAsStream returned a hard-coded null before this existed, so nothing + * can regress, only start working. + * + * A weak DEFINITION rather than a weak declaration: Mach-O will not link an + * undefined weak symbol without weak_import, while a weak definition is overridable + * on both Mach-O and ELF. + */ +__attribute__((weak)) const unsigned char* cn1FindResource(const char* name, int* lenOut) { + (void)name; + if(lenOut) { + *lenOut = 0; + } + return 0; +} + +JAVA_OBJECT java_lang_Class_cn1EmbeddedResource___java_lang_String_R_byte_1ARRAY(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT name) { + if(name == JAVA_NULL) { + return JAVA_NULL; + } + const char* n = stringToUTF8(threadStateData, name); + if(n == 0) { + return JAVA_NULL; + } + int len = 0; + const unsigned char* data = cn1FindResource(n, &len); + if(data == 0 || len <= 0) { + return JAVA_NULL; + } + JAVA_OBJECT arr = __NEW_ARRAY_JAVA_BYTE(threadStateData, len); + memcpy(((JAVA_ARRAY)arr)->data, data, len); + return arr; +} + JAVA_BOOLEAN java_lang_Class_isArray___R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT cls) { struct clazz* clz = (struct clazz*)cls; return clz->isArray; diff --git a/vm/JavaAPI/src/java/lang/Class.java b/vm/JavaAPI/src/java/lang/Class.java index 4e2f0be4ce3..ff1f5aa0f3e 100644 --- a/vm/JavaAPI/src/java/lang/Class.java +++ b/vm/JavaAPI/src/java/lang/Class.java @@ -50,6 +50,21 @@ public ClassLoader getClassLoader() { * following code fragment returns the runtime Class descriptor for the * class named java.lang.Thread: Classt= Class.forName("java.lang.Thread") */ + /** + * Returns the Class object for {@code className}. + * + * ParparVM links the whole program ahead of time, so there is no second class + * loader to consult and nothing to defer: both extra arguments are accepted and + * ignored, and the class is resolved exactly as the one-argument form resolves + * it. The overload exists because library bytecode calls it -- ASM's + * ClassWriter.getCommonSuperClass does -- and an absent overload is a link + * error in translated code, not a compile error here. + */ + public static java.lang.Class forName(java.lang.String className, boolean initialize, + ClassLoader loader) throws java.lang.ClassNotFoundException { + return forName(className); + } + public static java.lang.Class forName(java.lang.String className) throws java.lang.ClassNotFoundException { className = className.replace('$', '.'); Class c = forNameImpl(className); @@ -136,7 +151,70 @@ public static java.lang.Class forName(java.lang.String className) throws java.la * class upon which the getResourceAsStream method was called. */ public java.io.InputStream getResourceAsStream(java.lang.String name){ - return null; + if (name == null) { + return null; + } + String absolute = name; + if (!absolute.startsWith("/")) { + // Relative names resolve against this class's package, as the javadoc + // above describes. + String className = getName(); + int lastDot = className.lastIndexOf('.'); + absolute = lastDot < 0 ? "/" + name + : "/" + className.substring(0, lastDot).replace('.', '/') + "/" + name; + } + byte[] embedded = cn1EmbeddedResource(absolute); + if (embedded != null) { + return new java.io.ByteArrayInputStream(embedded); + } + return cn1FileResource(absolute); + } + + /** + * Resources linked into the executable, or null when there are none. + * + * The native side calls a weakly-linked {@code cn1FindResource}, which the + * generated resource table overrides on targets that embed resources. Where + * nothing provides it the weak symbol is null and this returns null, so a target + * that embeds nothing behaves exactly as it did before this existed. + */ + private static native byte[] cn1EmbeddedResource(String name); + + /** + * The filesystem half of {@link #getResourceAsStream}: looks the resource up + * under a search path, so a translated command-line program can read files that + * sit beside it rather than being linked into it. + * + * The path comes from CN1_RESOURCE_PATH, else a "cn1runtime" directory next to + * the executable. Entries are separated the way the platform separates path + * entries. + */ + private static java.io.InputStream cn1FileResource(String absolute) { + String path = System.getenv("CN1_RESOURCE_PATH"); + if (path == null || path.length() == 0) { + return null; + } + String relative = absolute.substring(1); + int from = 0; + while (from <= path.length()) { + int end = path.indexOf(java.io.File.pathSeparatorChar, from); + String root = end < 0 ? path.substring(from) : path.substring(from, end); + if (root.length() > 0) { + java.io.File candidate = new java.io.File(root, relative); + if (candidate.exists()) { + try { + return new java.io.FileInputStream(candidate); + } catch (java.io.IOException err) { + return null; + } + } + } + if (end < 0) { + break; + } + from = end + 1; + } + return null; } /** diff --git a/vm/JavaAPI/src/java/lang/Double.java b/vm/JavaAPI/src/java/lang/Double.java index a58612b9c38..05d12f67f31 100644 --- a/vm/JavaAPI/src/java/lang/Double.java +++ b/vm/JavaAPI/src/java/lang/Double.java @@ -88,6 +88,14 @@ public byte byteValue(){ * If the argument is NaN, the result is 0x7ff8000000000000L. * In all cases, the result is a long integer that, when given to the longBitsToDouble(long) method, will produce a floating-point value equal to the argument to doubleToLongBits. */ + /** + * The raw IEEE 754 bits of {@code value}, without collapsing NaN to the + * canonical NaN. See {@link Float#floatToRawIntBits} for why this delegates. + */ + public static long doubleToRawLongBits(double value) { + return doubleToLongBits(value); + } + public native static long doubleToLongBits(double value); /** diff --git a/vm/JavaAPI/src/java/lang/Float.java b/vm/JavaAPI/src/java/lang/Float.java index a0a32ba6971..93e9d560a81 100644 --- a/vm/JavaAPI/src/java/lang/Float.java +++ b/vm/JavaAPI/src/java/lang/Float.java @@ -119,6 +119,20 @@ public boolean equals(java.lang.Object obj){ * Returns the bit representation of a single-float value. The result is a representation of the floating-point argument according to the IEEE 754 floating-point "single precision" bit layout. Bit 31 (the bit that is selected by the mask 0x80000000) represents the sign of the floating-point number. Bits 30-23 (the bits that are selected by the mask 0x7f800000) represent the exponent. Bits 22-0 (the bits that are selected by the mask 0x007fffff) represent the significand (sometimes called the mantissa) of the floating-point number. If the argument is positive infinity, the result is 0x7f800000. If the argument is negative infinity, the result is 0xff800000. If the argument is NaN, the result is 0x7fc00000. In all cases, the result is an integer that, when given to the * method, will produce a floating-point value equal to the argument to floatToIntBits. */ + /** + * The raw IEEE 754 bits of {@code value}, without collapsing NaN to the + * canonical NaN. + * + * Delegates rather than declaring a second native. ParparVM's floatToIntBits + * is a bare union punt that does not collapse NaN to the canonical NaN -- so it + * is already the raw operation, and the two differ in the spec but not here. A + * separate native would be one more mangled symbol to get wrong, silently, for + * no behavioural difference. + */ + public static int floatToRawIntBits(float value) { + return floatToIntBits(value); + } + public native static int floatToIntBits(float value); /** diff --git a/vm/JavaAPI/src/java/lang/Integer.java b/vm/JavaAPI/src/java/lang/Integer.java index f9842015071..4a1cb1a85d0 100644 --- a/vm/JavaAPI/src/java/lang/Integer.java +++ b/vm/JavaAPI/src/java/lang/Integer.java @@ -359,6 +359,18 @@ public static int signum(int i) { return (i >> 31) | (-i >>> 31); // Hacker's delight 2-7 } + /** + * Rotates the two's-complement binary representation of {@code i} left by + * {@code distance} bits. + * + * The shift distance is used modulo 32 by the JLS shift rules, which is what + * makes the negation on the right half correct for every distance, including + * zero and multiples of 32. + */ + public static int rotateLeft(int i, int distance) { + return (i << distance) | (i >>> -distance); + } + public static int compare(int f1, int f2) { if (f1 > f2) return 1; diff --git a/vm/JavaAPI/src/java/lang/TypeNotPresentException.java b/vm/JavaAPI/src/java/lang/TypeNotPresentException.java new file mode 100644 index 00000000000..5320f04fa67 --- /dev/null +++ b/vm/JavaAPI/src/java/lang/TypeNotPresentException.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +package java.lang; + +/** + * Thrown when an application tries to access a type using a string naming the + * type, but no definition for that type can be found. + */ +public class TypeNotPresentException extends java.lang.RuntimeException { + private final String typeName; + + public TypeNotPresentException(String typeName, Throwable cause) { + super("Type " + typeName + " not present", cause); + this.typeName = typeName; + } + + /** + * The fully qualified name of the unavailable type. + */ + public String typeName() { + return typeName; + } +} diff --git a/vm/selfhost/README.md b/vm/selfhost/README.md new file mode 100644 index 00000000000..cea630bb893 --- /dev/null +++ b/vm/selfhost/README.md @@ -0,0 +1,93 @@ +# Self-hosting ParparVM + +Builds `ByteCodeTranslator` with ParparVM itself: the translator's own bytecode, +plus ASM's, is translated to C and compiled into a native binary. + +It buys two things: + +1. **Validation.** The translator is a ~37k-line real program that exercises + collections, strings, file I/O, exceptions and the GC at scale. Running the + native build and the JVM build over the same input and diffing the emitted C + is an end-to-end conformance test of the whole VM, and the corpus grows on its + own as the translator does. +2. **Performance and memory.** A translation is a short-lived, allocation-heavy + batch job -- the shape where AOT should beat a cold JVM. + +## `stubs/` + +The self-hosted binary does the `clean`/`ios`/`macos` translation and nothing +else, so a few classes are replaced by no-op stubs when it is built. They are +never selected at run time; they exist so the source set compiles without +dragging in API that ParparVM's JavaAPI deliberately lacks. + +| stub | why | +|---|---| +| `Javascript*` | the JavaScript target, ~12.5k lines. Needs `java.util.regex` and `ConcurrentHashMap`. | +| `ArchiveClassScanner` | `java.util.zip`. Reachable only from `NativeSignatureVerifier`'s command-line entry point; the translator itself never reads an archive. | +| `DebugSymbolCompressor` | `java.util.zip` again, for the on-device-debug symbol sidecar. | + +`java.util.zip` cannot simply be added to JavaAPI: JavaAPI is mirrored by +`Ports/CLDC11`, where the package does not belong. + +Everything else the translator needs was removed from the translator rather than +added to JavaAPI -- see `Util`'s `splitLiteral`, `collapseWhitespace`, +`rewriteLocalObjectRefs`, `getProperty`, `listFiles` and `writeBytes`. Adding +`String.split`/`replaceAll` to JavaAPI in particular would have collided with +`BytecodeComplianceMojo`, which rewrites those calls onto +`com.codename1.util.regex` precisely because JavaAPI does not declare them. + +## Building and verifying + +```bash +export JDK_8_HOME=/path/to/a/working/jdk8 +./build-selfhost.sh # -> target/parpar +./verify-selfhost.sh # gates D and A +``` + +`build-selfhost.sh` compiles the source set against JavaAPI alone, stages ASM as +class directories (the translator walks directories, never archives), translates, +and clangs the result. The `-fwrapv -fno-strict-aliasing -fno-builtin-fmod(f)` +flags are mandatory for generated C -- Java arithmetic wraps and clang -O3 +miscompiles without them. + +The binary finds the C runtime it has to copy into its output through +`Class.getResourceAsStream`, which now consults resources linked into the +executable and then a search path named by `CN1_RESOURCE_PATH`. Before this it +returned a hard-coded null on every ParparVM target. + +## State + +Gate D (the native translator against itself) passes. Gate A (JVM against native) +is at **245 of 247 files byte-identical** on a JavaAPI-sized corpus, and binaries +built from the two trees produce identical output. + +The two files that still differ are `java_util_HashMap.c` and `.h`: the native +translator's dead-code pass culls seven more methods than the JVM's +(`cn1PutSlot`, `cn1MaybeGrow`, `clearImpl`, `containsKeyImpl`, `getImpl`, +`putImpl`, `removeImpl`), and emits them as empty stubs. Both trees compile, link +and run correctly, so the extra culling is safe here, but the two runtimes should +not disagree and the cause is not yet found. What is already ruled out: it is not +nondeterminism -- gate D passes on both sides -- and it is not identity-hash +iteration order, which was tested directly by re-running the JVM under +`-XX:hashCode=2` and getting byte-identical output. + +## What self-hosting has already found + +Three defects that were invisible to every existing test, because each was +self-consistent on HotSpot: + +- **`Integer.TYPE` and the other wrapper `TYPE` fields were null.** `TYPE = + int.class` compiles to `getstatic TYPE; putstatic TYPE`. A `Map` keyed on them + collapsed onto the single null key. `Util`'s primitive-to-C-type maps are exactly + that shape. +- **C label names came from identity hash codes.** ASM's `Label.toString()` is + `"L" + System.identityHashCode(this)`. That made the emitted C irreproducible, + and on ParparVM -- whose identity hash is the object pointer narrowed to int, so + often negative -- it emitted `label_L-180306432001`, which C reads as a + subtraction. Every method with a try/catch failed to compile. +- **C local-variable declarations were emitted in `HashSet` iteration order**, so + the same input produced different C. `debugVarEntries` had already had to learn + this for the debug side-table; the declarations had the same defect. + +Only the first is a runtime bug. The other two are reproducible-build defects in +the translator that a second runtime made visible. diff --git a/vm/selfhost/build-selfhost.sh b/vm/selfhost/build-selfhost.sh new file mode 100755 index 00000000000..108185d8eb8 --- /dev/null +++ b/vm/selfhost/build-selfhost.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# Builds the ParparVM translator with ParparVM: its own bytecode, plus ASM's, is +# translated to C and compiled into a native binary. +# +# build-selfhost.sh [-O1|-O3] default -O1 +# +# Requirements: +# JDK_8_HOME a working JDK 8 (JavaAPI and the translator compile with it) +# clang, and maven on PATH the first time (to resolve ASM) +# +# The mandatory clang flags below are not negotiable for generated C: Java +# arithmetic wraps, and clang -O3 provably miscompiles without -fwrapv +# -fno-strict-aliasing -fno-builtin-fmod(f). See vm/benchmarks/README.md. +set -e +cd "$(dirname "$0")" +REPO="$(cd ../.. && pwd)" +OPT="${1:--O1}" +CC="${CN1_SELFHOST_CC:-clang}" +J8="${JDK_8_HOME:?set JDK_8_HOME to a working JDK 8}" +OUT="$REPO/vm/selfhost/target" +mkdir -p "$OUT" + +# 1. translator classes + ASM classpath, built once by maven and then cached. +TRANSLATOR="$REPO/vm/ByteCodeTranslator/target/classes" +if [ ! -f "$TRANSLATOR/com/codename1/tools/translator/ByteCodeTranslator.class" ]; then + (cd "$REPO/vm" && mvn -q -B -pl ByteCodeTranslator -am package -DskipTests) +fi +ASM_CP_FILE="$REPO/vm/ByteCodeTranslator/target/selfhost-asm-classpath.txt" +if [ ! -f "$ASM_CP_FILE" ]; then + (cd "$REPO/vm" && mvn -q -B -pl ByteCodeTranslator dependency:build-classpath \ + -Dmdep.outputFile=target/selfhost-asm-classpath.txt) +fi +ASM_CP="$(cat "$ASM_CP_FILE")" + +# 2. the C runtime the translator emits from its own classpath resources. +for f in cn1_globals.h cn1_globals.m nativeMethods.m cn1_intrinsics.h; do + cp "$REPO/vm/ByteCodeTranslator/src/$f" "$TRANSLATOR/$f" +done + +# 3. JavaAPI, rebuilt from source whenever the source set changed. +# +# The presence check alone is not enough, and it fails in a way that looks like a VM +# bug rather than a stale cache: a class compiled before a method stopped being +# native still declares it native, so the translator emits a call to a symbol nothing +# defines. Three things invalidate it and it takes all three -- `-newer` catches an +# edited or added source, but a DELETED one moves no remaining file's timestamp, so +# the sorted manifest is what catches removals. Comparing a file list rather than +# hashing timestamps keeps this portable; `stat` takes -f on BSD and -c on Linux. +JAVAAPI="$OUT/javaapi-classes" +STAMP="$OUT/javaapi-classes.stamp" +MANIFEST="$OUT/javaapi-classes.manifest" +find "$REPO/vm/JavaAPI/src" -name '*.java' | sort > "$MANIFEST.now" +if [ ! -f "$JAVAAPI/java/lang/Object.class" ] || [ ! -f "$STAMP" ] || [ ! -f "$MANIFEST" ] || \ + ! cmp -s "$MANIFEST" "$MANIFEST.now" || \ + [ -n "$(find "$REPO/vm/JavaAPI/src" -name '*.java' -newer "$STAMP" -print -quit 2>/dev/null)" ]; then + rm -rf "$JAVAAPI"; mkdir -p "$JAVAAPI" + "$J8/bin/javac" -nowarn -Xmaxerrs 10000 -source 1.8 -target 1.8 -d "$JAVAAPI" $(cat "$MANIFEST.now") + mv "$MANIFEST.now" "$MANIFEST" + touch "$STAMP" +else + rm -f "$MANIFEST.now" +fi + +# 4. the self-host source set: every translator source except the ones stubs replace, +# the JavaScript target, and CastSemanticsVerifier (its own command-line tool). +SRC="$REPO/vm/ByteCodeTranslator/src" +STUBS="$REPO/vm/selfhost/stubs" +STUBBED=$(cd "$STUBS" && find . -name '*.java' | sed 's|.*/||;s|\.java$||' | tr '\n' '|' | sed 's/|$//') +SRCLIST="$OUT/sources.txt" +find "$SRC" -name '*.java' \ + | grep -vE "/($STUBBED)\.java$" \ + | grep -vE '/Javascript[A-Za-z]*\.java$' \ + | grep -v '/CastSemanticsVerifier\.java$' \ + | grep -v '/NativeSignatureVerifierCli\.java$' > "$SRCLIST" +find "$STUBS" -name '*.java' >> "$SRCLIST" + +# 5. compile it against JavaAPI ALONE. -Xmaxerrs because javac's default cap of 100 +# silently truncates and makes a large gap look small. +rm -rf "$OUT/classes"; mkdir -p "$OUT/classes" +"$J8/bin/javac" -nowarn -Xmaxerrs 100000 -source 1.8 -target 1.8 \ + -bootclasspath "$JAVAAPI" -cp "$ASM_CP" -d "$OUT/classes" "@$SRCLIST" + +# 6. ASM as class files: the translator walks directories, never archives. +rm -rf "$OUT/asm-classes"; mkdir -p "$OUT/asm-classes" +for jar in $(echo "$ASM_CP" | tr ':' '\n' | grep -E 'asm.*\.jar$'); do + (cd "$OUT/asm-classes" && unzip -oq "$jar" -x 'module-info.class' 'META-INF/*') +done + +# 7. translate. The app name has to be the mangled main class: three classes in the +# set declare main, and ByteCodeClass.addMethod refuses to pick one otherwise. +APP=com_codename1_tools_translator_ByteCodeTranslator +rm -rf "$OUT/out"; mkdir -p "$OUT/out" +"$J8/bin/java" -Xmx4g -cp "$TRANSLATOR:$ASM_CP" com.codename1.tools.translator.ByteCodeTranslator \ + clean "$JAVAAPI;$OUT/asm-classes;$OUT/classes" "$OUT/out" \ + "$APP" com.codename1.tools.translator "$APP" 1.0 clean none \ + > "$OUT/translate.log" 2>&1 \ + || { echo "TRANSLATE FAILED"; tail -40 "$OUT/translate.log"; exit 1; } + +# 8. compile. The .S as well as the .c: the virtual-thread context switch is emitted +# beside the generated sources and the C half references it, so a *.c-only +# invocation links against a missing cn1VirtualThreadSwitch. +SRCDIR="$OUT/out/dist/$APP-src" +ASMS=$(ls "$SRCDIR"/*.S 2>/dev/null || true) +BIN="$OUT/parpar$( [ "$OPT" = "-O3" ] && echo "-O3" || echo "" )" +$CC $OPT -w -fwrapv -fno-strict-aliasing -fno-builtin-fmod -fno-builtin-fmodf \ + $CN1_SELFHOST_CFLAGS -I"$SRCDIR" "$SRCDIR"/*.c $ASMS -lm -lpthread -o "$BIN" \ + 2> "$OUT/cc.log" || { echo "COMPILE FAILED"; tail -40 "$OUT/cc.log"; exit 1; } +echo "built $BIN" diff --git a/vm/selfhost/stubs/com/codename1/tools/translator/ArchiveClassScanner.java b/vm/selfhost/stubs/com/codename1/tools/translator/ArchiveClassScanner.java new file mode 100644 index 00000000000..dd815624bde --- /dev/null +++ b/vm/selfhost/stubs/com/codename1/tools/translator/ArchiveClassScanner.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; +import java.io.File; +import java.io.IOException; +import java.util.List; + +/** + * Stub for the self-hosted translator build. See {@code vm/selfhost/README.md}. + * + * The real class reads a jar with java.util.zip, which JavaAPI has no business + * gaining -- it is mirrored by Ports/CLDC11, where the package does not belong. + * It is reachable only from NativeSignatureVerifier's offline command-line entry + * point; a translation never reads an archive, because every caller extracts a jar + * into a directory of class files first. + */ +final class ArchiveClassScanner { + private ArchiveClassScanner() { + } + + static void collect(File archive, List into) throws IOException { + throw new UnsupportedOperationException( + "archive scanning is not built into this translator; pass a directory of class files"); + } +} diff --git a/vm/selfhost/stubs/com/codename1/tools/translator/DebugSymbolCompressor.java b/vm/selfhost/stubs/com/codename1/tools/translator/DebugSymbolCompressor.java new file mode 100644 index 00000000000..b371d7d2afc --- /dev/null +++ b/vm/selfhost/stubs/com/codename1/tools/translator/DebugSymbolCompressor.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +/** + * Stub for the self-hosted translator build. See {@code vm/selfhost/README.md}. + * + * The real class gzips the on-device-debug symbol table with java.util.zip, which + * JavaAPI has no business gaining -- it is mirrored by Ports/CLDC11, where the + * package does not belong. Reached only when cn1.onDeviceDebug is set, which is + * off by default. + */ +final class DebugSymbolCompressor { + private DebugSymbolCompressor() { + } + + static byte[] gzip(ByteArrayOutputStream raw) throws IOException { + throw new UnsupportedOperationException( + "on-device-debug symbols are not built into this translator"); + } +} diff --git a/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptBundleWriter.java b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptBundleWriter.java new file mode 100644 index 00000000000..e29a755c8dd --- /dev/null +++ b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptBundleWriter.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +/** + * See {@code vm/selfhost/README.md}. + * + * Stub for the self-hosted translator build, which does the clean/ios/macos + * targets only. Replaced on the source path -- the real class is never compiled + * into that binary, and none of these methods is reachable in it. + * + * They throw rather than returning a plausible value: the JavaScript target is + * selected explicitly, so reaching one of these would mean the binary was asked + * for a target it was not built with, and that should be loud. + */ +final class JavascriptBundleWriter { + private JavascriptBundleWriter() { + } + + static void write(File outputDirectory, List classes) throws IOException { + throw new UnsupportedOperationException("JavaScript target not built into this translator"); + } +} diff --git a/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptMethodGenerator.java b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptMethodGenerator.java new file mode 100644 index 00000000000..113c3b99a6e --- /dev/null +++ b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptMethodGenerator.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import java.util.List; + +/** + * See {@code vm/selfhost/README.md}. + * + * Stub for the self-hosted translator build, which does the clean/ios/macos + * targets only. Replaced on the source path -- the real class is never compiled + * into that binary, and none of these methods is reachable in it. + * + * They throw rather than returning a plausible value: the JavaScript target is + * selected explicitly, so reaching one of these would mean the binary was asked + * for a target it was not built with, and that should be loud. + */ +final class JavascriptMethodGenerator { + private JavascriptMethodGenerator() { + } + + static String generateClassJavascript(ByteCodeClass cls, List allClasses) { + throw new UnsupportedOperationException("JavaScript target not built into this translator"); + } +} diff --git a/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptNativeRegistry.java b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptNativeRegistry.java new file mode 100644 index 00000000000..8cff0ae37d4 --- /dev/null +++ b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptNativeRegistry.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +/** + * See {@code vm/selfhost/README.md}. + * + * Stub for the self-hosted translator build, which does the clean/ios/macos + * targets only. Replaced on the source path -- the real class is never compiled + * into that binary, and none of these methods is reachable in it. + * + * They throw rather than returning a plausible value: the JavaScript target is + * selected explicitly, so reaching one of these would mean the binary was asked + * for a target it was not built with, and that should be loud. + * + * This one answers instead of throwing: Parser consults it on EVERY target, not + * just JavaScript, and the answer for a build with no JavaScript runtime in it is + * that nothing is delegated there. + */ +final class JavascriptNativeRegistry { + private JavascriptNativeRegistry() { + } + + static boolean isRuntimeDelegateTarget(String mangledClassName, String methodName) { + return false; + } +} diff --git a/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptReachability.java b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptReachability.java new file mode 100644 index 00000000000..bde73904500 --- /dev/null +++ b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptReachability.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import java.util.List; + +/** + * See {@code vm/selfhost/README.md}. + * + * Stub for the self-hosted translator build, which does the clean/ios/macos + * targets only. Replaced on the source path -- the real class is never compiled + * into that binary, and none of these methods is reachable in it. + * + * They throw rather than returning a plausible value: the JavaScript target is + * selected explicitly, so reaching one of these would mean the binary was asked + * for a target it was not built with, and that should be loud. + */ +final class JavascriptReachability { + private JavascriptReachability() { + } + + static int run(List classes, List classPool, + String[] nativeSources) { + throw new UnsupportedOperationException("JavaScript target not built into this translator"); + } +} diff --git a/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptSuspensionAnalysis.java b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptSuspensionAnalysis.java new file mode 100644 index 00000000000..a8db88412cf --- /dev/null +++ b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptSuspensionAnalysis.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import java.util.List; + +/** + * See {@code vm/selfhost/README.md}. + * + * Stub for the self-hosted translator build, which does the clean/ios/macos + * targets only. Replaced on the source path -- the real class is never compiled + * into that binary, and none of these methods is reachable in it. + * + * They throw rather than returning a plausible value: the JavaScript target is + * selected explicitly, so reaching one of these would mean the binary was asked + * for a target it was not built with, and that should be loud. + */ +final class JavascriptSuspensionAnalysis { + private JavascriptSuspensionAnalysis() { + } + + static int run(List classes) { + throw new UnsupportedOperationException("JavaScript target not built into this translator"); + } +} diff --git a/vm/selfhost/verify-selfhost.sh b/vm/selfhost/verify-selfhost.sh new file mode 100755 index 00000000000..6ab132c8122 --- /dev/null +++ b/vm/selfhost/verify-selfhost.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# Validation gates for the self-hosted translator. +# +# verify-selfhost.sh +# +# Compares the C emitted by the JVM-hosted translator against the C emitted by the +# native one. The comparison is on the emitted SOURCE, never on the compiled binary: +# clang is not what is under test, and gating on object code would fail for toolchain +# reasons that have nothing to do with the VM. +# +# Gate D runs first and is the cheap one: the native translator against itself. If it +# is not self-consistent, nothing downstream means anything, and the cause is VM +# nondeterminism rather than a difference between the two runtimes. +# +# Gate A is the headline: same program, different runtime, identical output. +# +# Both sides run into the SAME absolute output path, sequentially, with the tree +# moved aside between runs. The generated CMakeLists embeds +# srcRoot.getAbsolutePath(), so running in one place removes a whole class of false +# differences rather than normalizing it away afterwards. Both also run under a +# constructed environment: the translator reads its knobs from getenv (see +# Util.getProperty), so a stray CN1_* variable would change one side's output. +set -e +cd "$(dirname "$0")" +REPO="$(cd ../.. && pwd)" +J8="${JDK_8_HOME:?set JDK_8_HOME to a working JDK 8}" +CLASSES="${1:?usage: verify-selfhost.sh }" +APP="${2:?}" +PKG="${3:?}" + +PARPAR="$REPO/vm/selfhost/target/parpar" +[ -x "$PARPAR" ] || { echo "no $PARPAR -- run build-selfhost.sh first"; exit 1; } +JAPI="$REPO/vm/selfhost/target/javaapi-classes" +TR="$REPO/vm/ByteCodeTranslator/target/classes" +ASM="$(cat "$REPO/vm/ByteCodeTranslator/target/selfhost-asm-classpath.txt")" + +W="$REPO/vm/selfhost/target/verify" +rm -rf "$W"; mkdir -p "$W" +OUT="$W/out" + +run() { + local tag=$1; shift + mkdir -p "$OUT" + ( cd "$W" && env -i PATH=/usr/bin:/bin HOME="$HOME" TMPDIR=/tmp LC_ALL=C \ + CN1_RESOURCE_PATH="$REPO/vm/ByteCodeTranslator/src" "$@" ) > "$W/$tag.log" 2>&1 \ + || { echo "$tag FAILED"; tail -20 "$W/$tag.log"; exit 1; } + mv "$OUT" "$W/$tag-tree" +} + +jvm_args=( "$J8/bin/java" -cp "$TR:$ASM" com.codename1.tools.translator.ByteCodeTranslator ) +common=( clean "$JAPI;$CLASSES" "$OUT" "$APP" "$PKG" "$APP" 1.0 clean none ) + +run parpar1 "$PARPAR" "${common[@]}" +run parpar2 "$PARPAR" "${common[@]}" +run jvm "${jvm_args[@]}" "${common[@]}" + +files=$(find "$W/jvm-tree" -type f | wc -l | tr -d ' ') +bytes=$(find "$W/jvm-tree" -type f -exec cat {} + | wc -c | tr -d ' ') +# A comparison of two empty trees is not a passing comparison. +[ "$files" -gt 10 ] || { echo "VACUOUS: only $files files emitted"; exit 1; } +echo "corpus: $APP -- $files files, $bytes bytes" + +fail=0 +if diff -rq "$W/parpar1-tree" "$W/parpar2-tree" > "$W/gateD.txt" 2>&1; then + echo "GATE D (parpar vs parpar): PASS" +else + echo "GATE D (parpar vs parpar): FAIL -- $(grep -c . "$W/gateD.txt") paths"; fail=1 +fi +if diff -rq "$W/jvm-tree" "$W/parpar1-tree" > "$W/gateA.txt" 2>&1; then + echo "GATE A (jvm vs parpar): PASS -- $files files byte-identical" +else + echo "GATE A (jvm vs parpar): FAIL -- $(grep -c . "$W/gateA.txt") of $files paths differ" + sed 's|.*/'"$APP"'-src/||;s| and .*||' "$W/gateA.txt" | head -20 + fail=1 +fi + +# Negative control: a comparator nobody has watched fail is not a comparator. Flip one +# byte and require the comparison to notice, so a pass above cannot be a pass by +# accident (a mis-set path, an empty tree, a diff invocation that never ran). +victim=$(find "$W/parpar1-tree" -name '*.c' | sort | head -1) +cp "$victim" "$W/victim.bak" +printf 'x' | dd of="$victim" bs=1 seek=40 conv=notrunc status=none +if diff -rq "$W/jvm-tree" "$W/parpar1-tree" > /dev/null 2>&1; then + echo "NEGATIVE CONTROL: FAIL -- a corrupted tree still compared equal"; fail=1 +else + echo "NEGATIVE CONTROL: PASS -- corruption detected" +fi +cp "$W/victim.bak" "$victim" + +exit $fail diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/UtilStringHelperTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/UtilStringHelperTest.java new file mode 100644 index 00000000000..9e84582446a --- /dev/null +++ b/vm/tests/src/test/java/com/codename1/tools/translator/UtilStringHelperTest.java @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Holds {@link Util}'s hand-written string helpers to the JDK regex behaviour they + * replaced. + * + * The translator has to compile against ParparVM's JavaAPI in order to translate + * itself, and String.split/replaceAll are not declared there -- they are among the + * methods BytecodeComplianceMojo rewrites onto com.codename1.util.regex precisely + * because JavaAPI lacks them. The call sites lost the regex rather than JavaAPI + * gaining a second engine, so the risk is that a replacement quietly disagrees and + * changes generated C. These tests compare against the originals directly, so the + * JDK is the oracle rather than a hand-written expectation. + */ +class UtilStringHelperTest { + + private static final String LOCALS_REGEX = "locals\\[(\\d+)\\]\\.data\\.o"; + private static final String LOCALS_REPLACEMENT = "olocals_$1_"; + + @Test + void rewriteLocalObjectRefsMatchesReplaceAll() { + for (String s : localsCases()) { + assertEquals(s.replaceAll(LOCALS_REGEX, LOCALS_REPLACEMENT), + Util.rewriteLocalObjectRefs(s), + "rewriteLocalObjectRefs diverged on: " + s); + } + } + + @Test + void collapseWhitespaceMatchesReplaceAll() { + for (String s : whitespaceCases()) { + assertEquals(s.replaceAll("\\s+", " "), Util.collapseWhitespace(s), + "collapseWhitespace diverged on: " + escape(s)); + } + } + + @Test + void splitWhitespaceMatchesSplit() { + for (String s : whitespaceCases()) { + assertArrayEquals(s.split("\\s+"), Util.splitWhitespace(s), + "splitWhitespace diverged on: " + escape(s)); + } + } + + @Test + void splitLiteralMatchesSplit() { + String[] cases = { + "", ";", ";;", "a", "a;b", "a;b;c", ";a", "a;", "a;;b", ";;a;;b;;", + "a;b;", "a;b;;", " a ; b ", "one" + }; + for (String s : cases) { + assertArrayEquals(s.split(";"), Util.splitLiteral(s, ';'), + "splitLiteral diverged on: " + escape(s)); + } + } + + /** + * The generated-code shapes plus the near misses: a bracket with no digits, a + * digit run that is not followed by ".data.o", and a nested occurrence. These are + * where a hand-written scanner and a regex are most likely to part company. + */ + private List localsCases() { + List cases = new ArrayList(); + for (String s : new String[]{ + "", + "locals[0].data.o", + "locals[12].data.o", + "locals[0].data.o + locals[1].data.o", + "f(locals[3].data.o, locals[44].data.o)", + "locals[].data.o", + "locals[x].data.o", + "locals[0].data.i", + "locals[0].data", + "locals[", + "locals[0", + "locals[0]", + "prefix locals[7].data.o suffix", + "locals[locals[1].data.o].data.o", + "no match here at all", + "LOCALS[0].DATA.O" + }) { + cases.add(s); + } + // Randomised fuzz over the alphabet the pattern cares about, so the oracle + // sees inputs nobody thought to enumerate. + Random r = new Random(20260909L); + char[] alphabet = {'l', 'o', 'c', 'a', 's', '[', ']', '.', 'd', 't', '0', '1', '9', ' ', 'x'}; + for (int i = 0; i < 3000; i++) { + StringBuilder b = new StringBuilder(); + int len = r.nextInt(24); + for (int j = 0; j < len; j++) { + b.append(alphabet[r.nextInt(alphabet.length)]); + } + if (r.nextBoolean()) { + b.append("locals[").append(r.nextInt(200)).append("].data.o"); + } + cases.add(b.toString()); + } + return cases; + } + + private List whitespaceCases() { + List cases = new ArrayList(); + // 0x0B is the vertical tab: Java's \s includes it and Character.isWhitespace + // does not, which is the difference most likely to be got wrong. + String vt = String.valueOf((char) 0x0B); + for (String s : new String[]{ + "", " ", " ", "a", "a b", "a b", " a b ", "\ta\tb\t", "a\nb", + "a" + vt + "b", "a\fb", "a\r\nb", "JAVA_OBJECT me", " leading", "trailing ", + " both ", "a \t\n b" + }) { + cases.add(s); + } + Random r = new Random(20260910L); + char[] alphabet = {' ', '\t', '\n', 0x0B, '\f', '\r', 'a', 'b', '*'}; + for (int i = 0; i < 3000; i++) { + StringBuilder b = new StringBuilder(); + int len = r.nextInt(16); + for (int j = 0; j < len; j++) { + b.append(alphabet[r.nextInt(alphabet.length)]); + } + cases.add(b.toString()); + } + return cases; + } + + private String escape(String s) { + StringBuilder b = new StringBuilder(); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c < 0x20) { + b.append("\\x").append(Integer.toHexString(c)); + } else { + b.append(c); + } + } + return b.toString(); + } +} From 9ebb174826ba830f8ed8e1699d779dc209dee6d6 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 9 Sep 2026 22:09:18 +0300 Subject: [PATCH 03/66] Measure the self-hosted translator: it is 6x slower and uses 2.8x the memory bench-selfhost.sh runs both translators over the same corpus, interleaved, taking the minimum wall clock and the peak phys_footprint (never ps rss). It refuses to print ratios unless the two emitted identical C. On the self-hosting corpus -- ASM plus the translator's own classes, ~570 classes -- in the documented release shape (-O3 -flto=thin), against JDK 8: wall clock (min of 3) parpar 7.06s jdk8 1.17s jdk8 6.0x faster peak phys_footprint parpar 1434MB jdk8 509MB jdk8 2.8x smaller That is the opposite of the expectation on both axes, so it is worth being clear that it is a real measurement rather than a mistake. /usr/bin/time -l independently reports 1328 MB and 501 MB, agreeing with the sampled vmmap figures; building at -O1 rather than -O3 -flto=thin changes nothing measurable, so code quality is not the bottleneck; and the corpus is large enough that JVM startup is not carrying the result. The user-versus-real split locates most of the gap: parpar 6.29 real 7.31 user -> ~1.2x parallelism jdk8 1.13 real 6.09 user -> ~5.4x parallelism The two burn comparable CPU. HotSpot spends it across cores on JIT compiler threads and parallel GC, while the translated program is single-threaded, so the 6x is mostly concurrency ParparVM does not have rather than per-instruction code quality. Fixing an early error in the harness, since it is the kind that reads as a result: the memory sampler took $! from a subshell wrapper and reported the wrapper's ~1.3 MB footprint for both arms. It now execs the translator so the pid is the process being measured. Co-Authored-By: Claude Opus 5 (1M context) --- vm/selfhost/README.md | 41 ++++++++++++++ vm/selfhost/bench-selfhost.sh | 103 ++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100755 vm/selfhost/bench-selfhost.sh diff --git a/vm/selfhost/README.md b/vm/selfhost/README.md index cea630bb893..74034123a62 100644 --- a/vm/selfhost/README.md +++ b/vm/selfhost/README.md @@ -91,3 +91,44 @@ self-consistent on HotSpot: Only the first is a runtime bug. The other two are reproducible-build defects in the translator that a second runtime made visible. + +## Performance + +`bench-selfhost.sh` runs both translators over the same corpus, interleaved, and +reports the minimum wall clock and the peak `phys_footprint`. Ratios are refused +unless the two emitted identical C -- a speed number from a translator that emits +different output is meaningless. + +Translating the self-hosting corpus (ASM + the translator's own classes, ~570 +classes) on an M-series Mac, release shape (`-O3 -flto=thin`), against JDK 8: + +| | parpar | jdk8 | | +|---|---:|---:|---| +| wall clock (min of 3) | 7.06 s | 1.17 s | **jdk8 6.0x faster** | +| peak phys_footprint | 1434 MB | 509 MB | **jdk8 2.8x smaller** | + +**This is the opposite of what was hoped for, on both axes.** It is recorded here +rather than buried because it is reproducible and cross-checked: `/usr/bin/time -l` +independently reports 1328 MB and 501 MB, agreeing with the sampled `vmmap` +figures. Building at `-O1` instead of `-O3 -flto=thin` changes nothing measurable, +so code quality is not the bottleneck. + +The `user` versus `real` split says where the wall-clock gap comes from: + +``` +parpar 6.29 real 7.31 user -> ~1.2x parallelism +jdk8 1.13 real 6.09 user -> ~5.4x parallelism +``` + +The two burn comparable CPU. HotSpot spends it across cores -- JIT compiler +threads and parallel GC -- while the translated program is essentially +single-threaded. So most of the 6x is concurrency the JVM has and ParparVM does +not, rather than per-instruction code quality. + +Two things to be careful about before reading more into these numbers. The JVM's +memory figure is bounded by its own heap ergonomics: it collects to stay under a +default maximum, while the native binary has no such ceiling, so this compares +what each process actually used and not the live set. And this is one corpus on +one machine; `vm/benchmarks/run-benchmark.sh` measures tight compute loops, which +is a different shape from a large allocation-heavy graph walk, and the published +geomean-parity result there does not transfer to this workload. diff --git a/vm/selfhost/bench-selfhost.sh b/vm/selfhost/bench-selfhost.sh new file mode 100755 index 00000000000..0e3ea9dbb6e --- /dev/null +++ b/vm/selfhost/bench-selfhost.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# Wall clock and peak memory: the native translator against the JVM-hosted one. +# +# bench-selfhost.sh [rounds] +# +# Discipline copied from vm/benchmarks/run-benchmark.sh: +# +# - Arms are INTERLEAVED within each round. Sequential A-then-B on this hardware +# carries a thermal bias large enough to invent a result. +# - Time takes the MINIMUM of N: the floor is the machine's best, and noise only +# ever adds. Memory takes the MAXIMUM, because a peak is a max and a +# min-of-peaks would understate it. +# - Raw per-round samples are printed, not just the extremum, because a single +# min hides a bimodal distribution. +# - Ratios are refused unless the two translators emitted identical C. A speed +# number from a translator that emits different output is meaningless. +# +# Memory is phys_footprint via `vmmap --summary` on macOS, sampled while the child +# runs. NEVER ps rss: vm/CLAUDE.md records 151/207/219 MB measured for one +# unchanged binary. Timing and memory rounds are separate so the sampler cannot +# contaminate the clock. +set -e +cd "$(dirname "$0")" +REPO="$(cd ../.. && pwd)" +J8="${JDK_8_HOME:?set JDK_8_HOME to a working JDK 8}" +CLASSES="${1:?usage: bench-selfhost.sh [rounds]}" +APP="${2:?}"; PKG="${3:?}"; ROUNDS="${4:-5}" + +# -O3 -flto=thin is the documented release shape (vm/benchmarks/README.md); +# CN1_SELFHOST_BIN overrides it for an A/B against the -O1 diff-gate build. +PARPAR="${CN1_SELFHOST_BIN:-$REPO/vm/selfhost/target/parpar-O3}" +JAPI="$REPO/vm/selfhost/target/javaapi-classes" +TR="$REPO/vm/ByteCodeTranslator/target/classes" +ASM="$(cat "$REPO/vm/ByteCodeTranslator/target/selfhost-asm-classpath.txt")" +W="$REPO/vm/selfhost/target/bench"; rm -rf "$W"; mkdir -p "$W" + +runcmd() { # $1 out dir; rest ignored -- selects arm by $ARM + if [ "$ARM" = parpar ]; then + env CN1_RESOURCE_PATH="$REPO/vm/ByteCodeTranslator/src" \ + "$PARPAR" clean "$JAPI;$CLASSES" "$1" "$APP" "$PKG" "$APP" 1.0 clean none + else + "$J8/bin/java" -cp "$TR:$ASM" com.codename1.tools.translator.ByteCodeTranslator \ + clean "$JAPI;$CLASSES" "$1" "$APP" "$PKG" "$APP" 1.0 clean none + fi +} + +echo "corpus: $CLASSES rounds: $ROUNDS" +echo "memory metric: phys_footprint via vmmap --summary (macOS)" +declare -a t_parpar t_jvm +for r in $(seq 1 $ROUNDS); do + for ARM in parpar jvm; do + out="$W/$ARM-$r"; rm -rf "$out"; mkdir -p "$out" + s=$(python3 -c 'import time;print(time.monotonic())') + runcmd "$out" > "$W/$ARM-$r.log" 2>&1 + e=$(python3 -c 'import time;print(time.monotonic())') + d=$(python3 -c "print(f'{$e-$s:.3f}')") + if [ "$ARM" = parpar ]; then t_parpar+=("$d"); else t_jvm+=("$d"); fi + rm -rf "$out" + done +done +min() { printf '%s\n' "$@" | sort -n | head -1; } +mp=$(min "${t_parpar[@]}"); mj=$(min "${t_jvm[@]}") +echo "parpar times: ${t_parpar[*]} min=${mp}s" +echo "jvm8 times: ${t_jvm[*]} min=${mj}s" +python3 -c "print(f'TIME parpar/jvm8 = {$mp/$mj:.2f}x ({\"parpar faster\" if $mp<$mj else \"jvm faster\"})')" + +# memory, sampled in its own rounds +peak() { # $1 = arm -- peak phys_footprint in MB + local out="$W/mem-$1"; rm -rf "$out"; mkdir -p "$out" + # `exec` inside the subshell so $! is the translator's own pid. Without it the + # pid belongs to the subshell wrapper, and vmmap dutifully reports the wrapper's + # ~1 MB footprint for both arms -- a measurement that looks like a result. + if [ "$1" = parpar ]; then + ( exec env CN1_RESOURCE_PATH="$REPO/vm/ByteCodeTranslator/src" \ + "$PARPAR" clean "$JAPI;$CLASSES" "$out" "$APP" "$PKG" "$APP" 1.0 clean none \ + > /dev/null 2>&1 ) & + else + ( exec "$J8/bin/java" -cp "$TR:$ASM" com.codename1.tools.translator.ByteCodeTranslator \ + clean "$JAPI;$CLASSES" "$out" "$APP" "$PKG" "$APP" 1.0 clean none \ + > /dev/null 2>&1 ) & + fi + local pid=$! best=0 + # The kernel tracks the peak itself ("Physical footprint (peak)"), so a sample + # taken at any point reports the high-water mark so far rather than an instant + # -- sampling only has to catch the process alive at least once. + while kill -0 $pid 2>/dev/null; do + local raw + raw=$(vmmap --summary $pid 2>/dev/null | awk -F: '/Physical footprint \(peak\)/{gsub(/ /,"",$2); print $2; exit}') + if [ -n "$raw" ]; then + best=$(python3 -c " +v='$raw' +mult={'K':1/1024.0,'M':1.0,'G':1024.0}.get(v[-1:], 1/1048576.0) +n=float(v[:-1]) if v[-1:] in 'KMG' else float(v) +print(max($best, n*mult))") + fi + done + wait $pid 2>/dev/null || true + rm -rf "$out" + echo "$best" +} +pp=$(peak parpar); pj=$(peak jvm) +printf 'MEM parpar peak=%.1f MB jvm8 peak=%.1f MB\n' "$pp" "$pj" +python3 -c "print(f'MEM parpar/jvm8 = {$pp/$pj:.2f}x ({\"parpar smaller\" if $pp<$pj else \"jvm smaller\"})')" From 842bc63121b6620a861a737200d7c1f96ab6d919 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 10 Sep 2026 04:17:52 +0300 Subject: [PATCH 04/66] Stop stubbing JavascriptNativeRegistry; gates A and B are now byte-identical The self-hosted translator emitted seven java.util.HashMap methods as empty stubs that the JVM-hosted one emitted in full. The cause was not the VM: it was the blanket "Javascript*" exclusion in build-selfhost.sh, which stubbed JavascriptNativeRegistry along with the two classes that genuinely cannot compile against JavaAPI. That class compiles fine, and -- as the comment at its call site in Parser says -- it is consulted on EVERY target, not just JavaScript, because the C natives use some of the same methods as fallbacks. Its RUNTIME_DELEGATE_TARGETS lists java_util_HashMap's getImpl, putImpl, removeImpl, containsKeyImpl and clearImpl; answering false for them let the dead-code pass cull all five, and with them cn1PutSlot and cn1MaybeGrow, which nothing else calls. Found by instrumenting the cull decision and diffing the two runs: the five showed up as "examined jvm=0x parpar=1x" -- the JVM never even reached the cull check for them, because isRuntimeDelegateTarget had already made it `continue`. The stub list is now driven by what is actually in stubs/ rather than by a name pattern, so only the sources that cannot compile are replaced. Gate A (JavaAPI corpus, 247 files) byte-identical Gate A (self-hosting corpus, 797 files / 21.6 MB) byte-identical Gate D (native against itself), both corpora pass The second of those is the bootstrap gate, and it is a stronger statement than GCC's three-stage comparison: there is no foreign compiler in the loop, so the program really is identical and only the runtime executing it changed. Co-Authored-By: Claude Opus 5 (1M context) --- vm/selfhost/build-selfhost.sh | 14 ++++-- .../translator/JavascriptNativeRegistry.java | 47 ------------------- 2 files changed, 11 insertions(+), 50 deletions(-) delete mode 100644 vm/selfhost/stubs/com/codename1/tools/translator/JavascriptNativeRegistry.java diff --git a/vm/selfhost/build-selfhost.sh b/vm/selfhost/build-selfhost.sh index 108185d8eb8..8176ed868cc 100755 --- a/vm/selfhost/build-selfhost.sh +++ b/vm/selfhost/build-selfhost.sh @@ -61,15 +61,23 @@ else rm -f "$MANIFEST.now" fi -# 4. the self-host source set: every translator source except the ones stubs replace, -# the JavaScript target, and CastSemanticsVerifier (its own command-line tool). +# 4. The self-host source set: every translator source except the ones a stub +# replaces, plus the two classes that carry their own main(). +# +# Only the sources that CANNOT compile against JavaAPI are stubbed, and the list +# is driven by what is in stubs/ rather than by a name pattern. A blanket +# "Javascript*" exclusion is what stubbed JavascriptNativeRegistry, which +# compiles fine and -- as the comment at its call site in Parser warns -- is +# consulted on EVERY target, not just JavaScript. Answering false there culled +# java.util.HashMap's getImpl/putImpl/removeImpl/containsKeyImpl/clearImpl and +# the two helpers only they call, and the native translator emitted seven +# methods as empty stubs that the JVM one emitted in full. SRC="$REPO/vm/ByteCodeTranslator/src" STUBS="$REPO/vm/selfhost/stubs" STUBBED=$(cd "$STUBS" && find . -name '*.java' | sed 's|.*/||;s|\.java$||' | tr '\n' '|' | sed 's/|$//') SRCLIST="$OUT/sources.txt" find "$SRC" -name '*.java' \ | grep -vE "/($STUBBED)\.java$" \ - | grep -vE '/Javascript[A-Za-z]*\.java$' \ | grep -v '/CastSemanticsVerifier\.java$' \ | grep -v '/NativeSignatureVerifierCli\.java$' > "$SRCLIST" find "$STUBS" -name '*.java' >> "$SRCLIST" diff --git a/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptNativeRegistry.java b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptNativeRegistry.java deleted file mode 100644 index 8cff0ae37d4..00000000000 --- a/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptNativeRegistry.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ -package com.codename1.tools.translator; - -/** - * See {@code vm/selfhost/README.md}. - * - * Stub for the self-hosted translator build, which does the clean/ios/macos - * targets only. Replaced on the source path -- the real class is never compiled - * into that binary, and none of these methods is reachable in it. - * - * They throw rather than returning a plausible value: the JavaScript target is - * selected explicitly, so reaching one of these would mean the binary was asked - * for a target it was not built with, and that should be loud. - * - * This one answers instead of throwing: Parser consults it on EVERY target, not - * just JavaScript, and the answer for a build with no JavaScript runtime in it is - * that nothing is delegated there. - */ -final class JavascriptNativeRegistry { - private JavascriptNativeRegistry() { - } - - static boolean isRuntimeDelegateTarget(String mangledClassName, String methodName) { - return false; - } -} From 6263a35b37b998fc6f822353c127955cd2af30ac Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 10 Sep 2026 04:38:25 +0300 Subject: [PATCH 05/66] Profile the self-hosted translator: the 6x gap is one pacing clamp, not the GC The wall-clock gap to JDK 8 is almost entirely the allocator's backpressure throttle. `sample` on a default run puts 64% of the process's samples in a single stack, and the mutator is not marking or sweeping -- it is asleep: Ldc.getValueAsString -> cn1BibopAlloc -> cn1BibopMaybeGc -> cn1PacingPark (3491 of 5476 samples) -> usleep -> nanosleep -> __semwait_signal (3475) CN1_LOG_PACING_PARKS reports only TWO park events for the whole run, so each one is seconds long. Isolated by A/B, translating ~570 classes on a 64GB / 16-core host, release shape: as shipped 6.7-8.7s 6 cycles 2 parks 1434MB CN1_GC_TRIGGER_MB=32768 (no GC) 1.42s 3 cycles 0 parks CN1_GC_PACING_CAP_MB=4096 1.45s 4 cycles 0 parks growth clamp disarmed 1.39-1.52s 4 cycles 0 parks 1467MB Collection itself is nearly free: with the clamp disarmed the collector still runs its four cycles and the time matches disabling GC outright. Against JDK 8 that is 1.19x, ordinary AOT-versus-warmed-JIT territory, instead of 6x. The mechanism, from cn1BibopPacingCap: it computes fm/8 -- 4GB on this host -- and then clamps to `trigger * CN1_BIBOP_GC_MAX_CAP_MULTIPLIER` once cn1PacingPastGrowthFloor() is true, which is a process footprint over CN1_PACING_GROWTH_FLOOR_BYTES (512MB). Early in the run the trigger is still at its own 24MB floor, so the ceiling is 24 * 8 = 192MB, matching the observed minCapKb=196608 exactly. A program whose live set is ~1.4GB cannot stay inside a 192MB allocation window, so it parks against a collector that can never get under it. That bound is calibrated for phone-sized heaps and has no scaling for a 64GB host: it costs 5x throughput to save 2% of peak footprint here. Left alone, since what it should scale with is a policy call for the VM owners; the reproduction is one -DCN1_PACING_GROWTH_FLOOR_BYTES, documented in vm/selfhost/README.md. One real defect found alongside it IS fixed: cn1RefreshFreeMemCache() had exactly one caller, inside the mark cycle, so cn1CachedFreeMem stayed 0 until the first collection and the cap sat at its 72MB floor through the window with the least reason to throttle anything. Primed in cn1BibopDoInit now. ProcessBudgetPacingIntegrationTest's control arm reports minCapKb=4194304 with the fix and the 72MB floor without it, and its budget-bounded arm still engages backpressure (legacyParks=52, boundedChecks=771), so the ceiling that bound exists to enforce is untouched. Gates A and D still pass byte-identical on both corpora after the change. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.m | 18 +++++ vm/selfhost/README.md | 88 ++++++++++++++++++------- 2 files changed, 81 insertions(+), 25 deletions(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index 8825ffac59f..3128c7ff1e6 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -5556,6 +5556,24 @@ static void cn1BibopDoInit() { atomic_store_explicit(&bibopBypassGeneration[i], 0, memory_order_relaxed); bibopHighSurvivalStreak[i] = 0; } + // Prime the free-memory snapshot the pacing cap is computed from. + // + // Its only other caller is the mark cycle, so until the FIRST collection + // cn1CachedFreeMem was 0 and cn1BibopPacingCap's `fm / 8` evaluated to 0, leaving + // the cap at its floor of trigger * CN1_BIBOP_GC_HARD_CAP_MULTIPLIER = 72MB -- + // during exactly the window where there is least reason to throttle anything, + // since nothing has been collected yet. ProcessBudgetPacingIntegrationTest's + // control arm reports minCapKb=4194304 with this in place and the 72MB floor + // without it. + // + // This is NOT the whole story for an allocation-heavy program, and the rest is + // deliberately left alone: once the process passes CN1_PACING_GROWTH_FLOOR_BYTES + // (512MB) the growth bound below clamps the cap to trigger * 8, which is 192MB + // while the trigger is still at its own floor. Translating ~570 classes on a + // 64GB host, that clamp costs 6.7-8.7s against 1.4-1.5s with it disarmed, for + // 2% less peak footprint (1434MB vs 1467MB). Whether to scale it with host + // memory is a policy call, not a bug fix; see vm/selfhost/README.md. + cn1RefreshFreeMemCache(); } static void cn1BibopFormatPage(CN1BibopPage* p, int ci) { diff --git a/vm/selfhost/README.md b/vm/selfhost/README.md index 74034123a62..6f038fd1553 100644 --- a/vm/selfhost/README.md +++ b/vm/selfhost/README.md @@ -100,35 +100,73 @@ unless the two emitted identical C -- a speed number from a translator that emit different output is meaningless. Translating the self-hosting corpus (ASM + the translator's own classes, ~570 -classes) on an M-series Mac, release shape (`-O3 -flto=thin`), against JDK 8: +classes) on a 64 GB / 16-core Mac, release shape (`-O3 -flto=thin`), against JDK 8: -| | parpar | jdk8 | | -|---|---:|---:|---| -| wall clock (min of 3) | 7.06 s | 1.17 s | **jdk8 6.0x faster** | -| peak phys_footprint | 1434 MB | 509 MB | **jdk8 2.8x smaller** | +| | wall clock | peak footprint | +|---|---:|---:| +| jdk8 | 1.17 s | 509 MB | +| parpar, as shipped | 6.7 - 8.7 s | 1434 MB | +| parpar, pacing growth clamp disarmed | **1.39 - 1.52 s** | 1467 MB | -**This is the opposite of what was hoped for, on both axes.** It is recorded here -rather than buried because it is reproducible and cross-checked: `/usr/bin/time -l` -independently reports 1328 MB and 501 MB, agreeing with the sampled `vmmap` -figures. Building at `-O1` instead of `-O3 -flto=thin` changes nothing measurable, -so code quality is not the bottleneck. +**Nearly all of the wall-clock gap is one pacing policy, not collection work and +not code quality.** Building at `-O1` instead of `-O3 -flto=thin` measures the +same, and with the clamp disarmed the collector still runs its four cycles. -The `user` versus `real` split says where the wall-clock gap comes from: +### Where it goes + +`sample` on a default run puts 64% of the process's samples in one stack: ``` -parpar 6.29 real 7.31 user -> ~1.2x parallelism -jdk8 1.13 real 6.09 user -> ~5.4x parallelism +Ldc.getValueAsString -> cn1BibopAlloc -> cn1BibopMaybeGc + -> cn1PacingPark (3491 of 5476 samples) + -> usleep -> nanosleep -> __semwait_signal (3475) ``` -The two burn comparable CPU. HotSpot spends it across cores -- JIT compiler -threads and parallel GC -- while the translated program is essentially -single-threaded. So most of the 6x is concurrency the JVM has and ParparVM does -not, rather than per-instruction code quality. - -Two things to be careful about before reading more into these numbers. The JVM's -memory figure is bounded by its own heap ergonomics: it collects to stay under a -default maximum, while the native binary has no such ceiling, so this compares -what each process actually used and not the live set. And this is one corpus on -one machine; `vm/benchmarks/run-benchmark.sh` measures tight compute loops, which -is a different shape from a large allocation-heavy graph walk, and the published -geomean-parity result there does not transfer to this workload. +The mutator is not marking or sweeping. It is asleep in the allocator's +backpressure loop. `CN1_LOG_PACING_PARKS` reports only **two** park events for the +whole run, so those two parks are seconds long each. + +### Why + +`cn1BibopPacingCap` computes a generous cap -- `cn1CachedFreeMem / 8`, which is +4 GB on this host -- and then clamps it: + +```c +long capCeiling = trigger * CN1_BIBOP_GC_MAX_CAP_MULTIPLIER; /* 8 */ +if(cap > capCeiling && cn1PacingPastGrowthFloor()) cap = capCeiling; +``` + +`cn1PacingPastGrowthFloor()` is true once the process footprint passes +`CN1_PACING_GROWTH_FLOOR_BYTES`, which is **512 MB**. Early in the run the GC +trigger is still at its own floor of 24 MB, so the ceiling is 24 x 8 = **192 MB** +-- and `CN1_LOG_PACING_PARKS` reports exactly `minCapKb=196608`. A program whose +live set is ~1.4 GB cannot stay inside a 192 MB allocation window, so it parks +waiting for a collector that can never get under it. + +This is a policy calibrated for phone-sized heaps, where bounding RSS is worth +real throughput. It has no scaling for a host with 64 GB of RAM: **disarming it +cost 2% more memory (1434 -> 1467 MB) and returned 5x the speed.** Whether and how +to scale it -- with available RAM, with a process budget, or by letting the +trigger rise faster before the clamp engages -- is a policy decision for the VM +owners, not something this project should decide. The reproduction is one +`#define`: + +```bash +CN1_SELFHOST_CFLAGS="-flto=thin -DCN1_PACING_GROWTH_FLOOR_BYTES=1099511627776LL" \ + ./build-selfhost.sh -O3 +``` + +A related but secondary defect **is** fixed here: `cn1RefreshFreeMemCache()` had +exactly one caller, inside the mark cycle, so `cn1CachedFreeMem` was 0 until the +first collection and the cap fell to its 72 MB floor rather than 192 MB during the +window with the least reason to throttle anything. It is now primed in +`cn1BibopDoInit`. + +### What is left, once pacing is out of the way + +Against JDK 8: **1.19x slower** and **2.9x more memory**. The time is ordinary +AOT-versus-warmed-JIT territory. The memory gap is real and separate, and worth +noting that the JVM figure is bounded by its own heap ergonomics -- it collects to +stay under a default maximum, while the native binary has no such ceiling -- so +this compares what each process used, not the live set. + From fff6f538daae59d653c3979b2bfbf26d25abe41c Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 10 Sep 2026 05:26:08 +0300 Subject: [PATCH 06/66] Scale the GC pacing floor with available RAM, and index the constant pool Two fixes take the self-hosted translator from 6.0x slower than JDK 8 to 1.18x slower than JDK 25 -- and 1.24x FASTER than JDK 8 -- on the self-hosting corpus. Benchmarks now use JDK 25 as the reference; JDK 8 is kept only because it is what the builders fork. parpar 1.84s / 1443MB jdk25 1.56s / 516MB jdk8 2.27s / 502MB 1. The mutator slept instead of allocating, and would on any machine. cn1BibopPacingCap computes a cap from available memory (fm/8, 4GB here) and then clamps it to trigger * 8 once cn1PacingPastGrowthFloor() is true. That floor was a flat 512MB. Early in a run the trigger is still at its own 24MB floor, so the ceiling was 192MB -- confirmed by minCapKb=196608 -- and a program with a ~1.4GB live set cannot stay inside a 192MB allocation window. It parked against a collector that could never get under it: `sample` put 64% of samples in cn1PacingPark -> usleep, from just two park events, each seconds long. A fixed 512MB says the process has grown; it does not say the machine is under pressure, and the bound exists for pressure. The floor is now max(512MB, availableMemory/4). Where cn1_available_memory is the flat 100MB placeholder (Linux, Windows, non-Apple fallback) the absolute floor still wins and behaviour is bit-for-bit unchanged, and the floor can only rise, never fall, so no constrained host becomes more permissive than it was. This is the no-per-process-ceiling path only. Where a ceiling exists -- iOS's dirty-memory limit, or an explicit budget -- cn1PacingPark takes the bounded branch and never reaches cn1BibopPacingCap. ProcessBudgetPacingIntegrationTest covers both halves and still passes: control arm minCapKb=4194304 with zero parks, bounded arm holding a 120MB limit at a 60MB peak across 427 parks. cn1RefreshFreeMemCache() also had exactly one caller, inside the mark cycle, so cn1CachedFreeMem was 0 until the first collection and both the cap and this floor sat at their absolute minimums during the window with the least reason to throttle. Primed in cn1BibopDoInit. 2. Parser.addToConstantPool was O(n^2). With pacing out of the way the main thread's profile was dominated by constantPool.indexOf(s) -- a String.equals against every string already interned, and the pool holds ~200k of them on a self-hosting translation. It was 32% of main-thread samples across String.equals (11.2%), the list iterator (10.3%), indexOf (6.2%) and ArrayList.get (5.1%). A HashMap side index answers the same question; the list stays the source of truth so the emitted indices are unchanged. Gates A and D still pass byte-identical on both corpora after both changes. Still open: peak footprint is 2.8x JDK 25's. It is retained data, not garbage -- sweeping the trigger from 8MB to 256MB moves peak less than 15% -- and it scales with the object graph rather than being a fixed cost (2.37x on a hello-world corpus, 2.73x on the full one). The 16-byte object header and BiBOP size-class rounding do not account for it, and compact strings are not the explanation either since JDK 8 has none and still fits in ~500MB. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.m | 50 ++++- .../codename1/tools/translator/Parser.java | 24 ++- vm/selfhost/README.md | 129 ++++++------- vm/selfhost/bench-selfhost.sh | 174 ++++++++++-------- 4 files changed, 227 insertions(+), 150 deletions(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index 3128c7ff1e6..e2b33b4c21f 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -5566,13 +5566,9 @@ static void cn1BibopDoInit() { // control arm reports minCapKb=4194304 with this in place and the 72MB floor // without it. // - // This is NOT the whole story for an allocation-heavy program, and the rest is - // deliberately left alone: once the process passes CN1_PACING_GROWTH_FLOOR_BYTES - // (512MB) the growth bound below clamps the cap to trigger * 8, which is 192MB - // while the trigger is still at its own floor. Translating ~570 classes on a - // 64GB host, that clamp costs 6.7-8.7s against 1.4-1.5s with it disarmed, for - // 2% less peak footprint (1434MB vs 1467MB). Whether to scale it with host - // memory is a policy call, not a bug fix; see vm/selfhost/README.md. + // Priming it matters twice over: the run-ahead bound's own floor is scaled off + // the same reading (see cn1PacingGrowthFloorBytes), so a zero here would arm that + // bound at its absolute 512MB minimum no matter how much memory the host has. cn1RefreshFreeMemCache(); } @@ -6459,14 +6455,48 @@ static long long cn1PacingFootprintNow(void) { return fp; } +// The footprint at which the run-ahead bound starts applying, scaled to the memory +// this host actually has. +// +// A fixed 512MB says "this process has grown"; it does not say the machine is under +// any pressure, and the bound exists for pressure. On a host with tens of GB free, a +// process holding a couple of GB is nowhere near runaway, and clamping it there +// parks the mutator against a collector that cannot get under the ceiling: measured +// at 6.7-8.7s versus 1.4s for the same work, to save 2% of peak footprint. +// +// So take the larger of the absolute floor and a quarter of available memory. Two +// properties this has to keep: +// +// - Where cn1_available_memory is the flat 100MB placeholder (Linux, Windows, and +// the non-Apple fallback), fm/4 is 25MB, the absolute floor wins, and behaviour is +// bit-for-bit what it was. Nothing changes on a platform where we cannot measure. +// - It only ever RAISES the floor, so the bound can only engage later than before, +// never earlier. It cannot make a constrained host more permissive than it was. +// +// This is the no-per-process-ceiling path only. Where a ceiling exists -- iOS's dirty +// memory limit, or an explicit process budget -- cn1PacingPark takes the bounded +// branch instead and never reaches cn1BibopPacingCap, so none of this loosens the +// admission control that keeps an app inside its own limit. +static long long cn1PacingGrowthFloorBytes(void) { + long long floor = CN1_PACING_GROWTH_FLOOR_BYTES; + long fm = atomic_load_explicit(&cn1CachedFreeMem, memory_order_relaxed); + if(fm > 0) { + long long scaled = (long long)fm / 4; + if(scaled > floor) { + floor = scaled; + } + } + return floor; +} + static JAVA_BOOLEAN cn1PacingPastGrowthFloor(void) { + long long floor = cn1PacingGrowthFloorBytes(); // Once the cache is over the floor the bound is engaged and a syscall to re-confirm // it buys nothing, so this stays ahead of the probe. - if(atomic_load_explicit(&cn1CachedProcFootprint, memory_order_relaxed) - > CN1_PACING_GROWTH_FLOOR_BYTES) { + if(atomic_load_explicit(&cn1CachedProcFootprint, memory_order_relaxed) > floor) { return JAVA_TRUE; } - return cn1PacingFootprintNow() > CN1_PACING_GROWTH_FLOOR_BYTES; + return cn1PacingFootprintNow() > floor; } static long cn1BibopPacingCap(CODENAME_ONE_THREAD_STATE) { diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java index 555c9702d91..020ef07571b 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java @@ -446,6 +446,16 @@ public static NativeSymbolIndex getNativeSymbolIndex(String[] nativeSources) { } private static final ArrayList constantPool = new ArrayList<>(); + // Index of constantPool, so addToConstantPool does not have to scan it. + // + // The list stays the source of truth -- writeOutput emits it in order and the + // emitted indices are positions in it -- and this only answers "where is s", the + // question ArrayList.indexOf was answering with a String.equals against every + // entry already interned. On a self-hosting translation the pool holds ~200k + // strings and that scan was the single largest cost on the mutator thread: + // String.equals 11.2%, the iterator 10.3%, indexOf 6.2% and ArrayList.get 5.1% + // of samples, all of it here. + private static final Map constantPoolIndex = new HashMap(); // Name -> class index, replacing the O(N) linear scans that getClassObject / // getClassByName / ByteCodeClass.findClass used to do. Those run per dependency @@ -481,12 +491,14 @@ public static ByteCodeClass getClassObject(String name) { * Adds the given string to the hardcoded constant pool strings returns the offset in the pool */ public static int addToConstantPool(String s) { - int i = constantPool.indexOf(s); - if(i < 0) { - constantPool.add(s); - return constantPool.size() - 1; - } - return i; + Integer existing = constantPoolIndex.get(s); + if(existing != null) { + return existing.intValue(); + } + int index = constantPool.size(); + constantPool.add(s); + constantPoolIndex.put(s, Integer.valueOf(index)); + return index; } diff --git a/vm/selfhost/README.md b/vm/selfhost/README.md index 6f038fd1553..2a210eabb20 100644 --- a/vm/selfhost/README.md +++ b/vm/selfhost/README.md @@ -94,79 +94,84 @@ the translator that a second runtime made visible. ## Performance -`bench-selfhost.sh` runs both translators over the same corpus, interleaved, and -reports the minimum wall clock and the peak `phys_footprint`. Ratios are refused -unless the two emitted identical C -- a speed number from a translator that emits -different output is meaningless. +`bench-selfhost.sh` runs each arm over the same corpus, interleaved, and reports the +minimum wall clock and the peak `phys_footprint`. It refuses to print ratios unless +every arm emitted identical C. The reference JVM is **JDK 25** -- what HotSpot can +actually do; JDK 8 is kept only because it is what the builders currently fork. Translating the self-hosting corpus (ASM + the translator's own classes, ~570 -classes) on a 64 GB / 16-core Mac, release shape (`-O3 -flto=thin`), against JDK 8: +classes) on a 64 GB / 16-core Mac, release shape (`-O3 -flto=thin`): | | wall clock | peak footprint | |---|---:|---:| -| jdk8 | 1.17 s | 509 MB | -| parpar, as shipped | 6.7 - 8.7 s | 1434 MB | -| parpar, pacing growth clamp disarmed | **1.39 - 1.52 s** | 1467 MB | +| parpar | 1.84 s | 1443 MB | +| jdk25 | 1.56 s | 516 MB | +| jdk8 | 2.27 s | 502 MB | -**Nearly all of the wall-clock gap is one pacing policy, not collection work and -not code quality.** Building at `-O1` instead of `-O3 -flto=thin` measures the -same, and with the clamp disarmed the collector still runs its four cycles. +**vs JDK 25: 1.18x slower, 2.79x more memory. vs JDK 8: 1.24x faster.** -### Where it goes +Two fixes got it there from 6x slower; both are described below. Wall clock on this +machine is only meaningful when it is quiet -- at load 113 the same benchmark +produced samples from 3.6 s to 24 s for every arm, JVM included. CPU time +(`user+sys`) is far more robust to contention, and by that measure the two are +level or better: parpar 4.35 s against jdk25 4.82 s on a loaded host. -`sample` on a default run puts 64% of the process's samples in one stack: +### Fix 1: the mutator slept instead of allocating -``` -Ldc.getValueAsString -> cn1BibopAlloc -> cn1BibopMaybeGc - -> cn1PacingPark (3491 of 5476 samples) - -> usleep -> nanosleep -> __semwait_signal (3475) -``` - -The mutator is not marking or sweeping. It is asleep in the allocator's -backpressure loop. `CN1_LOG_PACING_PARKS` reports only **two** park events for the -whole run, so those two parks are seconds long each. - -### Why - -`cn1BibopPacingCap` computes a generous cap -- `cn1CachedFreeMem / 8`, which is -4 GB on this host -- and then clamps it: +`sample` on the original build put 64% of the process's samples in one stack, and +the mutator was not marking or sweeping -- it was asleep: -```c -long capCeiling = trigger * CN1_BIBOP_GC_MAX_CAP_MULTIPLIER; /* 8 */ -if(cap > capCeiling && cn1PacingPastGrowthFloor()) cap = capCeiling; ``` - -`cn1PacingPastGrowthFloor()` is true once the process footprint passes -`CN1_PACING_GROWTH_FLOOR_BYTES`, which is **512 MB**. Early in the run the GC -trigger is still at its own floor of 24 MB, so the ceiling is 24 x 8 = **192 MB** --- and `CN1_LOG_PACING_PARKS` reports exactly `minCapKb=196608`. A program whose -live set is ~1.4 GB cannot stay inside a 192 MB allocation window, so it parks -waiting for a collector that can never get under it. - -This is a policy calibrated for phone-sized heaps, where bounding RSS is worth -real throughput. It has no scaling for a host with 64 GB of RAM: **disarming it -cost 2% more memory (1434 -> 1467 MB) and returned 5x the speed.** Whether and how -to scale it -- with available RAM, with a process budget, or by letting the -trigger rise faster before the clamp engages -- is a policy decision for the VM -owners, not something this project should decide. The reproduction is one -`#define`: - -```bash -CN1_SELFHOST_CFLAGS="-flto=thin -DCN1_PACING_GROWTH_FLOOR_BYTES=1099511627776LL" \ - ./build-selfhost.sh -O3 +Ldc.getValueAsString -> cn1BibopAlloc -> cn1BibopMaybeGc + -> cn1PacingPark -> usleep -> nanosleep -> __semwait_signal ``` -A related but secondary defect **is** fixed here: `cn1RefreshFreeMemCache()` had -exactly one caller, inside the mark cycle, so `cn1CachedFreeMem` was 0 until the -first collection and the cap fell to its 72 MB floor rather than 192 MB during the -window with the least reason to throttle anything. It is now primed in -`cn1BibopDoInit`. - -### What is left, once pacing is out of the way - -Against JDK 8: **1.19x slower** and **2.9x more memory**. The time is ordinary -AOT-versus-warmed-JIT territory. The memory gap is real and separate, and worth -noting that the JVM figure is bounded by its own heap ergonomics -- it collects to -stay under a default maximum, while the native binary has no such ceiling -- so -this compares what each process used, not the live set. +`CN1_LOG_PACING_PARKS` reported only **two** park events for the whole run, so each +was seconds long. `cn1BibopPacingCap` computed a generous cap -- `cn1CachedFreeMem/8`, +4 GB here -- and then clamped it to `trigger * 8` once the footprint passed +`CN1_PACING_GROWTH_FLOOR_BYTES`. That floor was a flat **512 MB**, and early in the +run the trigger is still at its own 24 MB floor, so the ceiling was **192 MB** +(`minCapKb=196608` confirmed it). A program with a ~1.4 GB live set cannot stay +inside a 192 MB allocation window, so it parked against a collector that could +never get under it. + +A fixed 512 MB says "this process has grown"; it does not say the machine is under +pressure, and the bound exists for pressure. The floor now scales: +`max(512MB, availableMemory/4)`. Where `cn1_available_memory` is the flat 100 MB +placeholder (Linux, Windows, the non-Apple fallback) the absolute floor still wins +and behaviour is unchanged; the floor can only ever rise, never fall. This is the +no-per-process-ceiling path only -- where a ceiling exists (iOS's dirty-memory +limit, or an explicit budget) `cn1PacingPark` takes the bounded branch and never +reaches this code. `ProcessBudgetPacingIntegrationTest` confirms both halves: its +control arm reports `minCapKb=4194304` with no parks, and its budget-bounded arm +still holds a 120 MB limit at a 60 MB peak across 427 parks. + +`cn1RefreshFreeMemCache()` also had exactly one caller, inside the mark cycle, so +`cn1CachedFreeMem` was 0 until the first collection and both the cap and this floor +fell to their absolute minimums during the window with the least reason to throttle. +It is primed in `cn1BibopDoInit` now. + +### Fix 2: the constant pool was O(n^2) + +With pacing out of the way, the main thread's own profile was dominated by +`Parser.addToConstantPool`, which did `constantPool.indexOf(s)` -- a `String.equals` +against every string already interned. On a self-hosting translation the pool holds +~200k strings: `String.equals` 11.2%, the list iterator 10.3%, `indexOf` 6.2% and +`ArrayList.get` 5.1% of main-thread samples, all of it there. A `HashMap` side index +answers the same question directly; the list stays the source of truth, so the +emitted indices are unchanged and gate A still passes byte-identical. + +### What is left: memory, and it is live data + +The remaining gap is **2.8x peak footprint**, and it is not the collector's fault. +Sweeping the GC trigger from 8 MB to 256 MB -- from four cycles to two -- moves peak +by less than 15% and never below 1.27 GB, so this is retained data rather than +uncollected garbage. It is also not a fixed startup cost: the ratio is 2.37x on a +hello-world corpus and 2.73x on the full one, so it scales with the object graph. + +Two candidates are already ruled out. The object header is 16 bytes +(`struct JavaObjectPrototype`), comparable to HotSpot's. And compact strings are not +it: JDK 8 has none and still fits in ~500 MB. BiBOP size-class rounding (32, 48, 64, +80, 96, 112, 128, 160, 192, 224, 256, 320, 384, 448, 512) costs maybe 10-15%, not +180%. Finding the rest is the next piece of work. diff --git a/vm/selfhost/bench-selfhost.sh b/vm/selfhost/bench-selfhost.sh index 0e3ea9dbb6e..ff66b9dcb88 100755 --- a/vm/selfhost/bench-selfhost.sh +++ b/vm/selfhost/bench-selfhost.sh @@ -1,103 +1,133 @@ #!/bin/bash -# Wall clock and peak memory: the native translator against the JVM-hosted one. +# Wall clock and peak memory: the native translator against JVM-hosted ones. # # bench-selfhost.sh [rounds] # -# Discipline copied from vm/benchmarks/run-benchmark.sh: +# Reference JVMs come from SELFHOST_REF_JAVAS (comma-separated java binaries). +# The default is JDK 25 first, then JDK 8. JDK 25 is the honest headline -- it is +# what HotSpot can actually do -- and JDK 8 is kept only because it is what the +# builders currently fork. +# +# Discipline, following vm/benchmarks/run-benchmark.sh: # # - Arms are INTERLEAVED within each round. Sequential A-then-B on this hardware # carries a thermal bias large enough to invent a result. # - Time takes the MINIMUM of N: the floor is the machine's best, and noise only -# ever adds. Memory takes the MAXIMUM, because a peak is a max and a -# min-of-peaks would understate it. -# - Raw per-round samples are printed, not just the extremum, because a single -# min hides a bimodal distribution. -# - Ratios are refused unless the two translators emitted identical C. A speed -# number from a translator that emits different output is meaningless. +# ever adds. Memory takes the MAXIMUM, because a peak is a max. +# - Raw per-round samples are printed, not just the extremum: a lone minimum +# hides a bimodal distribution. +# - Ratios are refused unless every arm emitted identical C. A speed number from +# a translator that emits different output is meaningless. # -# Memory is phys_footprint via `vmmap --summary` on macOS, sampled while the child -# runs. NEVER ps rss: vm/CLAUDE.md records 151/207/219 MB measured for one -# unchanged binary. Timing and memory rounds are separate so the sampler cannot -# contaminate the clock. +# Memory is the peak phys_footprint reported by /usr/bin/time -l on macOS, which +# is the same quantity vmmap calls "Physical footprint (peak)". NEVER ps rss: +# vm/CLAUDE.md records 151/207/219 MB measured for one unchanged binary. set -e cd "$(dirname "$0")" REPO="$(cd ../.. && pwd)" -J8="${JDK_8_HOME:?set JDK_8_HOME to a working JDK 8}" CLASSES="${1:?usage: bench-selfhost.sh [rounds]}" APP="${2:?}"; PKG="${3:?}"; ROUNDS="${4:-5}" +T="$REPO/vm/selfhost/target" # -O3 -flto=thin is the documented release shape (vm/benchmarks/README.md); # CN1_SELFHOST_BIN overrides it for an A/B against the -O1 diff-gate build. -PARPAR="${CN1_SELFHOST_BIN:-$REPO/vm/selfhost/target/parpar-O3}" -JAPI="$REPO/vm/selfhost/target/javaapi-classes" +PARPAR="${CN1_SELFHOST_BIN:-$T/parpar-O3}" +JAPI="$T/javaapi-classes" TR="$REPO/vm/ByteCodeTranslator/target/classes" ASM="$(cat "$REPO/vm/ByteCodeTranslator/target/selfhost-asm-classpath.txt")" -W="$REPO/vm/selfhost/target/bench"; rm -rf "$W"; mkdir -p "$W" +DEFAULT_JAVAS="/Users/shai/Library/Java/JavaVirtualMachines/azul-25/Contents/Home/bin/java,${JDK_8_HOME:-}/bin/java" +IFS=',' read -r -a REF_JAVAS <<< "${SELFHOST_REF_JAVAS:-$DEFAULT_JAVAS}" + +W="$T/bench"; rm -rf "$W"; mkdir -p "$W" -runcmd() { # $1 out dir; rest ignored -- selects arm by $ARM - if [ "$ARM" = parpar ]; then - env CN1_RESOURCE_PATH="$REPO/vm/ByteCodeTranslator/src" \ - "$PARPAR" clean "$JAPI;$CLASSES" "$1" "$APP" "$PKG" "$APP" 1.0 clean none +# $1 = arm label, $2 = output dir; runs one translation +invoke() { + local arm=$1 out=$2 + if [ "$arm" = parpar ]; then + env CN1_RESOURCE_PATH="$REPO/vm/ByteCodeTranslator/src" "$PARPAR" \ + clean "$JAPI;$CLASSES" "$out" "$APP" "$PKG" "$APP" 1.0 clean none else - "$J8/bin/java" -cp "$TR:$ASM" com.codename1.tools.translator.ByteCodeTranslator \ - clean "$JAPI;$CLASSES" "$1" "$APP" "$PKG" "$APP" 1.0 clean none + "$arm" -cp "$TR:$ASM" com.codename1.tools.translator.ByteCodeTranslator \ + clean "$JAPI;$CLASSES" "$out" "$APP" "$PKG" "$APP" 1.0 clean none fi } -echo "corpus: $CLASSES rounds: $ROUNDS" -echo "memory metric: phys_footprint via vmmap --summary (macOS)" -declare -a t_parpar t_jvm -for r in $(seq 1 $ROUNDS); do - for ARM in parpar jvm; do - out="$W/$ARM-$r"; rm -rf "$out"; mkdir -p "$out" +# "25" -> jdk25, "1.8.0_372" -> jdk8. Taking the leading number alone turns 1.8.0 +# into "jdk1", which is why the second sed exists. +label() { + case "$1" in + parpar) echo parpar;; + *) "$1" -version 2>&1 | head -1 \ + | sed -e 's/.*version "\([0-9][0-9.]*\).*/\1/' \ + -e 's/^1\.\([0-9]*\).*/\1/' -e 's/\..*//' -e 's/^/jdk/';; + esac +} + +ARMS=(parpar "${REF_JAVAS[@]}") +declare -a NAMES +for a in "${ARMS[@]}"; do NAMES+=("$(label "$a")"); done +echo "corpus : $CLASSES" +echo "arms : ${NAMES[*]} rounds: $ROUNDS" +echo "memory : peak phys_footprint (/usr/bin/time -l)" + +# --- correctness precondition: every arm must emit the same C ------------------- +# Same absolute output path for all arms, sequentially, because the generated +# CMakeLists embeds srcRoot.getAbsolutePath(). +OUT="$W/out" +for i in "${!ARMS[@]}"; do + mkdir -p "$OUT" + invoke "${ARMS[$i]}" "$OUT" > "$W/${NAMES[$i]}.log" 2>&1 || { echo "${NAMES[$i]} FAILED"; tail -5 "$W/${NAMES[$i]}.log"; exit 1; } + mv "$OUT" "$W/tree-${NAMES[$i]}" +done +for i in "${!ARMS[@]}"; do + [ "$i" -eq 0 ] && continue + if ! diff -rq "$W/tree-${NAMES[0]}" "$W/tree-${NAMES[$i]}" > "$W/diff-${NAMES[$i]}.txt" 2>&1; then + echo "DIVERGENCE (${NAMES[0]} vs ${NAMES[$i]}) -- ratios would be meaningless:" + sed "s|.*/$APP-src/||;s| and .*||" "$W/diff-${NAMES[$i]}.txt" | head -5 + exit 1 + fi +done +files=$(find "$W/tree-${NAMES[0]}" -type f | wc -l | tr -d ' ') +[ "$files" -gt 10 ] || { echo "VACUOUS: only $files files"; exit 1; } +echo "output : $files files, identical across all arms" +echo + +# --- timing, interleaved -------------------------------------------------------- +declare -a SAMPLES +for r in $(seq 1 "$ROUNDS"); do + for i in "${!ARMS[@]}"; do + rm -rf "$W/run"; mkdir -p "$W/run" s=$(python3 -c 'import time;print(time.monotonic())') - runcmd "$out" > "$W/$ARM-$r.log" 2>&1 + invoke "${ARMS[$i]}" "$W/run" > /dev/null 2>&1 e=$(python3 -c 'import time;print(time.monotonic())') - d=$(python3 -c "print(f'{$e-$s:.3f}')") - if [ "$ARM" = parpar ]; then t_parpar+=("$d"); else t_jvm+=("$d"); fi - rm -rf "$out" + SAMPLES[$i]="${SAMPLES[$i]} $(python3 -c "print(f'{$e-$s:.2f}')")" done done -min() { printf '%s\n' "$@" | sort -n | head -1; } -mp=$(min "${t_parpar[@]}"); mj=$(min "${t_jvm[@]}") -echo "parpar times: ${t_parpar[*]} min=${mp}s" -echo "jvm8 times: ${t_jvm[*]} min=${mj}s" -python3 -c "print(f'TIME parpar/jvm8 = {$mp/$mj:.2f}x ({\"parpar faster\" if $mp<$mj else \"jvm faster\"})')" +declare -a MINS +for i in "${!ARMS[@]}"; do + MINS[$i]=$(printf '%s\n' ${SAMPLES[$i]} | sort -n | head -1) + printf "time %-8s min %6ss samples:%s\n" "${NAMES[$i]}" "${MINS[$i]}" "${SAMPLES[$i]}" +done -# memory, sampled in its own rounds -peak() { # $1 = arm -- peak phys_footprint in MB - local out="$W/mem-$1"; rm -rf "$out"; mkdir -p "$out" - # `exec` inside the subshell so $! is the translator's own pid. Without it the - # pid belongs to the subshell wrapper, and vmmap dutifully reports the wrapper's - # ~1 MB footprint for both arms -- a measurement that looks like a result. - if [ "$1" = parpar ]; then - ( exec env CN1_RESOURCE_PATH="$REPO/vm/ByteCodeTranslator/src" \ - "$PARPAR" clean "$JAPI;$CLASSES" "$out" "$APP" "$PKG" "$APP" 1.0 clean none \ - > /dev/null 2>&1 ) & +# --- memory, measured separately so the probe cannot perturb the clock ---------- +declare -a PEAKS +for i in "${!ARMS[@]}"; do + rm -rf "$W/run"; mkdir -p "$W/run" + if [ "${ARMS[$i]}" = parpar ]; then + env CN1_RESOURCE_PATH="$REPO/vm/ByteCodeTranslator/src" /usr/bin/time -l "$PARPAR" \ + clean "$JAPI;$CLASSES" "$W/run" "$APP" "$PKG" "$APP" 1.0 clean none 2>"$W/mem.txt" >/dev/null else - ( exec "$J8/bin/java" -cp "$TR:$ASM" com.codename1.tools.translator.ByteCodeTranslator \ - clean "$JAPI;$CLASSES" "$out" "$APP" "$PKG" "$APP" 1.0 clean none \ - > /dev/null 2>&1 ) & + /usr/bin/time -l "${ARMS[$i]}" -cp "$TR:$ASM" com.codename1.tools.translator.ByteCodeTranslator \ + clean "$JAPI;$CLASSES" "$W/run" "$APP" "$PKG" "$APP" 1.0 clean none 2>"$W/mem.txt" >/dev/null fi - local pid=$! best=0 - # The kernel tracks the peak itself ("Physical footprint (peak)"), so a sample - # taken at any point reports the high-water mark so far rather than an instant - # -- sampling only has to catch the process alive at least once. - while kill -0 $pid 2>/dev/null; do - local raw - raw=$(vmmap --summary $pid 2>/dev/null | awk -F: '/Physical footprint \(peak\)/{gsub(/ /,"",$2); print $2; exit}') - if [ -n "$raw" ]; then - best=$(python3 -c " -v='$raw' -mult={'K':1/1024.0,'M':1.0,'G':1024.0}.get(v[-1:], 1/1048576.0) -n=float(v[:-1]) if v[-1:] in 'KMG' else float(v) -print(max($best, n*mult))") - fi - done - wait $pid 2>/dev/null || true - rm -rf "$out" - echo "$best" -} -pp=$(peak parpar); pj=$(peak jvm) -printf 'MEM parpar peak=%.1f MB jvm8 peak=%.1f MB\n' "$pp" "$pj" -python3 -c "print(f'MEM parpar/jvm8 = {$pp/$pj:.2f}x ({\"parpar smaller\" if $pp<$pj else \"jvm smaller\"})')" + PEAKS[$i]=$(awk '/peak memory footprint/{print $1}' "$W/mem.txt") + printf "mem %-8s peak %8.0f MB\n" "${NAMES[$i]}" "$(python3 -c "print(${PEAKS[$i]}/1048576.0)")" +done + +echo +for i in "${!ARMS[@]}"; do + [ "$i" -eq 0 ] && continue + python3 -c " +t=${MINS[0]}/${MINS[$i]}; m=${PEAKS[0]}/${PEAKS[$i]} +print(f'vs ${NAMES[$i]}: time {t:.2f}x ({\"parpar faster\" if t<1 else \"parpar slower\"}), memory {m:.2f}x ({\"parpar smaller\" if m<1 else \"parpar larger\"})')" +done From 00776914417a0506972f0e447a2a42e7a8a1e53d Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 10 Sep 2026 07:52:10 +0300 Subject: [PATCH 07/66] Wire up the heap/allocation census, and make -O3 imply ThinLTO cn1HeapAccounting and cn1AllocCensus were written but called from nowhere, so nothing in the tree could answer "what is the footprint actually made of". They now run post-sweep (the same world-consistent point the GC verifier uses) and once at exit, under -DCN1_ALLOC_CENSUS plus CN1_HEAP_REPORT at run time, so an ordinary build is untouched. A batch program usually ends between collections, hence the atexit report as well as the per-cycle ones. The forward declarations sit outside the CN1_GC_VERIFY block: putting them next to cn1GcVerifyHeap looked natural and compiled to nothing in a census build, since that block is off. build-selfhost.sh: -O3 now implies -flto=thin. That IS the documented release shape, and measured over five interleaved rounds it is the only rung that beats -O1 -- 1.45s against 1.61s for -O1, 1.70s for -O2 and 1.73s for bare -O3. Benchmarking a plain -O3 binary and calling it the release build understates it, which is too easy to do when the flag is left to the caller to remember. First results on the self-hosting corpus (~570 classes), at exit: bibop pages=12029 reserved=751.81MB live=749.62MB slack=2.19MB legacy objects=729174 bytes=110.85MB JAVA TOTAL live=860.47MB process peak phys_footprint=1467MB Two things fall out immediately. Page-pool slack is 2.19MB, so fragmentation is not the memory story. And the Java heap is 860MB of a 1467MB process, so roughly 600MB is not the Java heap at all and needs its own answer. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.m | 35 +++++++++++++++++++++++++ vm/selfhost/build-selfhost.sh | 5 ++++ 2 files changed, 40 insertions(+) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index e2b33b4c21f..70a45875ba0 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -1791,6 +1791,14 @@ static void cn1DrainDeadThreadPending() { // walks the page registry and its slots before their definitions. static inline JAVA_OBJECT cn1BibopSlot(CN1BibopPage* p, int i); static CN1BibopPage* _Atomic bibopAllPages; +#ifdef CN1_ALLOC_CENSUS +// Defined far below, beside the BiBOP page structures they read. Declared up here +// because the post-sweep hook that calls them is compiled earlier -- and OUTSIDE the +// CN1_GC_VERIFY block just above, which is off in an ordinary census build. +void cn1HeapAccounting(const char* label); +void cn1AllocCensus(const char* label); +#endif + #ifdef CN1_GRACE_AUDIT static void cn1GraceAuditPreSweep(CODENAME_ONE_THREAD_STATE); #endif @@ -4865,6 +4873,14 @@ void codenameOneGCSweep() { // permanently broken. cn1GcVerifyHeap(threadStateData); #endif +#ifdef CN1_ALLOC_CENSUS + // Same reasoning as the verify hook above: post-sweep is when "live" means + // live. cn1HeapAccounting and cn1AllocCensus were written but never called + // from anywhere, so nothing could answer "what is the footprint made of". + if(getenv("CN1_HEAP_REPORT")) { + cn1HeapAccounting("post-sweep"); + } +#endif } JAVA_BOOLEAN removeObjectFromHeapCollection(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT o) { @@ -5525,6 +5541,10 @@ JAVA_INT java_lang_System_identityHashCode___java_lang_Object_R_int(CODENAME_ONE // Non-static: the inlined bump fast path (cn1_globals.h) reads bibopCurrent[ci]. __thread CN1BibopPage* bibopCurrent[CN1_BIBOP_NUM_CLASSES]; +#ifdef CN1_ALLOC_CENSUS +static void cn1BibopExitReport(void); +#endif + static void cn1BibopDoInit() { int ci = 0; // DIAGNOSTIC KNOB -- CN1_GC_TRIGGER_MB overrides how many uncollected bytes @@ -5570,8 +5590,23 @@ static void cn1BibopDoInit() { // the same reading (see cn1PacingGrowthFloorBytes), so a zero here would arm that // bound at its absolute 512MB minimum no matter how much memory the host has. cn1RefreshFreeMemCache(); +#ifdef CN1_ALLOC_CENSUS + if(getenv("CN1_HEAP_REPORT")) { + atexit(cn1BibopExitReport); + } +#endif } +#ifdef CN1_ALLOC_CENSUS +// Registered from cn1BibopDoInit under CN1_HEAP_REPORT. A batch program usually +// ends between collections, so the post-sweep reports alone never show the state +// the process actually died holding. +static void cn1BibopExitReport(void) { + cn1HeapAccounting("exit"); + cn1AllocCensus("exit"); +} +#endif + static void cn1BibopFormatPage(CN1BibopPage* p, int ci) { int slotSize = cn1BibopClassSize[ci]; // slot 0 starts after the page header, rounded up to 16-byte alignment so diff --git a/vm/selfhost/build-selfhost.sh b/vm/selfhost/build-selfhost.sh index 8176ed868cc..c3e8533af17 100755 --- a/vm/selfhost/build-selfhost.sh +++ b/vm/selfhost/build-selfhost.sh @@ -15,6 +15,11 @@ set -e cd "$(dirname "$0")" REPO="$(cd ../.. && pwd)" OPT="${1:--O1}" +# -O3 implies ThinLTO: that IS the documented release shape (vm/benchmarks/README.md), +# and measured here it is the only rung that beats -O1 -- 1.45s against 1.61s for -O1, +# 1.70s for -O2 and 1.73s for plain -O3. Benchmarking a bare -O3 binary and calling it +# the release build understates it, so the flag is not left to the caller to remember. +case "$OPT" in -O3) CN1_SELFHOST_CFLAGS="-flto=thin $CN1_SELFHOST_CFLAGS";; esac CC="${CN1_SELFHOST_CC:-clang}" J8="${JDK_8_HOME:?set JDK_8_HOME to a working JDK 8}" OUT="$REPO/vm/selfhost/target" From 82a710ace0bef42858d54aeefc10c62fcf4e4e3e Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 10 Sep 2026 08:16:43 +0300 Subject: [PATCH 08/66] Stop ArrayList and IdentityHashMap allocating for nothing Two defects found by the per-class allocation census, on a self-hosting translation of the ParparVM translator. IdentityHashMap allocated an Entry on EVERY next(), including key and value iteration, where the entry existed only to have one field read back out of it and then dropped: 1,366,140 of them, 43.7MB, all garbage. java.util.HashMap already had separate key/value/entry iterators for exactly this reason -- its key iterator reads the flat table directly -- and this map had been left on the older shape. It now has the same split, with the generic MapEntry.Type callback used only for entrySet, which is the one view where a caller can observe an Entry at all. ArrayList's no-arg constructor called this(10), so every list allocated a 128-byte slot up front, including one that is never added to. It now shares a zero-length array until the first growth. That growth allocates exactly ten, not the twelve the general growth path picks: ten keeps a one-to-ten element list in the size class it already occupied, and growing to twelve would have traded a win on empty lists for a loss on the common case. Measured together on the self-hosting corpus: allocations 10,160,401 objects / 991MB -> 8,706,929 / 940MB legacy-heap objects 729,174 -> 444,783 Java live heap 860MB -> 770MB process peak 1467MB -> 1324MB CollectionSemanticsIntegrationTest holds both against a real JDK rather than a hand-written expectation, since these fail at the edges and are invisible when they work: empty-list operations, all three growth paths, identity semantics, null keys and values through each of the three views, iterator removal, and a rehash. Confirmed to fail when the key iterator stops mapping the table's NULL_OBJECT sentinel back to null. HashMap was investigated and deliberately left alone. It eagerly allocates three arrays at capacity 16 and looks like the same defect, but the maps in this workload are populated rather than empty, so the table is not waste. Rebuilding with a default capacity of 1 -- the cheapest probe for how much of it is wasted -- made everything worse, because the maps then regrow repeatedly: default capacity 16 Object[] 1,324,987 int[] 213,725 live 770MB default capacity 1 Object[] 1,802,249 int[] 452,356 live 882MB Its growth is also post-insert by design, so the shared-empty-table trick that works for ArrayList would leave the put path writing into the shared table. Also recorded in vm/selfhost/README.md: String's `long nsString` field, which backs the Apple targets' direct NSString mapping, costs nothing anywhere else. sizeof(obj__java_lang_String) is 48 with the field at offset 40, and the fields before it end at 36, so half of those eight bytes were padding already. Without it the struct is 40 bytes, and BiBOP's size classes are 32, 48, 64 -- both land in the same 48-byte slot. Removing it would save zero bytes per String and cost the Apple targets a side table and a lookup. Full vm/tests suite: 36 classes, no failures. Co-Authored-By: Claude Opus 5 (1M context) --- vm/JavaAPI/src/java/util/ArrayList.java | 44 +++- vm/JavaAPI/src/java/util/IdentityHashMap.java | 53 +++-- vm/selfhost/README.md | 79 ++++++- .../CollectionSemanticsIntegrationTest.java | 216 ++++++++++++++++++ .../translator/CollectionSemanticsApp.java | 174 ++++++++++++++ 5 files changed, 540 insertions(+), 26 deletions(-) create mode 100644 vm/tests/src/test/java/com/codename1/tools/translator/CollectionSemanticsIntegrationTest.java create mode 100644 vm/tests/src/test/resources/com/codename1/tools/translator/CollectionSemanticsApp.java diff --git a/vm/JavaAPI/src/java/util/ArrayList.java b/vm/JavaAPI/src/java/util/ArrayList.java index 8a3c4129a15..e9bb981768b 100644 --- a/vm/JavaAPI/src/java/util/ArrayList.java +++ b/vm/JavaAPI/src/java/util/ArrayList.java @@ -38,8 +38,47 @@ public class ArrayList extends AbstractList implements List, RandomAcce /** * Constructs a new instance of {@code ArrayList} with ten capacity. */ + /** + * A shared zero-length array for a list created at the default capacity. + * + * A list that is never added to keeps this and allocates nothing: the eager + * {@code new Object[10]} that used to happen in this constructor cost a + * 128-byte slot for every empty list, and the translator alone builds hundreds + * of thousands of them. Sharing one instance is safe because the array is only + * ever REPLACED (by the grow methods), never written through. + * + * Its identity is what marks the "still at default capacity" state, so it must + * not be merged with any other empty array -- see allocateDefaultCapacity. + */ + private static final Object[] DEFAULT_EMPTY_ARRAY = new Object[0]; + + /** + * Capacity the first growth allocates when the list was created at the default. + * + * Ten, not the twelve the general growth path would pick, because ten keeps a + * small list inside the same allocation size class it occupied when the array + * was allocated eagerly. Growing straight to twelve would have made every list + * of one to ten elements LARGER than before, trading a win on empty lists for a + * loss on the common case. + */ + private static final int DEFAULT_CAPACITY = 10; + public ArrayList() { - this(10); + firstIndex = size = 0; + array = (E[]) DEFAULT_EMPTY_ARRAY; + } + + /** + * Replaces the shared empty array with a real one on the first growth. + * + * Called at the top of each grow method, which then proceeds exactly as it + * always did against a normally-sized array. + */ + private void allocateDefaultCapacity(int required) { + if (array == DEFAULT_EMPTY_ARRAY) { + array = newElementArray(required > DEFAULT_CAPACITY ? required : DEFAULT_CAPACITY); + firstIndex = 0; + } } public ArrayList(E... arr) { @@ -331,6 +370,7 @@ public E get(int location) { } private void growAtEnd(int required) { + allocateDefaultCapacity(required); if (array.length - size >= required) { // REVIEW: as growAtEnd, why not move size == 0 out as // special case @@ -362,6 +402,7 @@ private void growAtEnd(int required) { } private void growAtFront(int required) { + allocateDefaultCapacity(required); if (array.length - size >= required) { int newFirst = array.length - size; // REVIEW: as growAtEnd, why not move size == 0 out as @@ -391,6 +432,7 @@ private void growAtFront(int required) { } private void growForInsert(int location, int required) { + allocateDefaultCapacity(required); // REVIEW: we grow too quickly because we are called with the // size of the new collection to add without taking in // to account the free space we already have diff --git a/vm/JavaAPI/src/java/util/IdentityHashMap.java b/vm/JavaAPI/src/java/util/IdentityHashMap.java index 4010c21c92a..d8ed67b86af 100644 --- a/vm/JavaAPI/src/java/util/IdentityHashMap.java +++ b/vm/JavaAPI/src/java/util/IdentityHashMap.java @@ -125,12 +125,38 @@ static class IdentityHashMapIterator implements Iterator { final MapEntry.Type type; + /** + * Which of the three views this iterator serves. + * + * Keys and values come straight out of the table; only entrySet has to + * materialise an Entry, and only there can the caller observe one. The + * generic {@code type} callback cannot express that, because it takes a + * MapEntry -- so serving a key iterator through it allocated an Entry per + * next() purely to read one field back out and drop it. Measured on a + * self-hosting translation of the ParparVM translator: 1,366,140 such + * entries, 43.7MB, all garbage. java.util.HashMap already had separate + * key/value/entry iterators for exactly this reason; this one was missed. + */ + static final int KIND_ENTRY = 0; + static final int KIND_KEY = 1; + static final int KIND_VALUE = 2; + + final int kind; + boolean canRemove = false; IdentityHashMapIterator(MapEntry.Type value, IdentityHashMap hm) { associatedMap = hm; type = value; + kind = KIND_ENTRY; + expectedModCount = hm.modCount; + } + + IdentityHashMapIterator(int iteratorKind, IdentityHashMap hm) { + associatedMap = hm; + type = null; + kind = iteratorKind; expectedModCount = hm.modCount; } @@ -152,19 +178,26 @@ void checkConcurrentMod() throws ConcurrentModificationException { } } + @SuppressWarnings("unchecked") public E next() { checkConcurrentMod(); if (!hasNext()) { throw new NoSuchElementException(); } - IdentityHashMapEntry result = associatedMap - .getEntry(position); lastPosition = position; position += 2; - canRemove = true; - return type.get(result); + + if (kind == KIND_KEY) { + Object key = associatedMap.elementData[lastPosition]; + return (E) (key == NULL_OBJECT ? null : key); + } + if (kind == KIND_VALUE) { + Object value = associatedMap.elementData[lastPosition + 1]; + return (E) (value == NULL_OBJECT ? null : value); + } + return type.get(associatedMap.getEntry(lastPosition)); } public void remove() { @@ -687,11 +720,7 @@ public boolean remove(Object key) { @Override public Iterator iterator() { return new IdentityHashMapIterator( - new MapEntry.Type() { - public K get(MapEntry entry) { - return entry.key; - } - }, IdentityHashMap.this); + IdentityHashMapIterator.KIND_KEY, IdentityHashMap.this); } }; } @@ -739,11 +768,7 @@ public void clear() { @Override public Iterator iterator() { return new IdentityHashMapIterator( - new MapEntry.Type() { - public V get(MapEntry entry) { - return entry.value; - } - }, IdentityHashMap.this); + IdentityHashMapIterator.KIND_VALUE, IdentityHashMap.this); } @Override diff --git a/vm/selfhost/README.md b/vm/selfhost/README.md index 2a210eabb20..a91868bc7c7 100644 --- a/vm/selfhost/README.md +++ b/vm/selfhost/README.md @@ -161,17 +161,74 @@ against every string already interned. On a self-hosting translation the pool ho answers the same question directly; the list stays the source of truth, so the emitted indices are unchanged and gate A still passes byte-identical. -### What is left: memory, and it is live data +### What is left: memory + +The remaining gap is peak footprint. Sweeping the GC trigger from 8 MB to 256 MB -- +four cycles down to two -- moves peak by less than 15%, so this is retained data +rather than uncollected garbage, and page-pool slack is about 2 MB, so it is not +fragmentation either. `CN1_HEAP_REPORT` on a census build prints the split. + +Two allocation defects came out of the per-class census and are fixed: + +- **`IdentityHashMap` allocated an `Entry` on every `next()`**, even for key and + value iteration, where the entry was built only to read one field back out of it + and drop it. 1,366,140 of them, 43.7 MB, all garbage. `java.util.HashMap` already + had separate key/value/entry iterators for exactly this reason and this map had + been missed; it now has the same split. +- **`ArrayList()` eagerly allocated `Object[10]`**, a 128-byte slot for every list, + including one never added to. It now shares a zero-length array until the first + growth. The first growth allocates exactly ten and not the twelve the general + growth path would pick, because ten keeps a small list in the size class it + already occupied -- growing to twelve would have traded a win on empty lists for + a loss on every list of one to ten elements. + +Measured together on the self-hosting corpus: + +| | before | after | +|---|---:|---:| +| allocations | 10,160,401 objects / 991 MB | 8,706,929 / 940 MB | +| legacy-heap objects | 729,174 | 444,783 | +| Java live heap | 860 MB | 770 MB | +| process peak | 1467 MB | 1324 MB | + +`CollectionSemanticsIntegrationTest` holds both against a real JDK -- empty-list +operations, the three growth paths, identity semantics, null keys and values +through each of the three views, iterator removal, and a rehash. It was confirmed +to fail when the key iterator stops mapping the table's sentinel back to null. + +**`HashMap` was investigated and deliberately left alone.** It eagerly allocates +three arrays (keys, values, meta) at capacity 16, which looks like the same defect, +but the maps in this workload are populated rather than empty. Rebuilding with a +default capacity of 1 -- the cheapest probe for "how much of that table is wasted" +-- made everything worse, because the maps then regrow repeatedly: + +| default capacity | Object[] allocs | int[] allocs | Java live | +|---|---:|---:|---:| +| 16 (current) | 1,324,987 | 213,725 | 770 MB | +| 1 (probe) | 1,802,249 | 452,356 | 882 MB | -The remaining gap is **2.8x peak footprint**, and it is not the collector's fault. -Sweeping the GC trigger from 8 MB to 256 MB -- from four cycles to two -- moves peak -by less than 15% and never below 1.27 GB, so this is retained data rather than -uncollected garbage. It is also not a fixed startup cost: the ratio is 2.37x on a -hello-world corpus and 2.73x on the full one, so it scales with the object graph. +Growth there is also post-insert by design, so the shared-empty-table trick that +works for ArrayList would have the put path writing into the shared table. Not +worth it for an unmeasured win in the hottest class in the runtime. -Two candidates are already ruled out. The object header is 16 bytes -(`struct JavaObjectPrototype`), comparable to HotSpot's. And compact strings are not -it: JDK 8 has none and still fits in ~500 MB. BiBOP size-class rounding (32, 48, 64, -80, 96, 112, 128, 160, 192, 224, 256, 320, 384, 448, 512) costs maybe 10-15%, not -180%. Finding the rest is the next piece of work. +### String: the NSString field is free + +`java.lang.String` carries a `long nsString` for the Apple targets' direct NSString +mapping, and the obvious question is what that costs everywhere else. Measured: +nothing. + +``` +sizeof(obj__java_lang_String) = 48 nsString at offset 40 +``` +The fields before it end at 36 and the struct is 8-aligned, so four of those eight +bytes were padding already. Without the field the struct is 40 bytes -- and BiBOP's +size classes are 32, 48, 64, ..., so 40 and 48 both land in the same 48-byte slot. +Removing it would save zero bytes per String while costing the Apple targets a +side table and a lookup. Keep it. + +The strings themselves are still the largest single consumer (`char[]`, 368 MB +allocated). Note that a compact Latin-1 path already exists for the concat +fast path -- `cn1FusedLatin1Begin` allocates the String and a `byte[]` payload in +one BiBOP slot -- so the remaining `char[]` volume is strings built some other way. +That is the next thing to look at. diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/CollectionSemanticsIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/CollectionSemanticsIntegrationTest.java new file mode 100644 index 00000000000..b69e0314c4d --- /dev/null +++ b/vm/tests/src/test/java/com/codename1/tools/translator/CollectionSemanticsIntegrationTest.java @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import org.junit.jupiter.api.Test; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Pins ArrayList and IdentityHashMap against a real JDK after both were changed to + * stop allocating. + * + *

ArrayList no longer allocates a backing array in its no-arg constructor -- it + * shares a zero-length one until the first growth, which then allocates exactly ten + * so a small list stays in the size class it always occupied. IdentityHashMap's key + * and value iterators no longer build an Entry per step; only entrySet does, which is + * the only view where a caller can observe one. java.util.HashMap already had that + * split and this map had been missed.

+ * + *

Both changes are invisible when they work and produce wrong answers at the + * edges when they do not -- an empty list that reports the wrong size, a null key + * that reads back as the table's sentinel -- so the JDK is used as the oracle rather + * than a hand-written expectation.

+ */ +class CollectionSemanticsIntegrationTest { + + @Test + void collectionSemanticsMatchTheJvm() throws Exception { + Parser.cleanup(); + + Path sourceDir = Files.createTempDirectory("collection-semantics-sources"); + Path classesDir = Files.createTempDirectory("collection-semantics-classes"); + Path javaApiDir = Files.createTempDirectory("collection-semantics-java-api"); + + Path source = sourceDir.resolve("CollectionSemanticsApp.java"); + Files.write(source, loadAppSource().getBytes(StandardCharsets.UTF_8)); + + CompilerHelper.CompilerConfig config = selectCompiler(); + if (config == null) { + fail("No compatible compiler available for the collection semantics integration test"); + } + assertTrue(CompilerHelper.isJavaApiCompatible(config), + "JDK " + config.jdkVersion + " must target matching bytecode level for JavaAPI"); + + CompilerHelper.compileJavaAPI(javaApiDir, config); + + List compileArgs = new ArrayList<>(); + compileArgs.add("-source"); + compileArgs.add(config.targetVersion); + compileArgs.add("-target"); + compileArgs.add(config.targetVersion); + if (CompilerHelper.useClasspath(config)) { + compileArgs.add("-classpath"); + compileArgs.add(javaApiDir.toString()); + } else { + compileArgs.add("-bootclasspath"); + compileArgs.add(javaApiDir.toString()); + compileArgs.add("-Xlint:-options"); + } + compileArgs.add("-d"); + compileArgs.add(classesDir.toString()); + compileArgs.add(source.toString()); + + assertEquals(0, CompilerHelper.compile(config.jdkHome, compileArgs), + "CollectionSemanticsApp should compile against the JavaAPI"); + + Map expected = parseCases(runJavaMain(config, classesDir, javaApiDir)); + assertFalse(expected.isEmpty(), "JVM run should emit cases"); + + CompilerHelper.copyDirectory(javaApiDir, classesDir); + + Path outputDir = Files.createTempDirectory("collection-semantics-output"); + CleanTargetIntegrationTest.runTranslator(classesDir, outputDir, "CollectionSemanticsApp"); + + Path distDir = outputDir.resolve("dist"); + Path cmakeLists = distDir.resolve("CMakeLists.txt"); + assertTrue(Files.exists(cmakeLists), "Translator should emit a CMake project"); + CleanTargetIntegrationTest.replaceLibraryWithExecutableTarget(cmakeLists, "CollectionSemanticsApp-src"); + + Path buildDir = distDir.resolve("build"); + Files.createDirectories(buildDir); + CleanTargetIntegrationTest.runCommand(Arrays.asList( + "cmake", + "-S", distDir.toString(), + "-B", buildDir.toString(), + "-DCMAKE_C_COMPILER=clang", + "-DCMAKE_OBJC_COMPILER=clang" + ), distDir); + CleanTargetIntegrationTest.runCommand(Arrays.asList("cmake", "--build", buildDir.toString()), distDir); + + Path executable = buildDir.resolve("CollectionSemanticsApp"); + String parparOutput = CleanTargetIntegrationTest.runCommand( + Arrays.asList(executable.toString()), buildDir); + assertTrue(parparOutput.contains("DONE"), + "ParparVM run should complete. Output: " + parparOutput); + + Map actual = parseCases(parparOutput); + assertEquals(expected.keySet(), actual.keySet(), "ParparVM should emit the same cases"); + + List differences = new ArrayList<>(); + for (Map.Entry entry : expected.entrySet()) { + if (!entry.getValue().equals(actual.get(entry.getKey()))) { + differences.add(entry.getKey() + + "\n jvm : " + entry.getValue() + + "\n parparvm: " + actual.get(entry.getKey())); + } + } + assertTrue(differences.isEmpty(), + "Collection semantics diverged from the JVM:\n" + String.join("\n", differences)); + + // Named explicitly so a regression points at the change rather than at a + // generic diff. + assertEquals("0", actual.get("empty.size"), "a list never added to must be empty"); + assertEquals("IndexOutOfBounds", actual.get("empty.get0"), + "get(0) on an empty list must still throw"); + assertEquals("1", actual.get("ihm.keyNulls"), + "the key iterator must hand back the null key as null, not the table's sentinel"); + assertEquals("1", actual.get("ihm.valNulls"), + "the value iterator must hand back a null value as null"); + assertEquals("500", actual.get("ihm.bigSeen"), + "key iteration must survive a rehash"); + } + + private Map parseCases(String output) { + Map cases = new LinkedHashMap<>(); + for (String line : output.split("\\R")) { + if (!line.startsWith("CASE|")) { + continue; + } + String body = line.substring("CASE|".length()); + int separator = body.indexOf('|'); + assertTrue(separator > 0, "Malformed case line: " + line); + cases.put(body.substring(0, separator), body.substring(separator + 1)); + } + return cases; + } + + private String loadAppSource() throws Exception { + java.io.InputStream in = CollectionSemanticsIntegrationTest.class + .getResourceAsStream("/com/codename1/tools/translator/CollectionSemanticsApp.java"); + assertNotNull(in, "CollectionSemanticsApp.java test resource should exist"); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) { + return reader.lines().collect(Collectors.joining("\n")) + "\n"; + } + } + + private String runJavaMain(CompilerHelper.CompilerConfig config, Path classesDir, Path javaApiDir) + throws Exception { + String javaExe = config.jdkHome.resolve("bin").resolve(CompilerHelper.executableName("java")).toString(); + ProcessBuilder pb = new ProcessBuilder( + javaExe, + "-cp", + classesDir + System.getProperty("path.separator") + javaApiDir, + "CollectionSemanticsApp" + ); + pb.redirectErrorStream(true); + + Process process = pb.start(); + String output; + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8))) { + output = reader.lines().collect(Collectors.joining("\n")); + } + assertEquals(0, process.waitFor(), "JVM run should exit cleanly. Output: " + output); + return output; + } + + private CompilerHelper.CompilerConfig selectCompiler() { + String[] preferredTargets = {"11", "17", "21", "25", "1.8"}; + for (String target : preferredTargets) { + List configs = CompilerHelper.getAvailableCompilers(target); + for (CompilerHelper.CompilerConfig config : configs) { + if (CompilerHelper.isJavaApiCompatible(config)) { + return config; + } + } + } + return null; + } +} diff --git a/vm/tests/src/test/resources/com/codename1/tools/translator/CollectionSemanticsApp.java b/vm/tests/src/test/resources/com/codename1/tools/translator/CollectionSemanticsApp.java new file mode 100644 index 00000000000..e581a5f12b2 --- /dev/null +++ b/vm/tests/src/test/resources/com/codename1/tools/translator/CollectionSemanticsApp.java @@ -0,0 +1,174 @@ +import java.util.ArrayList; +import java.util.IdentityHashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Exercises the parts of ArrayList and IdentityHashMap that were changed to stop + * allocating: ArrayList no longer allocates a backing array until the first growth, + * and IdentityHashMap's key and value iterators no longer build an Entry per step. + * + * Every line is compared against a real JDK run, so the JDK is the oracle rather + * than a hand-written expectation. + */ +public class CollectionSemanticsApp { + static void emit(String k, Object v) { + System.out.println("CASE|" + k + "|" + v); + } + + public static void main(String[] args) { + // ---- an ArrayList that is never added to ------------------------------- + List empty = new ArrayList(); + emit("empty.size", empty.size()); + emit("empty.isEmpty", empty.isEmpty()); + emit("empty.contains", empty.contains("x")); + emit("empty.indexOf", empty.indexOf("x")); + emit("empty.iterHasNext", empty.iterator().hasNext()); + emit("empty.toArrayLen", empty.toArray().length); + emit("empty.toString", empty.toString()); + empty.clear(); + emit("empty.afterClear", empty.size()); + try { + empty.get(0); + emit("empty.get0", "no throw"); + } catch (IndexOutOfBoundsException err) { + emit("empty.get0", "IndexOutOfBounds"); + } + + // ---- first growth, and growth past it ---------------------------------- + List grow = new ArrayList(); + for (int i = 0; i < 40; i++) { + grow.add(Integer.valueOf(i)); + if (i < 3 || i == 9 || i == 10 || i == 11 || i == 12 || i == 39) { + emit("grow.size@" + i, grow.size() + ":" + grow.get(0) + ":" + grow.get(i)); + } + } + emit("grow.toString", grow.toString()); + emit("grow.indexOf37", grow.indexOf(Integer.valueOf(37))); + + // ---- add-at-front on a fresh list (the growAtFront path) --------------- + List front = new ArrayList(); + front.add(0, "b"); + front.add(0, "a"); + front.add("c"); + emit("front.toString", front.toString()); + emit("front.size", front.size()); + + // ---- insert into the middle of a fresh list (growForInsert) ------------ + List mid = new ArrayList(); + mid.add("x"); + mid.add("z"); + mid.add(1, "y"); + emit("mid.toString", mid.toString()); + + // ---- ensureCapacity on a fresh list ------------------------------------ + ArrayList ec = new ArrayList(); + ec.ensureCapacity(100); + ec.add("only"); + emit("ec.toString", ec.toString()); + + // ---- remove down to empty and re-add ----------------------------------- + List churn = new ArrayList(); + churn.add("p"); + churn.add("q"); + churn.remove("p"); + churn.remove(0); + emit("churn.emptyAgain", churn.size()); + churn.add("r"); + emit("churn.readd", churn.toString()); + + // ---- IdentityHashMap: identity semantics, and all three views ---------- + String k1 = new String("dup"); + String k2 = new String("dup"); + IdentityHashMap ihm = new IdentityHashMap(); + ihm.put(k1, "first"); + ihm.put(k2, "second"); + emit("ihm.size", ihm.size()); + emit("ihm.get1", ihm.get(k1)); + emit("ihm.get2", ihm.get(k2)); + emit("ihm.containsKey1", ihm.containsKey(k1)); + + // Null key and null value must survive the table's NULL_OBJECT sentinel in + // BOTH directions -- this is what the key/value iterators read directly now. + ihm.put(null, "nullkey"); + ihm.put("nullval", null); + emit("ihm.getNullKey", ihm.get(null)); + emit("ihm.getNullVal", String.valueOf(ihm.get("nullval"))); + emit("ihm.sizeWithNulls", ihm.size()); + + int keyNulls = 0, keyCount = 0; + for (Iterator it = ihm.keySet().iterator(); it.hasNext();) { + String k = it.next(); + keyCount++; + if (k == null) { + keyNulls++; + } + } + emit("ihm.keyCount", keyCount); + emit("ihm.keyNulls", keyNulls); + + int valNulls = 0, valCount = 0; + for (Iterator it = ihm.values().iterator(); it.hasNext();) { + String v = it.next(); + valCount++; + if (v == null) { + valNulls++; + } + } + emit("ihm.valCount", valCount); + emit("ihm.valNulls", valNulls); + + int entryCount = 0, entryKeyNulls = 0, entryValNulls = 0; + for (Map.Entry e : ihm.entrySet()) { + entryCount++; + if (e.getKey() == null) { + entryKeyNulls++; + } + if (e.getValue() == null) { + entryValNulls++; + } + } + emit("ihm.entryCount", entryCount); + emit("ihm.entryKeyNulls", entryKeyNulls); + emit("ihm.entryValNulls", entryValNulls); + + // keySet().contains and removal through the key view + Set keys = ihm.keySet(); + emit("ihm.keysContainsK1", keys.contains(k1)); + emit("ihm.keysRemoveK1", keys.remove(k1)); + emit("ihm.sizeAfterRemove", ihm.size()); + + // iterator removal + IdentityHashMap rem = new IdentityHashMap(); + String r1 = new String("r1"); + String r2 = new String("r2"); + rem.put(r1, "1"); + rem.put(r2, "2"); + for (Iterator it = rem.keySet().iterator(); it.hasNext();) { + if (it.next() == r1) { + it.remove(); + } + } + emit("ihm.afterIterRemove", rem.size() + ":" + rem.get(r2)); + + // a map big enough to force a rehash, iterated by key + IdentityHashMap big = new IdentityHashMap(); + Object[] held = new Object[500]; + for (int i = 0; i < held.length; i++) { + held[i] = new Object(); + big.put(held[i], Integer.valueOf(i)); + } + long sum = 0; + int seen = 0; + for (Object o : big.keySet()) { + sum += big.get(o).intValue(); + seen++; + } + emit("ihm.bigSeen", seen); + emit("ihm.bigSum", sum); + + System.out.println("DONE"); + } +} From 3f20c45ed50e8fa12cbc732934aae7b6c2b8ff53 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 10 Sep 2026 08:53:52 +0300 Subject: [PATCH 09/66] Add a live-heap census: what is IN the heap, not just what was allocated cn1AllocCensus answers churn -- what was allocated, which is what costs CPU. There was nothing that answered retention -- what is still here, which is what costs memory -- and those are different questions: an iterator allocated a million times retains nothing, a cache allocated once retains everything. cn1LiveCensus walks the BiBOP pages and the legacy heap after a sweep and reports the heap by class: occupied bytes, object count, bytes each, and how many of them the last mark proved reachable. Objects are charged what they OCCUPY -- a whole size-class slot, a whole malloc block -- so the rows add up to the footprint and rounding waste lands on the class that causes it. Classes are collected in a local pointer-keyed table rather than read from cn1ClazzSet, which only exists under CN1_CONSERVATIVE_GC_ROOTS. Occupied and reachable are reported separately on purpose: "a million live iterators" and "a million dead iterators still holding slots" call for opposite fixes. Reachability is only meaningful in the post-sweep report -- it means "carries the current mark", so at exit, long after the last cycle, almost everything reads as unreachable whether it is or not. That trap is real: the exit report says 6% reachable and the last post-sweep says 75%. What it says about a self-hosting translation, all of it recorded in vm/selfhost/README.md: - The heap is genuinely live, not uncollected garbage: 295MB occupied and 75% reachable at the last sweep. The run then ends at 769MB because only three or four cycles complete in 1.4s while the mark thread sits at 97% CPU. - Collecting harder does not fix it. -DCN1_GC_MARK_THREADS=4 more than doubles the cycles (4 -> 9) and is slightly faster, but peak moves 1256MB -> 1243MB. - vmmap puts essentially all of the process in malloc'd heap: MALLOC_LARGE 435.8MB dirty (the BiBOP arenas), MALLOC_SMALL 111.7MB, and 50.2MB of MALLOC_LARGE (empty) -- freed but not returned. An earlier claim in the README that ~600MB was "not the Java heap" was wrong; it compared an exit-time census against the whole-run peak. Two leads it opens without settling: per-object width against the JDK (String 73B here against ~32B there), and 295,907 SimpleListIterator objects reading 72% reachable at a fresh sweep, which a stack-local iterator should never be -- conservative stack scanning is on unconditionally and would explain it, but that is a hypothesis and not yet measured. Compile-gated on CN1_ALLOC_CENSUS and run-gated on CN1_HEAP_REPORT, so an ordinary build is untouched. Gates A and D still byte-identical. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.m | 140 ++++++++++++++++++++++++ vm/selfhost/README.md | 71 ++++++++++++ 2 files changed, 211 insertions(+) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index 70a45875ba0..c9fe664c36b 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -1797,6 +1797,7 @@ static void cn1DrainDeadThreadPending() { // CN1_GC_VERIFY block just above, which is off in an ordinary census build. void cn1HeapAccounting(const char* label); void cn1AllocCensus(const char* label); +void cn1LiveCensus(const char* label); #endif #ifdef CN1_GRACE_AUDIT @@ -4879,6 +4880,7 @@ void codenameOneGCSweep() { // from anywhere, so nothing could answer "what is the footprint made of". if(getenv("CN1_HEAP_REPORT")) { cn1HeapAccounting("post-sweep"); + cn1LiveCensus("post-sweep"); } #endif } @@ -5603,6 +5605,7 @@ static void cn1BibopDoInit() { // the process actually died holding. static void cn1BibopExitReport(void) { cn1HeapAccounting("exit"); + cn1LiveCensus("exit"); cn1AllocCensus("exit"); } #endif @@ -7387,6 +7390,143 @@ void cn1HeapAccounting(const char* label) { fflush(stderr); } +/** + * Prints the LIVE heap by class, biggest first. + * + * The twin of cn1AllocCensus and the one that answers a different question. + * cn1AllocCensus is a census of what was ALLOCATED -- churn, which is what costs + * CPU. This is a census of what is still HERE at the moment the sweep finished, + * which is what costs memory. A class can dominate one and not appear in the + * other: a short-lived iterator allocated a million times retains nothing, and a + * cache allocated once retains everything. + * + * Sizes are what the object OCCUPIES, not what it asked for: a BiBOP object is + * charged its whole size-class slot and a legacy object its whole malloc block, + * so the per-class totals add up to the footprint rather than to a smaller + * idealised number. Rounding waste therefore shows up against the class that + * causes it, which is the class that can be made to stop causing it. + * + * Classes are collected into a local open-addressed table keyed on the clazz + * pointer rather than read out of cn1ClazzSet, which only exists under + * CN1_CONSERVATIVE_GC_ROOTS. + * + * Must run where the marks are meaningful -- the post-sweep hook, the same point + * the GC verifier uses. + */ +#define CN1_LIVE_CENSUS_SLOTS 8192 +struct CN1LiveRow { struct clazz* c; long count; long long bytes; long reachable; }; +static struct CN1LiveRow cn1LiveRows[CN1_LIVE_CENSUS_SLOTS]; + +static void cn1LiveTally(struct clazz* c, long long bytes, int reachable) { + if(c == 0) { + return; + } + size_t h = (((uintptr_t)c) >> 4) & (CN1_LIVE_CENSUS_SLOTS - 1); + for(int probe = 0 ; probe < CN1_LIVE_CENSUS_SLOTS ; probe++) { + size_t i = (h + (size_t)probe) & (CN1_LIVE_CENSUS_SLOTS - 1); + if(cn1LiveRows[i].c == 0) { + cn1LiveRows[i].c = c; + } + if(cn1LiveRows[i].c == c) { + cn1LiveRows[i].count++; + cn1LiveRows[i].bytes += bytes; + cn1LiveRows[i].reachable += reachable; + return; + } + } + // Table full: 8192 slots against the ~170 classes a large program allocates, + // so this is unreachable short of a pathological program. Dropping the row is + // still better than looping forever, and the printed total will not match the + // per-class rows, which is the visible signal that it happened. +} + +void cn1LiveCensus(const char* label) { + memset(cn1LiveRows, 0, sizeof(cn1LiveRows)); + long long bibopBytes = 0, legacyBytes = 0; + long bibopObjs = 0, legacyObjs = 0, bibopReach = 0, legacyReach = 0; + + CN1BibopPage* p = atomic_load_explicit(&bibopAllPages, memory_order_acquire); + while(p != 0) { + int n = atomic_load_explicit(&p->bumpIndex, memory_order_acquire); + for(int i = 0 ; i < n ; i++) { + JAVA_OBJECT o = cn1BibopSlot(p, i); + int m = __atomic_load_n(&o->__codenameOneGcMark, __ATOMIC_ACQUIRE); + // Occupied, not "provably reachable": a slot awaiting collection is + // still holding memory, and this census is about what memory is being + // held. A slot on the page free-list is the one that costs nothing -- + // the same test cn1ConservativeResolve uses. (CN1_GC_POISON_MARK is + // deliberately not consulted: it is defined further down, inside the + // verifier's section, and exists only in a CN1_GC_VERIFY build.) + if(m == CN1_BIBOP_FREE_MARK) { + continue; + } + cn1LiveTally(o->__codenameOneParentClsReference, (long long)p->slotSize, + m == currentGcMarkValue ? 1 : 0); + bibopBytes += (long long)p->slotSize; + bibopObjs++; + if(m == currentGcMarkValue) { + bibopReach++; + } + } + p = atomic_load_explicit(&p->nextAll, memory_order_acquire); + } + + int nHeap = currentSizeOfAllObjectsInHeap; + for(int i = 0 ; i < nHeap ; i++) { + JAVA_OBJECT o = allObjectsInHeap[i]; + if(o == JAVA_NULL) { + continue; + } + // An adopted object lives in a BiBOP slot and was already charged by the + // page walk; malloc_size on it would read a block header that is not there. + if(o->__heapPosition == CN1_BIBOP_ADOPTED) { + continue; + } + long long sz = 0; +#if defined(__APPLE__) + sz = (long long)malloc_size((void*)o); +#endif + cn1LiveTally(o->__codenameOneParentClsReference, sz, + o->__codenameOneGcMark == currentGcMarkValue ? 1 : 0); + legacyBytes += sz; + legacyObjs++; + if(o->__codenameOneGcMark == currentGcMarkValue) { + legacyReach++; + } + } + + // OCCUPIED is what costs memory; REACHABLE is what the last mark actually + // proved live. The gap between them is garbage the collector has not got to, + // and telling them apart is the whole point -- "a million live iterators" and + // "a million dead iterators still holding slots" call for opposite fixes. + fprintf(stderr, "[LIVE:%s] occupied %ld objects %.2fMB | reachable %ld (%.0f%%) " + "| bibop %ld/%.2fMB legacy %ld/%.2fMB\n", + label, bibopObjs + legacyObjs, (bibopBytes + legacyBytes) / 1048576.0, + bibopReach + legacyReach, + 100.0 * (bibopReach + legacyReach) / ((bibopObjs + legacyObjs) > 0 ? (bibopObjs + legacyObjs) : 1), + bibopObjs, bibopBytes / 1048576.0, legacyObjs, legacyBytes / 1048576.0); + for(int shown = 0 ; shown < 30 ; shown++) { + int best = -1; + for(int i = 0 ; i < CN1_LIVE_CENSUS_SLOTS ; i++) { + if(cn1LiveRows[i].c != 0 && cn1LiveRows[i].bytes > 0 + && (best < 0 || cn1LiveRows[i].bytes > cn1LiveRows[best].bytes)) { + best = i; + } + } + if(best < 0) { + break; + } + fprintf(stderr, "[LIVE:%s] %8.2fMB %9ld objs %4lld B/obj reach %9ld (%3.0f%%) %s\n", + label, cn1LiveRows[best].bytes / 1048576.0, cn1LiveRows[best].count, + cn1LiveRows[best].bytes / (cn1LiveRows[best].count > 0 ? cn1LiveRows[best].count : 1), + cn1LiveRows[best].reachable, + 100.0 * cn1LiveRows[best].reachable / (cn1LiveRows[best].count > 0 ? cn1LiveRows[best].count : 1), + cn1LiveRows[best].c->clsName ? cn1LiveRows[best].c->clsName : "?"); + cn1LiveRows[best].bytes = 0; + } + fflush(stderr); +} + void cn1AllocCensus(const char* label) { struct Row { const char* name; long count; long bytes; }; static struct Row rows[4096]; diff --git a/vm/selfhost/README.md b/vm/selfhost/README.md index a91868bc7c7..6966bd0a873 100644 --- a/vm/selfhost/README.md +++ b/vm/selfhost/README.md @@ -211,6 +211,77 @@ Growth there is also post-insert by design, so the shared-empty-table trick that works for ArrayList would have the put path writing into the shared table. Not worth it for an unmeasured win in the hottest class in the runtime. +### Heap telemetry + +A census build answers "what is actually in the heap": + +```bash +CN1_SELFHOST_CFLAGS="-DCN1_ALLOC_CENSUS" ./build-selfhost.sh -O3 +CN1_HEAP_REPORT=1 ./target/parpar-O3 clean ... 2> report.txt +``` + +Three reports, after every sweep and once at exit: + +- `[JHEAP]` -- BiBOP pages reserved / live / slack, plus the legacy heap. Answers + "is this fragmentation?" (here: no, slack is ~2 MB of 715 MB). +- `[LIVE]` -- **the live heap by class**, occupied bytes, objects, bytes each, and + how many the last mark proved reachable. This is the one that was missing. +- `[ALLOC]` -- allocation volume by class. Churn, which costs CPU, as opposed to + retention, which costs memory. A class can dominate one and not the other. + +`[LIVE]` charges each object what it OCCUPIES -- a whole BiBOP size-class slot, a +whole malloc block -- so the per-class rows add up to the footprint and rounding +waste is charged to the class that causes it. + +**Read the post-sweep report, not the exit one, for reachability.** `reachable` +means "carries the current mark", so at exit -- long after the last cycle -- almost +everything looks unreachable whether it is or not. At exit that column says 6%; at +the last sweep, with fresh marks, it says 75%. + +### What the census says about this workload + +At the last completed sweep: 295 MB occupied, **75% of it reachable**. So the heap +is genuinely full of live data, not garbage the collector failed to reclaim. The +trajectory across cycles is 41 MB, 107 MB, 295 MB -- and then the run ends at +769 MB, because only three or four cycles complete in a 1.4 s program while the +mark thread sits at 97% CPU in `gcMarkObject`. + +Collecting harder does not fix it, which is the useful negative result: + +| mark threads | wall | cycles | peak | +|---|---:|---:|---:| +| 1 | 1.40 s | 4 | 1256 MB | +| 4 | 1.25 s | 9 | 1243 MB | +| 8 | 1.47 s | 8 | 1223 MB | + +`-DCN1_GC_MARK_THREADS=4` more than doubles the cycles and is slightly faster, but +peak barely moves. Combined with 75% reachability, that says the live set really is +this large rather than the collector being behind. + +Where the process memory sits, from `vmmap --summary` around peak: + +``` +MALLOC_LARGE 551.5M virtual / 435.8M dirty BiBOP arenas +MALLOC_LARGE (empty) 53.7M / 50.2M dirty freed, not returned +MALLOC_SMALL 232.0M / 111.7M dirty legacy heap +Stack 12.2M / 0.2M +``` + +It is all malloc'd heap; there is no large non-heap component. (An earlier note in +this file claimed ~600 MB was "not the Java heap" -- that compared an exit-time +census against the whole-run peak and was wrong.) + +Two leads the census opens and does not settle: + +- Per-object width against the JDK: `String` 73 B, `ArrayList` 48 B, `Object[]` + 100 B average. The JDK's String is ~32 B and its ArrayList ~40 B. +- 295,907 `SimpleListIterator` objects were 72% *reachable* at a fresh sweep, and + a stack-local iterator should be dead the moment its loop ends. ParparVM scans + stacks conservatively (`CN1_CONSERVATIVE_GC_ROOTS` is defined unconditionally), + so a stale stack word that looks like a pointer keeps its object alive -- which + a deeply recursive, allocation-heavy program produces a lot of. Plausible, not + proven, and worth measuring before acting on. + ### String: the NSString field is free `java.lang.String` carries a `long nsString` for the Apple targets' direct NSString From 3f1bee7a485a22aac43ec59eb4920ff4a99f7a7d Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 10 Sep 2026 09:15:27 +0300 Subject: [PATCH 10/66] Census the heap by WHY each slot is occupied, and answer it with experiments The live census reported "reachable", which conflated two different things and got the answer backwards. Read post-sweep, an object stamped live by the sweep's grace rule is indistinguishable from one the mark actually traced -- so a heap full of fresh garbage read as a heap full of live data. The census now runs PRE-sweep, the only point where the four reasons a slot is still occupied are still distinguishable, and reports all four per class: traced (the mark reached it), fresh (allocated since the mark, kept by grace), aging (known dead, kept one more cycle) and dead (this sweep returns it). On a self-hosting translation, at the last cycle: occupied 4,441,347 objects 349MB traced 47% fresh 30% aging 14% dead 9% Only 47% of the occupied heap is traced live; the rest is held by collector policy rather than by the program. Per class it is sharper -- char[] is 5% traced and 76% fresh, almost pure churn caught between cycles. experiments/PinProbe establishes the mechanism directly: three arms allocate and drop 200,000 objects each -- shallow, under a 400-deep recursion, and with the stack scrubbed -- and a dead object needs THREE cycles to have its slot returned (grace while fresh, then aging, then reclamation). A translation completes three or four cycles in 1.4s, so most of what it allocates is never eligible to be freed and the heap grows towards total allocation volume: 940MB allocated, 1.3GB peak, 150-300MB genuinely live. The same probe rules OUT the hypothesis it was built to test. Conservative stack scanning does not pin dead objects: the marks are precise and all three arms behave identically, so depth and stale stack words make no difference. Fragmentation is ruled out too, at ~2MB of page-pool slack in 715MB. Collecting faster helps but does not change the ratio, since the grace rule keeps everything allocated since the last mark whatever the rate: 1 mark thread 3 cycles peak 1320MB -DCN1_GC_MARK_THREADS=4 8 cycles peak 1172MB 4 threads + CN1_GC_TRIGGER_MB=24 7 cycles peak 1259MB So the dominant lever is allocation churn, and cutting one allocation removes about three cycles of occupancy rather than one object. The [ALLOC] census names where it is: char[] 368MB, Object[] 196MB, String 77MB, SimpleListIterator 40MB. Gates A and D still byte-identical; the census is compile-gated on CN1_ALLOC_CENSUS and run-gated on CN1_HEAP_REPORT, so an ordinary build is untouched. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.m | 100 ++++++++++++------ vm/selfhost/README.md | 79 +++++++++----- vm/selfhost/experiments/README.md | 47 ++++++++ .../experiments/src/com/exp/PinProbe.java | 100 ++++++++++++++++++ 4 files changed, 269 insertions(+), 57 deletions(-) create mode 100644 vm/selfhost/experiments/README.md create mode 100644 vm/selfhost/experiments/src/com/exp/PinProbe.java diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index c9fe664c36b..50001d2d941 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -4729,6 +4729,14 @@ static void cn1GcReportStaleIndexSkip(void) { void codenameOneGCSweep() { struct ThreadLocalData* threadStateData = getThreadLocalData(); +#ifdef CN1_ALLOC_CENSUS + // BEFORE the sweep on purpose. This is the only point where the four slot + // states are still distinguishable -- the sweep stamps every fresh object with + // the current mark, after which "traced" and "kept by grace" look identical. + if(getenv("CN1_HEAP_REPORT")) { + cn1LiveCensus("pre-sweep"); + } +#endif // THE MARK THIS SWEEP WOULD ACT ON MAY BE INCOMPLETE. cn1GcPageIndexStale says the // page index could not be rebuilt, so every reference into a page registered since // the last successful rebuild failed to resolve and its object was never marked -- @@ -7414,10 +7422,35 @@ void cn1HeapAccounting(const char* label) { * the GC verifier uses. */ #define CN1_LIVE_CENSUS_SLOTS 8192 -struct CN1LiveRow { struct clazz* c; long count; long long bytes; long reachable; }; +// Four states a slot can be in when the SWEEP is about to look at it. Read +// pre-sweep they are distinguishable; read post-sweep they are not, because the +// sweep stamps every fresh object live and that is exactly the population the +// question is about. +#define CN1_LB_TRACED 0 /* mark == currentGcMarkValue: traced live this cycle */ +#define CN1_LB_FRESH 1 /* mark == -1: allocated since the mark, gets one grace */ +#define CN1_LB_AGING 2 /* mark == V-1: not traced, kept one more cycle anyway */ +#define CN1_LB_DEAD 3 /* older: this sweep reclaims it */ +#define CN1_LB_COUNT 4 +struct CN1LiveRow { struct clazz* c; long count; long long bytes; long b[CN1_LB_COUNT]; }; static struct CN1LiveRow cn1LiveRows[CN1_LIVE_CENSUS_SLOTS]; -static void cn1LiveTally(struct clazz* c, long long bytes, int reachable) { +static int cn1LiveBucket(int m) { + // -1 must be tested before the "older than V-1" arm: it is numerically less + // than V-1 for any live epoch, so the ordering is what keeps a fresh object + // out of the reclaimable bucket. + if(m == -1) { + return CN1_LB_FRESH; + } + if(m == currentGcMarkValue) { + return CN1_LB_TRACED; + } + if(m == currentGcMarkValue - 1) { + return CN1_LB_AGING; + } + return CN1_LB_DEAD; +} + +static void cn1LiveTally(struct clazz* c, long long bytes, int bucket) { if(c == 0) { return; } @@ -7430,7 +7463,7 @@ static void cn1LiveTally(struct clazz* c, long long bytes, int reachable) { if(cn1LiveRows[i].c == c) { cn1LiveRows[i].count++; cn1LiveRows[i].bytes += bytes; - cn1LiveRows[i].reachable += reachable; + cn1LiveRows[i].b[bucket]++; return; } } @@ -7443,7 +7476,11 @@ static void cn1LiveTally(struct clazz* c, long long bytes, int reachable) { void cn1LiveCensus(const char* label) { memset(cn1LiveRows, 0, sizeof(cn1LiveRows)); long long bibopBytes = 0, legacyBytes = 0; - long bibopObjs = 0, legacyObjs = 0, bibopReach = 0, legacyReach = 0; + long bibopObjs = 0, legacyObjs = 0; + long totals[CN1_LB_COUNT]; + for(int i = 0 ; i < CN1_LB_COUNT ; i++) { + totals[i] = 0; + } CN1BibopPage* p = atomic_load_explicit(&bibopAllPages, memory_order_acquire); while(p != 0) { @@ -7460,13 +7497,11 @@ void cn1LiveCensus(const char* label) { if(m == CN1_BIBOP_FREE_MARK) { continue; } - cn1LiveTally(o->__codenameOneParentClsReference, (long long)p->slotSize, - m == currentGcMarkValue ? 1 : 0); + int bucket = cn1LiveBucket(m); + cn1LiveTally(o->__codenameOneParentClsReference, (long long)p->slotSize, bucket); bibopBytes += (long long)p->slotSize; bibopObjs++; - if(m == currentGcMarkValue) { - bibopReach++; - } + totals[bucket]++; } p = atomic_load_explicit(&p->nextAll, memory_order_acquire); } @@ -7486,25 +7521,26 @@ void cn1LiveCensus(const char* label) { #if defined(__APPLE__) sz = (long long)malloc_size((void*)o); #endif - cn1LiveTally(o->__codenameOneParentClsReference, sz, - o->__codenameOneGcMark == currentGcMarkValue ? 1 : 0); + int lbucket = cn1LiveBucket(o->__codenameOneGcMark); + cn1LiveTally(o->__codenameOneParentClsReference, sz, lbucket); legacyBytes += sz; legacyObjs++; - if(o->__codenameOneGcMark == currentGcMarkValue) { - legacyReach++; - } - } - - // OCCUPIED is what costs memory; REACHABLE is what the last mark actually - // proved live. The gap between them is garbage the collector has not got to, - // and telling them apart is the whole point -- "a million live iterators" and - // "a million dead iterators still holding slots" call for opposite fixes. - fprintf(stderr, "[LIVE:%s] occupied %ld objects %.2fMB | reachable %ld (%.0f%%) " - "| bibop %ld/%.2fMB legacy %ld/%.2fMB\n", - label, bibopObjs + legacyObjs, (bibopBytes + legacyBytes) / 1048576.0, - bibopReach + legacyReach, - 100.0 * (bibopReach + legacyReach) / ((bibopObjs + legacyObjs) > 0 ? (bibopObjs + legacyObjs) : 1), - bibopObjs, bibopBytes / 1048576.0, legacyObjs, legacyBytes / 1048576.0); + totals[lbucket]++; + } + + // OCCUPIED is what costs memory. The four buckets say WHY each object is still + // occupying a slot, and they call for different fixes: traced means the program + // really is holding it, fresh and aging mean the collector is holding it under + // the grace and aging rules, and dead means this sweep is about to return it. + long occupied = bibopObjs + legacyObjs; + fprintf(stderr, "[LIVE:%s] occupied %ld objects %.2fMB | traced %ld (%.0f%%) " + "fresh %ld (%.0f%%) aging %ld (%.0f%%) dead %ld (%.0f%%) | bibop %.2fMB legacy %.2fMB\n", + label, occupied, (bibopBytes + legacyBytes) / 1048576.0, + totals[CN1_LB_TRACED], 100.0 * totals[CN1_LB_TRACED] / (occupied > 0 ? occupied : 1), + totals[CN1_LB_FRESH], 100.0 * totals[CN1_LB_FRESH] / (occupied > 0 ? occupied : 1), + totals[CN1_LB_AGING], 100.0 * totals[CN1_LB_AGING] / (occupied > 0 ? occupied : 1), + totals[CN1_LB_DEAD], 100.0 * totals[CN1_LB_DEAD] / (occupied > 0 ? occupied : 1), + bibopBytes / 1048576.0, legacyBytes / 1048576.0); for(int shown = 0 ; shown < 30 ; shown++) { int best = -1; for(int i = 0 ; i < CN1_LIVE_CENSUS_SLOTS ; i++) { @@ -7516,11 +7552,15 @@ void cn1LiveCensus(const char* label) { if(best < 0) { break; } - fprintf(stderr, "[LIVE:%s] %8.2fMB %9ld objs %4lld B/obj reach %9ld (%3.0f%%) %s\n", + long rc = cn1LiveRows[best].count > 0 ? cn1LiveRows[best].count : 1; + fprintf(stderr, "[LIVE:%s] %8.2fMB %9ld objs %4lld B/obj traced %3.0f%% fresh %3.0f%% " + "aging %3.0f%% dead %3.0f%% %s\n", label, cn1LiveRows[best].bytes / 1048576.0, cn1LiveRows[best].count, - cn1LiveRows[best].bytes / (cn1LiveRows[best].count > 0 ? cn1LiveRows[best].count : 1), - cn1LiveRows[best].reachable, - 100.0 * cn1LiveRows[best].reachable / (cn1LiveRows[best].count > 0 ? cn1LiveRows[best].count : 1), + cn1LiveRows[best].bytes / rc, + 100.0 * cn1LiveRows[best].b[CN1_LB_TRACED] / rc, + 100.0 * cn1LiveRows[best].b[CN1_LB_FRESH] / rc, + 100.0 * cn1LiveRows[best].b[CN1_LB_AGING] / rc, + 100.0 * cn1LiveRows[best].b[CN1_LB_DEAD] / rc, cn1LiveRows[best].c->clsName ? cn1LiveRows[best].c->clsName : "?"); cn1LiveRows[best].bytes = 0; } diff --git a/vm/selfhost/README.md b/vm/selfhost/README.md index 6966bd0a873..30ff9e18cf4 100644 --- a/vm/selfhost/README.md +++ b/vm/selfhost/README.md @@ -240,23 +240,59 @@ the last sweep, with fresh marks, it says 75%. ### What the census says about this workload -At the last completed sweep: 295 MB occupied, **75% of it reachable**. So the heap -is genuinely full of live data, not garbage the collector failed to reclaim. The -trajectory across cycles is 41 MB, 107 MB, 295 MB -- and then the run ends at -769 MB, because only three or four cycles complete in a 1.4 s program while the -mark thread sits at 97% CPU in `gcMarkObject`. +The `[LIVE]` report is printed **pre-sweep**, which is the only point where the four +reasons a slot is still occupied are distinguishable: `traced` (the current mark +reached it), `fresh` (allocated since the mark, kept by the grace rule), `aging` +(known dead, kept one more cycle) and `dead` (this sweep returns it). Post-sweep +the grace stamp makes the first two identical, and the first version of this census +reported one as the other. -Collecting harder does not fix it, which is the useful negative result: +At the last cycle of a self-hosting translation: -| mark threads | wall | cycles | peak | +``` +occupied 4,441,347 objects 349MB + traced 47% fresh 30% aging 14% dead 9% +``` + +**Only 47% of the occupied heap is traced live. The rest is held by collector +policy, not by the program.** Per class the split is sharper still -- `char[]` is +**5% traced and 76% fresh**, i.e. almost pure churn caught between cycles: + +``` + 68.84MB 726070 objs 99 B/obj traced 49% fresh 20% aging 19% dead 12% java.lang.Object[] + 51.17MB 483395 objs 110 B/obj traced 5% fresh 76% aging 13% dead 6% char[] + 26.12MB 363289 objs 75 B/obj traced 57% fresh 27% aging 10% dead 5% java.lang.String + 15.09MB 240879 objs 65 B/obj traced 5% fresh 62% aging 21% dead 13% boolean[] +``` + +The mechanism is the sweep's own rule, confirmed directly by +`experiments/PinProbe`: a dead object needs **three cycles** to have its slot +returned -- one of grace while it is fresh, one of aging, then reclamation. A +translation completes three or four cycles in 1.4s, so most of what it allocates is +never eligible to be freed and the heap grows towards total allocation volume +(940MB allocated, 1.3GB peak, ~150-300MB genuinely live). + +Collecting faster helps, but does not change the ratio, because the grace rule +keeps everything allocated since the last mark whatever the rate: + +| | cycles | peak | traced at last cycle | |---|---:|---:|---:| -| 1 | 1.40 s | 4 | 1256 MB | -| 4 | 1.25 s | 9 | 1243 MB | -| 8 | 1.47 s | 8 | 1223 MB | +| 1 mark thread | 3 | 1320 MB | 47% | +| `-DCN1_GC_MARK_THREADS=4` | 8 | **1172 MB** | 25% | +| 4 threads + `CN1_GC_TRIGGER_MB=24` | 7 | 1259 MB | 39% | + +So the dominant lever is **allocation churn**, and the `[ALLOC]` census names it: +`char[]` 368MB, `Object[]` 196MB, `String` 77MB, `SimpleListIterator` 40MB. Cutting +an allocation removes roughly three cycles of occupancy, not one object. -`-DCN1_GC_MARK_THREADS=4` more than doubles the cycles and is slightly faster, but -peak barely moves. Combined with 75% reachability, that says the live set really is -this large rather than the collector being behind. +Two hypotheses this ruled OUT, both of which looked plausible: + +- **Conservative stack roots pinning dead objects.** `experiments/PinProbe` shows + the marks are precise and depth makes no difference: a dropped batch reads 100% + kept on the cycle after it is allocated (the grace stamp) and 0% on the next, + identically whether it was allocated in a shallow frame, under a 400-deep + recursion, or with the stack scrubbed afterwards. +- **Fragmentation.** `[JHEAP]` puts page-pool slack at ~2MB of 715MB. Where the process memory sits, from `vmmap --summary` around peak: @@ -267,20 +303,9 @@ MALLOC_SMALL 232.0M / 111.7M dirty legacy heap Stack 12.2M / 0.2M ``` -It is all malloc'd heap; there is no large non-heap component. (An earlier note in -this file claimed ~600 MB was "not the Java heap" -- that compared an exit-time -census against the whole-run peak and was wrong.) - -Two leads the census opens and does not settle: - -- Per-object width against the JDK: `String` 73 B, `ArrayList` 48 B, `Object[]` - 100 B average. The JDK's String is ~32 B and its ArrayList ~40 B. -- 295,907 `SimpleListIterator` objects were 72% *reachable* at a fresh sweep, and - a stack-local iterator should be dead the moment its loop ends. ParparVM scans - stacks conservatively (`CN1_CONSERVATIVE_GC_ROOTS` is defined unconditionally), - so a stale stack word that looks like a pointer keeps its object alive -- which - a deeply recursive, allocation-heavy program produces a lot of. Plausible, not - proven, and worth measuring before acting on. +It is all malloc'd heap; there is no large non-heap component. (An earlier note +here claimed ~600MB was "not the Java heap" -- that compared an exit-time census +against the whole-run peak and was wrong.) ### String: the NSString field is free diff --git a/vm/selfhost/experiments/README.md b/vm/selfhost/experiments/README.md new file mode 100644 index 00000000000..a61efa8d5c5 --- /dev/null +++ b/vm/selfhost/experiments/README.md @@ -0,0 +1,47 @@ +# Heap experiments + +Small programs that answer one question each about the collector, read through the +census (`-DCN1_ALLOC_CENSUS` + `CN1_HEAP_REPORT=1`, see ../README.md). + +## PinProbe + +**Question: does the conservative stack scan pin objects that are provably dead?** + +Three arms, three distinct classes so one run compares them in one census: +`PinShallow` allocated and dropped in a shallow frame, `PinDeep` allocated at the +bottom of a 400-deep recursion, `PinScrub` the same but with the stack overwritten +before collecting. Nothing holds a reference to any of them, so a precise collector +reclaims all three and any survivor is a stale stack word mistaken for a pointer. + +Build and run: + +```bash +javac -bootclasspath -d /tmp/exp/classes src/com/exp/PinProbe.java +java -cp : com.codename1.tools.translator.ByteCodeTranslator \ + clean ";/tmp/exp/classes" /tmp/exp/out PinProbe com.exp PinProbe 1.0 clean none +clang -O3 -flto=thin -w -fwrapv -fno-strict-aliasing -fno-builtin-fmod -fno-builtin-fmodf \ + -DCN1_ALLOC_CENSUS -I /*.c /*.S -lm -lpthread -o /tmp/exp/pinprobe +CN1_HEAP_REPORT=1 /tmp/exp/pinprobe 2>&1 | grep -E 'PROBE|Pin(Shallow|Deep|Scrub)' +``` + +**Answer: no.** All three arms behave identically, and the marks are precise -- a +batch reads 100% live on the cycle after it is allocated and 0% on the next, with +no difference between the shallow, deep and scrubbed arms. What the probe found +instead is the reclamation LATENCY: a dead object needs **three cycles** to have +its slot returned. + +``` +cycle 1: PinShallow 200,000 occupied, 100% kept <- grace: fresh objects are stamped live +cycle 2: PinShallow 200,000 occupied, 0% kept <- known dead, still occupying +cycle 3: PinShallow 196,320 occupied <- reclaimed +``` + +That is the sweep's own rule: `m == -1` (fresh) gets one cycle of grace, `m == V-1` +is kept for another, and only `m < V-1` is reclaimed. It is why a short program +retains nearly everything it allocates -- the ParparVM translator completes three +or four cycles in 1.4s, so most of what it allocates is never eligible. + +This probe is also the reason the census reports its four buckets **pre-sweep**: +read post-sweep, the grace stamp makes "traced live" and "kept because it is fresh" +indistinguishable, and the first version of the census reported the second as the +first. diff --git a/vm/selfhost/experiments/src/com/exp/PinProbe.java b/vm/selfhost/experiments/src/com/exp/PinProbe.java new file mode 100644 index 00000000000..15990187ead --- /dev/null +++ b/vm/selfhost/experiments/src/com/exp/PinProbe.java @@ -0,0 +1,100 @@ +package com.exp; + +/** + * Does ParparVM's conservative stack scan pin objects that are provably dead? + * + * Three arms, three distinct classes so one run compares them in one census: + * + * PinShallow allocated and dropped in a shallow frame + * PinDeep allocated at the bottom of a deep recursion, then unwound + * PinScrub same as PinDeep, then the stack is overwritten before collecting + * + * Every instance is unreachable by the time the collector runs -- nothing holds a + * reference. So a correct precise collector reclaims all of them, and any survivor + * is something the conservative scan mistook a stale stack word for. If Deep >> + * Shallow the depth is what pins; if Scrub << Deep the stale words are the + * mechanism and overwriting them frees the objects. + */ +public class PinProbe { + static final int BATCH = 200000; + static final int DEPTH = 400; + + // Sinks so the allocations cannot be optimised away, without retaining anything. + static int shallowSink, deepSink, scrubSink, scrubberSink; + + static class PinShallow { int a; } + static class PinDeep { int a; } + static class PinScrub { int a; } + + static void allocShallow() { + for (int i = 0; i < BATCH; i++) { + PinShallow p = new PinShallow(); + p.a = i; + shallowSink += p.a; + } + } + + static void allocDeep(int depth) { + if (depth > 0) { + allocDeep(depth - 1); + return; + } + for (int i = 0; i < BATCH; i++) { + PinDeep p = new PinDeep(); + p.a = i; + deepSink += p.a; + } + } + + static void allocScrub(int depth) { + if (depth > 0) { + allocScrub(depth - 1); + return; + } + for (int i = 0; i < BATCH; i++) { + PinScrub p = new PinScrub(); + p.a = i; + scrubSink += p.a; + } + } + + /** + * Walks back down to the same depth writing non-pointer values into locals, so + * every stack slot the allocation loops left behind is overwritten with an + * integer that cannot be mistaken for a heap address. + */ + static void scrub(int depth) { + int a = depth * 3 + 1, b = depth * 5 + 2, c = depth * 7 + 3, d = depth * 11 + 4; + int e = depth * 13 + 5, f = depth * 17 + 6, g = depth * 19 + 7, h = depth * 23 + 8; + if (depth > 0) { + scrub(depth - 1); + } + scrubberSink += a + b + c + d + e + f + g + h; + } + + static void collect(String label) throws Exception { + System.gc(); + // gc() only signals the collector; give it room to finish a cycle so the + // census that follows is reading fresh marks. + Thread.sleep(1500); + System.err.println("[PROBE] after " + label); + } + + public static void main(String[] args) throws Exception { + System.err.println("[PROBE] batch=" + BATCH + " depth=" + DEPTH); + + allocShallow(); + collect("shallow"); + + allocDeep(DEPTH); + collect("deep"); + + allocScrub(DEPTH); + scrub(DEPTH); + collect("deep+scrub"); + + System.err.println("[PROBE] sinks " + shallowSink + " " + deepSink + " " + + scrubSink + " " + scrubberSink); + System.out.println("DONE"); + } +} From d12892d76b12b6f0508173d6fa9b577c004e3a6a Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 10 Sep 2026 09:42:44 +0300 Subject: [PATCH 11/66] Make the second grace cycle optional: it is a 2014 pre-SATB workaround A dead object needs three GC cycles to have its slot returned, because the sweep keeps it twice: once as fresh (never marked) and once as aging (mark == V-1, not traced this cycle). The first is load-bearing. The second is not obviously anything, and the history says what it is -- November 2014, 31528ecfa6: - if(o->__codenameOneGcMark != currentGcMarkValue) { + if(o->__codenameOneGcMark < currentGcMarkValue - 1) { with the message "Delayed GCing of elements to prevent them from being collected due to a race condition with the GC thread". That collector had NO SATB barrier -- zero matches for satb or snapshot in the file at that commit -- so a mutator could hide a reference from the mark, and keeping an extra generation made the resulting lost-object race improbable rather than impossible. It has been inherited ever since, including by the BiBOP sweep, without a rationale anywhere in the tree. The cases that would need it today have their own guards. A page missing from the page index for one cycle is covered by the fresh grace rule, since its objects are mark == -1; the repeated miss that aging could not save either is exactly what cn1GcPageIndexStale skips the whole reclaim for, and that comment says so. CN1_GC_NO_AGING compiles the second cycle out. Evidence: - run-gc-verify.sh GREEN, and its three self-tests still detect their injected faults -- including the injected EARLY-FREE fault, which is precisely the failure this change could cause, so the gate is not vacuous for it - run-gauntlet.sh GREEN: 12 torture suites byte-identical to the host JVM, plus GC stress in cooperative and forced-signal thread-stop modes - self-hosting gates A and D byte-identical over 793 files - peak footprint 1334 -> 1322 MB and 1349 -> 1302 MB, about 2-3% Deliberately NOT the default. The win here is small because aging is only 14-16% of the occupied heap while fresh is 26-36%, so removing the second cycle moves those objects one cycle earlier in a run that only has three or four; a long-running application whose heap reaches a steady state would see closer to the full 15%. And vm/CLAUDE.md is explicit that a green verifier is necessary rather than sufficient around the SATB window: it could not open the residual window even with the barrier deliberately compiled out. Also recorded, since dropping it for the non-GUI Apple targets is an obvious thing to try: String's `long nsString` costs nothing to keep. sizeof(obj__java_lang_String) is 48 with it and really does fall to 40 without, but BiBOP's size classes are 32/48/64 so both land in the same 48-byte slot. Three runs each way put peak at 1240/1379/1340 MB with the field and 1290/1345/1341 MB without -- ranges that overlap completely. There is no effect to find, and removing it would cost the Apple targets a side table and a lookup for nothing. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.m | 35 +++++++++++++++++++++-- vm/selfhost/README.md | 37 +++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index 50001d2d941..7b8172c941b 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -646,6 +646,37 @@ static void cn1ReportPacingParks(void) { // rebuilds and includes them. It is the REPEAT that is fatal: on the second miss those // objects are no longer fresh, still do not resolve, and age into the sweep's // m < V - 1 reclamation while a live field still points at them. +// AGING: the sweep keeps an object for one cycle AFTER the mark stopped reaching it. +// +// CN1_GC_NO_AGING compiles that second cycle out, so a dead object is reclaimed by +// the first sweep that does not mark it -- the fresh-object grace rule is untouched +// either way. +// +// It is a diagnostic switch, not a supported setting, because what it is really +// asking is whether a 2014 workaround is still load-bearing. The rule arrived in +// 31528ecfa6, "Delayed GCing of elements to prevent them from being collected due to +// a race condition with the GC thread", which turned `mark != currentGcMarkValue` +// into `mark < currentGcMarkValue - 1`. That collector had no SATB barrier at all -- +// a mutator could hide a reference from the mark, and keeping an extra generation +// made the resulting lost-object race improbable rather than impossible. The cases +// that would need it today have their own guards: a page missing from the index for +// one cycle is covered by grace (its objects are mark == -1), and the repeat miss +// that aging cannot save either is what cn1GcPageIndexStale skips the whole reclaim +// for. +// +// It costs a third of the retention latency. A dead object needs three cycles to +// have its slot returned -- grace, aging, reclaim -- and a program that completes +// three or four cycles therefore frees almost nothing it allocates. +// +// Before this becomes a default it needs more than a green verifier: vm/CLAUDE.md +// records that the verifier could not open the residual SATB window even with the +// barrier deliberately compiled out, so a pass is necessary and not sufficient. +#ifdef CN1_GC_NO_AGING +#define CN1_GC_SLOT_IS_DEAD(mark, epoch) ((mark) < (epoch)) +#else +#define CN1_GC_SLOT_IS_DEAD(mark, epoch) ((mark) < (epoch) - 1) +#endif + static JAVA_BOOLEAN cn1GcPageIndexStale = JAVA_FALSE; // Page-heap bytes allocated across the whole run, charged cycle by cycle. Divided by // the cycle count it says how far the mutator ran ahead of the collector, which is what @@ -4768,7 +4799,7 @@ void codenameOneGCSweep() { JAVA_OBJECT o = allObjectsInHeap[iter]; if(o != JAVA_NULL) { if(o->__codenameOneGcMark != -1) { - if(o->__codenameOneGcMark < currentGcMarkValue - 1) { + if(CN1_GC_SLOT_IS_DEAD(o->__codenameOneGcMark, currentGcMarkValue)) { if (o->__codenameOneGcMark <= 0) { #if defined(__APPLE__) && defined(__OBJC__) #if TARGET_OS_SIMULATOR @@ -8323,7 +8354,7 @@ static void cn1BibopSweep(CODENAME_ONE_THREAD_STATE) { if(o->__codenameOneParentClsReference != 0 && o->__codenameOneParentClsReference->finalizerFunction != 0) needsReclaim = JAVA_TRUE; #endif - } else if(m < V - 1) { + } else if(CN1_GC_SLOT_IS_DEAD(m, V)) { cn1BibopReclaimSlot(threadStateData, o); #ifdef CN1_GC_VERIFY { extern long cn1GcVerifyFreedSlots; cn1GcVerifyFreedSlots++; } diff --git a/vm/selfhost/README.md b/vm/selfhost/README.md index 30ff9e18cf4..0d6fa949989 100644 --- a/vm/selfhost/README.md +++ b/vm/selfhost/README.md @@ -307,6 +307,43 @@ It is all malloc'd heap; there is no large non-heap component. (An earlier note here claimed ~600MB was "not the Java heap" -- that compared an exit-time census against the whole-run peak and was wrong.) +### The second grace cycle is a 2014 workaround, and it is worth ~2% + +A dead object needs three cycles because the sweep keeps it twice: once as `fresh` +(never marked) and once as `aging` (`mark == V-1`, not traced this cycle). The first +is load-bearing. The second traces to November 2014, commit `31528ecfa6`: + +``` +- if(o->__codenameOneGcMark != currentGcMarkValue) { // free what was not marked ++ if(o->__codenameOneGcMark < currentGcMarkValue - 1) { // keep one extra generation +``` + +whose message reads "Delayed GCing of elements to prevent them from being collected +due to a race condition with the GC thread". **That collector had no SATB barrier** +-- zero matches for satb or snapshot in the file at that commit -- so a mutator +could hide a reference from the mark, and keeping an extra generation made the +resulting lost-object race improbable rather than impossible. The cases that would +need it today have their own guards: a page missing from the index for one cycle is +covered by grace (its objects are `mark == -1`), and the repeated miss that aging +cannot save either is what `cn1GcPageIndexStale` skips the entire reclaim for. + +`-DCN1_GC_NO_AGING` compiles the second cycle out. Evidence gathered so far: + +| | result | +|---|---| +| `run-gc-verify.sh` | GREEN, and its three self-tests still detect their injected faults -- including the injected **early-free** fault, which is the exact failure this change could cause | +| `run-gauntlet.sh` | GREEN -- 12 torture suites byte-identical to the host JVM, plus GC stress in cooperative and forced-signal modes | +| self-hosting gates A and D | byte-identical, 793 files | +| peak footprint | 1334 -> 1322 MB and 1349 -> 1302 MB, about **2-3%** | + +**It is not the default, and the small win is why.** In this workload `aging` is only +14-16% of the occupied heap while `fresh` is 26-36%, so removing the second cycle +moves those objects one cycle earlier in a run that only has three or four. A long +running application, where the heap reaches a steady state instead of growing the +whole time, would see closer to the full 15%. And `vm/CLAUDE.md` is explicit that a +green verifier is necessary rather than sufficient here: it could not open the +residual SATB window even with the barrier deliberately compiled out. + ### String: the NSString field is free `java.lang.String` carries a `long nsString` for the Apple targets' direct NSString From 6069535eb441af16e1b5ac7b2c4ecc4e1e2d0297 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 10 Sep 2026 09:58:10 +0300 Subject: [PATCH 12/66] Revert the aging switch: the rule is load-bearing, and it was only worth 2-3% Backs out d12892d76b entirely -- both the CN1_GC_NO_AGING switch and the change behind it. Keeping a compile switch for this was the wrong shape regardless: it is not debug code, so it would just be a second collector policy nobody runs. The history in that commit still stands: the second grace cycle is a 2014 pre-SATB workaround (31528ecfa6, "Delayed GCing of elements to prevent them from being collected due to a race condition with the GC thread") and nothing in the tree records a reason for it. What the switch missed is that four later mechanisms have since been built ON the rule: - two java.lang.ref clearing sites that must use EXACTLY the sweep's liveness test; their comment spells out the failure as "FAILING to clear one the sweep frees hands get() a dangling pointer" - the fast-sweep page shortcut, whose gcGraceEpoch < V-1 bound is derived from the per-slot rule, and whose comment records issue 5425 when the two disagreed: "testing != V let it drop whole pages holding V-1 slots... 26,924 slots in one run. That is what left kept objects pointing into reclaimed memory" - the legacy and BiBOP sweeps ageing in step, so a matured Hashtable.Entry at V-1 is never kept while its page-resident payload at V-1 has already been freed The measurement that made it look safe was itself inconsistent: it changed the sweep and left the ref-clearing sites on the old rule, which IS the dangling-get() bug, and run-gc-verify.sh still came back green. So the verifier does not cover this coupling and a green result there was never sufficient evidence. Removing the rule properly means changing all four together and re-deriving the fast-sweep bound, for a measured 2-3% of peak. Not worth it; the allocation churn is where the memory is. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.m | 35 ++--------------------- vm/selfhost/README.md | 37 ------------------------- 2 files changed, 2 insertions(+), 70 deletions(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index 7b8172c941b..50001d2d941 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -646,37 +646,6 @@ static void cn1ReportPacingParks(void) { // rebuilds and includes them. It is the REPEAT that is fatal: on the second miss those // objects are no longer fresh, still do not resolve, and age into the sweep's // m < V - 1 reclamation while a live field still points at them. -// AGING: the sweep keeps an object for one cycle AFTER the mark stopped reaching it. -// -// CN1_GC_NO_AGING compiles that second cycle out, so a dead object is reclaimed by -// the first sweep that does not mark it -- the fresh-object grace rule is untouched -// either way. -// -// It is a diagnostic switch, not a supported setting, because what it is really -// asking is whether a 2014 workaround is still load-bearing. The rule arrived in -// 31528ecfa6, "Delayed GCing of elements to prevent them from being collected due to -// a race condition with the GC thread", which turned `mark != currentGcMarkValue` -// into `mark < currentGcMarkValue - 1`. That collector had no SATB barrier at all -- -// a mutator could hide a reference from the mark, and keeping an extra generation -// made the resulting lost-object race improbable rather than impossible. The cases -// that would need it today have their own guards: a page missing from the index for -// one cycle is covered by grace (its objects are mark == -1), and the repeat miss -// that aging cannot save either is what cn1GcPageIndexStale skips the whole reclaim -// for. -// -// It costs a third of the retention latency. A dead object needs three cycles to -// have its slot returned -- grace, aging, reclaim -- and a program that completes -// three or four cycles therefore frees almost nothing it allocates. -// -// Before this becomes a default it needs more than a green verifier: vm/CLAUDE.md -// records that the verifier could not open the residual SATB window even with the -// barrier deliberately compiled out, so a pass is necessary and not sufficient. -#ifdef CN1_GC_NO_AGING -#define CN1_GC_SLOT_IS_DEAD(mark, epoch) ((mark) < (epoch)) -#else -#define CN1_GC_SLOT_IS_DEAD(mark, epoch) ((mark) < (epoch) - 1) -#endif - static JAVA_BOOLEAN cn1GcPageIndexStale = JAVA_FALSE; // Page-heap bytes allocated across the whole run, charged cycle by cycle. Divided by // the cycle count it says how far the mutator ran ahead of the collector, which is what @@ -4799,7 +4768,7 @@ void codenameOneGCSweep() { JAVA_OBJECT o = allObjectsInHeap[iter]; if(o != JAVA_NULL) { if(o->__codenameOneGcMark != -1) { - if(CN1_GC_SLOT_IS_DEAD(o->__codenameOneGcMark, currentGcMarkValue)) { + if(o->__codenameOneGcMark < currentGcMarkValue - 1) { if (o->__codenameOneGcMark <= 0) { #if defined(__APPLE__) && defined(__OBJC__) #if TARGET_OS_SIMULATOR @@ -8354,7 +8323,7 @@ static void cn1BibopSweep(CODENAME_ONE_THREAD_STATE) { if(o->__codenameOneParentClsReference != 0 && o->__codenameOneParentClsReference->finalizerFunction != 0) needsReclaim = JAVA_TRUE; #endif - } else if(CN1_GC_SLOT_IS_DEAD(m, V)) { + } else if(m < V - 1) { cn1BibopReclaimSlot(threadStateData, o); #ifdef CN1_GC_VERIFY { extern long cn1GcVerifyFreedSlots; cn1GcVerifyFreedSlots++; } diff --git a/vm/selfhost/README.md b/vm/selfhost/README.md index 0d6fa949989..30ff9e18cf4 100644 --- a/vm/selfhost/README.md +++ b/vm/selfhost/README.md @@ -307,43 +307,6 @@ It is all malloc'd heap; there is no large non-heap component. (An earlier note here claimed ~600MB was "not the Java heap" -- that compared an exit-time census against the whole-run peak and was wrong.) -### The second grace cycle is a 2014 workaround, and it is worth ~2% - -A dead object needs three cycles because the sweep keeps it twice: once as `fresh` -(never marked) and once as `aging` (`mark == V-1`, not traced this cycle). The first -is load-bearing. The second traces to November 2014, commit `31528ecfa6`: - -``` -- if(o->__codenameOneGcMark != currentGcMarkValue) { // free what was not marked -+ if(o->__codenameOneGcMark < currentGcMarkValue - 1) { // keep one extra generation -``` - -whose message reads "Delayed GCing of elements to prevent them from being collected -due to a race condition with the GC thread". **That collector had no SATB barrier** --- zero matches for satb or snapshot in the file at that commit -- so a mutator -could hide a reference from the mark, and keeping an extra generation made the -resulting lost-object race improbable rather than impossible. The cases that would -need it today have their own guards: a page missing from the index for one cycle is -covered by grace (its objects are `mark == -1`), and the repeated miss that aging -cannot save either is what `cn1GcPageIndexStale` skips the entire reclaim for. - -`-DCN1_GC_NO_AGING` compiles the second cycle out. Evidence gathered so far: - -| | result | -|---|---| -| `run-gc-verify.sh` | GREEN, and its three self-tests still detect their injected faults -- including the injected **early-free** fault, which is the exact failure this change could cause | -| `run-gauntlet.sh` | GREEN -- 12 torture suites byte-identical to the host JVM, plus GC stress in cooperative and forced-signal modes | -| self-hosting gates A and D | byte-identical, 793 files | -| peak footprint | 1334 -> 1322 MB and 1349 -> 1302 MB, about **2-3%** | - -**It is not the default, and the small win is why.** In this workload `aging` is only -14-16% of the occupied heap while `fresh` is 26-36%, so removing the second cycle -moves those objects one cycle earlier in a run that only has three or four. A long -running application, where the heap reaches a steady state instead of growing the -whole time, would see closer to the full 15%. And `vm/CLAUDE.md` is explicit that a -green verifier is necessary rather than sufficient here: it could not open the -residual SATB window even with the barrier deliberately compiled out. - ### String: the NSString field is free `java.lang.String` carries a `long nsString` for the Apple targets' direct NSString From f44235f9d16a59e4e10d645eb0e74b411d9e2f3d Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 10 Sep 2026 09:58:44 +0300 Subject: [PATCH 13/66] Record why the second grace cycle stays: four mechanisms now depend on it Companion to the revert. The rule is vestigial in origin -- a 2014 pre-SATB workaround -- but the java.lang.ref clearing sites, the fast-sweep page bound and the legacy/BiBOP pairing have all been built on it since, and issue 5425 is what happened when two of them disagreed. Measured upside for removing it was 2-3% of peak. Co-Authored-By: Claude Opus 5 (1M context) --- vm/selfhost/README.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/vm/selfhost/README.md b/vm/selfhost/README.md index 30ff9e18cf4..90852587d9f 100644 --- a/vm/selfhost/README.md +++ b/vm/selfhost/README.md @@ -307,6 +307,46 @@ It is all malloc'd heap; there is no large non-heap component. (An earlier note here claimed ~600MB was "not the Java heap" -- that compared an exit-time census against the whole-run peak and was wrong.) +### The second grace cycle: vestigial in origin, load-bearing today + +A dead object needs three cycles because the sweep keeps it twice -- once as `fresh` +(never marked) and once as `aging` (`mark == V-1`). The first is load-bearing. The +second arrived in November 2014, commit `31528ecfa6`: + +``` +- if(o->__codenameOneGcMark != currentGcMarkValue) { // free what was not marked ++ if(o->__codenameOneGcMark < currentGcMarkValue - 1) { // keep one extra generation +``` + +message: "Delayed GCing of elements to prevent them from being collected due to a +race condition with the GC thread". **That collector had no SATB barrier** -- zero +matches for satb or snapshot at that commit -- so keeping an extra generation made a +lost-object race improbable rather than impossible. + +**It was removed, measured, and put back.** Removing it is verifier-green and +gauntlet-green and gives byte-identical self-hosting output, and it is worth about +**2-3% of peak** (1334 -> 1322 MB, 1349 -> 1302 MB). Not worth it, because four later +mechanisms have since been built on the rule: + +- Two `java.lang.ref` clearing sites that must use **exactly** the sweep's liveness + test. Their comment spells out the failure: "FAILING to clear one the sweep frees + hands get() a dangling pointer", and on ParparVM a dangling read is a native crash + no Java catch can see. +- The fast-sweep page shortcut, whose `gcGraceEpoch < V-1` bound is derived from the + per-slot rule. Its comment records what happened when the two disagreed: "testing + != V let it drop whole pages holding V-1 slots... 26,924 slots in one run. That is + what left kept objects pointing into reclaimed memory" -- issue 5425. +- The legacy and BiBOP sweeps ageing in step, so a matured `Hashtable.Entry` at V-1 + is never kept while its page-resident payload at V-1 has already gone. + +And the verifier does not cover the coupling: the measurement above changed the sweep +without changing the ref-clearing sites, which is precisely the dangling-`get()` bug, +and it still came back green. + +So the rule started as a band-aid and is now structural. Removing it means changing +all four together and re-deriving the fast-sweep bound, for 2-3%. The churn is worth +more and risks nothing. + ### String: the NSString field is free `java.lang.String` carries a `long nsString` for the Apple targets' direct NSString From 118044dc128bbca6499a822ad9bf108d132097db Mon Sep 17 00:00:00 2001 From: Shai Almog Date: Thu, 10 Sep 2026 21:00:36 +0300 Subject: [PATCH 14/66] ParparVM: make the translator self-hosting, and fix what that exposed Translating ByteCodeTranslator with itself turns a 37.6k-line real program into a VM conformance test: the emitted C is a byte-exact expected value that costs nothing to maintain, because it is whatever the JVM produced from the same inputs. A defect that changes behaviour rather than crashing shows up as a diff instead of passing silently. `.github/workflows/parparvm-selfhost.yml` runs it nightly, on demand, and on a PR carrying the `selfhost` label. The gates found a real VM bug and the profiler found several API implementations that were slow for reasons that had nothing to do with the VM. CORRECTNESS Every generated __STATIC_INITIALIZER_X was textbook-broken double-checked locking: __X_LOADED__ read with a plain load, written with a plain store OUTSIDE the monitor, and class__X.initialized read the same way by inline guards that skip the initialiser entirely and so never take the monitor. On arm64 a second thread can observe the flag set while the stores that filled the vtable and the classToInterfaceMap rows are still invisible, and then dispatch through a NULL row. Observed as three identical SIGSEGVs at classToInterfaceMap_java_util_NavigableMap[classId] + 0x8, reached from TreeSet.clear; the translator is single-threaded in its own code but shares the process with the GC thread, which also runs Java and so also runs initialisers. Both flags are now release-stored and acquire-loaded, across all 391 classes. The interface maps are calloc'd rather than malloc'd so a class id with no row reads NULL instead of whatever the allocator last left there. THROUGHPUT (5782-class corpus, min of 3 interleaved reps, phys_footprint) GC run-ahead ceiling in cn1BibopPacingCap. The cap was a fraction of AVAILABLE MACHINE RAM, and the trigger-derived clamp sat at 192MB (24MB x 8) for most of a run, so the mutator parked on a cycle it could not help finish. Bounded from both ends near 1GB, where the benefit saturates: 62.8s -> 27.4s, kernel time 33.4s -> 9.9s. Proportionate, so a phone or container is unaffected. Class-init checks were unconditional CALLS at 74% of 2116 sites; the callee's own first line already returns when the flag is set. Inline-guarded now that the flag has acquire/release: 7.21% -> 0.12% of mutator self-time. javac's `a + b` StringBuilder idiom is lowered to String.cn1ConcatN, the fused path invokedynamic concat already used. Two allocations and no byte<->char conversion against the builder's four plus two conversions. Only JDK 9+ emits the indy form, so everything built at source 8 -- the core, every port, every cn1lib -- reached none of it. 898 chains fused, StringBuilder allocation sites 3519 -> 2041. API IMPLEMENTATION AbstractList.SimpleListIterator.next had a try/catch per element to turn one exception into another. ParparVM has no zero-cost exception tables, so that is a setjmp per element in the hottest loop in the program, on top of virtual size() and get() calls and an index recomputed as size() - numLeft. ArrayList now has a direct-array iterator: iteration path 25.5% -> 12.4% of mutator self-time, ArrayList.get 7.42% -> 0.55%, _setjmp to zero. IdentityHashMap's iterator reached checkConcurrentMod() and hasNext() through two more non-inlined calls per element, making four with the interface dispatches. Inlined: checkConcurrentMod 2.69% -> 0.00%. StringBuilder grew by 1.5x (inherited from Harmony) where OpenJDK doubles. Growing to N chars costs N*r/(r-1) in abandoned arrays: 3N against 2N. String.equals and String.compareTo had their fast path INVERTED -- memcmp only when BOTH strings were UTF-16, the rare case, while two compact ASCII strings took a per-character loop calling a helper that re-derived the base pointer and re-tested the backing array's class every character. Corrected. Measured no improvement: the cost there is call overhead, not the comparison. Kept because the old structure was backwards, not because it is a win. ByteCodeClass.generateCCode built each file in a fresh StringBuilder, and 95 copies of x.replace('/','_').replace('$','_') re-mangled the same owner per emitted instruction. Reused buffer and a memo: char[] 1530MB -> 837MB. ALSO BytecodeInstructionIntegrationTest reflected on readFileAsStringBuilder, which no longer exists: replaceInFile works on a String since the translator had to compile against ParparVM's own JavaAPI, whose StringBuilder has no indexOf/replace. Pointed at readFileAsString. 45/45 green. VERIFIED Gate D, Gate A and the negative control pass on the 797-file self-hosting corpus, byte-identical. GC heap verifier clean over ~1e9 references. check-cast-semantics and check-native-signatures clean. KNOWN, NOT FIXED A rare (~1/14) crash remains in interface dispatch, now a clean NULL-row fault rather than silent garbage because of the calloc above. It is a bad class id reaching a registered-row lookup, it predates this branch as far as the evidence goes, and it is not reproducible under a debugger. Tracked separately. The remaining iterator cost is two interface dispatches per element, each a four-load pointer chase. Removing those means not making the calls -- lowering for-each to an indexed loop in the translator -- which is a control-flow rewrite and is deliberately left for its own change. --- .github/workflows/parparvm-selfhost.yml | 119 +++++++++++ vm/ByteCodeTranslator/src/cn1_globals.h | 4 +- vm/ByteCodeTranslator/src/cn1_globals.m | 70 ++++++ .../tools/translator/ByteCodeClass.java | 71 ++++++- .../tools/translator/BytecodeMethod.java | 199 ++++++++++++++++++ .../codename1/tools/translator/Parser.java | 20 ++ .../com/codename1/tools/translator/Util.java | 33 +++ .../translator/bytecodes/CustomInvoke.java | 22 +- .../tools/translator/bytecodes/Field.java | 30 +-- .../bytecodes/FusedConstructor.java | 5 +- .../tools/translator/bytecodes/Invoke.java | 26 +-- .../tools/translator/bytecodes/Ldc.java | 24 ++- .../translator/bytecodes/TypeInstruction.java | 10 +- vm/ByteCodeTranslator/src/nativeMethods.m | 41 +++- vm/JavaAPI/src/java/lang/StringBuilder.java | 19 +- vm/JavaAPI/src/java/util/ArrayList.java | 69 ++++++ vm/JavaAPI/src/java/util/IdentityHashMap.java | 49 +++-- .../BytecodeInstructionIntegrationTest.java | 12 +- 18 files changed, 739 insertions(+), 84 deletions(-) create mode 100644 .github/workflows/parparvm-selfhost.yml diff --git a/.github/workflows/parparvm-selfhost.yml b/.github/workflows/parparvm-selfhost.yml new file mode 100644 index 00000000000..0ca9db49e4d --- /dev/null +++ b/.github/workflows/parparvm-selfhost.yml @@ -0,0 +1,119 @@ +name: ParparVM Self-Hosting + +# Translates the ByteCodeTranslator with itself and compares the result against +# the same translation run on a JVM. +# +# What this buys that the existing suites do not: ByteCodeTranslator is a 37.6k +# line real program that hammers collections, strings, exceptions, file I/O and +# the GC at a scale no unit test reaches, and the emitted C is a byte-exact +# expected value that costs nothing to maintain -- it is whatever the JVM +# produced from the same inputs. A VM defect that changes behaviour rather than +# crashing (a wrong hash order, a dropped write barrier, a mis-mangled symbol) +# shows up as a diff instead of passing silently. +# +# Gates, cheapest first: +# D native vs native, two fresh processes, same input. If the native side is +# not self-consistent nothing else means anything, so it runs first. +# A JVM vs native over the same corpus. The headline. +# Negative control: after a green comparison one emitted byte is flipped and +# the comparator MUST report exactly that file. A comparator nobody has +# watched fail is not a comparator. +# +# Not on the PR leg by default: a full run builds the translator twice and +# translates a large corpus several times. It runs nightly, on demand, and on a +# PR that opts in with the `selfhost` label. + +on: + schedule: + # 04:20 UTC daily, off the hour to avoid the runner rush. + - cron: '20 4 * * *' + workflow_dispatch: + pull_request: + types: [ opened, synchronize, reopened, labeled ] + paths: + - 'vm/**' + - '.github/workflows/parparvm-selfhost.yml' + - '!vm/**/README.md' + - '!vm/**/docs/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + CN1_NATIVE_VERIFY: strict + +jobs: + selfhost: + # On a pull_request only when the author asked for it; the schedule and + # workflow_dispatch legs always run. + if: >- + github.event_name != 'pull_request' || + contains(github.event.pull_request.labels.*.name, 'selfhost') + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Install native build tools + run: | + bash scripts/ci/apt-get-update.sh + sudo apt-get install -y clang + + - name: Set up JDK 8 + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '8' + cache: 'maven' + - name: Save JDK 8 path + run: echo "JDK_8_HOME=$JAVA_HOME" >> $GITHUB_ENV + + # The translator has to exist as classes before it can translate itself. + - name: Build the translator + run: scripts/ci/retry.sh mvn -q -B -pl ByteCodeTranslator -am package -DskipTests + working-directory: vm + + - name: Resolve the ASM classpath + run: >- + scripts/ci/retry.sh mvn -q -B -pl ByteCodeTranslator + dependency:build-classpath + -Dmdep.outputFile=target/selfhost-asm-classpath.txt + working-directory: vm + + # -O1: the diff gates care about the EMITTED C, not about how well clang + # optimised the binary that emitted it, and -O1 links several times faster. + # Mark threads are set explicitly rather than left to the source default, + # which resolves to a single marker and makes a large corpus take hours. + - name: Build the self-hosted translator + run: vm/selfhost/build-selfhost.sh + env: + CN1_SELFHOST_CFLAGS: -DCN1_GC_MARK_THREADS=4 + + # The corpus is the translator's OWN classes plus ASM. verify-selfhost.sh + # prepends vm/selfhost/target/javaapi-classes itself, so it is not repeated + # here. Absolute paths: the script runs both sides under `env -i` into one + # fixed output directory, so a relative path would not survive. + - name: Gate D and Gate A, with the negative control + run: | + vm/selfhost/verify-selfhost.sh \ + "$PWD/vm/selfhost/target/asm-classes;$PWD/vm/selfhost/target/classes" \ + com_codename1_tools_translator_ByteCodeTranslator \ + com.codename1.tools.translator + + # Both trees, so a divergence can be inspected rather than guessed at from + # a one-line summary. + - name: Upload the compared trees on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: selfhost-trees + path: | + vm/selfhost/target/verify/jvm-tree + vm/selfhost/target/verify/parpar1-tree + vm/selfhost/target/verify/parpar2-tree + vm/selfhost/target/verify/*.txt + vm/selfhost/target/verify/*.log + retention-days: 7 + if-no-files-found: ignore diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index 6f0517b7073..6813f4ea164 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -2185,7 +2185,7 @@ static inline JAVA_OBJECT cn1BibopFastAllocNoZero(CODENAME_ONE_THREAD_STATE, int // because bibopCurrent[] is shared across all classes of the same size class). #if !defined(CN1_DISABLE_INLINE_ALLOC) && !defined(CN1_DISABLE_BIBOP) #define CN1_FAST_NEW(X) ({ \ - if(__builtin_expect(!class__##X.initialized, 0)) __STATIC_INITIALIZER_##X(threadStateData); \ + if(__builtin_expect(!__atomic_load_n(&class__##X.initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_##X(threadStateData); \ JAVA_OBJECT __cn1fo = cn1BibopFastAlloc(threadStateData, sizeof(struct obj__##X), &class__##X, CN1_BIBOP_CIDX(sizeof(struct obj__##X))); \ if(__builtin_expect(__cn1fo == (JAVA_OBJECT)0, 0)) __cn1fo = __NEW_##X(threadStateData); \ __cn1fo; }) @@ -2193,7 +2193,7 @@ static inline JAVA_OBJECT cn1BibopFastAllocNoZero(CODENAME_ONE_THREAD_STATE, int // still fully zeroes (calloc) -- correct, just un-elided on the rare page-full // path. #define CN1_FAST_NEW_NOZERO(X) ({ \ - if(__builtin_expect(!class__##X.initialized, 0)) __STATIC_INITIALIZER_##X(threadStateData); \ + if(__builtin_expect(!__atomic_load_n(&class__##X.initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_##X(threadStateData); \ JAVA_OBJECT __cn1fo = cn1BibopFastAllocNoZero(threadStateData, sizeof(struct obj__##X), &class__##X, CN1_BIBOP_CIDX(sizeof(struct obj__##X))); \ if(__builtin_expect(__cn1fo == (JAVA_OBJECT)0, 0)) __cn1fo = __NEW_##X(threadStateData); \ __cn1fo; }) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index 50001d2d941..211984981d9 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -6328,6 +6328,11 @@ static inline JAVA_OBJECT cn1BibopSlot(CN1BibopPage* p, int i) { #ifndef CN1_PACING_GROWTH_FLOOR_BYTES #define CN1_PACING_GROWTH_FLOOR_BYTES (512LL*1024*1024) #endif +// Ceiling on how far a mutator may run ahead of a cycle in flight, regardless of +// how much RAM the host has. See the measurement table in cn1BibopPacingCap. +#ifndef CN1_BIBOP_PACING_MAX_RUNAHEAD_BYTES +#define CN1_BIBOP_PACING_MAX_RUNAHEAD_BYTES (1024L*1024*1024) +#endif // How stale a below-floor footprint reading may be before the bound re-probes it. The // probe is task_info on Apple and one /proc read on Linux -- a microsecond or two -- and // it is taken at most once per interval across the whole process, and only when the bound @@ -6602,10 +6607,75 @@ static long cn1BibopPacingCap(CODENAME_ONE_THREAD_STATE) { if(capCeiling < base) { capCeiling = base; } + // FLOOR the clamp at the point where run-ahead stops paying, when the host + // can afford it. + // + // capCeiling is derived from the TRIGGER, and the trigger spends most of a + // run at its 24MB minimum, so this clamp lands at 24*8 = 192MB. Confirmed + // at runtime, not inferred: `[PACING] minCapKb=196608`. That is what + // actually throttles the mutator -- NOT the fm/8 and fm/2 figures above, + // which never bind on a large host. It is also why the diagnostic knob + // CN1_GC_PACING_CAP_MB appears to work miracles: returning early, it + // bypasses this clamp entirely. + // + // MEASURED, 5782-class hellocodenameone translation, min of 3 interleaved + // reps, phys_footprint: + // + // cap in force wall peak + // 192MB 46.3s 9736MB <- this clamp, as it stood + // 1024MB 23.8s 8325MB + // 2048MB 22.9s 12870MB <- 2 more seconds for 4GB + // + // Run-ahead saturates near 1GB: below it the mutator parks waiting on a + // cycle it cannot help finish, and the resulting bigger heap costs kernel + // time faulting pages in, so tightening this clamp lost on BOTH axes. + // + // Kept proportionate rather than absolute: on a host where fm/8 is already + // under the saturation point -- a phone, a container, the flat 100MB + // placeholder off Apple -- the floor follows fm/8 and nothing loosens. + { + long runAhead = CN1_BIBOP_PACING_MAX_RUNAHEAD_BYTES; + if(fm > 0 && runAhead > fm / 8) { + runAhead = fm / 8; + } + if(capCeiling < runAhead) { + capCeiling = runAhead; + } + } if(cap > capCeiling && cn1PacingPastGrowthFloor()) { cap = capCeiling; } } + // FINAL absolute bound on run-ahead. Applied last, after the trigger-derived + // clamp above, because the two failure modes are opposite and BOTH were + // measured on this workload: + // + // - the clamp alone drove cap down to 192MB (trigger 24MB x 8), which parks + // the mutator on a cycle it cannot help finish: 46.3s / 9736MB. + // - flooring the clamp without bounding the top left cap at fm/8 = 4GB (or + // fm/2 = 16GB for a thread flagged high-throughput), so the heap ran to + // 11848MB and the run took 48.0s -- worse on both axes. + // + // Pinning run-ahead near 1GB gives 23.8s / 8325MB. The saturation is real: at + // 2GB the run is 22.9s but the footprint is 12870MB, i.e. 2 more GB per second + // saved. So the useful range is narrow and this is its top. + // + // Proportionate, not absolute: on a host where fm/8 is already below the + // saturation point -- a phone, a container, the flat 100MB placeholder off + // Apple -- this follows fm/8 and nothing is loosened. `base` is still honoured + // so a build with a large static trigger keeps the admission it had. + { + long runAhead = CN1_BIBOP_PACING_MAX_RUNAHEAD_BYTES; + if(fm > 0 && runAhead > fm / 8) { + runAhead = fm / 8; + } + if(cap > runAhead) { + cap = runAhead; + } + if(cap < base) { + cap = base; + } + } if(cn1PacingTraceOn()) { long seen = atomic_load_explicit(&cn1PacingMinCap, memory_order_relaxed); while(cap < seen && diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java index 8c80e1903e3..55333ebff86 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java @@ -656,9 +656,24 @@ public static void addArrayType(String type, int dimenstions) { + // One reusable emit buffer for the whole output pass, reset per class rather + // than reallocated. Parser.writeOutput -> writeFile -> generateCCode is a + // single sequential loop with no executor and one call site, so there is no + // concurrent or re-entrant use to guard against. + // + // This is not micro-tuning. A fresh StringBuilder starts at capacity 16 and + // JavaAPI grows by 1.5x ((len>>1)+len+2), so building N chars allocates about + // 3N chars = 6N bytes in abandoned intermediate arrays. Across 5897 emitted + // files totalling 245MB that is roughly 1.4GB of pure churn, and MEASURED on + // ParparVM the emit phase allocated 2518MB in a single GC cycle against a + // 24MB trigger. Keeping the capacity across classes means the growth series + // runs only until the buffer reaches the largest class, then never again. + private static final StringBuilder EMIT_BUFFER = new StringBuilder(1 << 20); + public String generateCCode(List allClasses) { - StringBuilder b = new StringBuilder(); + StringBuilder b = EMIT_BUFFER; + b.setLength(0); b.append("#include \""); b.append(clsName); @@ -952,7 +967,14 @@ public String generateCCode(List allClasses) { b.append(clsName); b.append("_"); b.append(bf.getFieldName().replace('$', '_')); - b.append("() {\n __STATIC_INITIALIZER_"); + // Inline-guard rather than call: the initialiser's own first + // line already returns when the flag is set, so the call was a + // no-op after the first time -- but a CALL, on a path that runs + // per static-field access. MEASURED: __STATIC_INITIALIZER_* was + // 7.2% of mutator self-time, java.util.Iterator's alone 6.26%. + // Safe as an ACQUIRE load now that the flag is release-stored. + b.append("() {\n if(__builtin_expect(!__atomic_load_n(&class__").append(bf.getClsName()) + .append(".initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_"); b.append(bf.getClsName()); if (bf.isVolatile()) { b.append("(getThreadLocalData());\n return atomic_load_explicit(&STATIC_FIELD_"); @@ -978,7 +1000,8 @@ public String generateCCode(List allClasses) { b.append("CODENAME_ONE_THREAD_STATE, "); } b.append(bf.getCDefinition()); - b.append(" __cn1StaticVal) {\n __STATIC_INITIALIZER_"); + b.append(" __cn1StaticVal) {\n if(__builtin_expect(!__atomic_load_n(&class__").append(bf.getClsName()) + .append(".initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_"); b.append(bf.getClsName()); if (bf.isObjectType()) { b.append("(threadStateData);\n "); @@ -1230,7 +1253,8 @@ public String generateCCode(List allClasses) { if(!isInterface && !isAbstract) { b.append("JAVA_OBJECT __NEW_"); b.append(clsName); - b.append("(CODENAME_ONE_THREAD_STATE) {\n __STATIC_INITIALIZER_"); + b.append("(CODENAME_ONE_THREAD_STATE) {\n if(__builtin_expect(!__atomic_load_n(&class__").append(clsName) + .append(".initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_"); b.append(clsName); b.append("(threadStateData);\n JAVA_OBJECT o = codenameOneGcMalloc(threadStateData, sizeof(struct obj__"); b.append(clsName); @@ -1241,7 +1265,8 @@ public String generateCCode(List allClasses) { if(hasDefaultConstructor()) { b.append("JAVA_OBJECT __NEW_INSTANCE_"); b.append(clsName); - b.append("(CODENAME_ONE_THREAD_STATE) {\n __STATIC_INITIALIZER_"); + b.append("(CODENAME_ONE_THREAD_STATE) {\n if(__builtin_expect(!__atomic_load_n(&class__").append(clsName) + .append(".initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_"); b.append(clsName); b.append("(threadStateData);\n JAVA_OBJECT o = codenameOneGcMalloc(threadStateData, sizeof(struct obj__"); b.append(clsName); @@ -1514,7 +1539,22 @@ public String generateCCode(List allClasses) { b.append("static int __").append(clsName).append("_LOADED__=0;\n"); b.append("void __STATIC_INITIALIZER_"); b.append(clsName); - b.append("(CODENAME_ONE_THREAD_STATE) {\n if(__").append(clsName).append("_LOADED__) return;\n\n "); + // ACQUIRE, not a plain load. This is the fast path of a double-checked + // initialisation: the completing store below is a RELEASE, and the two + // together are what make the writes this function performed -- the + // vtable, and every classToInterfaceMap_[classId] row -- visible + // to a thread that observes the flag set. + // + // With plain accesses on arm64 a second thread could see LOADED==1 while + // those table stores were still invisible, then index a row that read as + // NULL. OBSERVED: three identical SIGSEGVs at + // classToInterfaceMap_java_util_NavigableMap[classId] + 0x8, reached from + // TreeSet.clear -> the interface dispatch for NavigableMap.clear, in a + // translator that is single-threaded in its own code but shares the + // process with the GC thread, which also runs Java and so also runs + // class initialisers. + b.append("(CODENAME_ONE_THREAD_STATE) {\n if(__atomic_load_n(&__") + .append(clsName).append("_LOADED__, __ATOMIC_ACQUIRE)) return;\n\n "); // Block-registered enter/exit (the synchronized-method pattern): if the @@ -1563,7 +1603,10 @@ public String generateCCode(List allClasses) { b.append(".vtable = initVtableForInterface();\n"); b.append(" classToInterfaceMap_"); b.append(clsName); - b.append(" = malloc(sizeof(int*) * cn1_array_start_offset);\n"); + // calloc, not malloc: rows are filled only for classes that implement + // this interface, so an id that does not read as a registered row must + // read as NULL rather than as whatever the allocator last left there. + b.append(" = calloc(cn1_array_start_offset, sizeof(int*));\n"); for(ByteCodeClass cls : allClasses) { if(!cls.isInterface) { if(cls.doesImplement(this)) { @@ -1600,9 +1643,14 @@ public String generateCCode(List allClasses) { b.append(".vtable);\n"); } - b.append(" class__"); + b.append(" __atomic_store_n(&class__"); b.append(clsName); - b.append(".initialized = JAVA_TRUE;\n"); + // RELEASE store, matching the one on __X_LOADED__ above. Readers outside + // this monitor (the inline guards emitted at allocation and static-access + // sites) do a plain-or-acquire load of this flag and SKIP the call + // entirely when it is set, so the monitor's own release is not enough on + // its own -- the guard never takes the monitor. + b.append(".initialized, JAVA_TRUE, __ATOMIC_RELEASE);\n"); // init static fields and invoke the static initializer code block if(clInitMethod != null) { b.append(" "); @@ -1613,7 +1661,10 @@ public String generateCCode(List allClasses) { b.append(clsName); b.append(");\n"); - b.append("__").append(clsName).append("_LOADED__=1;\n"); + // RELEASE: pairs with the acquire on the fast path above, so everything + // this initialiser wrote happens-before another thread's early return. + b.append("__atomic_store_n(&__").append(clsName) + .append("_LOADED__, 1, __ATOMIC_RELEASE);\n"); b.append("}\n\n"); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java index babf4d9fd49..74a66264d15 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java @@ -2189,6 +2189,8 @@ public void appendVirtualMethodC(String cls, StringBuilder b, String offset, boo } if(includeStaticInitializer) { + b.append("if(__builtin_expect(!__atomic_load_n(&class__").append(cls) + .append(".initialized, __ATOMIC_ACQUIRE), 0)) "); b.append("__STATIC_INITIALIZER_"); b.append(cls); b.append("(threadStateData);\n "); @@ -4246,6 +4248,203 @@ private void removeRepeatedCheckcasts() { } } + /** + * Route the javac string-concatenation idiom to the SAME fused path that + * invokedynamic concat already uses. + * + * `a + b` compiles two different ways depending on the source/target level. + * JDK 9+ emits `invokedynamic makeConcat(WithConstants)`, which + * Parser.visitInvokeDynamicInsn already rewrites to String.cn1ConcatN when + * every part is String-typed -- two allocations and no conversion, against + * the StringBuilder's four plus a byte->char decode per append and a + * char->byte re-encode in toString (StringBuilder is char[]-backed while + * Strings are compact byte[]). + * + * Anything compiled at source/target 8 emits the StringBuilder idiom + * directly instead, and reached NONE of that. That is not a corner: the + * Codename One core, every port and every cn1lib are built that way, so the + * fallback was what nearly all linked code paid, no matter which JDK built + * the application on top. MEASURED on the 5782-class hellocodenameone + * corpus: 3058 StringBuilder-idiom sites against 499 invokedynamic ones. + * + * The rewrite is a deletion, because the stack discipline already lines up: + * + * NEW/DUP/ -> [sb] + * -> [sb, a] + * append -> [sb] (consumes sb and a, returns sb) + * -> [sb, b] + * append -> [sb] + * toString -> [String] + * + * Drop the NEW, the DUP, the constructor and every append, and what is left + * is `` leaving exactly [a, b] -- the argument shape + * cn1Concat2 wants. Only the terminating toString is replaced, by the static + * call. No new runtime: cn1Concat2..5 and their cn1FusedConcatN natives are + * the ones the invokedynamic path has been using. + * + * Conservative on purpose; every bail-out below is a case where a naive + * deletion would change behaviour: + * - only all-String append chains, because cn1ConcatN takes Strings. An + * append(int) renders digits straight into the builder, and routing it + * here would mean materialising an intermediate String, which is not + * obviously cheaper. Those chains are left alone. + * - only 2..5 parts, matching the cn1ConcatN arity that exists. + * - a control-flow join inside the chain ends it (srNextRealNoJoin, and + * the explicit isJumpTarget check): control could enter mid-chain, so + * the builder would not be the one this NEW created. + * - a nested `new StringBuilder` inside the chain ends it, so the inner + * concat of `"a" + (x + y)` is not mistaken for the outer one. The inner + * site is rewritten on its own, and the outer becomes eligible on a + * later pass -- hence the fixpoint loop in the caller. + * - any other StringBuilder method (charAt, reverse, ...), or a store or + * return of the builder, ends it: the builder escapes the chain. + * + * @return true when at least one chain was rewritten + */ + private boolean fuseStringBuilderConcatOnce() { + final String SB = "java/lang/StringBuilder"; + final String APPEND_STR = "(Ljava/lang/String;)Ljava/lang/StringBuilder;"; + for (int i = 0; i < instructions.size(); i++) { + Instruction in = instructions.get(i); + if (!(in instanceof TypeInstruction) || in.getOpcode() != Opcodes.NEW + || !SB.equals(((TypeInstruction) in).getTypeName())) { + continue; + } + int iDup = srNextRealNoJoin(i + 1); + if (iDup < 0 || instructions.get(iDup).getOpcode() != Opcodes.DUP) { + continue; + } + int iInit = srNextRealNoJoin(iDup + 1); + if (iInit < 0) { + continue; + } + Instruction initIns = instructions.get(iInit); + if (!(initIns instanceof Invoke) || initIns.getOpcode() != Opcodes.INVOKESPECIAL) { + continue; + } + Invoke init = (Invoke) initIns; + if (!SB.equals(init.getOwner()) || !"".equals(init.getName()) + || !"()V".equals(init.getDesc())) { + continue; + } + + java.util.List appends = new java.util.ArrayList(); + int toStringIdx = -1; + boolean ok = true; + for (int j = iInit + 1; j < instructions.size(); j++) { + Instruction c = instructions.get(j); + if (c instanceof LabelInstruction) { + if (LabelInstruction.isJumpTarget(((LabelInstruction) c).getLabel())) { + ok = false; + } + if (!ok) { + break; + } + continue; + } + if (c instanceof LineNumber || c instanceof LocalVariable) { + continue; + } + if (c instanceof Jump) { + ok = false; + break; + } + if (c instanceof TypeInstruction && c.getOpcode() == Opcodes.NEW + && SB.equals(((TypeInstruction) c).getTypeName())) { + ok = false; + break; + } + int op = c.getOpcode(); + if (op == Opcodes.ASTORE || op == Opcodes.PUTFIELD || op == Opcodes.PUTSTATIC + || op == Opcodes.AASTORE || op == Opcodes.ARETURN) { + ok = false; + break; + } + if (c instanceof Invoke) { + Invoke ci = (Invoke) c; + if (SB.equals(ci.getOwner())) { + if ("append".equals(ci.getName()) && APPEND_STR.equals(ci.getDesc())) { + appends.add(Integer.valueOf(j)); + continue; + } + if ("toString".equals(ci.getName()) && "()Ljava/lang/String;".equals(ci.getDesc())) { + toStringIdx = j; + break; + } + ok = false; + break; + } + } + } + int n = appends.size(); + if (!ok || toStringIdx < 0 || n < 2 || n > 5) { + continue; + } + + StringBuilder sig = new StringBuilder("("); + for (int k = 0; k < n; k++) { + sig.append("Ljava/lang/String;"); + } + sig.append(")Ljava/lang/String;"); + Invoke fused = new Invoke(Opcodes.INVOKESTATIC, "java/lang/String", + "cn1Concat" + n, sig.toString(), false); + instructions.set(toStringIdx, fused); + // Register it exactly as addInstruction() would. Setting the list entry + // alone leaves the new call with no owning method, no class dependency + // and -- the one that bites -- NO EDGE IN THE DEPENDENCY GRAPH, so the + // unused-method cull cannot see that String.cn1ConcatN is now called. + fused.setMethod(this); + fused.addDependencies(dependentClasses); + if (dependencyGraph != null) { + String fusedUses = fused.getMethodUsed(); + if (fusedUses != null) { + dependencyGraph.recordMethodCall(this, fusedUses); + } + } + for (int k = n - 1; k >= 0; k--) { + instructions.remove(appends.get(k).intValue()); + } + instructions.remove(iInit); + instructions.remove(iDup); + instructions.remove(i); + // GROW the frame; do not clamp it. + // + // maxStack sizes the emitted C stack array (DEFINE_METHOD_STACK), so a + // value that is too small writes PAST that array. The failure is silent + // and arrives far away: the first symptom here was a SIGSEGV in + // java_io_File_getParentFile, called from File.mkdirs, nowhere near any + // concat. + // + // While evaluating the LAST part the builder held one persistent slot + // (sb) under that part's own working set; the fused form instead holds + // n-1 finished parts under it. So the requirement rises by n-2 over + // whatever the chain needed before. An earlier `if (maxStack < n + 1)` + // was a no-op for every real method, because maxStack is essentially + // always already larger than 6. + // + // Adding n (rather than the n-2 strictly implied) buys a slot of margin + // for a couple of pointers per frame, which is the right trade against a + // memory-corrupting underestimate. + maxStack += n; + cn1ConcatFused++; + return true; + } + return false; + } + + /** Count of chains rewritten by {@link #fuseStringBuilderConcatOnce}, for reporting. */ + static int cn1ConcatFused; + + void fuseStringBuilderConcat() { + // Fixpoint: rewriting an inner concat makes the outer one all-String and + // free of the nested NEW that had disqualified it. Bounded so a bug here + // cannot hang a build. + int guard = 0; + while (guard++ < 64 && fuseStringBuilderConcatOnce()) { + // keep going + } + } + boolean optimize() { // FUSED OBJECTS, constructor side: rewrite each planned // `ALOAD 0; ; NEWARRAY T; PUTFIELD f` quadruple into the diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java index 020ef07571b..7a0216424f4 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java @@ -804,6 +804,26 @@ public static void writeOutput(File outputDirectory) throws Exception { neliminated++; } + // Fuse all-String StringBuilder concat chains into String.cn1ConcatN + // BEFORE the cull, not during code generation. + // + // The cull decides what to keep from the dependency graph, and the graph + // is only told about a call when the instruction is added. A rewrite that + // runs later -- inside BytecodeMethod.optimize(), which happens during + // generateCCode -- inserts calls to methods the cull has already deleted, + // and a deleted method is emitted as `return 0;`. That is not a build + // error: cn1ConcatN silently answered null, java.io.File got a null path, + // and the translator died in File.getParentFile with a SIGSEGV nowhere + // near a concat. Running here, the references exist before anything is + // eliminated. See BytecodeMethod.fuseStringBuilderConcat. + if (BytecodeMethod.optimizerOn) { + for (ByteCodeClass fuseCls : classes) { + for (BytecodeMethod fuseMtd : fuseCls.getMethods()) { + fuseMtd.fuseStringBuilderConcat(); + } + } + } + // loop over methods and start eliminating the body of unused methods if (BytecodeMethod.optimizerOn) { if(ByteCodeTranslator.verbose) { diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Util.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Util.java index c8da001f38e..1f05d960e38 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Util.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Util.java @@ -626,6 +626,39 @@ public static String rewriteLocalObjectRefs(String s) { * NativeSignatureVerifier.mode() already reached for getenv for exactly this * reason; this generalizes it rather than adding a second convention. */ + /** + * Memoized ParparVM name mangling: '/' and '$' both become '_'. + * + * The tree contains 95 hand-written copies of + * {@code x.replace('/', '_').replace('$', '_')}, 54 of them in the + * per-instruction emit classes (Invoke, Field, CustomInvoke, Ldc), so the + * SAME owner string is re-mangled once per emitted instruction. The distinct + * inputs are bounded by the class count (5782 on the hellocodenameone + * corpus) while the calls run into the millions. + * + * String.replace already returns {@code this} when the character is absent, + * so the '$' pass is usually free; the '/' pass is the one that allocates a + * char[] and a String every time. Caching turns that into one lookup. + * + * Not synchronized: the translator parses and emits on a single thread -- + * Parser.writeOutput is one sequential loop with no executor and a single + * writeFile call site. + */ + private static final java.util.Map MANGLE_CACHE = + new java.util.HashMap(); + + public static String mangle(String name) { + if (name == null) { + return null; + } + String m = MANGLE_CACHE.get(name); + if (m == null) { + m = name.replace('/', '_').replace('$', '_'); + MANGLE_CACHE.put(name, m); + } + return m; + } + public static String getProperty(String key, String defaultValue) { String value = System.getProperty(key); if (value == null) { diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomInvoke.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomInvoke.java index 67bf006a8c1..b753a9357bc 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomInvoke.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomInvoke.java @@ -102,7 +102,7 @@ public String getMethodUsed() { public void addDependencies(List dependencyList) { String dependencyOwner = owner; if (origOpcode == Opcodes.INVOKEVIRTUAL) { - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); String resolvedConcreteOwner = resolveConcreteInvokeOwner(bc, true); if (resolvedConcreteOwner != null) { dependencyOwner = resolvedConcreteOwner; @@ -131,7 +131,7 @@ public void addDependencies(List dependencyList) { if(origOpcode != Opcodes.INVOKEINTERFACE && origOpcode != Opcodes.INVOKEVIRTUAL) { return; } - bld.append(owner.replace('/', '_').replace('$', '_')); + bld.append(Util.mangle(owner)); bld.append("_"); if(name.equals("")) { bld.append("__INIT__"); @@ -177,7 +177,7 @@ private String resolveConcreteInvokeOwner(ByteCodeClass ownerClass, boolean allo if (currentClass != null && (ownerName.equals(currentClass) || currentClass.startsWith(ownerName + "_"))) { return null; } - ByteCodeClass concreteClass = Parser.getClassObject(ownerClass.getConcreteClass().replace('/', '_').replace('$', '_')); + ByteCodeClass concreteClass = Parser.getClassObject(Util.mangle(ownerClass.getConcreteClass())); // The nearest class in the concrete type's own hierarchy that actually // declares the method -- which is what the runtime would dispatch to for // an instance of it. Resolving against concreteClass's declarations alone @@ -288,7 +288,7 @@ public boolean appendExpression(StringBuilder b) { // so we need to check boolean isVirtual = true; if (origOpcode == Opcodes.INVOKEVIRTUAL) { - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); if (bc == null) { System.err.println("WARNING: Failed to find class object for owner "+owner+" when rendering virtual method "+name); } else { @@ -321,13 +321,13 @@ public boolean appendExpression(StringBuilder b) { if(origOpcode == Opcodes.INVOKESTATIC) { // find the actual class of the static method to work around javac not defining it correctly - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); invokeOwner = findActualOwner(bc); } if (invokeOwner.startsWith("[")) { bld.append("java_lang_Object"); } else{ - bld.append(invokeOwner.replace('/', '_').replace('$', '_')); + bld.append(Util.mangle(invokeOwner)); } bld.append("_"); if(name.equals("")) { @@ -442,7 +442,7 @@ private boolean tryAppendInlinedConstructor(StringBuilder b) { // Memset elimination: allocate into a temp, build fully, THEN publish. // Literal-arg ctor with the receiver on-stack (from NEW;DUP): the // survivor sits one slot below the receiver (SP[-2]); pop the receiver. - String cType = owner.replace('/', '_').replace('$', '_'); + String cType = Util.mangle(owner); inlineCtorPlan.appendInitBeforePublish(b, cType, argExprs, argCats, 2, 1); return true; } @@ -489,7 +489,7 @@ private void appendFusedAllocBlock(StringBuilder b) { for (int i = 0; i < kids.size(); i++) { lenExprs[i] = kids.get(i).siteLengthExpr(temps); } - String cType = owner.replace('/', '_').replace('$', '_'); + String cType = Util.mangle(owner); fusedPlan.appendFusedAlloc(b, cType, lenExprs, 1, 2); // NOTE: the enclosing brace is closed AFTER the ordinary call emission by // appendInstruction (the temps must stay in scope for the call). @@ -540,7 +540,7 @@ public void appendInstruction(StringBuilder b) { // so we need to check boolean isVirtual = true; if (origOpcode == Opcodes.INVOKEVIRTUAL) { - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); if (bc == null) { System.err.println("WARNING: Failed to find class object for owner "+owner+" when rendering virtual method "+name); } else { @@ -573,13 +573,13 @@ public void appendInstruction(StringBuilder b) { if(origOpcode == Opcodes.INVOKESTATIC) { // find the actual class of the static method to work around javac not defining it correctly - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); invokeOwner = findActualOwner(bc); } if (invokeOwner.startsWith("[")) { bld.append("java_lang_Object"); } else{ - bld.append(invokeOwner.replace('/', '_').replace('$', '_')); + bld.append(Util.mangle(invokeOwner)); } bld.append("_"); if(name.equals("")) { diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Field.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Field.java index f535414024a..95147ccf367 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Field.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Field.java @@ -23,6 +23,8 @@ package com.codename1.tools.translator.bytecodes; +import com.codename1.tools.translator.Util; + import java.util.List; import org.objectweb.asm.Opcodes; @@ -79,7 +81,7 @@ public void addDependencies(List dependencyList) { } public String getFieldFromThis() { - return "get_field_" + owner.replace('/', '_').replace('$', '_') + + return "get_field_" + Util.mangle(owner) + "_" + name + "(__cn1ThisObject)"; } @@ -88,14 +90,14 @@ public String setFieldFromThis(int arg) { // Instance field setters only need value/target operands. // special case for this if(arg == 0) { - return " set_field_" + owner.replace('/', '_').replace('$', '_') + + return " set_field_" + Util.mangle(owner) + "_" + name + "(__cn1ThisObject, __cn1ThisObject);\n"; } if(isObject()) { - return " set_field_" + owner.replace('/', '_').replace('$', '_') + + return " set_field_" + Util.mangle(owner) + "_" + name + "(__cn1Arg" + arg + ", __cn1ThisObject);\n"; } - return " set_field_" + owner.replace('/', '_').replace('$', '_') + + return " set_field_" + Util.mangle(owner) + "_" + name + "(__cn1Arg" + arg + ", __cn1ThisObject);\n"; } @@ -124,7 +126,7 @@ public String pushFieldFromThis() { break; } b.append("(get_field_"); - b.append(owner.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(owner)); b.append("_"); b.append(name); b.append("(__cn1ThisObject));\n"); @@ -143,14 +145,14 @@ public boolean assignTo(String varName, StringBuilder sb) { } if (opcode == Opcodes.GETSTATIC) { b.append("get_static_"); - b.append(owner.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(owner)); b.append("_"); - b.append(name.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(name)); b.append("()"); } else { b.append("get_field_"); - b.append(owner.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(owner)); b.append("_"); b.append(name); StringBuilder sb3 = new StringBuilder(); @@ -224,17 +226,17 @@ public void appendInstruction(StringBuilder sbOut) { break; } b.append("(get_static_"); - b.append(owner.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(owner)); b.append("_"); - b.append(name.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(name)); b.append("());\n"); break; case Opcodes.PUTSTATIC: { //b.append("SAFE_RETAIN(1);\n "); b.append("set_static_"); - b.append(owner.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(owner)); b.append("_"); - b.append(name.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(name)); if (isObject()) { b.append("(threadStateData, "); } else { @@ -300,7 +302,7 @@ public void appendInstruction(StringBuilder sbOut) { } b.append("(get_field_"); - b.append(owner.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(owner)); b.append("_"); b.append(name); @@ -317,7 +319,7 @@ public void appendInstruction(StringBuilder sbOut) { case Opcodes.PUTFIELD: { //b.append("SAFE_RETAIN(1);\n "); b.append("set_field_"); - b.append(owner.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(owner)); b.append("_"); b.append(name); b.append("("); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedConstructor.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedConstructor.java index f722f2fcab6..8157fb040e3 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedConstructor.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedConstructor.java @@ -554,8 +554,9 @@ private static boolean descMatchesArrayType(String fieldDesc, int arrayType) { public void appendFusedAlloc(StringBuilder b, String cType, String[] lenExprs, int recvSlot, int survSlot) { b.append(" { /* FUSED construction of ").append(cType).append(" */\n"); - b.append(" if(__builtin_expect(!class__").append(cType) - .append(".initialized, 0)) __STATIC_INITIALIZER_").append(cType).append("(threadStateData);\n"); + // ACQUIRE; see the note in TypeInstruction. + b.append(" if(__builtin_expect(!__atomic_load_n(&class__").append(cType) + .append(".initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_").append(cType).append("(threadStateData);\n"); for (int i = 0; i < children.size(); i++) { b.append(" int __fLen").append(i).append(" = ").append(lenExprs[i]).append(";\n"); } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Invoke.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Invoke.java index ba0a5c67f6e..7d590fbdac3 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Invoke.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Invoke.java @@ -91,7 +91,7 @@ private String getCMethodName() { public void addDependencies(List dependencyList) { String dependencyOwner = owner; if (opcode == Opcodes.INVOKEVIRTUAL) { - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); String resolvedConcreteOwner = resolveConcreteInvokeOwner(bc, true); if (resolvedConcreteOwner != null) { dependencyOwner = resolvedConcreteOwner; @@ -121,7 +121,7 @@ public void addDependencies(List dependencyList) { if(opcode != Opcodes.INVOKEINTERFACE && opcode != Opcodes.INVOKEVIRTUAL) { return; } - bld.append(owner.replace('/', '_').replace('$', '_')); + bld.append(Util.mangle(owner)); bld.append("_"); if(name.equals("")) { bld.append("__INIT__"); @@ -167,7 +167,7 @@ private String resolveConcreteInvokeOwner(ByteCodeClass ownerClass, boolean allo if (currentClass != null && (ownerName.equals(currentClass) || currentClass.startsWith(ownerName + "_"))) { return null; } - ByteCodeClass concreteClass = Parser.getClassObject(ownerClass.getConcreteClass().replace('/', '_').replace('$', '_')); + ByteCodeClass concreteClass = Parser.getClassObject(Util.mangle(ownerClass.getConcreteClass())); // The nearest class in the concrete type's own hierarchy that actually // declares the method -- which is what the runtime would dispatch to for // an instance of it. Resolving against concreteClass's declarations alone @@ -227,7 +227,7 @@ private void appendFusedAllocBlock(StringBuilder b) { for (int i = 0; i < kids.size(); i++) { lenExprs[i] = kids.get(i).siteLengthExpr(argExprByParam); } - String cType = owner.replace('/', '_').replace('$', '_'); + String cType = Util.mangle(owner); fusedPlan.appendFusedAlloc(b, cType, lenExprs, n + 1, n + 2); } @@ -261,7 +261,7 @@ private boolean tryAppendInlinedConstructor(StringBuilder b) { // argCats == null: every argExpr here is a pure SP[-k].data.x read // (the args were evaluated onto the operand stack BEFORE this ), // so no temp hoisting is needed. - String cType = owner.replace('/', '_').replace('$', '_'); + String cType = Util.mangle(owner); inlineCtorPlan.appendInitBeforePublish(b, cType, argExprs, null, n + 2, n + 1); return true; } @@ -307,7 +307,7 @@ public void appendInstruction(StringBuilder b) { // if it is. boolean isVirtual = true; if (opcode == Opcodes.INVOKEVIRTUAL) { - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); if (bc == null) { System.err.println("WARNING: Failed to find class object for owner "+owner+" when rendering virtual method "+name); } else { @@ -340,7 +340,7 @@ public void appendInstruction(StringBuilder b) { if(opcode == Opcodes.INVOKESTATIC) { // find the actual class of the static method to work around javac not defining it correctly - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); invokeOwner = findActualOwner(bc); } if (invokeOwner.startsWith("[")) { @@ -348,7 +348,7 @@ public void appendInstruction(StringBuilder b) { // as an owner. We'll just change this to java_lang_Object instead. bld.append("java_lang_Object"); } else{ - bld.append(invokeOwner.replace('/', '_').replace('$', '_')); + bld.append(Util.mangle(invokeOwner)); } bld.append("_"); if(name.equals("")) { @@ -523,7 +523,7 @@ public Field asInlinableFieldAccess() { if (desc.length() < 3 || desc.charAt(0) != '(' || desc.charAt(1) != ')' || desc.charAt(2) == 'V') { return null; } - BytecodeMethod target = findMethodUp(Parser.getClassObject(owner.replace('/', '_').replace('$', '_'))); + BytecodeMethod target = findMethodUp(Parser.getClassObject(Util.mangle(owner))); if (target == null || !target.isStatic()) { return null; } @@ -586,10 +586,10 @@ public Field asInlinableFieldAccess() { */ private BytecodeMethod resolveDirectTarget() { if (opcode == Opcodes.INVOKESPECIAL) { - return findMethodUp(Parser.getClassObject(owner.replace('/', '_').replace('$', '_'))); + return findMethodUp(Parser.getClassObject(Util.mangle(owner))); } // INVOKEVIRTUAL - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); if (bc == null) { return null; } @@ -601,7 +601,7 @@ private BytecodeMethod resolveDirectTarget() { if (rc == null) { return null; // genuinely virtual -> target not fixed -> unsafe to inline } - return findMethodUp(Parser.getClassObject(rc.replace('/', '_').replace('$', '_'))); + return findMethodUp(Parser.getClassObject(Util.mangle(rc))); } /** @@ -629,7 +629,7 @@ private static BytecodeMethod trivialStaticForwarderTarget(BytecodeMethod m) { if (rc != Opcodes.IRETURN && rc != Opcodes.LRETURN && rc != Opcodes.FRETURN && rc != Opcodes.DRETURN && rc != Opcodes.ARETURN) return null; BytecodeMethod t = inner.findMethodUp(Parser.getClassObject( - inner.owner.replace('/', '_').replace('$', '_'))); + Util.mangle(inner.owner))); return (t != null && t.isStatic()) ? t : null; } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Ldc.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Ldc.java index 9f9e297d37b..551dc3629e9 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Ldc.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Ldc.java @@ -23,6 +23,8 @@ package com.codename1.tools.translator.bytecodes; +import com.codename1.tools.translator.Util; + import com.codename1.tools.translator.ByteCodeClass; import com.codename1.tools.translator.Parser; import java.util.List; @@ -57,7 +59,7 @@ public void addDependencies(List dependencyList) { int sort = ((Type) cst).getSort(); Type tp = (Type) cst; if (sort == Type.OBJECT) { - String t = tp.getInternalName().replace('/', '_').replace('$', '_'); + String t = Util.mangle(tp.getInternalName()); if(!dependencyList.contains(t)) { dependencyList.add(t); } @@ -75,7 +77,7 @@ public void addDependencies(List dependencyList) { case Type.SHORT: return; } - String t = ttt.getInternalName().replace('/', '_').replace('$', '_'); + String t = Util.mangle(ttt.getInternalName()); ByteCodeClass.addArrayType(t, tp.getDimensions()); if(!dependencyList.contains(t)) { dependencyList.add(t); @@ -159,15 +161,15 @@ public String getValueAsString() { Type tp = (Type) cst; if (sort == Type.OBJECT) { //b.append("/* LDC: '"); - //b.append(tp.getInternalName().replace('/', '_').replace('$', '_')); + //b.append(Util.mangle(tp.getInternalName())); //b.append("'*/\n PUSH_POINTER((JAVA_OBJECT)&class__"); - //b.append(tp.getInternalName().replace('/', '_').replace('$', '_')); + //b.append(Util.mangle(tp.getInternalName())); //b.append(");\n"); b.append("(JAVA_OBJECT)&class__"); - b.append(tp.getInternalName().replace('/', '_').replace('$', '_')); + b.append(Util.mangle(tp.getInternalName())); } else if (sort == Type.ARRAY) { //b.append("/* LDC Array: '"); - //b.append(tp.getInternalName().replace('/', '_').replace('$', '_')); + //b.append(Util.mangle(tp.getInternalName())); //b.append("'*/\n PUSH_POINTER((JAVA_OBJECT)&class_array"); b.append("(JAVA_OBJECT)&class_array"); b.append(tp.getDimensions()); @@ -199,7 +201,7 @@ public String getValueAsString() { b.append("JAVA_SHORT"); break; default: - b.append(ttt.getInternalName().replace('/', '_').replace('$', '_')); + b.append(Util.mangle(ttt.getInternalName())); break; } //b.append(");\n"); @@ -283,13 +285,13 @@ public void appendInstruction(StringBuilder b) { Type tp = (Type) cst; if (sort == Type.OBJECT) { b.append("/* LDC: '"); - b.append(tp.getInternalName().replace('/', '_').replace('$', '_')); + b.append(Util.mangle(tp.getInternalName())); b.append("'*/\n PUSH_POINTER((JAVA_OBJECT)&class__"); - b.append(tp.getInternalName().replace('/', '_').replace('$', '_')); + b.append(Util.mangle(tp.getInternalName())); b.append(");\n"); } else if (sort == Type.ARRAY) { b.append("/* LDC Array: '"); - b.append(tp.getInternalName().replace('/', '_').replace('$', '_')); + b.append(Util.mangle(tp.getInternalName())); b.append("'*/\n PUSH_POINTER((JAVA_OBJECT)&class_array"); b.append(tp.getDimensions()); b.append("__"); @@ -320,7 +322,7 @@ public void appendInstruction(StringBuilder b) { b.append("JAVA_SHORT"); break; default: - b.append(ttt.getInternalName().replace('/', '_').replace('$', '_')); + b.append(Util.mangle(ttt.getInternalName())); break; } b.append(");\n"); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java index 8eb39cf8a35..0538da67be6 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java @@ -254,9 +254,15 @@ public void appendInstruction(StringBuilder b, List l) { // reaches it as a root (its pointer rides the operand stack) and // scans its fields, so any heap objects it references stay live. // It is never freed; it simply dies when the frame unwinds. - b.append("if(__builtin_expect(!class__"); + // ACQUIRE: this guard SKIPS the initialiser when the flag is + // set, so it never takes the class monitor and cannot rely on + // the monitor's release. Pairs with the __ATOMIC_RELEASE store + // in ByteCodeClass. A plain load here let a thread see the flag + // set while the vtable / classToInterfaceMap rows it describes + // were still invisible. + b.append("if(__builtin_expect(!__atomic_load_n(&class__"); b.append(type); - b.append(".initialized, 0)) __STATIC_INITIALIZER_"); + b.append(".initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_"); b.append(type); b.append("(threadStateData); memset(&__cn1stk_"); b.append(stackAllocId); diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index da508632c4e..4b7f8d1259b 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -423,6 +423,26 @@ JAVA_BOOLEAN java_lang_String_equals___java_lang_Object_R_boolean(CODENAME_ONE_T // Fast path: both backing arrays are char[] -- byte-equality of UTF-16 code // units == string equality; libc memcmp is the SIMD-optimized comparison on // every target. + // BOTH LATIN-1 -- the overwhelmingly common case, and until now the SLOW one. + // + // The char[] path below already had a memcmp; the compact byte[] path did + // not, so two ASCII strings (every class name, method name and descriptor + // this translator compares) fell into the per-character loop at the bottom, + // which calls cn1StrCharAtRaw TWICE per character. That helper reloads + // `value` and `offset` and branches on the backing array's class pointer + // EVERY time, so the common case paid a branch and two field loads per char + // where a single memcmp would do. + // + // Latin-1 stores each char as its raw 0..255 byte, so memcmp's unsigned byte + // ordering is exactly char ordering; equality is bit-identical. + // + // MEASURED before the fix: java_lang_String_equals was 6.78% of mutator + // self-time on the 5782-class hellocodenameone translation. + if(cn1StrIsLatin1(__cn1ThisObject) && cn1StrIsLatin1(__cn1Arg1)) { + JAVA_ARRAY_BYTE* ta = ((JAVA_ARRAY_BYTE*)((JAVA_ARRAY)t->java_lang_String_value)->data) + t->java_lang_String_offset; + JAVA_ARRAY_BYTE* oa = ((JAVA_ARRAY_BYTE*)((JAVA_ARRAY)o->java_lang_String_value)->data) + o->java_lang_String_offset; + return memcmp(ta, oa, (size_t)t->java_lang_String_count) == 0 ? JAVA_TRUE : JAVA_FALSE; + } if(!cn1StrIsLatin1(__cn1ThisObject) && !cn1StrIsLatin1(__cn1Arg1)) { JAVA_ARRAY_CHAR* oa = ((JAVA_ARRAY_CHAR*)((JAVA_ARRAY)o->java_lang_String_value)->data) + o->java_lang_String_offset; JAVA_ARRAY_CHAR* ta = ((JAVA_ARRAY_CHAR*)((JAVA_ARRAY)t->java_lang_String_value)->data) + t->java_lang_String_offset; @@ -471,8 +491,25 @@ JAVA_INT java_lang_String_compareTo___java_lang_String_R_int(CODENAME_ONE_THREAD } return tc - oc; } - // Coder-aware path: at least one string is Latin-1 (byte[]); compare logical - // chars. Same UTF-16 code-unit ordering, bit-identical to the char[] path. + // BOTH Latin-1: hoist the coder test and the field reloads OUT of the loop. + // cn1StrCharAtRaw re-derives the base pointer and re-tests the backing array's + // class on every character, twice per iteration; with both coders known the + // loop is two raw byte pointers. Ordering is unchanged -- Latin-1 bytes are + // the char values 0..255. + if(cn1StrIsLatin1(__cn1ThisObject) && cn1StrIsLatin1(__cn1Arg1)) { + struct obj__java_lang_String* ts = (struct obj__java_lang_String*)__cn1ThisObject; + struct obj__java_lang_String* os = (struct obj__java_lang_String*)__cn1Arg1; + const JAVA_ARRAY_BYTE* tb = ((JAVA_ARRAY_BYTE*)((JAVA_ARRAY)ts->java_lang_String_value)->data) + ts->java_lang_String_offset; + const JAVA_ARRAY_BYTE* ob = ((JAVA_ARRAY_BYTE*)((JAVA_ARRAY)os->java_lang_String_value)->data) + os->java_lang_String_offset; + for(JAVA_INT k = 0; k < minL; k++) { + int d = (int)(tb[k] & 0xff) - (int)(ob[k] & 0xff); + if(d) { + return d; + } + } + return tc - oc; + } + // Mixed coders: one Latin-1, one UTF-16. Rare; keep the general helper. for(JAVA_INT k = 0; k < minL; k++) { int d = (int)cn1StrCharAtRaw(__cn1ThisObject, k) - (int)cn1StrCharAtRaw(__cn1Arg1, k); if(d) { diff --git a/vm/JavaAPI/src/java/lang/StringBuilder.java b/vm/JavaAPI/src/java/lang/StringBuilder.java index 4c5fd18d310..c031b06fc03 100644 --- a/vm/JavaAPI/src/java/lang/StringBuilder.java +++ b/vm/JavaAPI/src/java/lang/StringBuilder.java @@ -100,7 +100,24 @@ private StringBuilder(char[] data, int offset, int charCount) { } private void enlargeBuffer(int min) { - int newCount = ((value.length >> 1) + value.length) + 2; + // Double, as OpenJDK's AbstractStringBuilder does, rather than the 1.5x + // ((len>>1)+len+2) inherited from Harmony. + // + // Growing to N chars costs sum(capacity) in ABANDONED intermediate + // arrays, and that sum is N*r/(r-1): 3N at r=1.5, 2N at r=2. The + // difference is pure garbage, and on ParparVM garbage is expensive in a + // way it is not on a generational JVM -- the collector is a concurrent + // mark/sweep with no nursery, so a dead intermediate array occupies its + // slot until a later cycle sweeps it. + // + // MEASURED on the 5782-class hellocodenameone translation, where the + // emit phase is StringBuilder-bound: char[] occupancy 1529.84MB and the + // legacy (large-array) heap 1533.43MB before the emit-buffer reuse fix. + // + // The cost is peak overshoot: a buffer can now be up to 2x the chars + // actually needed rather than 1.5x. That is bounded and transient, where + // the reallocation garbage is unbounded in the number of appends. + int newCount = (value.length << 1) + 2; char[] newData = new char[min > newCount ? min : newCount]; System.arraycopy(value, 0, newData, 0, count); value = newData; diff --git a/vm/JavaAPI/src/java/util/ArrayList.java b/vm/JavaAPI/src/java/util/ArrayList.java index e9bb981768b..1c3d83d7899 100644 --- a/vm/JavaAPI/src/java/util/ArrayList.java +++ b/vm/JavaAPI/src/java/util/ArrayList.java @@ -361,6 +361,75 @@ public void ensureCapacity(int minimumCapacity) { } } + /** + * Direct-array iterator, overriding AbstractList's generic SimpleListIterator. + * + * The inherited one was the single hottest method in a large translation -- + * 16.45% of mutator self-time on the 5782-class hellocodenameone corpus, more + * than twice the next entry. Three costs per element, none inherent: + * + * - a try/catch around the body, to turn IndexOutOfBoundsException into + * NoSuchElementException. ParparVM has no zero-cost exception tables, so a + * try block is a setjmp -- once per element, in the hottest loop in the + * program. An explicit bounds test costs a compare. + * - size() and get() as VIRTUAL calls on the outer list, with no JIT to + * inline them. + * - the index recomputed as size() - numLeft every iteration instead of + * being carried in a cursor. + * + * MEASURED after: the iteration path fell from 25.5% of mutator self-time to + * 12.4%, ArrayList.get from 7.42% to 0.55%, and _setjmp from 1.61% to zero. + * + * Semantics are unchanged: same ConcurrentModificationException on structural + * modification, same NoSuchElementException past the end, remove() still + * works. Reads array[firstIndex + i] exactly as get(int) does. + * + * Applies to every `for (x : list)` in every translated application whatever + * the loop's static type, because dispatch lands on the concrete ArrayList. + */ + private class ArrayListIterator implements Iterator { + private int cursor; + private int lastReturned = -1; + private int expectedModCount = modCount; + + public boolean hasNext() { + return cursor < size; + } + + public E next() { + if (modCount != expectedModCount) { + throw new ConcurrentModificationException(); + } + int i = cursor; + if (i >= size) { + throw new NoSuchElementException(); + } + cursor = i + 1; + lastReturned = i; + return array[firstIndex + i]; + } + + public void remove() { + if (lastReturned < 0) { + throw new IllegalStateException(); + } + if (modCount != expectedModCount) { + throw new ConcurrentModificationException(); + } + ArrayList.this.remove(lastReturned); + if (lastReturned < cursor) { + cursor--; + } + lastReturned = -1; + expectedModCount = modCount; + } + } + + @Override + public Iterator iterator() { + return new ArrayListIterator(); + } + @Override public E get(int location) { if (location < 0 || location >= size) { diff --git a/vm/JavaAPI/src/java/util/IdentityHashMap.java b/vm/JavaAPI/src/java/util/IdentityHashMap.java index d8ed67b86af..188a307ac74 100644 --- a/vm/JavaAPI/src/java/util/IdentityHashMap.java +++ b/vm/JavaAPI/src/java/util/IdentityHashMap.java @@ -161,15 +161,16 @@ static class IdentityHashMapIterator implements Iterator { } public boolean hasNext() { - while (position < associatedMap.elementData.length) { - // if this is an empty spot, go to the next one - if (associatedMap.elementData[position] == null) { - position += 2; - } else { - return true; - } + // elementData hoisted into a local: it was re-loaded from the outer map + // on every comparison AND on every array access, twice per probe step. + Object[] data = associatedMap.elementData; + int p = position; + int len = data.length; + while (p < len && data[p] == null) { + p += 2; } - return false; + position = p; + return p < len; } void checkConcurrentMod() throws ConcurrentModificationException { @@ -180,13 +181,37 @@ void checkConcurrentMod() throws ConcurrentModificationException { @SuppressWarnings("unchecked") public E next() { - checkConcurrentMod(); - if (!hasNext()) { + // The concurrent-modification test and the null-skipping scan are + // INLINED here rather than reached through checkConcurrentMod() and + // hasNext(). + // + // An enhanced-for already pays two interface dispatches per element + // (hasNext then next); routing next() through two more non-inlined + // calls made it four, and ParparVM has no JIT to fold them away. + // MEASURED on the 5782-class hellocodenameone translation: + // IdentityHashMapIterator.next 6.43% of mutator self-time with + // checkConcurrentMod a further 1.84%, second only to the ArrayList + // iterator. + // + // Behaviour is unchanged: same ConcurrentModificationException on a + // structural change, same NoSuchElementException past the end, and + // position still advances past empty slots exactly as hasNext() did. + if (expectedModCount != associatedMap.modCount) { + throw new ConcurrentModificationException(); + } + Object[] data = associatedMap.elementData; + int p = position; + int len = data.length; + while (p < len && data[p] == null) { + p += 2; + } + if (p >= len) { + position = p; throw new NoSuchElementException(); } - lastPosition = position; - position += 2; + lastPosition = p; + position = p + 2; canRemove = true; if (kind == KIND_KEY) { diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/BytecodeInstructionIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/BytecodeInstructionIntegrationTest.java index 0e88b50b8aa..6087a921a36 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/BytecodeInstructionIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/BytecodeInstructionIntegrationTest.java @@ -972,15 +972,19 @@ void handleDefaultOutputWritesOutput(CompilerHelper.CompilerConfig config) throw } @Test - void readFileAsStringBuilderReadsContent() throws Exception { + void readFileAsStringReadsContent() throws Exception { File temp = File.createTempFile("readfile", ".txt"); Files.write(temp.toPath(), "Hello World".getBytes(StandardCharsets.UTF_8)); - Method m = ByteCodeTranslator.class.getDeclaredMethod("readFileAsStringBuilder", File.class); + // readFileAsStringBuilder until the translator had to compile against + // ParparVM's own JavaAPI in order to translate itself: StringBuilder there + // has no indexOf/replace, so replaceInFile works on a String instead and + // this helper returns one. + Method m = ByteCodeTranslator.class.getDeclaredMethod("readFileAsString", File.class); m.setAccessible(true); - StringBuilder sb = (StringBuilder) m.invoke(null, temp); + String contents = (String) m.invoke(null, temp); - assertEquals("Hello World", sb.toString()); + assertEquals("Hello World", contents); temp.delete(); } From 01643b78c916aef8e3a9e9679741918d2dc35ff5 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 10 Sep 2026 21:27:25 +0300 Subject: [PATCH 15/66] Review fixes: guard on the completion flag, refuse stack manipulation, keep the pacing bound off placeholder hosts Three defects, two of them found by review. CLASS-INIT GUARDS TESTED THE WRONG FLAG (P1, review) class__X.initialized is stored BEFORE __CLINIT__ runs, because it doubles as the recursion guard for a that touches its own statics. A guard on it can therefore skip the initializer while is still executing and hand back a default for a static field that has not been assigned yet -- and the guards were added to the static accessors, which is exactly where that is observable. __X_LOADED__ is stored after __CLINIT__ returns and is the only flag meaning "finished". Every guard emitted from ByteCodeClass and BytecodeMethod now tests it. It is file-local, so generateCCode forward-declares it above the accessors; the initializer block later in the same file is the definition. The two guards emitted from TypeInstruction and FusedConstructor still test initialized: they name a DIFFERENT class, whose flag is not visible from the emitting translation unit. That is pre-existing, it is now written down where the guard is emitted, and closing it needs a globally visible completion flag on struct clazz. THE CONCAT MATCHER TRUSTED THE OWNER, NOT THE STACK (P1, review) It recognised appends by owner rather than by tracking which object was on the stack, so it accepted new StringBuilder(); POP; return existing.append(a).append(b).toString(); -- valid bytecode -- and took the appends on `existing` for appends on the builder it had just allocated. Removing the allocation and the appends would then leave the POP: an operand-stack underflow and a concat of the wrong operands. The whole DUP/POP/SWAP family now ends the chain. Refused rather than reasoned about, because a missed fusion is slower and a wrong one is memory corruption. Cost: one site out of 898. THE RUN-AHEAD BOUND SCALED OFF A NUMBER THAT IS NOT A MEASUREMENT (CI) cn1_available_memory answers a flat 100MB wherever it cannot measure -- Linux, Windows, the non-Apple fallback. cn1PacingGrowthFloorBytes only ever RAISES its floor from that value, so a placeholder host is bit-for-bit unchanged. The new run-ahead bound only ever LOWERS the cap, so scaling it by the placeholder tightened pacing on precisely the hosts we know nothing about. BibopPageFloorIntegrationTest went red on arm64 Linux, where fm/8 is 12.5MB, while the same code passed on macOS where fm is real. The bound now applies only where fm is a genuine reading, and answers 0 -- "leave the cap alone" -- elsewhere. macOS keeps the measured win: 25.1s. ALSO ArrayList.java joins the copyright exclusions as Apache Harmony source retaining its Apache-2.0 notice, beside the other Harmony files. Swapping in the Codename One header, which is what the gate was asking for, would have relicensed third-party code. The three genuinely new files got the real header. VERIFIED Gate D, Gate A and the negative control pass byte-identical on 797 files. BibopPageFloor and GcHeapIntegrity green. Copyright, control-character and ASCII gates clean. --- scripts/copyright-header-exclusions.txt | 1 + vm/ByteCodeTranslator/src/cn1_globals.m | 52 ++++++++++++++----- .../tools/translator/ByteCodeClass.java | 28 +++++++--- .../tools/translator/BytecodeMethod.java | 30 ++++++++++- .../translator/bytecodes/TypeInstruction.java | 10 ++++ .../experiments/src/com/exp/PinProbe.java | 22 ++++++++ .../translator/CollectionSemanticsApp.java | 22 ++++++++ .../tools/translator/PrimitiveTypeApp.java | 22 ++++++++ 8 files changed, 163 insertions(+), 24 deletions(-) diff --git a/scripts/copyright-header-exclusions.txt b/scripts/copyright-header-exclusions.txt index 5c9b569b096..d3a3e6ee084 100644 --- a/scripts/copyright-header-exclusions.txt +++ b/scripts/copyright-header-exclusions.txt @@ -33,3 +33,4 @@ vm/JavaAPI/src/java/util/Collections.java | Apache Harmony source retaining its vm/JavaAPI/src/java/util/HashMap.java | Apache Harmony source retaining its original Apache-2.0 notice vm/JavaAPI/src/java/util/Hashtable.java | Apache Harmony source retaining its original Apache-2.0 notice vm/JavaAPI/src/java/util/IdentityHashMap.java | Apache Harmony source retaining its original Apache-2.0 notice +vm/JavaAPI/src/java/util/ArrayList.java | Apache Harmony source retaining its original Apache-2.0 notice diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index 211984981d9..5d032e631ee 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -6333,6 +6333,34 @@ static inline JAVA_OBJECT cn1BibopSlot(CN1BibopPage* p, int i) { #ifndef CN1_BIBOP_PACING_MAX_RUNAHEAD_BYTES #define CN1_BIBOP_PACING_MAX_RUNAHEAD_BYTES (1024L*1024*1024) #endif +// cn1_available_memory answers a flat 100MB on every platform where it cannot +// measure: Linux, Windows, and the non-Apple fallback. That number is not a +// reading, and a bound DERIVED from it is not a bound -- it is a constant that +// happens to look like one. +// +// This matters asymmetrically. cn1PacingGrowthFloorBytes above only ever RAISES +// its floor from fm, so on a placeholder host the absolute floor wins and +// behaviour is bit-for-bit unchanged. The run-ahead bound below only ever LOWERS +// the cap, so scaling it by a placeholder TIGHTENS pacing on exactly the hosts we +// know nothing about. It did: BibopPageFloorIntegrationTest went red on arm64 +// Linux, where fm/8 is 12.5MB, while the same code passed on macOS where fm is +// real. +// +// So the bound applies only where fm is a genuine reading. Returns 0 to mean +// "not measurable here, leave the cap alone". +#ifndef CN1_PACING_PLACEHOLDER_FREE_MEM +#define CN1_PACING_PLACEHOLDER_FREE_MEM (1024L*1024*100) +#endif +static long cn1PacingRunAheadBound(long fm) { + if(fm <= CN1_PACING_PLACEHOLDER_FREE_MEM) { + return 0; + } + long bound = CN1_BIBOP_PACING_MAX_RUNAHEAD_BYTES; + if(bound > fm / 8) { + bound = fm / 8; + } + return bound; +} // How stale a below-floor footprint reading may be before the bound re-probes it. The // probe is task_info on Apple and one /proc read on Linux -- a microsecond or two -- and // it is taken at most once per interval across the whole process, and only when the bound @@ -6634,11 +6662,8 @@ static long cn1BibopPacingCap(CODENAME_ONE_THREAD_STATE) { // under the saturation point -- a phone, a container, the flat 100MB // placeholder off Apple -- the floor follows fm/8 and nothing loosens. { - long runAhead = CN1_BIBOP_PACING_MAX_RUNAHEAD_BYTES; - if(fm > 0 && runAhead > fm / 8) { - runAhead = fm / 8; - } - if(capCeiling < runAhead) { + long runAhead = cn1PacingRunAheadBound(fm); + if(runAhead > 0 && capCeiling < runAhead) { capCeiling = runAhead; } } @@ -6665,15 +6690,14 @@ static long cn1BibopPacingCap(CODENAME_ONE_THREAD_STATE) { // Apple -- this follows fm/8 and nothing is loosened. `base` is still honoured // so a build with a large static trigger keeps the admission it had. { - long runAhead = CN1_BIBOP_PACING_MAX_RUNAHEAD_BYTES; - if(fm > 0 && runAhead > fm / 8) { - runAhead = fm / 8; - } - if(cap > runAhead) { - cap = runAhead; - } - if(cap < base) { - cap = base; + long runAhead = cn1PacingRunAheadBound(fm); + if(runAhead > 0) { + if(cap > runAhead) { + cap = runAhead; + } + if(cap < base) { + cap = base; + } } } if(cn1PacingTraceOn()) { diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java index 55333ebff86..7622e869b9e 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java @@ -679,6 +679,18 @@ public String generateCCode(List allClasses) { b.append(".h\"\n"); + // Forward-declare the COMPLETION flag so the inline class-init guards below + // can test it. Tentative definition; the initializer block later in this + // same file defines it with = 0. + // + // The guards must NOT test class__X.initialized: that flag is set BEFORE + // __CLINIT__ runs, because it doubles as the recursion guard for a + // that touches its own statics. A guard on it can therefore skip the + // initializer while is still executing and hand back a default for + // a static field that has not been assigned yet. __X_LOADED__ is stored + // after returns and is the only flag that means "finished". + b.append("static int __").append(clsName).append("_LOADED__;\n"); + for(String s : dependsClassesInterfaces) { if (exportsClassesInterfaces.contains(s)) { continue; @@ -973,8 +985,8 @@ public String generateCCode(List allClasses) { // per static-field access. MEASURED: __STATIC_INITIALIZER_* was // 7.2% of mutator self-time, java.util.Iterator's alone 6.26%. // Safe as an ACQUIRE load now that the flag is release-stored. - b.append("() {\n if(__builtin_expect(!__atomic_load_n(&class__").append(bf.getClsName()) - .append(".initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_"); + b.append("() {\n if(__builtin_expect(!__atomic_load_n(&__").append(bf.getClsName()) + .append("_LOADED__, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_"); b.append(bf.getClsName()); if (bf.isVolatile()) { b.append("(getThreadLocalData());\n return atomic_load_explicit(&STATIC_FIELD_"); @@ -1000,8 +1012,8 @@ public String generateCCode(List allClasses) { b.append("CODENAME_ONE_THREAD_STATE, "); } b.append(bf.getCDefinition()); - b.append(" __cn1StaticVal) {\n if(__builtin_expect(!__atomic_load_n(&class__").append(bf.getClsName()) - .append(".initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_"); + b.append(" __cn1StaticVal) {\n if(__builtin_expect(!__atomic_load_n(&__").append(bf.getClsName()) + .append("_LOADED__, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_"); b.append(bf.getClsName()); if (bf.isObjectType()) { b.append("(threadStateData);\n "); @@ -1253,8 +1265,8 @@ public String generateCCode(List allClasses) { if(!isInterface && !isAbstract) { b.append("JAVA_OBJECT __NEW_"); b.append(clsName); - b.append("(CODENAME_ONE_THREAD_STATE) {\n if(__builtin_expect(!__atomic_load_n(&class__").append(clsName) - .append(".initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_"); + b.append("(CODENAME_ONE_THREAD_STATE) {\n if(__builtin_expect(!__atomic_load_n(&__").append(clsName) + .append("_LOADED__, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_"); b.append(clsName); b.append("(threadStateData);\n JAVA_OBJECT o = codenameOneGcMalloc(threadStateData, sizeof(struct obj__"); b.append(clsName); @@ -1265,8 +1277,8 @@ public String generateCCode(List allClasses) { if(hasDefaultConstructor()) { b.append("JAVA_OBJECT __NEW_INSTANCE_"); b.append(clsName); - b.append("(CODENAME_ONE_THREAD_STATE) {\n if(__builtin_expect(!__atomic_load_n(&class__").append(clsName) - .append(".initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_"); + b.append("(CODENAME_ONE_THREAD_STATE) {\n if(__builtin_expect(!__atomic_load_n(&__").append(clsName) + .append("_LOADED__, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_"); b.append(clsName); b.append("(threadStateData);\n JAVA_OBJECT o = codenameOneGcMalloc(threadStateData, sizeof(struct obj__"); b.append(clsName); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java index 74a66264d15..d58deab2bf3 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java @@ -2189,8 +2189,12 @@ public void appendVirtualMethodC(String cls, StringBuilder b, String offset, boo } if(includeStaticInitializer) { - b.append("if(__builtin_expect(!__atomic_load_n(&class__").append(cls) - .append(".initialized, __ATOMIC_ACQUIRE), 0)) "); + // Completion flag, not class__X.initialized: that one is set BEFORE + // __CLINIT__ runs (it is also the recursion guard), so a guard on it can + // skip the initializer mid-. See the forward declaration in + // ByteCodeClass.generateCCode. + b.append("if(__builtin_expect(!__atomic_load_n(&__").append(cls) + .append("_LOADED__, __ATOMIC_ACQUIRE), 0)) "); b.append("__STATIC_INITIALIZER_"); b.append(cls); b.append("(threadStateData);\n "); @@ -4355,6 +4359,28 @@ private boolean fuseStringBuilderConcatOnce() { break; } int op = c.getOpcode(); + // Any opcode that can MOVE OR DISCARD the builder reference ends the + // chain, not just the ones that store it somewhere. + // + // The matcher recognises appends by owner, not by tracking which + // object is on the stack, so without this it accepts + // new StringBuilder(); POP; return existing.append(a).append(b).toString(); + // -- valid bytecode -- and mistakes the appends on `existing` for + // appends on the builder it just allocated. Deleting the allocation + // and the appends would then leave the POP behind: an operand-stack + // underflow, and a concat of the wrong operands. + // + // The whole DUP/POP/SWAP family is refused rather than reasoned + // about. This costs coverage on chains whose argument expressions + // happen to contain one, which is the right trade: a missed fusion + // is slower, a wrong one is memory corruption. The pattern's own DUP + // sits before the scan window and is unaffected. + if (op == Opcodes.POP || op == Opcodes.POP2 || op == Opcodes.SWAP + || op == Opcodes.DUP || op == Opcodes.DUP_X1 || op == Opcodes.DUP_X2 + || op == Opcodes.DUP2 || op == Opcodes.DUP2_X1 || op == Opcodes.DUP2_X2) { + ok = false; + break; + } if (op == Opcodes.ASTORE || op == Opcodes.PUTFIELD || op == Opcodes.PUTSTATIC || op == Opcodes.AASTORE || op == Opcodes.ARETURN) { ok = false; diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java index 0538da67be6..feb8f2095aa 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java @@ -254,6 +254,16 @@ public void appendInstruction(StringBuilder b, List l) { // reaches it as a root (its pointer rides the operand stack) and // scans its fields, so any heap objects it references stay live. // It is never freed; it simply dies when the frame unwinds. + // NOTE: this guard necessarily tests class__X.initialized rather + // than __X_LOADED__, because X is a DIFFERENT class from the one + // being emitted and __X_LOADED__ is file-local to X's own + // translation unit. initialized is set before __CLINIT__ runs, so + // this can still enter the allocation while X's is in + // flight -- pre-existing, and the reason the guards emitted from + // ByteCodeClass (same translation unit) use the completion flag + // instead. Closing it here needs a globally visible completion + // flag on struct clazz, which is a larger change than this. + // // ACQUIRE: this guard SKIPS the initialiser when the flag is // set, so it never takes the class monitor and cannot rely on // the monitor's release. Pairs with the __ATOMIC_RELEASE store diff --git a/vm/selfhost/experiments/src/com/exp/PinProbe.java b/vm/selfhost/experiments/src/com/exp/PinProbe.java index 15990187ead..f4ccd7baa4b 100644 --- a/vm/selfhost/experiments/src/com/exp/PinProbe.java +++ b/vm/selfhost/experiments/src/com/exp/PinProbe.java @@ -1,3 +1,25 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ package com.exp; /** diff --git a/vm/tests/src/test/resources/com/codename1/tools/translator/CollectionSemanticsApp.java b/vm/tests/src/test/resources/com/codename1/tools/translator/CollectionSemanticsApp.java index e581a5f12b2..667062c9344 100644 --- a/vm/tests/src/test/resources/com/codename1/tools/translator/CollectionSemanticsApp.java +++ b/vm/tests/src/test/resources/com/codename1/tools/translator/CollectionSemanticsApp.java @@ -1,3 +1,25 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ import java.util.ArrayList; import java.util.IdentityHashMap; import java.util.Iterator; diff --git a/vm/tests/src/test/resources/com/codename1/tools/translator/PrimitiveTypeApp.java b/vm/tests/src/test/resources/com/codename1/tools/translator/PrimitiveTypeApp.java index fd3753208dc..5d61c920ed9 100644 --- a/vm/tests/src/test/resources/com/codename1/tools/translator/PrimitiveTypeApp.java +++ b/vm/tests/src/test/resources/com/codename1/tools/translator/PrimitiveTypeApp.java @@ -1,3 +1,25 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ import java.util.HashMap; import java.util.Map; From a57b3ecdbcaa97232d3078cca3016ed29d24db73 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Fri, 11 Sep 2026 01:55:27 +0300 Subject: [PATCH 16/66] Apple targets: stop the translated java.io.File overwriting the port's native handleAppleOutput copied the runtime native to srcRoot/java_io_File.m and then called Parser.writeOutput, which on an Apple target emits one .m per surviving class. A retained java.io.File therefore lands on exactly the name the copy had just used and CLOBBERS the native. What is left declares exists() calling existsImpl and has no definition of it, so the link fails: Undefined symbols: _java_io_File_existsImpl___java_lang_String_R_boolean, referenced from _java_io_File_exists___R_boolean in java_io_File.o on build-ios, build-ios-tv and build-ios-metal. build-macos passed in the same run, which is what identified the mechanism: MacOSNativeBuilder sets -DconcatenateFiles=true, routing class output into one buffer so the colliding name is never written. IPhoneBuilder sets it only under ios.superfastBuild, so the collision is live by default there. The comment in MacOSNativeBuilder has described this hazard for as long as that flag has been passed; the flag hides it rather than fixing it. The clean target already writes the same resource as java_io_File_runtime.c precisely so the two can coexist. This does the same on the Apple path. The generated class keeps java_io_File.m; the native becomes java_io_File_runtime.m; both are compiled, and the symbols resolve. Nothing else needed changing. The Xcode project collects sources by extension rather than from a fixed list, so the renamed file is picked up. And NativeSignatureVerifier reads the RESOURCE "/java_io_File.m" off the classpath, not this output path, so its scan is unaffected -- deliberately, per the comment on bundledRuntimeSources. VERIFIED Translating for the ios target now emits BOTH java_io_File.m (the generated class, referencing existsImpl) and java_io_File_runtime.m (the native, defining it); before, only the former survived. Gate D, Gate A and the negative control still pass byte-identical on the 797-file self-hosting corpus. --- .../tools/translator/ByteCodeTranslator.java | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java index 64760b89ae2..95053649725 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java @@ -804,7 +804,29 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_globals.m"), new FileOutputStream(cn1GlobalsM)); File nativeMethods = new File(srcRoot, "nativeMethods.m"); copy(ByteCodeTranslator.class.getResourceAsStream("/nativeMethods.m"), new FileOutputStream(nativeMethods)); - File javaIoFileM = new File(srcRoot, "java_io_File.m"); + // java_io_File_RUNTIME.m, not java_io_File.m. + // + // Parser.writeOutput below emits one file per surviving class, and on an + // Apple target that file is .m -- so a retained java.io.File is + // written to exactly the name this copy just used, and the generated class + // OVERWRITES the runtime native. The result links only if nothing calls a + // java.io.File native: the generated class still declares exists() calling + // existsImpl, but existsImpl's only definition has been clobbered. + // + // OBSERVED as `Undefined symbols: _java_io_File_existsImpl... referenced + // from _java_io_File_exists___R_boolean in java_io_File.o` on build-ios and + // build-ios-tv, while build-macos passed -- MacOSNativeBuilder sets + // -DconcatenateFiles=true, which routes class output into one buffer and so + // never writes the colliding name. IPhoneBuilder sets it only under + // ios.superfastBuild, so the collision is live by default there. + // + // The clean target already avoids this by writing the same resource as + // java_io_File_runtime.c; this is the same fix for the Apple path. The + // Xcode project collects sources by extension (see the .m/.c glob in the + // project writer), so the renamed file is compiled without further change, + // and NativeSignatureVerifier reads the RESOURCE "/java_io_File.m" off the + // classpath rather than this output path, so it is unaffected. + File javaIoFileM = new File(srcRoot, "java_io_File_runtime.m"); copy(ByteCodeTranslator.class.getResourceAsStream("/java_io_File.m"), new FileOutputStream(javaIoFileM)); if (Util.getProperty("USE_RPMALLOC", "false").equals("true")) { From b236ee39c2b75699d5f7399bbde9b34ba6327c4d Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Fri, 11 Sep 2026 02:59:39 +0300 Subject: [PATCH 17/66] Revert the inline class-init guards; keep the memory-ordering fix The guards tested __X_LOADED__, which is the only flag that means " finished" -- that part was right, and the review comment that prompted it stands. What is wrong is what happens when a class never reaches the store. The initializer returns early, WITHOUT setting __X_LOADED__, whenever it finds class__X.initialized already true: the re-entrant case, and the case where another thread is mid-. It also never reaches the store if __CLINIT__ throws. Any class left in that state has __X_LOADED__ == 0 permanently, and with the guards in place EVERY subsequent static access and every allocation calls the initializer, takes the class monitor, finds initialized true and returns. Not a hang -- a monitor acquire on a path that used to be a predicted-not-taken load. MEASURED: the hello screenshot suite stops after 145 of 166 screenshots on Linux, in three runs across x64 and musl, at 699s and 1052s elapsed, with no crash, no OOM and no bad_alloc in the log. A passing master run reports CN1_HELLO_SUITE_PNGS=166. The stop lands in a different test each time but always at the same count, which is what a uniform slowdown looks like rather than a hang at one place. I had previously attributed those failures to the pre-existing flake in that workflow. That was wrong: the flake is real and does hit other branches, but it stops at a different count (82 on master), and matching on the symptom string hid a regression of my own. So the guards come out. What stays is the fix they were built on top of, which is independent and still wanted: __X_LOADED__ and class__X.initialized are release-stored and acquire-loaded in all 391 classes, and the interface maps are calloc'd. Those close the double-checked-initialization race that produced the SIGSEGV at classToInterfaceMap_java_util_NavigableMap[classId] + 0x8. The 7.2% of mutator self-time the guards were worth needs a design that cannot leave the flag unset -- a third state, or setting it on the already-initialized path once "another thread finished" can be told apart from "this thread is re-entrant". That belongs in its own change, with the suite as its gate. VERIFIED 0 inline guards emitted; 391 acquire fast paths and 391 release stores retained; 39 interface maps still calloc'd. Gate D, Gate A and the negative control pass byte-identical on the 797-file self-hosting corpus. --- .../tools/translator/ByteCodeClass.java | 23 ++++--------------- .../tools/translator/BytecodeMethod.java | 6 ----- 2 files changed, 4 insertions(+), 25 deletions(-) diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java index 7622e869b9e..ab8caee4ac4 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java @@ -679,17 +679,6 @@ public String generateCCode(List allClasses) { b.append(".h\"\n"); - // Forward-declare the COMPLETION flag so the inline class-init guards below - // can test it. Tentative definition; the initializer block later in this - // same file defines it with = 0. - // - // The guards must NOT test class__X.initialized: that flag is set BEFORE - // __CLINIT__ runs, because it doubles as the recursion guard for a - // that touches its own statics. A guard on it can therefore skip the - // initializer while is still executing and hand back a default for - // a static field that has not been assigned yet. __X_LOADED__ is stored - // after returns and is the only flag that means "finished". - b.append("static int __").append(clsName).append("_LOADED__;\n"); for(String s : dependsClassesInterfaces) { if (exportsClassesInterfaces.contains(s)) { @@ -985,8 +974,7 @@ public String generateCCode(List allClasses) { // per static-field access. MEASURED: __STATIC_INITIALIZER_* was // 7.2% of mutator self-time, java.util.Iterator's alone 6.26%. // Safe as an ACQUIRE load now that the flag is release-stored. - b.append("() {\n if(__builtin_expect(!__atomic_load_n(&__").append(bf.getClsName()) - .append("_LOADED__, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_"); + b.append("() {\n __STATIC_INITIALIZER_"); b.append(bf.getClsName()); if (bf.isVolatile()) { b.append("(getThreadLocalData());\n return atomic_load_explicit(&STATIC_FIELD_"); @@ -1012,8 +1000,7 @@ public String generateCCode(List allClasses) { b.append("CODENAME_ONE_THREAD_STATE, "); } b.append(bf.getCDefinition()); - b.append(" __cn1StaticVal) {\n if(__builtin_expect(!__atomic_load_n(&__").append(bf.getClsName()) - .append("_LOADED__, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_"); + b.append(" __cn1StaticVal) {\n __STATIC_INITIALIZER_"); b.append(bf.getClsName()); if (bf.isObjectType()) { b.append("(threadStateData);\n "); @@ -1265,8 +1252,7 @@ public String generateCCode(List allClasses) { if(!isInterface && !isAbstract) { b.append("JAVA_OBJECT __NEW_"); b.append(clsName); - b.append("(CODENAME_ONE_THREAD_STATE) {\n if(__builtin_expect(!__atomic_load_n(&__").append(clsName) - .append("_LOADED__, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_"); + b.append("(CODENAME_ONE_THREAD_STATE) {\n __STATIC_INITIALIZER_"); b.append(clsName); b.append("(threadStateData);\n JAVA_OBJECT o = codenameOneGcMalloc(threadStateData, sizeof(struct obj__"); b.append(clsName); @@ -1277,8 +1263,7 @@ public String generateCCode(List allClasses) { if(hasDefaultConstructor()) { b.append("JAVA_OBJECT __NEW_INSTANCE_"); b.append(clsName); - b.append("(CODENAME_ONE_THREAD_STATE) {\n if(__builtin_expect(!__atomic_load_n(&__").append(clsName) - .append("_LOADED__, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_"); + b.append("(CODENAME_ONE_THREAD_STATE) {\n __STATIC_INITIALIZER_"); b.append(clsName); b.append("(threadStateData);\n JAVA_OBJECT o = codenameOneGcMalloc(threadStateData, sizeof(struct obj__"); b.append(clsName); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java index d58deab2bf3..c309e3c4b27 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java @@ -2189,12 +2189,6 @@ public void appendVirtualMethodC(String cls, StringBuilder b, String offset, boo } if(includeStaticInitializer) { - // Completion flag, not class__X.initialized: that one is set BEFORE - // __CLINIT__ runs (it is also the recursion guard), so a guard on it can - // skip the initializer mid-. See the forward declaration in - // ByteCodeClass.generateCCode. - b.append("if(__builtin_expect(!__atomic_load_n(&__").append(cls) - .append("_LOADED__, __ATOMIC_ACQUIRE), 0)) "); b.append("__STATIC_INITIALIZER_"); b.append(cls); b.append("(threadStateData);\n "); From 28e62d6692eef1d683a82dab52e61344d2bce4fa Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Fri, 11 Sep 2026 05:36:59 +0300 Subject: [PATCH 18/66] Park at the safepoint after a pacing assist, and drop a synthetic bridge Two independent fixes in the allocation-pacing and collection paths. cn1PacingPark tested threadBlockedByGC, then ran a mutator assist, then looped with a bare `continue`. The assist can take a long time, and a GC that requests a stop while it is running found the thread still marked active with no safepoint ahead of it on that path -- so the collector waited out its 250ms timeout and force-stopped the thread instead. That is what the iOS packaging leg was reporting: the screenshot suite ran to completion and then could not emit SUITE:FINISHED, with [GC] force-stopped thread 3 after 250000us at a safepoint it never reached (2 so far) ... (16 so far) and the child finally killed with SIGTERM. Re-check the flag once the assist returns and park properly if it is set. Separately, ArrayListIterator was declared private. A private inner class whose constructor is reached from the outer class makes javac synthesise an access bridge and an ArrayList$1 marker type, so every iterator() paid an extra class plus an aconst_null for the bridge argument. Package private is invisible outside java.util either way. The self-hosting corpus drops from 797 emitted files to 795 -- the .c/.h pair for the synthetic that no longer exists. Gates D and A stay byte-identical over the 795-file corpus, with the negative control still detecting an injected corruption. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.m | 25 +++++++++++++++++++++++++ vm/JavaAPI/src/java/util/ArrayList.java | 7 ++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index 5d032e631ee..e16ca1cc202 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -6934,6 +6934,31 @@ static void cn1PacingPark(CODENAME_ONE_THREAD_STATE, int which, long long pendin // sleep-until-done park. See cn1GcMutatorAssist. if(!threadStateData->threadBlockedByGC && cn1GcMutatorAssist(threadStateData) > 0) { + // HONOUR A STOP REQUESTED WHILE WE WERE ASSISTING. + // + // The test above is taken BEFORE the assist, and the assist marks a + // batch, so the collector can raise threadBlockedByGC while this + // thread is inside it. Without the check below this path continues + // with threadActive still TRUE and never passes the safepoint wait + // further down, so a thread with marking work available can loop + // here indefinitely: the collector waits out its handshake and then + // force-stops it. + // + // OBSERVED on the iOS simulator, where the app finished its suite + // and then hung without emitting the completion marker: + // [GC] force-stopped thread 3 after 250000us at a safepoint it + // never reached (2 so far) ... (16 so far) + // The hazard predates the run-ahead bound; tightening the cap keeps + // `volume > cap` true for longer, which is what made it reachable. + if(threadStateData->threadBlockedByGC) { + threadStateData->threadActive = JAVA_FALSE; + while(threadStateData->threadBlockedByGC) { + if(!cn1VirtualThreadYieldIfVirtual()) { + usleep((JAVA_INT)(500)); + } + } + threadStateData->threadActive = JAVA_TRUE; + } continue; } threadStateData->threadActive = JAVA_FALSE; diff --git a/vm/JavaAPI/src/java/util/ArrayList.java b/vm/JavaAPI/src/java/util/ArrayList.java index 1c3d83d7899..d15e8f0a6f1 100644 --- a/vm/JavaAPI/src/java/util/ArrayList.java +++ b/vm/JavaAPI/src/java/util/ArrayList.java @@ -387,7 +387,12 @@ public void ensureCapacity(int minimumCapacity) { * Applies to every `for (x : list)` in every translated application whatever * the loop's static type, because dispatch lands on the concrete ArrayList. */ - private class ArrayListIterator implements Iterator { + // Package-private, not private: a private inner class whose constructor is + // reached from the outer class makes javac synthesise an access bridge and a + // ArrayList$1 marker type, so every iterator() paid an extra class and an + // aconst_null for the bridge argument. Nothing outside java.util can see it + // either way. + class ArrayListIterator implements Iterator { private int cursor; private int lastReturned = -1; private int expectedModCount = modCount; From f0718e633956ac114c9c6cc68407ed34e3c22a2b Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Fri, 11 Sep 2026 05:58:26 +0300 Subject: [PATCH 19/66] Give a for-each the concrete Iterator type its collection returns A for-each compiles to Iterator.hasNext()/next() through INVOKEINTERFACE, the most expensive dispatch this VM has -- a lookup in the owning class's interface map before the vtable read -- and it runs twice per element. Neither the emitter's closed-world devirtualization nor ThinLTO can touch it, because both start from a concrete owner and an interface call has none: java.util.Iterator has 27 implementors here. The concrete type is recoverable locally even though the translator has no general stack-type inference. When the collection's iterator() has exactly one reachable implementation, and that implementation's whole body is `return new T(...)`, the object the following ASTORE writes is a T -- no inference required. Retyping the Iterator calls on that local to INVOKEVIRTUAL on T is then enough by itself: the existing devirt in Invoke.appendInstruction carries any virtual call with no reachable override the rest of the way to a direct call, which ThinLTO can inline. Soundness rests on the local being assigned exactly once. A slot written twice could hold an iterator of another class at the same ALOAD, and a virtual call on the wrong class reads fields out of an object that does not have them -- silent here, since ParparVM's CHECKCAST is unchecked. allocatedReturnType() is deliberately strict for the same reason: a body that could return an object it did not just allocate is rejected rather than guessed at. Like the concat fusion, the pass runs before the unused-method cull so its new edges exist while reachability is computed. Yield is bounded by what the static types admit, and it is worth being explicit about: over the translator's own corpus, 294 for-each sites, 11 resolve, 7 lower. 190 are java/util/List and 74 more are Set or Collection -- all interface-typed receivers, which closed-world analysis cannot reduce to one implementation. Reaching those needs a runtime guarded specialization, which is a separate and much larger change. This pass is the part that is provable, and it fires for any collection whose iterator() is monomorphic rather than for a hard-coded list of types. Also guard gate A against a stale JVM side. It runs ByteCodeTranslator/target/classes, which neither build-selfhost.sh nor verify-selfhost.sh builds, so a translator edit that has not been through `mvn package` makes the gate compare the new translator against the old one and report the intended change as a VM divergence -- which is exactly what it did here, pointing convincingly at java_util_ArrayDeque.c. Maven is no help: it answered "Nothing to compile - all classes are up to date" for a source three hours newer than its class, so the guard compares the trees itself. CI was never affected; it runs mvn package first. Gates D and A byte-identical over 795 files, negative control still detecting an injected corruption, and vm/tests 562 passed 0 failed. Co-Authored-By: Claude Opus 5 (1M context) --- .../tools/translator/BytecodeMethod.java | 197 ++++++++++++++++++ .../codename1/tools/translator/Parser.java | 28 +++ vm/selfhost/verify-selfhost.sh | 18 ++ 3 files changed, 243 insertions(+) diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java index c309e3c4b27..d680d232a69 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java @@ -2455,6 +2455,203 @@ public String getDesc() { return desc; } + /** + * The type this method allocates and hands straight back -- NEW T, DUP, the + * constructor arguments, T.<init>, ARETURN -- or null for any other shape. + * The point of being this strict is that the caller uses the answer as a + * certainty about the returned object's concrete class, so a body that could + * return something it did not just allocate has to be rejected rather than + * guessed at. + */ + public String allocatedReturnType() { + List real = new ArrayList(); + for (Instruction i : instructions) { + if (i instanceof LabelInstruction || i instanceof LineNumber || i instanceof TryCatch) { + continue; + } + real.add(i); + } + if (real.size() < 4) { + return null; + } + Instruction first = real.get(0); + if (!(first instanceof TypeInstruction) || first.getOpcode() != Opcodes.NEW) { + return null; + } + String type = ((TypeInstruction) first).getTypeName(); + if (type == null || real.get(1).getOpcode() != Opcodes.DUP) { + return null; + } + if (real.get(real.size() - 1).getOpcode() != Opcodes.ARETURN) { + return null; + } + Instruction ctor = real.get(real.size() - 2); + if (!(ctor instanceof Invoke) || ctor.getOpcode() != Opcodes.INVOKESPECIAL) { + return null; + } + Invoke ci = (Invoke) ctor; + if (!"".equals(ci.getName()) || !type.equals(ci.getOwner())) { + return null; + } + // Everything between the DUP and the constructor has to be a plain local + // read. Anything with a side effect could leave a different object under + // the ARETURN, and then the type above would be a lie. + for (int i = 2; i < real.size() - 2; i++) { + Instruction a = real.get(i); + if (!(a instanceof VarOp) || !isLoadOpcode(a.getOpcode())) { + return null; + } + } + return type; + } + + private static boolean isLoadOpcode(int op) { + return op == Opcodes.ALOAD || op == Opcodes.ILOAD || op == Opcodes.LLOAD + || op == Opcodes.FLOAD || op == Opcodes.DLOAD; + } + + private int nextExecutable(int from) { + for (int i = from; i < instructions.size(); i++) { + Instruction ins = instructions.get(i); + if (ins instanceof LabelInstruction || ins instanceof LineNumber || ins instanceof TryCatch) { + continue; + } + return i; + } + return -1; + } + + private int prevExecutable(int from) { + for (int i = from; i >= 0; i--) { + Instruction ins = instructions.get(i); + if (ins instanceof LabelInstruction || ins instanceof LineNumber || ins instanceof TryCatch) { + continue; + } + return i; + } + return -1; + } + + private int countStoresTo(int slot) { + int n = 0; + for (Instruction ins : instructions) { + if (ins instanceof VarOp && ins.getOpcode() == Opcodes.ASTORE + && ((VarOp) ins).getIndex() == slot) { + n++; + } + } + return n; + } + + /** + * ITERATOR LOWERING: give a for-each loop the concrete Iterator type its + * collection really returns, so the calls stop going through the interface. + * + * A for-each compiles to Iterator.hasNext()/next() through INVOKEINTERFACE, + * which is the most expensive dispatch the VM has -- a lookup in the owning + * class's interface map before the vtable read -- and it runs twice per + * element. Neither the emitter's closed-world devirtualization nor ThinLTO + * can touch it, because both start from a concrete owner and an interface + * call does not have one: java.util.Iterator has 27 implementors here. + * + * The concrete type is recoverable locally even though the translator has no + * general stack-type inference. If the collection's iterator() has exactly + * one reachable implementation, and that implementation's whole body is + * `return new T(...)`, then the object stored by the ASTORE that follows the + * call is a T -- no inference needed. Retyping the calls to INVOKEVIRTUAL on + * T is then enough on its own: the existing devirtualization in + * Invoke.appendInstruction takes any virtual call with no reachable override + * the rest of the way to a direct one, which ThinLTO can inline. + * + * The single-assignment requirement on the local is what makes this sound + * without dataflow. If a slot were written twice, a second iterator of some + * other class could reach the same ALOAD, and a virtual call on the wrong + * class reads its fields out of an object that does not have them -- silent + * on this VM, since ParparVM's CHECKCAST is unchecked. + * + * Like the concat fusion this must run BEFORE the unused-method cull, so the + * newly created edges exist while reachability is computed. + */ + public void lowerIteratorCalls() { + for (int i = 0; i < instructions.size(); i++) { + Instruction ins = instructions.get(i); + if (!(ins instanceof Invoke)) { + continue; + } + Invoke inv = (Invoke) ins; + int op = inv.getOpcode(); + if (op != Opcodes.INVOKEINTERFACE && op != Opcodes.INVOKEVIRTUAL) { + continue; + } + if (!"iterator".equals(inv.getName()) || !"()Ljava/util/Iterator;".equals(inv.getDesc())) { + continue; + } + ByteCodeClass coll = Parser.getClassObject(Util.mangle(inv.getOwner())); + String itType = Parser.resolveConcreteIteratorType(coll); + if (itType == null) { + continue; + } + int st = nextExecutable(i + 1); + if (st < 0) { + continue; + } + Instruction store = instructions.get(st); + if (!(store instanceof VarOp) || store.getOpcode() != Opcodes.ASTORE) { + continue; + } + int slot = ((VarOp) store).getIndex(); + if (countStoresTo(slot) != 1) { + continue; + } + retypeIteratorUses(slot, itType); + } + } + + private void retypeIteratorUses(int slot, String itType) { + ByteCodeClass itClass = Parser.getClassObject(Util.mangle(itType)); + if (itClass == null) { + return; + } + for (int i = 0; i < instructions.size(); i++) { + Instruction ins = instructions.get(i); + if (!(ins instanceof Invoke) || ins.getOpcode() != Opcodes.INVOKEINTERFACE) { + continue; + } + Invoke inv = (Invoke) ins; + if (!"java/util/Iterator".equals(inv.getOwner())) { + continue; + } + int r = prevExecutable(i - 1); + if (r < 0) { + continue; + } + Instruction recv = instructions.get(r); + if (!(recv instanceof VarOp) || recv.getOpcode() != Opcodes.ALOAD + || ((VarOp) recv).getIndex() != slot) { + continue; + } + // The concrete class has to actually resolve the method, and resolve it + // monomorphically -- otherwise the retyped call has nothing to bind to. + if (Parser.resolveDevirtualizedOwner(itClass, inv.getName(), inv.getDesc()) == null) { + continue; + } + Invoke direct = new Invoke(Opcodes.INVOKEVIRTUAL, itType, inv.getName(), inv.getDesc(), false); + instructions.set(i, direct); + // Register it exactly as addInstruction() would: the list entry alone + // leaves the call with no owning method, no class dependency and no + // edge in the dependency graph, so the cull would not see the concrete + // iterator's methods being called. + direct.setMethod(this); + direct.addDependencies(dependentClasses); + if (dependencyGraph != null) { + String uses = direct.getMethodUsed(); + if (uses != null) { + dependencyGraph.recordMethodCall(this, uses); + } + } + } + } + public Set getLocalVariables() { return localVariables; } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java index 7a0216424f4..d5f84574329 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java @@ -138,6 +138,33 @@ public static synchronized String resolveDevirtualizedOwner(ByteCodeClass owner, } return null; } + /** + * The concrete Iterator class a for-each over this collection type will + * really get, or null when that cannot be established with certainty. + * + * Two things have to hold. The collection's iterator() must have exactly one + * reachable implementation -- resolveDevirtualizedOwner answers that -- and + * that implementation must do nothing but allocate and return, so the class + * it allocates is the class the caller receives. Anything else answers null + * and the call site is left as the interface call it was. + */ + public static synchronized String resolveConcreteIteratorType(ByteCodeClass owner) { + String decl = resolveDevirtualizedOwner(owner, "iterator", "()Ljava/util/Iterator;"); + if (decl == null) { + return null; + } + ByteCodeClass dc = getClassObject(Util.mangle(decl)); + if (dc == null) { + return null; + } + for (BytecodeMethod m : dc.getMethods()) { + if ("iterator".equals(m.getMethodName()) && "()Ljava/util/Iterator;".equals(m.getDesc())) { + return m.allocatedReturnType(); + } + } + return null; + } + private static final MethodDependencyGraph dependencyGraph = new MethodDependencyGraph(); private int lambdaCounter; private int stringConcatCounter; @@ -820,6 +847,7 @@ public static void writeOutput(File outputDirectory) throws Exception { for (ByteCodeClass fuseCls : classes) { for (BytecodeMethod fuseMtd : fuseCls.getMethods()) { fuseMtd.fuseStringBuilderConcat(); + fuseMtd.lowerIteratorCalls(); } } } diff --git a/vm/selfhost/verify-selfhost.sh b/vm/selfhost/verify-selfhost.sh index 6ab132c8122..a1ff647a745 100755 --- a/vm/selfhost/verify-selfhost.sh +++ b/vm/selfhost/verify-selfhost.sh @@ -34,6 +34,24 @@ JAPI="$REPO/vm/selfhost/target/javaapi-classes" TR="$REPO/vm/ByteCodeTranslator/target/classes" ASM="$(cat "$REPO/vm/ByteCodeTranslator/target/selfhost-asm-classpath.txt")" +# The JVM side of gate A runs target/classes, which nothing in this script builds +# -- build-selfhost.sh compiles the translator only for the NATIVE side. A source +# edit that has not been through `mvn package` therefore makes gate A compare the +# new translator against the old one, and it reports the intended change as a VM +# divergence. That has happened; the diff pointed at java_util_ArrayDeque.c and +# looked exactly like a real one. Maven's own incremental check does not save us +# here either -- it answered "Nothing to compile - all classes are up to date" +# for a source three hours newer than its class, so this compares the trees +# directly rather than trusting it. +newest_src="$(find "$REPO/vm/ByteCodeTranslator/src" -name '*.java' -newer "$TR" -print -quit 2>/dev/null || true)" +if [ -n "$newest_src" ]; then + echo "STALE: $TR is older than $newest_src" >&2 + echo "gate A would compare the new translator against the old one. Run:" >&2 + echo " (cd $REPO/vm && mvn -q -B -pl ByteCodeTranslator clean package -DskipTests)" >&2 + echo "and restore target/selfhost-asm-classpath.txt, which clean removes." >&2 + exit 1 +fi + W="$REPO/vm/selfhost/target/verify" rm -rf "$W"; mkdir -p "$W" OUT="$W/out" From 68503d5f1b84f042b0693553650cbac495aad83c Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Fri, 11 Sep 2026 08:20:08 +0300 Subject: [PATCH 20/66] Collect real crash evidence, and withdraw the lazy ArrayList allocation The suite stops after exactly 145 of 166 screenshots on every target except glibc-x64 -- arm64, musl and Windows x64/arm64 all short by the same 21 -- and master is green on the same jobs (10 consecutive runs of the Windows leg). The app log shows what stops it, and it is not a clean failure: AIOOBE 89 at Display.callSeriallyOnIdle:1174 (pendingIdleSerialCalls.add) AIOOBE -1 at Display.edtLoopImpl:1813 NPE at java_util_ArrayList.get:443 get() is array[firstIndex + location], so an NPE there means the backing array reference itself is null. That is corrupt list state, not a bounds mistake. Two things are ruled out rather than assumed. The list logic is correct: a differential fuzz of this exact source against java.util.ArrayList ran 3000 seeds x 200 random add/insert/remove/set/clear/trim/ensureCapacity/ iterate/iterator-remove/addAll operations with no divergence. And there is no Java-level race: every access to pendingIdleSerialCalls in Display is inside synchronized(lock). The GC's lifetime rules are also unchanged by this PR -- the cn1_globals.m diff is 433 added lines of census behind CN1_ALLOC_CENSUS and three removed pacing lines. So the corruption is below Java, and the change that most alters what the runtime's most-allocated class does is withdrawn here: the lazy default-capacity allocation replaced an eager new Object[10] with a process-wide SHARED static zero-length array, which also gave java.util.ArrayList a it had never had (master's only static is a compile-time serialVersionUID, so no static initializer was emitted at all). Verified back to zero after this change. The generated initializer sets class.initialized before running __CLINIT__, so a class that newly acquires one newly acquires that window too. This is an isolating experiment, not a proven mechanism: if the suite goes green the cause is established, and if it does not, a major suspect is eliminated and the run now produces evidence. The iterator -- the change that carried the measured win, iteration 25.5% -> 12.4% of mutator self-time -- is kept. The evidence half, because the last round produced none: - The musl leg had NO crash wiring whatsoever, so a suite that died mid-run was indistinguishable from one that hung. It now enables core dumps (core_pattern is global to the host kernel, but the kernel writes the file in the crashing process's mount namespace, so an absolute path under the bind-mounted /cn1 lands in the uploaded workspace), installs gdb, post-mortems any core, and states explicitly when no core was written. - Both legs now ship the UNSTRIPPED binary beside the logs. A core or a raw backtrace is addresses and nothing else once the runner is gone. - The live-stack dump swallowed an absent gdb through a catch-all and left hang-stacks.txt holding nothing but its sample headers -- which reads as "we sampled and all was well". It now records why it could not collect, and the gdb install is verified rather than silenced. Gates D and A stay byte-identical over 795 files with the negative control still detecting an injected corruption. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/linux-build-run.yml | 58 ++++++++++++++++- vm/JavaAPI/src/java/util/ArrayList.java | 65 +++++++------------ .../CleanTargetLinuxIntegrationTest.java | 44 +++++++++++-- 3 files changed, 114 insertions(+), 53 deletions(-) diff --git a/.github/workflows/linux-build-run.yml b/.github/workflows/linux-build-run.yml index 3a87ad3b1c6..aaaf82b2941 100644 --- a/.github/workflows/linux-build-run.yml +++ b/.github/workflows/linux-build-run.yml @@ -271,7 +271,16 @@ jobs: LIBGL_ALWAYS_SOFTWARE: '1' run: | set -e + # Installed up front and VERIFIED. This used to be silenced entirely, so + # a runner image without gdb produced a hang-stacks.txt holding nothing + # but sample headers -- which reads as "we sampled and all was well". + # A diagnostic that cannot run has to say so. bash scripts/ci/apt-get-install.sh gdb >/dev/null 2>&1 || true + if command -v gdb >/dev/null 2>&1; then + echo "gdb: $(gdb --version | head -1)" + else + echo "WARNING: gdb is NOT available on this runner; hang and crash stacks will be empty" + fi Xvfb :99 -screen 0 1200x1600x24 >/tmp/xvfb-run.log 2>&1 & export DISPLAY=:99 sleep 2 @@ -321,6 +330,15 @@ jobs: # frames (e.g. a stack overflow's recursion / the caller that ran CN1 on a small # native stack). The suite binary is not stripped, so addr2line resolves them. elf="$(/usr/bin/find /tmp -maxdepth 4 -name LinuxHelloMain -type f 2>/dev/null | head -1)" + # Ship the unstripped binary with the artifact: a core or a raw backtrace + # is addresses and nothing else once the runner is gone. + if [ -n "$elf" ]; then + cp "$elf" "$(dirname "$CN1_APP_LOG_TEE")/LinuxHelloMain" 2>/dev/null || true + fi + for core in /tmp/cn1-cores/core.*; do + [ -f "$core" ] || continue + cp "$core" "$(dirname "$CN1_APP_LOG_TEE")/" 2>/dev/null || true + done if [ -n "$elf" ] && [ -f "$CN1_APP_LOG_TEE" ]; then { echo "=== addr2line of CN1 backtrace addresses (from app log) ===" @@ -371,9 +389,21 @@ jobs: name: linux-suite-classes - name: Unpack suite classes run: tar xzf suite-classes.tgz + # The musl leg had no crash wiring at all: when the suite died mid-run the + # only artifact was a short screenshot directory, which is indistinguishable + # from a hang. core_pattern is global to the host kernel rather than per + # namespace, but the kernel writes the file in the CRASHING process's mount + # namespace -- so an absolute path under /cn1 lands inside the container, and + # /cn1 is the bind-mounted workspace, which means the core is uploaded with + # everything else. + - name: Enable core dumps for the container + run: | + mkdir -p artifacts/linux-port/raw-musl + sudo sysctl -w kernel.core_pattern='/cn1/artifacts/linux-port/raw-musl/core.%e.%p' || true + - name: Translate + build + run the suite in Alpine (musl, end-to-end) run: | - docker run --rm \ + docker run --rm --ulimit core=-1 \ -v "$GITHUB_WORKSPACE":/cn1 -w /cn1 \ -e CN1_SHOT_OUTPUT_DIR=/cn1/artifacts/linux-port/raw-musl \ -e CN1_APP_LOG_TEE=/cn1/artifacts/linux-port/raw-musl/app-output.log \ @@ -381,7 +411,7 @@ jobs: -e LIBGL_ALWAYS_SOFTWARE=1 \ docker.io/library/alpine:3.20 sh -ec ' sed -i "s|^#\(.*/community\)|\1|" /etc/apk/repositories - apk add --no-cache build-base cmake samurai pkgconf bash git openjdk8 openjdk17 maven \ + apk add --no-cache build-base cmake samurai pkgconf bash git openjdk8 openjdk17 maven gdb \ gtk+3.0-dev cairo-dev pango-dev gdk-pixbuf-dev glib-dev fontconfig-dev freetype-dev \ curl-dev openssl-dev libepoxy-dev mesa-dev mesa-gles mesa-egl mesa-gbm mesa-dri-gallium \ webkit2gtk-4.1-dev gstreamer-dev gst-plugins-base-dev \ @@ -400,9 +430,31 @@ jobs: # core + Linux port (musl JDK8); the default cc on Alpine already links musl. cd /cn1/maven && mvn -B -pl linux -am -DskipTests -Dmaven.javadoc.skip=true -Plocal-dev-javase install cd /cn1/vm && mvn -B clean package -pl JavaAPI -am -DskipTests + ulimit -c unlimited || true + rc=0 mvn -B test -pl tests -am \ -Dtest=CleanTargetLinuxIntegrationTest#capturesHelloSuiteOverWebSocketLinux \ - -Dsurefire.failIfNoSpecifiedTests=false + -Dsurefire.failIfNoSpecifiedTests=false || rc=$? + # Keep the UNSTRIPPED binary next to any core: without it a core names + # addresses and nothing else, and the container is gone by the time the + # artifact is looked at. + out=/cn1/artifacts/linux-port/raw-musl + mkdir -p "$out" + elf="$(find /tmp -maxdepth 4 -name LinuxHelloMain -type f 2>/dev/null | head -1)" + if [ -n "$elf" ]; then cp "$elf" "$out/LinuxHelloMain" || true; fi + for core in "$out"/core.*; do + [ -f "$core" ] || continue + { + echo "=== post-mortem of $core (elf=$elf) ===" + gdb "$elf" "$core" -batch -ex "set pagination off" \ + -ex "thread apply all bt" -ex "thread 1" -ex "bt full" 2>&1 + } >> "$out/crash-stacks.txt" || true + done + if [ ! -f "$out/crash-stacks.txt" ]; then + echo "no core file was written -- the suite stopped without a fatal signal" \ + > "$out/crash-stacks.txt" + fi + exit $rc ' - name: Upload musl screenshots if: always() diff --git a/vm/JavaAPI/src/java/util/ArrayList.java b/vm/JavaAPI/src/java/util/ArrayList.java index d15e8f0a6f1..8774c398204 100644 --- a/vm/JavaAPI/src/java/util/ArrayList.java +++ b/vm/JavaAPI/src/java/util/ArrayList.java @@ -38,47 +38,29 @@ public class ArrayList extends AbstractList implements List, RandomAcce /** * Constructs a new instance of {@code ArrayList} with ten capacity. */ - /** - * A shared zero-length array for a list created at the default capacity. - * - * A list that is never added to keeps this and allocates nothing: the eager - * {@code new Object[10]} that used to happen in this constructor cost a - * 128-byte slot for every empty list, and the translator alone builds hundreds - * of thousands of them. Sharing one instance is safe because the array is only - * ever REPLACED (by the grow methods), never written through. - * - * Its identity is what marks the "still at default capacity" state, so it must - * not be merged with any other empty array -- see allocateDefaultCapacity. - */ - private static final Object[] DEFAULT_EMPTY_ARRAY = new Object[0]; - - /** - * Capacity the first growth allocates when the list was created at the default. - * - * Ten, not the twelve the general growth path would pick, because ten keeps a - * small list inside the same allocation size class it occupied when the array - * was allocated eagerly. Growing straight to twelve would have made every list - * of one to ten elements LARGER than before, trading a win on empty lists for a - * loss on the common case. - */ - private static final int DEFAULT_CAPACITY = 10; - + // ISOLATION (PR #5766): the lazy default-capacity allocation that used to sit + // here is withdrawn. It replaced the eager new Object[10] with a SHARED static + // zero-length array, which also gave java.util.ArrayList a it had + // never had -- master's only static is a compile-time serialVersionUID, so the + // class previously emitted no static initializer at all. + // + // The suite then began stopping after exactly 145 of 166 screenshots on every + // target except glibc-x64, with ArrayList state corrupt at the point of + // failure: AIOOBE 89 inside pendingIdleSerialCalls.add, then AIOOBE -1, then a + // NullPointerException inside ArrayList.get, which only happens when the + // backing array reference itself is null. + // + // The list logic is NOT at fault: a differential fuzz of this exact source + // against java.util.ArrayList ran 3000 seeds x 200 random operations with no + // divergence, and every access to the corrupted list in Display is inside + // synchronized(lock). The corruption is therefore below Java, which makes the + // new and the process-wide shared array the part worth removing + // before anything subtler is blamed. + // + // The iterator below is the change that carried the measured win (iteration + // 25.5% -> 12.4% of mutator self-time) and is kept. public ArrayList() { - firstIndex = size = 0; - array = (E[]) DEFAULT_EMPTY_ARRAY; - } - - /** - * Replaces the shared empty array with a real one on the first growth. - * - * Called at the top of each grow method, which then proceeds exactly as it - * always did against a normally-sized array. - */ - private void allocateDefaultCapacity(int required) { - if (array == DEFAULT_EMPTY_ARRAY) { - array = newElementArray(required > DEFAULT_CAPACITY ? required : DEFAULT_CAPACITY); - firstIndex = 0; - } + this(10); } public ArrayList(E... arr) { @@ -444,7 +426,6 @@ public E get(int location) { } private void growAtEnd(int required) { - allocateDefaultCapacity(required); if (array.length - size >= required) { // REVIEW: as growAtEnd, why not move size == 0 out as // special case @@ -476,7 +457,6 @@ private void growAtEnd(int required) { } private void growAtFront(int required) { - allocateDefaultCapacity(required); if (array.length - size >= required) { int newFirst = array.length - size; // REVIEW: as growAtEnd, why not move size == 0 out as @@ -506,7 +486,6 @@ private void growAtFront(int required) { } private void growForInsert(int location, int required) { - allocateDefaultCapacity(required); // REVIEW: we grow too quickly because we are called with the // size of the new collection to add without taking in // to account the free space we already have diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java index 5689af3ccc9..f4b4ea8149c 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java @@ -637,11 +637,38 @@ private static int runGdbAttach(java.io.File out, String pid, boolean viaSudo) t cmd.add("set pagination off"); cmd.add("-ex"); cmd.add("thread apply all bt"); - Process gdb = new ProcessBuilder(cmd) - .redirectErrorStream(true) - .redirectOutput(ProcessBuilder.Redirect.appendTo(out)) - .start(); - return gdb.waitFor(); + try { + Process gdb = new ProcessBuilder(cmd) + .redirectErrorStream(true) + .redirectOutput(ProcessBuilder.Redirect.appendTo(out)) + .start(); + int rc = gdb.waitFor(); + if (rc != 0) { + note(out, "gdb " + (viaSudo ? "(sudo) " : "") + "exited " + rc + + " -- no stacks from this sample"); + } + return rc; + } catch (java.io.IOException notInstalled) { + // An absent gdb used to throw here and be swallowed by the caller's + // catch-all, leaving hang-stacks.txt holding nothing but its sample + // headers. That reads as "we looked and the process was fine", which is + // the opposite of what happened, and it cost a full CI round to notice + // the file was empty rather than uninformative. Say so in the file. + note(out, "gdb " + (viaSudo ? "(sudo) " : "") + "could not be started: " + + notInstalled + " -- install gdb on this runner to get stacks"); + return -1; + } + } + + /// Appends one diagnostic line to the dump file, so a failure to collect + /// evidence is itself recorded as evidence. + private static void note(java.io.File out, String msg) { + try (java.io.PrintWriter w = new java.io.PrintWriter( + new java.io.FileWriter(out, true), true)) { + w.println(" !! " + msg); + } catch (java.io.IOException ignore) { + // nothing further we can do from a diagnostic path + } } /// Dumps every thread's native stack from the still-running suite process. @@ -682,8 +709,11 @@ private static void dumpLiveThreadStacks() { } System.out.println("CN1SS:HARNESS: wrote live thread stacks for pid " + pid.trim() + " to " + out); - } catch (Exception ignore) { - // A missing gdb or a denied ptrace must not mask the real failure. + } catch (Exception e) { + // A missing gdb or a denied ptrace must not mask the real failure -- but + // it must not be invisible either, or an empty dump file gets read as a + // clean sample. Print it; the harness output is captured in the job log. + System.out.println("CN1SS:HARNESS: live stack dump failed: " + e); } } From 2ad038a92872a4ee107c5604a7fd86cf4d2102d4 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Fri, 11 Sep 2026 09:21:36 +0300 Subject: [PATCH 21/66] Root cause: the fused concat natives allocate with the source strings unrooted The core dump the previous commit's wiring collected names the fault exactly: #0 cn1_set_array_element_int cn1_globals.h:2537 ((JAVA_ARRAY_INT*)(*(JAVA_ARRAY)array).data)[index] = value; #1 com_codename1_ui_Display_edtLoopImpl__ Display.c:4072 #2 com_codename1_ui_Display_mainEDTLoop__ Display.c:4072 is Display.java:1813, actualStack[actualStack.length - 1] = Integer.MAX_VALUE; which is also the line the Windows run reported as AIOOBE -1. Index -1 means actualStack.length read 0. That array is inputEventStackTmp, which is new int[1000] and is only ever swapped with another int[1000] or a new int[qt.length] -- a zero length is unreachable in correct Java, so the array header itself was clobbered. The -1 write is the consequence, not the cause, and on Linux -O3 (unchecked stores) it segfaults instead of throwing. The cause is in this PR's own fused concat natives. cn1FusedConcat2..5 take raw interior pointers into the source Strings' byte[]s, and hold those plus the JAVA_OBJECT arguments in plain C locals, ACROSS cn1FusedLatin1Begin -- which allocates, and therefore can collect. None of those are roots: CN1_CONSERVATIVE_GC_ROOTS is defined by no build in this tree (it survives only in one comment), so the native stack is not scanned and enteringNativeAllocations() is live rather than a no-op. A collection inside that allocation can reclaim the byte[]s the copy loop then reads, and the freed block can be handed to another allocation while the loop is still walking it -- which is how an unrelated int[] ends up with a zeroed header. Every other native allocation path in this file brackets for exactly this reason, including cn1ConcatFallback and newStringFromAsciiLen two functions above. The fused concat was the one that did not. The bracket now spans the whole body, since the references are held across both the fused attempt and the fallback, and cn1ConcatFallback no longer sets it itself -- doing so would clear it on return while its caller still holds those pointers. How the alternatives were eliminated rather than assumed: the ArrayList logic is correct (3000 seeds x 200 random operations differentially fuzzed against java.util.ArrayList, no divergence); there is no Java-level race (every access to the corrupted list and to the event stack is inside synchronized(lock)); the GC's lifetime rules are unchanged by this PR (433 added lines in cn1_globals.m are census behind CN1_ALLOC_CENSUS, the 3 removed lines are pacing); the classId == -1 primitive-class sentinel only reaches an array subscript under DEBUG_GC_OBJECTS_IN_HEAP; and withdrawing the lazy ArrayList allocation in the previous commit did not fix it. Gates D and A stay byte-identical over 795 files -- a corpus that builds strings constantly -- with the negative control still detecting an injected corruption. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/nativeMethods.m | 75 +++++++++++++++++++++-- 1 file changed, 69 insertions(+), 6 deletions(-) diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 4b7f8d1259b..0b0314a58b1 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -1766,8 +1766,12 @@ JAVA_OBJECT java_lang_Long_toString___long_int_R_java_lang_String(CODENAME_ONE_T #define CN1_SB_PTR(s) ((JAVA_ARRAY_BYTE*)((JAVA_ARRAY)((struct obj__java_lang_String*)(s))->java_lang_String_value)->data + ((struct obj__java_lang_String*)(s))->java_lang_String_offset) #define CN1_SB_LEN(s) (((struct obj__java_lang_String*)(s))->java_lang_String_count) +// The caller owns the enteringNativeAllocations() bracket: it holds the source +// Strings and raw interior pointers into their byte[]s across BOTH this call and +// the fused attempt before it, so the protected region has to start there, not +// here. Setting it again here would clear it on return while the caller is still +// holding those references. static JAVA_OBJECT cn1ConcatFallback(CODENAME_ONE_THREAD_STATE, JAVA_ARRAY_BYTE* const* parts, const int* lens, int n, int total) { - enteringNativeAllocations(); JAVA_ARRAY dat = (JAVA_ARRAY)allocArray(threadStateData, total, &class_array1__JAVA_BYTE, sizeof(JAVA_ARRAY_BYTE), 1); JAVA_ARRAY_BYTE* d = (JAVA_ARRAY_BYTE*) (*dat).data; int o = 0; @@ -1777,11 +1781,20 @@ static JAVA_OBJECT cn1ConcatFallback(CODENAME_ONE_THREAD_STATE, JAVA_ARRAY_BYTE* struct obj__java_lang_String* ss = (struct obj__java_lang_String*)so; ss->java_lang_String_value = (JAVA_OBJECT)dat; ss->java_lang_String_count = total; - finishedNativeAllocations(); return so; } JAVA_OBJECT java_lang_String_cn1FusedConcat2___java_lang_String_java_lang_String_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT a, JAVA_OBJECT b) { + enteringNativeAllocations(); + // GC SAFETY: p[] are raw interior pointers into the source Strings' byte[]s, + // and a, b, ... are plain C locals. cn1FusedLatin1Begin ALLOCATES, and this + // build has precise roots -- CN1_CONSERVATIVE_GC_ROOTS is defined by no build + // in the tree, so the native stack is NOT scanned and none of those are roots. + // A collection during that allocation could therefore reclaim the very byte[]s + // the copy below reads, and the freed block can be handed to another + // allocation while this loop is still walking it. Every other native + // allocation path in this file brackets for exactly this reason; the fused + // concat was the one that did not. JAVA_ARRAY_BYTE* p[2] = { CN1_SB_PTR(a), CN1_SB_PTR(b) }; int l[2] = { CN1_SB_LEN(a), CN1_SB_LEN(b) }; int total = l[0] + l[1]; @@ -1791,12 +1804,27 @@ JAVA_OBJECT java_lang_String_cn1FusedConcat2___java_lang_String_java_lang_String int o = 0; for(int q = 0 ; q < 2 ; q++) { for(int i = 0 ; i < l[q] ; i++) dst[o + i] = p[q][i]; o += l[q]; } cn1FusedLatin1End(so, total); + finishedNativeAllocations(); return so; } - return cn1ConcatFallback(threadStateData, p, l, 2, total); + { + JAVA_OBJECT fb = cn1ConcatFallback(threadStateData, p, l, 2, total); + finishedNativeAllocations(); + return fb; + } } JAVA_OBJECT java_lang_String_cn1FusedConcat3___java_lang_String_java_lang_String_java_lang_String_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT a, JAVA_OBJECT b, JAVA_OBJECT c) { + enteringNativeAllocations(); + // GC SAFETY: p[] are raw interior pointers into the source Strings' byte[]s, + // and a, b, ... are plain C locals. cn1FusedLatin1Begin ALLOCATES, and this + // build has precise roots -- CN1_CONSERVATIVE_GC_ROOTS is defined by no build + // in the tree, so the native stack is NOT scanned and none of those are roots. + // A collection during that allocation could therefore reclaim the very byte[]s + // the copy below reads, and the freed block can be handed to another + // allocation while this loop is still walking it. Every other native + // allocation path in this file brackets for exactly this reason; the fused + // concat was the one that did not. JAVA_ARRAY_BYTE* p[3] = { CN1_SB_PTR(a), CN1_SB_PTR(b), CN1_SB_PTR(c) }; int l[3] = { CN1_SB_LEN(a), CN1_SB_LEN(b), CN1_SB_LEN(c) }; int total = l[0] + l[1] + l[2]; @@ -1806,12 +1834,27 @@ JAVA_OBJECT java_lang_String_cn1FusedConcat3___java_lang_String_java_lang_String int o = 0; for(int q = 0 ; q < 3 ; q++) { for(int i = 0 ; i < l[q] ; i++) dst[o + i] = p[q][i]; o += l[q]; } cn1FusedLatin1End(so, total); + finishedNativeAllocations(); return so; } - return cn1ConcatFallback(threadStateData, p, l, 3, total); + { + JAVA_OBJECT fb = cn1ConcatFallback(threadStateData, p, l, 3, total); + finishedNativeAllocations(); + return fb; + } } JAVA_OBJECT java_lang_String_cn1FusedConcat4___java_lang_String_java_lang_String_java_lang_String_java_lang_String_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT a, JAVA_OBJECT b, JAVA_OBJECT c, JAVA_OBJECT d) { + enteringNativeAllocations(); + // GC SAFETY: p[] are raw interior pointers into the source Strings' byte[]s, + // and a, b, ... are plain C locals. cn1FusedLatin1Begin ALLOCATES, and this + // build has precise roots -- CN1_CONSERVATIVE_GC_ROOTS is defined by no build + // in the tree, so the native stack is NOT scanned and none of those are roots. + // A collection during that allocation could therefore reclaim the very byte[]s + // the copy below reads, and the freed block can be handed to another + // allocation while this loop is still walking it. Every other native + // allocation path in this file brackets for exactly this reason; the fused + // concat was the one that did not. JAVA_ARRAY_BYTE* p[4] = { CN1_SB_PTR(a), CN1_SB_PTR(b), CN1_SB_PTR(c), CN1_SB_PTR(d) }; int l[4] = { CN1_SB_LEN(a), CN1_SB_LEN(b), CN1_SB_LEN(c), CN1_SB_LEN(d) }; int total = l[0] + l[1] + l[2] + l[3]; @@ -1821,12 +1864,27 @@ JAVA_OBJECT java_lang_String_cn1FusedConcat4___java_lang_String_java_lang_String int o = 0; for(int q = 0 ; q < 4 ; q++) { for(int i = 0 ; i < l[q] ; i++) dst[o + i] = p[q][i]; o += l[q]; } cn1FusedLatin1End(so, total); + finishedNativeAllocations(); return so; } - return cn1ConcatFallback(threadStateData, p, l, 4, total); + { + JAVA_OBJECT fb = cn1ConcatFallback(threadStateData, p, l, 4, total); + finishedNativeAllocations(); + return fb; + } } JAVA_OBJECT java_lang_String_cn1FusedConcat5___java_lang_String_java_lang_String_java_lang_String_java_lang_String_java_lang_String_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT a, JAVA_OBJECT b, JAVA_OBJECT c, JAVA_OBJECT d, JAVA_OBJECT e) { + enteringNativeAllocations(); + // GC SAFETY: p[] are raw interior pointers into the source Strings' byte[]s, + // and a, b, ... are plain C locals. cn1FusedLatin1Begin ALLOCATES, and this + // build has precise roots -- CN1_CONSERVATIVE_GC_ROOTS is defined by no build + // in the tree, so the native stack is NOT scanned and none of those are roots. + // A collection during that allocation could therefore reclaim the very byte[]s + // the copy below reads, and the freed block can be handed to another + // allocation while this loop is still walking it. Every other native + // allocation path in this file brackets for exactly this reason; the fused + // concat was the one that did not. JAVA_ARRAY_BYTE* p[5] = { CN1_SB_PTR(a), CN1_SB_PTR(b), CN1_SB_PTR(c), CN1_SB_PTR(d), CN1_SB_PTR(e) }; int l[5] = { CN1_SB_LEN(a), CN1_SB_LEN(b), CN1_SB_LEN(c), CN1_SB_LEN(d), CN1_SB_LEN(e) }; int total = l[0] + l[1] + l[2] + l[3] + l[4]; @@ -1836,9 +1894,14 @@ JAVA_OBJECT java_lang_String_cn1FusedConcat5___java_lang_String_java_lang_String int o = 0; for(int q = 0 ; q < 5 ; q++) { for(int i = 0 ; i < l[q] ; i++) dst[o + i] = p[q][i]; o += l[q]; } cn1FusedLatin1End(so, total); + finishedNativeAllocations(); return so; } - return cn1ConcatFallback(threadStateData, p, l, 5, total); + { + JAVA_OBJECT fb = cn1ConcatFallback(threadStateData, p, l, 5, total); + finishedNativeAllocations(); + return fb; + } } JAVA_DOUBLE java_lang_Math_cos___double_R_double(CODENAME_ONE_THREAD_STATE, JAVA_DOUBLE a) { From a5ec1879dfd579ce2a1968a53787990ada43f7bd Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Fri, 11 Sep 2026 11:17:23 +0300 Subject: [PATCH 22/66] Revert the fused-concat GC bracket: the premise was wrong 2ad038a claimed the fused concat natives allocate with their source Strings unrooted, on the grounds that CN1_CONSERVATIVE_GC_ROOTS "is defined by no build in this tree". That is false. cn1_globals.h:72 does #ifndef CN1_DISABLE_CONSERVATIVE_GC_ROOTS #define CN1_CONSERVATIVE_GC_ROOTS #endif so conservative roots are ON by default, exactly as vm/CLAUDE.md says in its first paragraph on tagged immediates. I reached the opposite conclusion by grepping for the symbol with .h files EXCLUDED, which filtered out the one line that defines it. With conservative roots the native stack is scanned, so a, b and the interior pointers in p[] are roots like any other local, and enteringNativeAllocations() expands to do {} while(0). The commit was therefore a no-op that compiled to nothing, and its comments asserted the opposite of what the build does -- worse than useless in a file where the next reader has no PR thread to check it against. Measured rather than argued: a torture driving all four fused arities (cn1Concat2..5 all present in the generated C, ~400k calls per round) while re-verifying 256 live int[1000] arrays element by element reports corrupt=0 and the HotSpot checksum, identically with and without the bracket. So the 145-screenshot stop is still unexplained, and the suspect list is back to what it was before 2ad038a. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/nativeMethods.m | 75 ++--------------------- 1 file changed, 6 insertions(+), 69 deletions(-) diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 0b0314a58b1..4b7f8d1259b 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -1766,12 +1766,8 @@ JAVA_OBJECT java_lang_Long_toString___long_int_R_java_lang_String(CODENAME_ONE_T #define CN1_SB_PTR(s) ((JAVA_ARRAY_BYTE*)((JAVA_ARRAY)((struct obj__java_lang_String*)(s))->java_lang_String_value)->data + ((struct obj__java_lang_String*)(s))->java_lang_String_offset) #define CN1_SB_LEN(s) (((struct obj__java_lang_String*)(s))->java_lang_String_count) -// The caller owns the enteringNativeAllocations() bracket: it holds the source -// Strings and raw interior pointers into their byte[]s across BOTH this call and -// the fused attempt before it, so the protected region has to start there, not -// here. Setting it again here would clear it on return while the caller is still -// holding those references. static JAVA_OBJECT cn1ConcatFallback(CODENAME_ONE_THREAD_STATE, JAVA_ARRAY_BYTE* const* parts, const int* lens, int n, int total) { + enteringNativeAllocations(); JAVA_ARRAY dat = (JAVA_ARRAY)allocArray(threadStateData, total, &class_array1__JAVA_BYTE, sizeof(JAVA_ARRAY_BYTE), 1); JAVA_ARRAY_BYTE* d = (JAVA_ARRAY_BYTE*) (*dat).data; int o = 0; @@ -1781,20 +1777,11 @@ static JAVA_OBJECT cn1ConcatFallback(CODENAME_ONE_THREAD_STATE, JAVA_ARRAY_BYTE* struct obj__java_lang_String* ss = (struct obj__java_lang_String*)so; ss->java_lang_String_value = (JAVA_OBJECT)dat; ss->java_lang_String_count = total; + finishedNativeAllocations(); return so; } JAVA_OBJECT java_lang_String_cn1FusedConcat2___java_lang_String_java_lang_String_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT a, JAVA_OBJECT b) { - enteringNativeAllocations(); - // GC SAFETY: p[] are raw interior pointers into the source Strings' byte[]s, - // and a, b, ... are plain C locals. cn1FusedLatin1Begin ALLOCATES, and this - // build has precise roots -- CN1_CONSERVATIVE_GC_ROOTS is defined by no build - // in the tree, so the native stack is NOT scanned and none of those are roots. - // A collection during that allocation could therefore reclaim the very byte[]s - // the copy below reads, and the freed block can be handed to another - // allocation while this loop is still walking it. Every other native - // allocation path in this file brackets for exactly this reason; the fused - // concat was the one that did not. JAVA_ARRAY_BYTE* p[2] = { CN1_SB_PTR(a), CN1_SB_PTR(b) }; int l[2] = { CN1_SB_LEN(a), CN1_SB_LEN(b) }; int total = l[0] + l[1]; @@ -1804,27 +1791,12 @@ JAVA_OBJECT java_lang_String_cn1FusedConcat2___java_lang_String_java_lang_String int o = 0; for(int q = 0 ; q < 2 ; q++) { for(int i = 0 ; i < l[q] ; i++) dst[o + i] = p[q][i]; o += l[q]; } cn1FusedLatin1End(so, total); - finishedNativeAllocations(); return so; } - { - JAVA_OBJECT fb = cn1ConcatFallback(threadStateData, p, l, 2, total); - finishedNativeAllocations(); - return fb; - } + return cn1ConcatFallback(threadStateData, p, l, 2, total); } JAVA_OBJECT java_lang_String_cn1FusedConcat3___java_lang_String_java_lang_String_java_lang_String_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT a, JAVA_OBJECT b, JAVA_OBJECT c) { - enteringNativeAllocations(); - // GC SAFETY: p[] are raw interior pointers into the source Strings' byte[]s, - // and a, b, ... are plain C locals. cn1FusedLatin1Begin ALLOCATES, and this - // build has precise roots -- CN1_CONSERVATIVE_GC_ROOTS is defined by no build - // in the tree, so the native stack is NOT scanned and none of those are roots. - // A collection during that allocation could therefore reclaim the very byte[]s - // the copy below reads, and the freed block can be handed to another - // allocation while this loop is still walking it. Every other native - // allocation path in this file brackets for exactly this reason; the fused - // concat was the one that did not. JAVA_ARRAY_BYTE* p[3] = { CN1_SB_PTR(a), CN1_SB_PTR(b), CN1_SB_PTR(c) }; int l[3] = { CN1_SB_LEN(a), CN1_SB_LEN(b), CN1_SB_LEN(c) }; int total = l[0] + l[1] + l[2]; @@ -1834,27 +1806,12 @@ JAVA_OBJECT java_lang_String_cn1FusedConcat3___java_lang_String_java_lang_String int o = 0; for(int q = 0 ; q < 3 ; q++) { for(int i = 0 ; i < l[q] ; i++) dst[o + i] = p[q][i]; o += l[q]; } cn1FusedLatin1End(so, total); - finishedNativeAllocations(); return so; } - { - JAVA_OBJECT fb = cn1ConcatFallback(threadStateData, p, l, 3, total); - finishedNativeAllocations(); - return fb; - } + return cn1ConcatFallback(threadStateData, p, l, 3, total); } JAVA_OBJECT java_lang_String_cn1FusedConcat4___java_lang_String_java_lang_String_java_lang_String_java_lang_String_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT a, JAVA_OBJECT b, JAVA_OBJECT c, JAVA_OBJECT d) { - enteringNativeAllocations(); - // GC SAFETY: p[] are raw interior pointers into the source Strings' byte[]s, - // and a, b, ... are plain C locals. cn1FusedLatin1Begin ALLOCATES, and this - // build has precise roots -- CN1_CONSERVATIVE_GC_ROOTS is defined by no build - // in the tree, so the native stack is NOT scanned and none of those are roots. - // A collection during that allocation could therefore reclaim the very byte[]s - // the copy below reads, and the freed block can be handed to another - // allocation while this loop is still walking it. Every other native - // allocation path in this file brackets for exactly this reason; the fused - // concat was the one that did not. JAVA_ARRAY_BYTE* p[4] = { CN1_SB_PTR(a), CN1_SB_PTR(b), CN1_SB_PTR(c), CN1_SB_PTR(d) }; int l[4] = { CN1_SB_LEN(a), CN1_SB_LEN(b), CN1_SB_LEN(c), CN1_SB_LEN(d) }; int total = l[0] + l[1] + l[2] + l[3]; @@ -1864,27 +1821,12 @@ JAVA_OBJECT java_lang_String_cn1FusedConcat4___java_lang_String_java_lang_String int o = 0; for(int q = 0 ; q < 4 ; q++) { for(int i = 0 ; i < l[q] ; i++) dst[o + i] = p[q][i]; o += l[q]; } cn1FusedLatin1End(so, total); - finishedNativeAllocations(); return so; } - { - JAVA_OBJECT fb = cn1ConcatFallback(threadStateData, p, l, 4, total); - finishedNativeAllocations(); - return fb; - } + return cn1ConcatFallback(threadStateData, p, l, 4, total); } JAVA_OBJECT java_lang_String_cn1FusedConcat5___java_lang_String_java_lang_String_java_lang_String_java_lang_String_java_lang_String_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT a, JAVA_OBJECT b, JAVA_OBJECT c, JAVA_OBJECT d, JAVA_OBJECT e) { - enteringNativeAllocations(); - // GC SAFETY: p[] are raw interior pointers into the source Strings' byte[]s, - // and a, b, ... are plain C locals. cn1FusedLatin1Begin ALLOCATES, and this - // build has precise roots -- CN1_CONSERVATIVE_GC_ROOTS is defined by no build - // in the tree, so the native stack is NOT scanned and none of those are roots. - // A collection during that allocation could therefore reclaim the very byte[]s - // the copy below reads, and the freed block can be handed to another - // allocation while this loop is still walking it. Every other native - // allocation path in this file brackets for exactly this reason; the fused - // concat was the one that did not. JAVA_ARRAY_BYTE* p[5] = { CN1_SB_PTR(a), CN1_SB_PTR(b), CN1_SB_PTR(c), CN1_SB_PTR(d), CN1_SB_PTR(e) }; int l[5] = { CN1_SB_LEN(a), CN1_SB_LEN(b), CN1_SB_LEN(c), CN1_SB_LEN(d), CN1_SB_LEN(e) }; int total = l[0] + l[1] + l[2] + l[3] + l[4]; @@ -1894,14 +1836,9 @@ JAVA_OBJECT java_lang_String_cn1FusedConcat5___java_lang_String_java_lang_String int o = 0; for(int q = 0 ; q < 5 ; q++) { for(int i = 0 ; i < l[q] ; i++) dst[o + i] = p[q][i]; o += l[q]; } cn1FusedLatin1End(so, total); - finishedNativeAllocations(); return so; } - { - JAVA_OBJECT fb = cn1ConcatFallback(threadStateData, p, l, 5, total); - finishedNativeAllocations(); - return fb; - } + return cn1ConcatFallback(threadStateData, p, l, 5, total); } JAVA_DOUBLE java_lang_Math_cos___double_R_double(CODENAME_ONE_THREAD_STATE, JAVA_DOUBLE a) { From 7d1b503d0cca8b1b21605d5039da7f15ec859792 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Fri, 11 Sep 2026 13:54:01 +0300 Subject: [PATCH 23/66] BISECT PROBE: turn off both bytecode-rewriting passes Not a fix. The concat fusion and the iterator lowering are the only changes in this PR that rewrite bytecode, and the failure is heap corruption: an int[] whose header reads length 0, which then takes an unchecked [-1] store in Display.edtLoopImpl and segfaults at cn1_set_array_element_int under -O3. A wrong stack depth or a wrong receiver type out of a rewrite is the most plausible way to produce that, so this splits the suspect space -- green means the cause is an emitted-code rewrite, red means it is in the JavaAPI or the C runtime. Both passes get restored once the answer is in. Reaching for this because the cheap local instruments are exhausted and came back clean: - run-gauntlet.sh GREEN (MapTorture, IdmTorture, HtTorture, SbTorture, StrCmp, FusedTest, IbpTest, ExcTest, ThreadChurn, SoeTest, TaggedSync, BoxEdge, GcStress+MtStress on both stop modes). - run-gc-verify.sh GREEN, with all three injected-fault self-tests still firing. One full-gate run reported ThreadChurn as a VACUOUS failure -- 0 verify passes, the workload completing no GC cycle under load -- and it is clean with 2 verify passes when run alone, so that was the gate correctly refusing to score a run that proved nothing, not a finding. - A torture driving all four fused arities (~400k calls per round, cn1Concat2..5 all confirmed present in the generated C) while re-verifying 256 live int[1000] arrays element by element: corrupt=0. - Self-hosting gates D and A byte-identical over 795 files, vm/tests 562 passed. A local macOS reproduction is not available: build-macos-app.sh needs the workspace toolchain, and tools/env.sh points into a TMPDIR that has been reaped. setup-workspace.sh would rebuild it, but under a shared $TMPDIR/codenameone-tools path that the sibling checkouts also use, so it is not safe to run from here. Worth noting separately that build-macos-app.sh exits 0 when it finds neither Xcode nor the toolchain -- a build step that reports success having done nothing. Co-Authored-By: Claude Opus 5 (1M context) --- .../com/codename1/tools/translator/Parser.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java index d5f84574329..0fcbab9952d 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java @@ -846,8 +846,20 @@ public static void writeOutput(File outputDirectory) throws Exception { if (BytecodeMethod.optimizerOn) { for (ByteCodeClass fuseCls : classes) { for (BytecodeMethod fuseMtd : fuseCls.getMethods()) { - fuseMtd.fuseStringBuilderConcat(); - fuseMtd.lowerIteratorCalls(); + // BISECT PROBE (PR #5766, not a fix): both passes are off. + // These are the only changes in this PR that REWRITE bytecode, + // and the failure under investigation is heap corruption -- an + // int[] header reading length 0, which then takes an unchecked + // [-1] store in Display.edtLoopImpl. Wrong stack depth or a + // wrong receiver type from a rewrite is the most plausible + // source of that, so turning both off splits the suspect space + // in half: green means the cause is an emitted-code rewrite, + // red means it is in the JavaAPI or the C runtime instead. + // Restore both once the answer is in. + if (false) { + fuseMtd.fuseStringBuilderConcat(); + fuseMtd.lowerIteratorCalls(); + } } } } From 718aa7b19b49d14f65589dd6ffb11b9383707c3b Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:22:42 +0300 Subject: [PATCH 24/66] BISECT PROBE 2: revert the whole runtime side to the merge base Not a fix, and it stacks on probe 1 (both bytecode-rewriting passes are still off). vm/JavaAPI, cn1_globals.{m,h} and nativeMethods.m go back to 0cd4328. What remains of the PR is the translator plumbing, the self-hosting harness and the CI work. Probe 1 answered only half a question. build-ios-tv went from pass=6/fail=117 to fully green with the rewrites off, but every Linux and Windows leg failed again with the SAME signature as before -- pngs=145, stopping in AccessibilityTest (x64, musl) or MutableImageReadbackTest (arm64). An unchanged signature under a changed input means the rewrites are not what those legs are dying on. That the stop point is fixed at 145 while the AIOOBE index varies (69 on Windows, 89 on Linux arm64) says the corruption itself is nondeterministic but its trigger is not: 145 is simply where AccessibilityTest sits in the run order. If this probe is green the cause is in the reverted runtime, and the remaining candidates are small enough to bisect one at a time: the primitive TYPE statics added to nine wrapper classes (Boolean, Short and Float gain the field outright, and vm/CLAUDE.md notes Byte and Boolean are deliberately NOT tagged), the IdentityHashMap iterator inlining, the StringBuilder growth change, the String.equals/compareTo fast path, and the pacing edits. If it is red the cause is in the translator plumbing that this probe keeps. Recorded so it is not re-litigated: master is green 10/10 on the Windows leg, so that leg is a clean signal; the Linux leg has its own pre-existing flake on master (5 failures in 20 runs) whose signature differs run to run (pngs=136, NativeMapFallbackScreenshotTest), which is NOT this. JavaAPI and the translator both compile at 0 errors after the revert. The self-hosting gates cannot run in this state -- Util's ctype map needs the primitive TYPE statics -- and that job is label-gated and not running on this PR anyway. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.h | 32 +- vm/ByteCodeTranslator/src/cn1_globals.m | 436 +----------------- vm/ByteCodeTranslator/src/nativeMethods.m | 135 +----- vm/JavaAPI/src/java/lang/Boolean.java | 6 - vm/JavaAPI/src/java/lang/Byte.java | 2 +- vm/JavaAPI/src/java/lang/Character.java | 2 +- vm/JavaAPI/src/java/lang/Class.java | 110 +---- vm/JavaAPI/src/java/lang/Double.java | 10 +- vm/JavaAPI/src/java/lang/Float.java | 20 - vm/JavaAPI/src/java/lang/Integer.java | 14 +- vm/JavaAPI/src/java/lang/Long.java | 2 +- vm/JavaAPI/src/java/lang/Short.java | 6 - vm/JavaAPI/src/java/lang/StringBuilder.java | 19 +- vm/JavaAPI/src/java/lang/Void.java | 2 +- vm/JavaAPI/src/java/util/ArrayList.java | 95 ---- vm/JavaAPI/src/java/util/IdentityHashMap.java | 102 ++-- 16 files changed, 51 insertions(+), 942 deletions(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index 6813f4ea164..b12ce93136d 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -2185,7 +2185,7 @@ static inline JAVA_OBJECT cn1BibopFastAllocNoZero(CODENAME_ONE_THREAD_STATE, int // because bibopCurrent[] is shared across all classes of the same size class). #if !defined(CN1_DISABLE_INLINE_ALLOC) && !defined(CN1_DISABLE_BIBOP) #define CN1_FAST_NEW(X) ({ \ - if(__builtin_expect(!__atomic_load_n(&class__##X.initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_##X(threadStateData); \ + if(__builtin_expect(!class__##X.initialized, 0)) __STATIC_INITIALIZER_##X(threadStateData); \ JAVA_OBJECT __cn1fo = cn1BibopFastAlloc(threadStateData, sizeof(struct obj__##X), &class__##X, CN1_BIBOP_CIDX(sizeof(struct obj__##X))); \ if(__builtin_expect(__cn1fo == (JAVA_OBJECT)0, 0)) __cn1fo = __NEW_##X(threadStateData); \ __cn1fo; }) @@ -2193,7 +2193,7 @@ static inline JAVA_OBJECT cn1BibopFastAllocNoZero(CODENAME_ONE_THREAD_STATE, int // still fully zeroes (calloc) -- correct, just un-elided on the rare page-full // path. #define CN1_FAST_NEW_NOZERO(X) ({ \ - if(__builtin_expect(!__atomic_load_n(&class__##X.initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_##X(threadStateData); \ + if(__builtin_expect(!class__##X.initialized, 0)) __STATIC_INITIALIZER_##X(threadStateData); \ JAVA_OBJECT __cn1fo = cn1BibopFastAllocNoZero(threadStateData, sizeof(struct obj__##X), &class__##X, CN1_BIBOP_CIDX(sizeof(struct obj__##X))); \ if(__builtin_expect(__cn1fo == (JAVA_OBJECT)0, 0)) __cn1fo = __NEW_##X(threadStateData); \ __cn1fo; }) @@ -2827,34 +2827,6 @@ extern struct clazz class_array1__JAVA_DOUBLE; extern struct clazz class_array2__JAVA_DOUBLE; extern struct clazz class_array3__JAVA_DOUBLE; -/** - * The nine scalar primitive class objects -- int.class, Integer.TYPE and friends. - * - * javac lowers a primitive class literal to a read of the boxed type's own TYPE - * field, so `TYPE = int.class` inside Integer's initializer compiles to - * `getstatic TYPE; putstatic TYPE` -- it reads the field it is initializing and - * leaves it null. Every wrapper that declared TYPE that way had a null one, and - * a Map keyed on them collapsed to a single entry, so a lookup for int answered - * with whatever type was stored last. Nothing threw. The wrappers now go through - * java_lang_Class_getPrimitiveClass, which hands back one of these. - * - * classId is CN1_PRIMITIVE_CLASS_ID for all nine: these never take part in an - * instanceof, and instanceofFunction indexes tables by classId, so the callers - * that could reach one (isAssignableFrom, isInstance) test primitiveType first - * rather than indexing with a value no table has a row for. - */ -#define CN1_PRIMITIVE_CLASS_ID (-1) - -extern struct clazz cn1_primitive_class_int; -extern struct clazz cn1_primitive_class_long; -extern struct clazz cn1_primitive_class_short; -extern struct clazz cn1_primitive_class_byte; -extern struct clazz cn1_primitive_class_char; -extern struct clazz cn1_primitive_class_float; -extern struct clazz cn1_primitive_class_double; -extern struct clazz cn1_primitive_class_boolean; -extern struct clazz cn1_primitive_class_void; - extern JAVA_OBJECT newString(CODENAME_ONE_THREAD_STATE, int length, JAVA_CHAR data[]); /** * Like newStringFromCString but DECODES, in the PLATFORM's encoding, instead of diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index e16ca1cc202..c4a93106b1a 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -956,54 +956,6 @@ static void init_gc_thresholds() { //#define DEBUG_GC_OBJECTS_IN_HEAP -/** - * Scalar primitive class objects. See the comment on CN1_PRIMITIVE_CLASS_ID in - * cn1_globals.h for why these exist and why their classId is a sentinel. - * - * baseClass is 0 because int.class.getSuperclass() is null, which - * java_lang_Class_getSuperclass already returns for a null baseClass. isArray is - * false and arrayType is 0: these are the scalar types, not the array classes, - * which already exist as class_arrayN__JAVA_*. - * - * Designated initializers, unlike the positional generated ones beside them, so - * that a future field added to struct clazz cannot silently shift every value. - */ -/* - * __codenameOneParentClsReference is the class OF this object. Every generated - * clazz sets it to class__java_lang_Class, and CN1_CLASS_OF reads it to find the - * vtable when a clazz is used as an ordinary object -- which is what happens the - * moment one becomes a Map key. Leaving it zero segfaults on the first - * hashCode(), well away from anything that names it. - * - * The comment sits outside the macro on purpose: backslash-newline splicing - * happens before comments are removed, so an unbackslashed comment line inside - * the macro would silently end the definition. - */ -#define CN1_DEFINE_PRIMITIVE_CLASS(cname, jname) \ -struct clazz cn1_primitive_class_##cname = { \ - .__codenameOneParentClsReference = &class__java_lang_Class, \ - .classId = CN1_PRIMITIVE_CLASS_ID, \ - .clsName = jname, \ - .isArray = JAVA_FALSE, \ - .dimensions = 0, \ - .arrayType = 0, \ - .primitiveType = JAVA_TRUE, \ - .baseClass = 0, \ - .baseInterfaces = EMPTY_INTERFACES, \ - .baseInterfaceCount = 0, \ - .initialized = JAVA_TRUE \ -} - -CN1_DEFINE_PRIMITIVE_CLASS(int, "int"); -CN1_DEFINE_PRIMITIVE_CLASS(long, "long"); -CN1_DEFINE_PRIMITIVE_CLASS(short, "short"); -CN1_DEFINE_PRIMITIVE_CLASS(byte, "byte"); -CN1_DEFINE_PRIMITIVE_CLASS(char, "char"); -CN1_DEFINE_PRIMITIVE_CLASS(float, "float"); -CN1_DEFINE_PRIMITIVE_CLASS(double, "double"); -CN1_DEFINE_PRIMITIVE_CLASS(boolean, "boolean"); -CN1_DEFINE_PRIMITIVE_CLASS(void, "void"); - struct clazz class_array1__JAVA_BOOLEAN = { DEBUG_GC_INIT 0, 0, 0, 0, 0, 0, 0, cn1_array_1_id_JAVA_BOOLEAN, "boolean[]", JAVA_TRUE, 1, &class__java_lang_Boolean, JAVA_TRUE, &class__java_lang_Object, EMPTY_INTERFACES, 0, 0, 0 }; @@ -1791,15 +1743,6 @@ static void cn1DrainDeadThreadPending() { // walks the page registry and its slots before their definitions. static inline JAVA_OBJECT cn1BibopSlot(CN1BibopPage* p, int i); static CN1BibopPage* _Atomic bibopAllPages; -#ifdef CN1_ALLOC_CENSUS -// Defined far below, beside the BiBOP page structures they read. Declared up here -// because the post-sweep hook that calls them is compiled earlier -- and OUTSIDE the -// CN1_GC_VERIFY block just above, which is off in an ordinary census build. -void cn1HeapAccounting(const char* label); -void cn1AllocCensus(const char* label); -void cn1LiveCensus(const char* label); -#endif - #ifdef CN1_GRACE_AUDIT static void cn1GraceAuditPreSweep(CODENAME_ONE_THREAD_STATE); #endif @@ -4729,14 +4672,6 @@ static void cn1GcReportStaleIndexSkip(void) { void codenameOneGCSweep() { struct ThreadLocalData* threadStateData = getThreadLocalData(); -#ifdef CN1_ALLOC_CENSUS - // BEFORE the sweep on purpose. This is the only point where the four slot - // states are still distinguishable -- the sweep stamps every fresh object with - // the current mark, after which "traced" and "kept by grace" look identical. - if(getenv("CN1_HEAP_REPORT")) { - cn1LiveCensus("pre-sweep"); - } -#endif // THE MARK THIS SWEEP WOULD ACT ON MAY BE INCOMPLETE. cn1GcPageIndexStale says the // page index could not be rebuilt, so every reference into a page registered since // the last successful rebuild failed to resolve and its object was never marked -- @@ -4882,15 +4817,6 @@ void codenameOneGCSweep() { // permanently broken. cn1GcVerifyHeap(threadStateData); #endif -#ifdef CN1_ALLOC_CENSUS - // Same reasoning as the verify hook above: post-sweep is when "live" means - // live. cn1HeapAccounting and cn1AllocCensus were written but never called - // from anywhere, so nothing could answer "what is the footprint made of". - if(getenv("CN1_HEAP_REPORT")) { - cn1HeapAccounting("post-sweep"); - cn1LiveCensus("post-sweep"); - } -#endif } JAVA_BOOLEAN removeObjectFromHeapCollection(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT o) { @@ -5551,10 +5477,6 @@ JAVA_INT java_lang_System_identityHashCode___java_lang_Object_R_int(CODENAME_ONE // Non-static: the inlined bump fast path (cn1_globals.h) reads bibopCurrent[ci]. __thread CN1BibopPage* bibopCurrent[CN1_BIBOP_NUM_CLASSES]; -#ifdef CN1_ALLOC_CENSUS -static void cn1BibopExitReport(void); -#endif - static void cn1BibopDoInit() { int ci = 0; // DIAGNOSTIC KNOB -- CN1_GC_TRIGGER_MB overrides how many uncollected bytes @@ -5586,38 +5508,8 @@ static void cn1BibopDoInit() { atomic_store_explicit(&bibopBypassGeneration[i], 0, memory_order_relaxed); bibopHighSurvivalStreak[i] = 0; } - // Prime the free-memory snapshot the pacing cap is computed from. - // - // Its only other caller is the mark cycle, so until the FIRST collection - // cn1CachedFreeMem was 0 and cn1BibopPacingCap's `fm / 8` evaluated to 0, leaving - // the cap at its floor of trigger * CN1_BIBOP_GC_HARD_CAP_MULTIPLIER = 72MB -- - // during exactly the window where there is least reason to throttle anything, - // since nothing has been collected yet. ProcessBudgetPacingIntegrationTest's - // control arm reports minCapKb=4194304 with this in place and the 72MB floor - // without it. - // - // Priming it matters twice over: the run-ahead bound's own floor is scaled off - // the same reading (see cn1PacingGrowthFloorBytes), so a zero here would arm that - // bound at its absolute 512MB minimum no matter how much memory the host has. - cn1RefreshFreeMemCache(); -#ifdef CN1_ALLOC_CENSUS - if(getenv("CN1_HEAP_REPORT")) { - atexit(cn1BibopExitReport); - } -#endif } -#ifdef CN1_ALLOC_CENSUS -// Registered from cn1BibopDoInit under CN1_HEAP_REPORT. A batch program usually -// ends between collections, so the post-sweep reports alone never show the state -// the process actually died holding. -static void cn1BibopExitReport(void) { - cn1HeapAccounting("exit"); - cn1LiveCensus("exit"); - cn1AllocCensus("exit"); -} -#endif - static void cn1BibopFormatPage(CN1BibopPage* p, int ci) { int slotSize = cn1BibopClassSize[ci]; // slot 0 starts after the page header, rounded up to 16-byte alignment so @@ -6328,39 +6220,6 @@ static inline JAVA_OBJECT cn1BibopSlot(CN1BibopPage* p, int i) { #ifndef CN1_PACING_GROWTH_FLOOR_BYTES #define CN1_PACING_GROWTH_FLOOR_BYTES (512LL*1024*1024) #endif -// Ceiling on how far a mutator may run ahead of a cycle in flight, regardless of -// how much RAM the host has. See the measurement table in cn1BibopPacingCap. -#ifndef CN1_BIBOP_PACING_MAX_RUNAHEAD_BYTES -#define CN1_BIBOP_PACING_MAX_RUNAHEAD_BYTES (1024L*1024*1024) -#endif -// cn1_available_memory answers a flat 100MB on every platform where it cannot -// measure: Linux, Windows, and the non-Apple fallback. That number is not a -// reading, and a bound DERIVED from it is not a bound -- it is a constant that -// happens to look like one. -// -// This matters asymmetrically. cn1PacingGrowthFloorBytes above only ever RAISES -// its floor from fm, so on a placeholder host the absolute floor wins and -// behaviour is bit-for-bit unchanged. The run-ahead bound below only ever LOWERS -// the cap, so scaling it by a placeholder TIGHTENS pacing on exactly the hosts we -// know nothing about. It did: BibopPageFloorIntegrationTest went red on arm64 -// Linux, where fm/8 is 12.5MB, while the same code passed on macOS where fm is -// real. -// -// So the bound applies only where fm is a genuine reading. Returns 0 to mean -// "not measurable here, leave the cap alone". -#ifndef CN1_PACING_PLACEHOLDER_FREE_MEM -#define CN1_PACING_PLACEHOLDER_FREE_MEM (1024L*1024*100) -#endif -static long cn1PacingRunAheadBound(long fm) { - if(fm <= CN1_PACING_PLACEHOLDER_FREE_MEM) { - return 0; - } - long bound = CN1_BIBOP_PACING_MAX_RUNAHEAD_BYTES; - if(bound > fm / 8) { - bound = fm / 8; - } - return bound; -} // How stale a below-floor footprint reading may be before the bound re-probes it. The // probe is task_info on Apple and one /proc read on Linux -- a microsecond or two -- and // it is taken at most once per interval across the whole process, and only when the bound @@ -6534,48 +6393,14 @@ static long long cn1PacingFootprintNow(void) { return fp; } -// The footprint at which the run-ahead bound starts applying, scaled to the memory -// this host actually has. -// -// A fixed 512MB says "this process has grown"; it does not say the machine is under -// any pressure, and the bound exists for pressure. On a host with tens of GB free, a -// process holding a couple of GB is nowhere near runaway, and clamping it there -// parks the mutator against a collector that cannot get under the ceiling: measured -// at 6.7-8.7s versus 1.4s for the same work, to save 2% of peak footprint. -// -// So take the larger of the absolute floor and a quarter of available memory. Two -// properties this has to keep: -// -// - Where cn1_available_memory is the flat 100MB placeholder (Linux, Windows, and -// the non-Apple fallback), fm/4 is 25MB, the absolute floor wins, and behaviour is -// bit-for-bit what it was. Nothing changes on a platform where we cannot measure. -// - It only ever RAISES the floor, so the bound can only engage later than before, -// never earlier. It cannot make a constrained host more permissive than it was. -// -// This is the no-per-process-ceiling path only. Where a ceiling exists -- iOS's dirty -// memory limit, or an explicit process budget -- cn1PacingPark takes the bounded -// branch instead and never reaches cn1BibopPacingCap, so none of this loosens the -// admission control that keeps an app inside its own limit. -static long long cn1PacingGrowthFloorBytes(void) { - long long floor = CN1_PACING_GROWTH_FLOOR_BYTES; - long fm = atomic_load_explicit(&cn1CachedFreeMem, memory_order_relaxed); - if(fm > 0) { - long long scaled = (long long)fm / 4; - if(scaled > floor) { - floor = scaled; - } - } - return floor; -} - static JAVA_BOOLEAN cn1PacingPastGrowthFloor(void) { - long long floor = cn1PacingGrowthFloorBytes(); // Once the cache is over the floor the bound is engaged and a syscall to re-confirm // it buys nothing, so this stays ahead of the probe. - if(atomic_load_explicit(&cn1CachedProcFootprint, memory_order_relaxed) > floor) { + if(atomic_load_explicit(&cn1CachedProcFootprint, memory_order_relaxed) + > CN1_PACING_GROWTH_FLOOR_BYTES) { return JAVA_TRUE; } - return cn1PacingFootprintNow() > floor; + return cn1PacingFootprintNow() > CN1_PACING_GROWTH_FLOOR_BYTES; } static long cn1BibopPacingCap(CODENAME_ONE_THREAD_STATE) { @@ -6635,71 +6460,10 @@ static long cn1BibopPacingCap(CODENAME_ONE_THREAD_STATE) { if(capCeiling < base) { capCeiling = base; } - // FLOOR the clamp at the point where run-ahead stops paying, when the host - // can afford it. - // - // capCeiling is derived from the TRIGGER, and the trigger spends most of a - // run at its 24MB minimum, so this clamp lands at 24*8 = 192MB. Confirmed - // at runtime, not inferred: `[PACING] minCapKb=196608`. That is what - // actually throttles the mutator -- NOT the fm/8 and fm/2 figures above, - // which never bind on a large host. It is also why the diagnostic knob - // CN1_GC_PACING_CAP_MB appears to work miracles: returning early, it - // bypasses this clamp entirely. - // - // MEASURED, 5782-class hellocodenameone translation, min of 3 interleaved - // reps, phys_footprint: - // - // cap in force wall peak - // 192MB 46.3s 9736MB <- this clamp, as it stood - // 1024MB 23.8s 8325MB - // 2048MB 22.9s 12870MB <- 2 more seconds for 4GB - // - // Run-ahead saturates near 1GB: below it the mutator parks waiting on a - // cycle it cannot help finish, and the resulting bigger heap costs kernel - // time faulting pages in, so tightening this clamp lost on BOTH axes. - // - // Kept proportionate rather than absolute: on a host where fm/8 is already - // under the saturation point -- a phone, a container, the flat 100MB - // placeholder off Apple -- the floor follows fm/8 and nothing loosens. - { - long runAhead = cn1PacingRunAheadBound(fm); - if(runAhead > 0 && capCeiling < runAhead) { - capCeiling = runAhead; - } - } if(cap > capCeiling && cn1PacingPastGrowthFloor()) { cap = capCeiling; } } - // FINAL absolute bound on run-ahead. Applied last, after the trigger-derived - // clamp above, because the two failure modes are opposite and BOTH were - // measured on this workload: - // - // - the clamp alone drove cap down to 192MB (trigger 24MB x 8), which parks - // the mutator on a cycle it cannot help finish: 46.3s / 9736MB. - // - flooring the clamp without bounding the top left cap at fm/8 = 4GB (or - // fm/2 = 16GB for a thread flagged high-throughput), so the heap ran to - // 11848MB and the run took 48.0s -- worse on both axes. - // - // Pinning run-ahead near 1GB gives 23.8s / 8325MB. The saturation is real: at - // 2GB the run is 22.9s but the footprint is 12870MB, i.e. 2 more GB per second - // saved. So the useful range is narrow and this is its top. - // - // Proportionate, not absolute: on a host where fm/8 is already below the - // saturation point -- a phone, a container, the flat 100MB placeholder off - // Apple -- this follows fm/8 and nothing is loosened. `base` is still honoured - // so a build with a large static trigger keeps the admission it had. - { - long runAhead = cn1PacingRunAheadBound(fm); - if(runAhead > 0) { - if(cap > runAhead) { - cap = runAhead; - } - if(cap < base) { - cap = base; - } - } - } if(cn1PacingTraceOn()) { long seen = atomic_load_explicit(&cn1PacingMinCap, memory_order_relaxed); while(cap < seen && @@ -6934,31 +6698,6 @@ static void cn1PacingPark(CODENAME_ONE_THREAD_STATE, int which, long long pendin // sleep-until-done park. See cn1GcMutatorAssist. if(!threadStateData->threadBlockedByGC && cn1GcMutatorAssist(threadStateData) > 0) { - // HONOUR A STOP REQUESTED WHILE WE WERE ASSISTING. - // - // The test above is taken BEFORE the assist, and the assist marks a - // batch, so the collector can raise threadBlockedByGC while this - // thread is inside it. Without the check below this path continues - // with threadActive still TRUE and never passes the safepoint wait - // further down, so a thread with marking work available can loop - // here indefinitely: the collector waits out its handshake and then - // force-stops it. - // - // OBSERVED on the iOS simulator, where the app finished its suite - // and then hung without emitting the completion marker: - // [GC] force-stopped thread 3 after 250000us at a safepoint it - // never reached (2 so far) ... (16 so far) - // The hazard predates the run-ahead bound; tightening the cap keeps - // `volume > cap` true for longer, which is what made it reachable. - if(threadStateData->threadBlockedByGC) { - threadStateData->threadActive = JAVA_FALSE; - while(threadStateData->threadBlockedByGC) { - if(!cn1VirtualThreadYieldIfVirtual()) { - usleep((JAVA_INT)(500)); - } - } - threadStateData->threadActive = JAVA_TRUE; - } continue; } threadStateData->threadActive = JAVA_FALSE; @@ -7517,175 +7256,6 @@ void cn1HeapAccounting(const char* label) { fflush(stderr); } -/** - * Prints the LIVE heap by class, biggest first. - * - * The twin of cn1AllocCensus and the one that answers a different question. - * cn1AllocCensus is a census of what was ALLOCATED -- churn, which is what costs - * CPU. This is a census of what is still HERE at the moment the sweep finished, - * which is what costs memory. A class can dominate one and not appear in the - * other: a short-lived iterator allocated a million times retains nothing, and a - * cache allocated once retains everything. - * - * Sizes are what the object OCCUPIES, not what it asked for: a BiBOP object is - * charged its whole size-class slot and a legacy object its whole malloc block, - * so the per-class totals add up to the footprint rather than to a smaller - * idealised number. Rounding waste therefore shows up against the class that - * causes it, which is the class that can be made to stop causing it. - * - * Classes are collected into a local open-addressed table keyed on the clazz - * pointer rather than read out of cn1ClazzSet, which only exists under - * CN1_CONSERVATIVE_GC_ROOTS. - * - * Must run where the marks are meaningful -- the post-sweep hook, the same point - * the GC verifier uses. - */ -#define CN1_LIVE_CENSUS_SLOTS 8192 -// Four states a slot can be in when the SWEEP is about to look at it. Read -// pre-sweep they are distinguishable; read post-sweep they are not, because the -// sweep stamps every fresh object live and that is exactly the population the -// question is about. -#define CN1_LB_TRACED 0 /* mark == currentGcMarkValue: traced live this cycle */ -#define CN1_LB_FRESH 1 /* mark == -1: allocated since the mark, gets one grace */ -#define CN1_LB_AGING 2 /* mark == V-1: not traced, kept one more cycle anyway */ -#define CN1_LB_DEAD 3 /* older: this sweep reclaims it */ -#define CN1_LB_COUNT 4 -struct CN1LiveRow { struct clazz* c; long count; long long bytes; long b[CN1_LB_COUNT]; }; -static struct CN1LiveRow cn1LiveRows[CN1_LIVE_CENSUS_SLOTS]; - -static int cn1LiveBucket(int m) { - // -1 must be tested before the "older than V-1" arm: it is numerically less - // than V-1 for any live epoch, so the ordering is what keeps a fresh object - // out of the reclaimable bucket. - if(m == -1) { - return CN1_LB_FRESH; - } - if(m == currentGcMarkValue) { - return CN1_LB_TRACED; - } - if(m == currentGcMarkValue - 1) { - return CN1_LB_AGING; - } - return CN1_LB_DEAD; -} - -static void cn1LiveTally(struct clazz* c, long long bytes, int bucket) { - if(c == 0) { - return; - } - size_t h = (((uintptr_t)c) >> 4) & (CN1_LIVE_CENSUS_SLOTS - 1); - for(int probe = 0 ; probe < CN1_LIVE_CENSUS_SLOTS ; probe++) { - size_t i = (h + (size_t)probe) & (CN1_LIVE_CENSUS_SLOTS - 1); - if(cn1LiveRows[i].c == 0) { - cn1LiveRows[i].c = c; - } - if(cn1LiveRows[i].c == c) { - cn1LiveRows[i].count++; - cn1LiveRows[i].bytes += bytes; - cn1LiveRows[i].b[bucket]++; - return; - } - } - // Table full: 8192 slots against the ~170 classes a large program allocates, - // so this is unreachable short of a pathological program. Dropping the row is - // still better than looping forever, and the printed total will not match the - // per-class rows, which is the visible signal that it happened. -} - -void cn1LiveCensus(const char* label) { - memset(cn1LiveRows, 0, sizeof(cn1LiveRows)); - long long bibopBytes = 0, legacyBytes = 0; - long bibopObjs = 0, legacyObjs = 0; - long totals[CN1_LB_COUNT]; - for(int i = 0 ; i < CN1_LB_COUNT ; i++) { - totals[i] = 0; - } - - CN1BibopPage* p = atomic_load_explicit(&bibopAllPages, memory_order_acquire); - while(p != 0) { - int n = atomic_load_explicit(&p->bumpIndex, memory_order_acquire); - for(int i = 0 ; i < n ; i++) { - JAVA_OBJECT o = cn1BibopSlot(p, i); - int m = __atomic_load_n(&o->__codenameOneGcMark, __ATOMIC_ACQUIRE); - // Occupied, not "provably reachable": a slot awaiting collection is - // still holding memory, and this census is about what memory is being - // held. A slot on the page free-list is the one that costs nothing -- - // the same test cn1ConservativeResolve uses. (CN1_GC_POISON_MARK is - // deliberately not consulted: it is defined further down, inside the - // verifier's section, and exists only in a CN1_GC_VERIFY build.) - if(m == CN1_BIBOP_FREE_MARK) { - continue; - } - int bucket = cn1LiveBucket(m); - cn1LiveTally(o->__codenameOneParentClsReference, (long long)p->slotSize, bucket); - bibopBytes += (long long)p->slotSize; - bibopObjs++; - totals[bucket]++; - } - p = atomic_load_explicit(&p->nextAll, memory_order_acquire); - } - - int nHeap = currentSizeOfAllObjectsInHeap; - for(int i = 0 ; i < nHeap ; i++) { - JAVA_OBJECT o = allObjectsInHeap[i]; - if(o == JAVA_NULL) { - continue; - } - // An adopted object lives in a BiBOP slot and was already charged by the - // page walk; malloc_size on it would read a block header that is not there. - if(o->__heapPosition == CN1_BIBOP_ADOPTED) { - continue; - } - long long sz = 0; -#if defined(__APPLE__) - sz = (long long)malloc_size((void*)o); -#endif - int lbucket = cn1LiveBucket(o->__codenameOneGcMark); - cn1LiveTally(o->__codenameOneParentClsReference, sz, lbucket); - legacyBytes += sz; - legacyObjs++; - totals[lbucket]++; - } - - // OCCUPIED is what costs memory. The four buckets say WHY each object is still - // occupying a slot, and they call for different fixes: traced means the program - // really is holding it, fresh and aging mean the collector is holding it under - // the grace and aging rules, and dead means this sweep is about to return it. - long occupied = bibopObjs + legacyObjs; - fprintf(stderr, "[LIVE:%s] occupied %ld objects %.2fMB | traced %ld (%.0f%%) " - "fresh %ld (%.0f%%) aging %ld (%.0f%%) dead %ld (%.0f%%) | bibop %.2fMB legacy %.2fMB\n", - label, occupied, (bibopBytes + legacyBytes) / 1048576.0, - totals[CN1_LB_TRACED], 100.0 * totals[CN1_LB_TRACED] / (occupied > 0 ? occupied : 1), - totals[CN1_LB_FRESH], 100.0 * totals[CN1_LB_FRESH] / (occupied > 0 ? occupied : 1), - totals[CN1_LB_AGING], 100.0 * totals[CN1_LB_AGING] / (occupied > 0 ? occupied : 1), - totals[CN1_LB_DEAD], 100.0 * totals[CN1_LB_DEAD] / (occupied > 0 ? occupied : 1), - bibopBytes / 1048576.0, legacyBytes / 1048576.0); - for(int shown = 0 ; shown < 30 ; shown++) { - int best = -1; - for(int i = 0 ; i < CN1_LIVE_CENSUS_SLOTS ; i++) { - if(cn1LiveRows[i].c != 0 && cn1LiveRows[i].bytes > 0 - && (best < 0 || cn1LiveRows[i].bytes > cn1LiveRows[best].bytes)) { - best = i; - } - } - if(best < 0) { - break; - } - long rc = cn1LiveRows[best].count > 0 ? cn1LiveRows[best].count : 1; - fprintf(stderr, "[LIVE:%s] %8.2fMB %9ld objs %4lld B/obj traced %3.0f%% fresh %3.0f%% " - "aging %3.0f%% dead %3.0f%% %s\n", - label, cn1LiveRows[best].bytes / 1048576.0, cn1LiveRows[best].count, - cn1LiveRows[best].bytes / rc, - 100.0 * cn1LiveRows[best].b[CN1_LB_TRACED] / rc, - 100.0 * cn1LiveRows[best].b[CN1_LB_FRESH] / rc, - 100.0 * cn1LiveRows[best].b[CN1_LB_AGING] / rc, - 100.0 * cn1LiveRows[best].b[CN1_LB_DEAD] / rc, - cn1LiveRows[best].c->clsName ? cn1LiveRows[best].c->clsName : "?"); - cn1LiveRows[best].bytes = 0; - } - fflush(stderr); -} - void cn1AllocCensus(const char* label) { struct Row { const char* name; long count; long bytes; }; static struct Row rows[4096]; diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 4b7f8d1259b..4e88a1d8ebc 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -423,26 +423,6 @@ JAVA_BOOLEAN java_lang_String_equals___java_lang_Object_R_boolean(CODENAME_ONE_T // Fast path: both backing arrays are char[] -- byte-equality of UTF-16 code // units == string equality; libc memcmp is the SIMD-optimized comparison on // every target. - // BOTH LATIN-1 -- the overwhelmingly common case, and until now the SLOW one. - // - // The char[] path below already had a memcmp; the compact byte[] path did - // not, so two ASCII strings (every class name, method name and descriptor - // this translator compares) fell into the per-character loop at the bottom, - // which calls cn1StrCharAtRaw TWICE per character. That helper reloads - // `value` and `offset` and branches on the backing array's class pointer - // EVERY time, so the common case paid a branch and two field loads per char - // where a single memcmp would do. - // - // Latin-1 stores each char as its raw 0..255 byte, so memcmp's unsigned byte - // ordering is exactly char ordering; equality is bit-identical. - // - // MEASURED before the fix: java_lang_String_equals was 6.78% of mutator - // self-time on the 5782-class hellocodenameone translation. - if(cn1StrIsLatin1(__cn1ThisObject) && cn1StrIsLatin1(__cn1Arg1)) { - JAVA_ARRAY_BYTE* ta = ((JAVA_ARRAY_BYTE*)((JAVA_ARRAY)t->java_lang_String_value)->data) + t->java_lang_String_offset; - JAVA_ARRAY_BYTE* oa = ((JAVA_ARRAY_BYTE*)((JAVA_ARRAY)o->java_lang_String_value)->data) + o->java_lang_String_offset; - return memcmp(ta, oa, (size_t)t->java_lang_String_count) == 0 ? JAVA_TRUE : JAVA_FALSE; - } if(!cn1StrIsLatin1(__cn1ThisObject) && !cn1StrIsLatin1(__cn1Arg1)) { JAVA_ARRAY_CHAR* oa = ((JAVA_ARRAY_CHAR*)((JAVA_ARRAY)o->java_lang_String_value)->data) + o->java_lang_String_offset; JAVA_ARRAY_CHAR* ta = ((JAVA_ARRAY_CHAR*)((JAVA_ARRAY)t->java_lang_String_value)->data) + t->java_lang_String_offset; @@ -491,25 +471,8 @@ JAVA_INT java_lang_String_compareTo___java_lang_String_R_int(CODENAME_ONE_THREAD } return tc - oc; } - // BOTH Latin-1: hoist the coder test and the field reloads OUT of the loop. - // cn1StrCharAtRaw re-derives the base pointer and re-tests the backing array's - // class on every character, twice per iteration; with both coders known the - // loop is two raw byte pointers. Ordering is unchanged -- Latin-1 bytes are - // the char values 0..255. - if(cn1StrIsLatin1(__cn1ThisObject) && cn1StrIsLatin1(__cn1Arg1)) { - struct obj__java_lang_String* ts = (struct obj__java_lang_String*)__cn1ThisObject; - struct obj__java_lang_String* os = (struct obj__java_lang_String*)__cn1Arg1; - const JAVA_ARRAY_BYTE* tb = ((JAVA_ARRAY_BYTE*)((JAVA_ARRAY)ts->java_lang_String_value)->data) + ts->java_lang_String_offset; - const JAVA_ARRAY_BYTE* ob = ((JAVA_ARRAY_BYTE*)((JAVA_ARRAY)os->java_lang_String_value)->data) + os->java_lang_String_offset; - for(JAVA_INT k = 0; k < minL; k++) { - int d = (int)(tb[k] & 0xff) - (int)(ob[k] & 0xff); - if(d) { - return d; - } - } - return tc - oc; - } - // Mixed coders: one Latin-1, one UTF-16. Rare; keep the general helper. + // Coder-aware path: at least one string is Latin-1 (byte[]); compare logical + // chars. Same UTF-16 code-unit ordering, bit-identical to the char[] path. for(JAVA_INT k = 0; k < minL; k++) { int d = (int)cn1StrCharAtRaw(__cn1ThisObject, k) - (int)cn1StrCharAtRaw(__cn1Arg1, k); if(d) { @@ -1500,6 +1463,16 @@ JAVA_LONG java_lang_Double_doubleToLongBits___double_R_long(CODENAME_ONE_THREAD_ return u.l; } +JAVA_LONG java_lang_Double_doubleToRawLongBits___double_R_long(CODENAME_ONE_THREAD_STATE, JAVA_DOUBLE n1) { + union { + JAVA_DOUBLE d; + JAVA_LONG l; + } u; + + u.d = n1; + return u.l; +} + JAVA_FLOAT java_lang_Float_intBitsToFloat___int_R_float(CODENAME_ONE_THREAD_STATE, JAVA_INT n1) { union { @@ -2000,81 +1973,6 @@ JAVA_OBJECT java_lang_Class_getName___R_java_lang_String(CODENAME_ONE_THREAD_STA return newStringFromCString(threadStateData, clz->clsName); } -/** - * Backs Integer.TYPE and the eight other wrapper TYPE fields. The JDK needs a - * native here for the same reason we do: `TYPE = int.class` cannot initialize the - * field, because javac lowers a primitive class literal to a read of that very - * field (getstatic TYPE; putstatic TYPE), leaving it null. - * - * Takes an int code rather than the JDK's String name deliberately. This runs - * inside the wrapper class initializers, which are among the earliest code in the - * process, and decoding a Java String here would drag in String.getBytes and the - * charset machinery during Integer's own clinit. An int argument allocates - * nothing and initializes nothing. - * - * The codes are an implementation detail shared only with java/lang/Class.java; - * they are matched by CN1_PRIM_* there. - */ -JAVA_OBJECT java_lang_Class_getPrimitiveClass___int_R_java_lang_Class(CODENAME_ONE_THREAD_STATE, JAVA_INT typeCode) { - switch(typeCode) { - case 0: return (JAVA_OBJECT)&cn1_primitive_class_int; - case 1: return (JAVA_OBJECT)&cn1_primitive_class_long; - case 2: return (JAVA_OBJECT)&cn1_primitive_class_short; - case 3: return (JAVA_OBJECT)&cn1_primitive_class_byte; - case 4: return (JAVA_OBJECT)&cn1_primitive_class_char; - case 5: return (JAVA_OBJECT)&cn1_primitive_class_float; - case 6: return (JAVA_OBJECT)&cn1_primitive_class_double; - case 7: return (JAVA_OBJECT)&cn1_primitive_class_boolean; - case 8: return (JAVA_OBJECT)&cn1_primitive_class_void; - } - // Only java/lang/Class.java calls this, always with one of its own constants, - // so this is unreachable short of the two files disagreeing. Returning null - // would restore exactly the silent null TYPE this code exists to remove. - fprintf(stderr, "getPrimitiveClass: unknown primitive type code %d\n", (int)typeCode); - exit(1); - return JAVA_NULL; -} - -/** - * Resources linked into the executable, backing Class.getResourceAsStream. - * - * cn1FindResource has a weak definition here that finds nothing. A target that - * embeds resources emits a strong one (the generated cn1_resources_table.c) and - * overrides it; everywhere else this one stands and getResourceAsStream falls - * through to the filesystem. That keeps every existing target unchanged -- - * getResourceAsStream returned a hard-coded null before this existed, so nothing - * can regress, only start working. - * - * A weak DEFINITION rather than a weak declaration: Mach-O will not link an - * undefined weak symbol without weak_import, while a weak definition is overridable - * on both Mach-O and ELF. - */ -__attribute__((weak)) const unsigned char* cn1FindResource(const char* name, int* lenOut) { - (void)name; - if(lenOut) { - *lenOut = 0; - } - return 0; -} - -JAVA_OBJECT java_lang_Class_cn1EmbeddedResource___java_lang_String_R_byte_1ARRAY(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT name) { - if(name == JAVA_NULL) { - return JAVA_NULL; - } - const char* n = stringToUTF8(threadStateData, name); - if(n == 0) { - return JAVA_NULL; - } - int len = 0; - const unsigned char* data = cn1FindResource(n, &len); - if(data == 0 || len <= 0) { - return JAVA_NULL; - } - JAVA_OBJECT arr = __NEW_ARRAY_JAVA_BYTE(threadStateData, len); - memcpy(((JAVA_ARRAY)arr)->data, data, len); - return arr; -} - JAVA_BOOLEAN java_lang_Class_isArray___R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT cls) { struct clazz* clz = (struct clazz*)cls; return clz->isArray; @@ -2090,12 +1988,6 @@ JAVA_BOOLEAN java_lang_Class_isArray___R_boolean(CODENAME_ONE_THREAD_STATE, JAVA JAVA_BOOLEAN java_lang_Class_isAssignableFrom___java_lang_Class_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT cls, JAVA_OBJECT cls2) { struct clazz* clz1 = (struct clazz*)cls; struct clazz* clz2 = (struct clazz*)cls2; - // A primitive class carries CN1_PRIMITIVE_CLASS_ID, which indexes no row of - // the instanceof tables, so it must never reach instanceofFunction. The JDK - // rule is also simply identity: int is assignable only from int. - if(clz1->primitiveType || clz2->primitiveType) { - return clz1 == clz2 ? JAVA_TRUE : JAVA_FALSE; - } // A.isAssignableFrom(B): target is A, the class under test is B. return instanceofFunction(clz1->classId, clz2->classId); } @@ -2103,9 +1995,6 @@ JAVA_BOOLEAN java_lang_Class_isAssignableFrom___java_lang_Class_R_boolean(CODENA JAVA_BOOLEAN java_lang_Class_isInstance___java_lang_Object_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT cls, JAVA_OBJECT obj) { if(obj == JAVA_NULL) { return JAVA_FALSE; } struct clazz* clz1 = (struct clazz*)cls; - // No object is ever an instance of a primitive class, and its sentinel - // classId indexes no instanceof table row -- see isAssignableFrom above. - if(((struct clazz*)cls)->primitiveType) { return JAVA_FALSE; } struct clazz* clz2 = (struct clazz*)CN1_CLASS_OF(obj); // tag-aware: a tagged Integer has no header // A.isInstance(o): target is A, the class under test is o's class. These were // reversed, so isInstance searched the TARGET's supertype table for the diff --git a/vm/JavaAPI/src/java/lang/Boolean.java b/vm/JavaAPI/src/java/lang/Boolean.java index 2f56aa21520..043fee9956d 100644 --- a/vm/JavaAPI/src/java/lang/Boolean.java +++ b/vm/JavaAPI/src/java/lang/Boolean.java @@ -27,12 +27,6 @@ * Since: JDK1.0, CLDC 1.0 */ public final class Boolean implements Comparable { - - /** - * The class object for the primitive type this class wraps. - */ - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_BOOLEAN); - /** * The Boolean object corresponding to the primitive value false. */ diff --git a/vm/JavaAPI/src/java/lang/Byte.java b/vm/JavaAPI/src/java/lang/Byte.java index b7b5ff186f0..9a7fa9d99e9 100644 --- a/vm/JavaAPI/src/java/lang/Byte.java +++ b/vm/JavaAPI/src/java/lang/Byte.java @@ -28,7 +28,7 @@ */ public final class Byte extends Number implements Comparable { - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_BYTE); + public static final Class TYPE = byte.class; public static final int SIZE = 8; /** diff --git a/vm/JavaAPI/src/java/lang/Character.java b/vm/JavaAPI/src/java/lang/Character.java index 448f24d6a51..93ce6f67946 100644 --- a/vm/JavaAPI/src/java/lang/Character.java +++ b/vm/JavaAPI/src/java/lang/Character.java @@ -155,7 +155,7 @@ public final class Character implements Comparable{ //public static final int MAX_RADIX = 36; //public static final char MIN_VALUE = '\0'; //public static final char MAX_VALUE = '\uFFFF'; - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_CHAR); + public static final Class TYPE = char.class; public static final byte UNASSIGNED = 0; public static final byte UPPERCASE_LETTER = 1; public static final byte LOWERCASE_LETTER = 2; diff --git a/vm/JavaAPI/src/java/lang/Class.java b/vm/JavaAPI/src/java/lang/Class.java index ff1f5aa0f3e..00b5f6466ec 100644 --- a/vm/JavaAPI/src/java/lang/Class.java +++ b/vm/JavaAPI/src/java/lang/Class.java @@ -50,21 +50,6 @@ public ClassLoader getClassLoader() { * following code fragment returns the runtime Class descriptor for the * class named java.lang.Thread: Classt= Class.forName("java.lang.Thread") */ - /** - * Returns the Class object for {@code className}. - * - * ParparVM links the whole program ahead of time, so there is no second class - * loader to consult and nothing to defer: both extra arguments are accepted and - * ignored, and the class is resolved exactly as the one-argument form resolves - * it. The overload exists because library bytecode calls it -- ASM's - * ClassWriter.getCommonSuperClass does -- and an absent overload is a link - * error in translated code, not a compile error here. - */ - public static java.lang.Class forName(java.lang.String className, boolean initialize, - ClassLoader loader) throws java.lang.ClassNotFoundException { - return forName(className); - } - public static java.lang.Class forName(java.lang.String className) throws java.lang.ClassNotFoundException { className = className.replace('$', '.'); Class c = forNameImpl(className); @@ -151,102 +136,9 @@ public static java.lang.Class forName(java.lang.String className) throws java.la * class upon which the getResourceAsStream method was called. */ public java.io.InputStream getResourceAsStream(java.lang.String name){ - if (name == null) { - return null; - } - String absolute = name; - if (!absolute.startsWith("/")) { - // Relative names resolve against this class's package, as the javadoc - // above describes. - String className = getName(); - int lastDot = className.lastIndexOf('.'); - absolute = lastDot < 0 ? "/" + name - : "/" + className.substring(0, lastDot).replace('.', '/') + "/" + name; - } - byte[] embedded = cn1EmbeddedResource(absolute); - if (embedded != null) { - return new java.io.ByteArrayInputStream(embedded); - } - return cn1FileResource(absolute); - } - - /** - * Resources linked into the executable, or null when there are none. - * - * The native side calls a weakly-linked {@code cn1FindResource}, which the - * generated resource table overrides on targets that embed resources. Where - * nothing provides it the weak symbol is null and this returns null, so a target - * that embeds nothing behaves exactly as it did before this existed. - */ - private static native byte[] cn1EmbeddedResource(String name); - - /** - * The filesystem half of {@link #getResourceAsStream}: looks the resource up - * under a search path, so a translated command-line program can read files that - * sit beside it rather than being linked into it. - * - * The path comes from CN1_RESOURCE_PATH, else a "cn1runtime" directory next to - * the executable. Entries are separated the way the platform separates path - * entries. - */ - private static java.io.InputStream cn1FileResource(String absolute) { - String path = System.getenv("CN1_RESOURCE_PATH"); - if (path == null || path.length() == 0) { - return null; - } - String relative = absolute.substring(1); - int from = 0; - while (from <= path.length()) { - int end = path.indexOf(java.io.File.pathSeparatorChar, from); - String root = end < 0 ? path.substring(from) : path.substring(from, end); - if (root.length() > 0) { - java.io.File candidate = new java.io.File(root, relative); - if (candidate.exists()) { - try { - return new java.io.FileInputStream(candidate); - } catch (java.io.IOException err) { - return null; - } - } - } - if (end < 0) { - break; - } - from = end + 1; - } - return null; + return null; } - /** - * Type codes for {@link #getPrimitiveClass(int)}. Shared only with - * nativeMethods.m, which switches on the same values. - */ - static final int CN1_PRIM_INT = 0; - static final int CN1_PRIM_LONG = 1; - static final int CN1_PRIM_SHORT = 2; - static final int CN1_PRIM_BYTE = 3; - static final int CN1_PRIM_CHAR = 4; - static final int CN1_PRIM_FLOAT = 5; - static final int CN1_PRIM_DOUBLE = 6; - static final int CN1_PRIM_BOOLEAN = 7; - static final int CN1_PRIM_VOID = 8; - - /** - * Returns the class object for a primitive type, e.g. the one - * {@code int.class} and {@link Integer#TYPE} denote. - * - * The wrapper classes cannot initialize their {@code TYPE} fields with a - * primitive class literal: javac lowers {@code int.class} to a read of - * {@code Integer.TYPE} itself, so {@code TYPE = int.class} compiles to - * {@code getstatic TYPE; putstatic TYPE} and leaves the field null. The JDK - * declares an equivalent native for the same reason. - * - * Takes an int code rather than a name so that it allocates nothing and - * decodes nothing: it runs inside the wrapper class initializers, which are - * among the earliest code in the process. - */ - static native Class getPrimitiveClass(int typeCode); - /** * Determines if this Class object represents an array class. */ diff --git a/vm/JavaAPI/src/java/lang/Double.java b/vm/JavaAPI/src/java/lang/Double.java index 05d12f67f31..22f161feb5d 100644 --- a/vm/JavaAPI/src/java/lang/Double.java +++ b/vm/JavaAPI/src/java/lang/Double.java @@ -29,7 +29,7 @@ */ public final class Double extends Number implements Comparable { - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_DOUBLE); + public static final Class TYPE = double.class; /** * The largest positive finite value of type double. It is equal to the value returned by Double.longBitsToDouble(0x7fefffffffffffffL) * See Also:Constant Field Values @@ -88,14 +88,6 @@ public byte byteValue(){ * If the argument is NaN, the result is 0x7ff8000000000000L. * In all cases, the result is a long integer that, when given to the longBitsToDouble(long) method, will produce a floating-point value equal to the argument to doubleToLongBits. */ - /** - * The raw IEEE 754 bits of {@code value}, without collapsing NaN to the - * canonical NaN. See {@link Float#floatToRawIntBits} for why this delegates. - */ - public static long doubleToRawLongBits(double value) { - return doubleToLongBits(value); - } - public native static long doubleToLongBits(double value); /** diff --git a/vm/JavaAPI/src/java/lang/Float.java b/vm/JavaAPI/src/java/lang/Float.java index 93e9d560a81..5b257d00f64 100644 --- a/vm/JavaAPI/src/java/lang/Float.java +++ b/vm/JavaAPI/src/java/lang/Float.java @@ -28,12 +28,6 @@ * Since: JDK1.0, CLDC 1.1 */ public final class Float extends Number implements Comparable { - - /** - * The class object for the primitive type this class wraps. - */ - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_FLOAT); - /** * The largest positive value of type float. It is equal to the value returned by Float.intBitsToFloat(0x7f7fffff). * See Also:Constant Field Values @@ -119,20 +113,6 @@ public boolean equals(java.lang.Object obj){ * Returns the bit representation of a single-float value. The result is a representation of the floating-point argument according to the IEEE 754 floating-point "single precision" bit layout. Bit 31 (the bit that is selected by the mask 0x80000000) represents the sign of the floating-point number. Bits 30-23 (the bits that are selected by the mask 0x7f800000) represent the exponent. Bits 22-0 (the bits that are selected by the mask 0x007fffff) represent the significand (sometimes called the mantissa) of the floating-point number. If the argument is positive infinity, the result is 0x7f800000. If the argument is negative infinity, the result is 0xff800000. If the argument is NaN, the result is 0x7fc00000. In all cases, the result is an integer that, when given to the * method, will produce a floating-point value equal to the argument to floatToIntBits. */ - /** - * The raw IEEE 754 bits of {@code value}, without collapsing NaN to the - * canonical NaN. - * - * Delegates rather than declaring a second native. ParparVM's floatToIntBits - * is a bare union punt that does not collapse NaN to the canonical NaN -- so it - * is already the raw operation, and the two differ in the spec but not here. A - * separate native would be one more mangled symbol to get wrong, silently, for - * no behavioural difference. - */ - public static int floatToRawIntBits(float value) { - return floatToIntBits(value); - } - public native static int floatToIntBits(float value); /** diff --git a/vm/JavaAPI/src/java/lang/Integer.java b/vm/JavaAPI/src/java/lang/Integer.java index 4a1cb1a85d0..0bcf391a733 100644 --- a/vm/JavaAPI/src/java/lang/Integer.java +++ b/vm/JavaAPI/src/java/lang/Integer.java @@ -29,7 +29,7 @@ */ public final class Integer extends Number implements Comparable { - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_INT); + public static final Class TYPE = int.class; private static final char[] DIGITS = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', @@ -359,18 +359,6 @@ public static int signum(int i) { return (i >> 31) | (-i >>> 31); // Hacker's delight 2-7 } - /** - * Rotates the two's-complement binary representation of {@code i} left by - * {@code distance} bits. - * - * The shift distance is used modulo 32 by the JLS shift rules, which is what - * makes the negation on the right half correct for every distance, including - * zero and multiples of 32. - */ - public static int rotateLeft(int i, int distance) { - return (i << distance) | (i >>> -distance); - } - public static int compare(int f1, int f2) { if (f1 > f2) return 1; diff --git a/vm/JavaAPI/src/java/lang/Long.java b/vm/JavaAPI/src/java/lang/Long.java index fce50a48abd..0e938265153 100644 --- a/vm/JavaAPI/src/java/lang/Long.java +++ b/vm/JavaAPI/src/java/lang/Long.java @@ -29,7 +29,7 @@ */ public final class Long extends Number implements Comparable { - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_LONG); + public static Class TYPE = long.class; /** * The largest value of type long. diff --git a/vm/JavaAPI/src/java/lang/Short.java b/vm/JavaAPI/src/java/lang/Short.java index f0800e1f219..233a1698268 100644 --- a/vm/JavaAPI/src/java/lang/Short.java +++ b/vm/JavaAPI/src/java/lang/Short.java @@ -27,12 +27,6 @@ * Since: JDK1.1, CLDC 1.0 */ public final class Short extends Number implements Comparable { - - /** - * The class object for the primitive type this class wraps. - */ - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_SHORT); - /** * The maximum value a Short can have. * See Also:Constant Field Values diff --git a/vm/JavaAPI/src/java/lang/StringBuilder.java b/vm/JavaAPI/src/java/lang/StringBuilder.java index c031b06fc03..4c5fd18d310 100644 --- a/vm/JavaAPI/src/java/lang/StringBuilder.java +++ b/vm/JavaAPI/src/java/lang/StringBuilder.java @@ -100,24 +100,7 @@ private StringBuilder(char[] data, int offset, int charCount) { } private void enlargeBuffer(int min) { - // Double, as OpenJDK's AbstractStringBuilder does, rather than the 1.5x - // ((len>>1)+len+2) inherited from Harmony. - // - // Growing to N chars costs sum(capacity) in ABANDONED intermediate - // arrays, and that sum is N*r/(r-1): 3N at r=1.5, 2N at r=2. The - // difference is pure garbage, and on ParparVM garbage is expensive in a - // way it is not on a generational JVM -- the collector is a concurrent - // mark/sweep with no nursery, so a dead intermediate array occupies its - // slot until a later cycle sweeps it. - // - // MEASURED on the 5782-class hellocodenameone translation, where the - // emit phase is StringBuilder-bound: char[] occupancy 1529.84MB and the - // legacy (large-array) heap 1533.43MB before the emit-buffer reuse fix. - // - // The cost is peak overshoot: a buffer can now be up to 2x the chars - // actually needed rather than 1.5x. That is bounded and transient, where - // the reallocation garbage is unbounded in the number of appends. - int newCount = (value.length << 1) + 2; + int newCount = ((value.length >> 1) + value.length) + 2; char[] newData = new char[min > newCount ? min : newCount]; System.arraycopy(value, 0, newData, 0, count); value = newData; diff --git a/vm/JavaAPI/src/java/lang/Void.java b/vm/JavaAPI/src/java/lang/Void.java index 96dbd87a71e..c1391f982e0 100644 --- a/vm/JavaAPI/src/java/lang/Void.java +++ b/vm/JavaAPI/src/java/lang/Void.java @@ -27,5 +27,5 @@ * @author Shai Almog */ public final class Void { - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_VOID); + public static final Class TYPE = Void.class; } diff --git a/vm/JavaAPI/src/java/util/ArrayList.java b/vm/JavaAPI/src/java/util/ArrayList.java index 8774c398204..8a3c4129a15 100644 --- a/vm/JavaAPI/src/java/util/ArrayList.java +++ b/vm/JavaAPI/src/java/util/ArrayList.java @@ -38,27 +38,6 @@ public class ArrayList extends AbstractList implements List, RandomAcce /** * Constructs a new instance of {@code ArrayList} with ten capacity. */ - // ISOLATION (PR #5766): the lazy default-capacity allocation that used to sit - // here is withdrawn. It replaced the eager new Object[10] with a SHARED static - // zero-length array, which also gave java.util.ArrayList a it had - // never had -- master's only static is a compile-time serialVersionUID, so the - // class previously emitted no static initializer at all. - // - // The suite then began stopping after exactly 145 of 166 screenshots on every - // target except glibc-x64, with ArrayList state corrupt at the point of - // failure: AIOOBE 89 inside pendingIdleSerialCalls.add, then AIOOBE -1, then a - // NullPointerException inside ArrayList.get, which only happens when the - // backing array reference itself is null. - // - // The list logic is NOT at fault: a differential fuzz of this exact source - // against java.util.ArrayList ran 3000 seeds x 200 random operations with no - // divergence, and every access to the corrupted list in Display is inside - // synchronized(lock). The corruption is therefore below Java, which makes the - // new and the process-wide shared array the part worth removing - // before anything subtler is blamed. - // - // The iterator below is the change that carried the measured win (iteration - // 25.5% -> 12.4% of mutator self-time) and is kept. public ArrayList() { this(10); } @@ -343,80 +322,6 @@ public void ensureCapacity(int minimumCapacity) { } } - /** - * Direct-array iterator, overriding AbstractList's generic SimpleListIterator. - * - * The inherited one was the single hottest method in a large translation -- - * 16.45% of mutator self-time on the 5782-class hellocodenameone corpus, more - * than twice the next entry. Three costs per element, none inherent: - * - * - a try/catch around the body, to turn IndexOutOfBoundsException into - * NoSuchElementException. ParparVM has no zero-cost exception tables, so a - * try block is a setjmp -- once per element, in the hottest loop in the - * program. An explicit bounds test costs a compare. - * - size() and get() as VIRTUAL calls on the outer list, with no JIT to - * inline them. - * - the index recomputed as size() - numLeft every iteration instead of - * being carried in a cursor. - * - * MEASURED after: the iteration path fell from 25.5% of mutator self-time to - * 12.4%, ArrayList.get from 7.42% to 0.55%, and _setjmp from 1.61% to zero. - * - * Semantics are unchanged: same ConcurrentModificationException on structural - * modification, same NoSuchElementException past the end, remove() still - * works. Reads array[firstIndex + i] exactly as get(int) does. - * - * Applies to every `for (x : list)` in every translated application whatever - * the loop's static type, because dispatch lands on the concrete ArrayList. - */ - // Package-private, not private: a private inner class whose constructor is - // reached from the outer class makes javac synthesise an access bridge and a - // ArrayList$1 marker type, so every iterator() paid an extra class and an - // aconst_null for the bridge argument. Nothing outside java.util can see it - // either way. - class ArrayListIterator implements Iterator { - private int cursor; - private int lastReturned = -1; - private int expectedModCount = modCount; - - public boolean hasNext() { - return cursor < size; - } - - public E next() { - if (modCount != expectedModCount) { - throw new ConcurrentModificationException(); - } - int i = cursor; - if (i >= size) { - throw new NoSuchElementException(); - } - cursor = i + 1; - lastReturned = i; - return array[firstIndex + i]; - } - - public void remove() { - if (lastReturned < 0) { - throw new IllegalStateException(); - } - if (modCount != expectedModCount) { - throw new ConcurrentModificationException(); - } - ArrayList.this.remove(lastReturned); - if (lastReturned < cursor) { - cursor--; - } - lastReturned = -1; - expectedModCount = modCount; - } - } - - @Override - public Iterator iterator() { - return new ArrayListIterator(); - } - @Override public E get(int location) { if (location < 0 || location >= size) { diff --git a/vm/JavaAPI/src/java/util/IdentityHashMap.java b/vm/JavaAPI/src/java/util/IdentityHashMap.java index 188a307ac74..4010c21c92a 100644 --- a/vm/JavaAPI/src/java/util/IdentityHashMap.java +++ b/vm/JavaAPI/src/java/util/IdentityHashMap.java @@ -125,52 +125,25 @@ static class IdentityHashMapIterator implements Iterator { final MapEntry.Type type; - /** - * Which of the three views this iterator serves. - * - * Keys and values come straight out of the table; only entrySet has to - * materialise an Entry, and only there can the caller observe one. The - * generic {@code type} callback cannot express that, because it takes a - * MapEntry -- so serving a key iterator through it allocated an Entry per - * next() purely to read one field back out and drop it. Measured on a - * self-hosting translation of the ParparVM translator: 1,366,140 such - * entries, 43.7MB, all garbage. java.util.HashMap already had separate - * key/value/entry iterators for exactly this reason; this one was missed. - */ - static final int KIND_ENTRY = 0; - static final int KIND_KEY = 1; - static final int KIND_VALUE = 2; - - final int kind; - boolean canRemove = false; IdentityHashMapIterator(MapEntry.Type value, IdentityHashMap hm) { associatedMap = hm; type = value; - kind = KIND_ENTRY; - expectedModCount = hm.modCount; - } - - IdentityHashMapIterator(int iteratorKind, IdentityHashMap hm) { - associatedMap = hm; - type = null; - kind = iteratorKind; expectedModCount = hm.modCount; } public boolean hasNext() { - // elementData hoisted into a local: it was re-loaded from the outer map - // on every comparison AND on every array access, twice per probe step. - Object[] data = associatedMap.elementData; - int p = position; - int len = data.length; - while (p < len && data[p] == null) { - p += 2; + while (position < associatedMap.elementData.length) { + // if this is an empty spot, go to the next one + if (associatedMap.elementData[position] == null) { + position += 2; + } else { + return true; + } } - position = p; - return p < len; + return false; } void checkConcurrentMod() throws ConcurrentModificationException { @@ -179,50 +152,19 @@ void checkConcurrentMod() throws ConcurrentModificationException { } } - @SuppressWarnings("unchecked") public E next() { - // The concurrent-modification test and the null-skipping scan are - // INLINED here rather than reached through checkConcurrentMod() and - // hasNext(). - // - // An enhanced-for already pays two interface dispatches per element - // (hasNext then next); routing next() through two more non-inlined - // calls made it four, and ParparVM has no JIT to fold them away. - // MEASURED on the 5782-class hellocodenameone translation: - // IdentityHashMapIterator.next 6.43% of mutator self-time with - // checkConcurrentMod a further 1.84%, second only to the ArrayList - // iterator. - // - // Behaviour is unchanged: same ConcurrentModificationException on a - // structural change, same NoSuchElementException past the end, and - // position still advances past empty slots exactly as hasNext() did. - if (expectedModCount != associatedMap.modCount) { - throw new ConcurrentModificationException(); - } - Object[] data = associatedMap.elementData; - int p = position; - int len = data.length; - while (p < len && data[p] == null) { - p += 2; - } - if (p >= len) { - position = p; + checkConcurrentMod(); + if (!hasNext()) { throw new NoSuchElementException(); } - lastPosition = p; - position = p + 2; - canRemove = true; + IdentityHashMapEntry result = associatedMap + .getEntry(position); + lastPosition = position; + position += 2; - if (kind == KIND_KEY) { - Object key = associatedMap.elementData[lastPosition]; - return (E) (key == NULL_OBJECT ? null : key); - } - if (kind == KIND_VALUE) { - Object value = associatedMap.elementData[lastPosition + 1]; - return (E) (value == NULL_OBJECT ? null : value); - } - return type.get(associatedMap.getEntry(lastPosition)); + canRemove = true; + return type.get(result); } public void remove() { @@ -745,7 +687,11 @@ public boolean remove(Object key) { @Override public Iterator iterator() { return new IdentityHashMapIterator( - IdentityHashMapIterator.KIND_KEY, IdentityHashMap.this); + new MapEntry.Type() { + public K get(MapEntry entry) { + return entry.key; + } + }, IdentityHashMap.this); } }; } @@ -793,7 +739,11 @@ public void clear() { @Override public Iterator iterator() { return new IdentityHashMapIterator( - IdentityHashMapIterator.KIND_VALUE, IdentityHashMap.this); + new MapEntry.Type() { + public V get(MapEntry entry) { + return entry.value; + } + }, IdentityHashMap.this); } @Override From 036daff1849a4a7a6e025bf822ba76c1c870ad78 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Fri, 11 Sep 2026 16:06:21 +0300 Subject: [PATCH 25/66] BISECT PROBE 3: the cause is in the runtime; narrow it to java/util Probe 2 answered the question it was pushed for. With vm/JavaAPI, cn1_globals.{m,h} and nativeMethods.m back at the merge base, the legs that had been failing all went green: Linux native build + run (GTK3, x64 + arm64) success ParparVM Java Tests (Windows) success Windows cross-build + run success The only failure was vm-tests, and that is this probe's own collateral: PrimitiveTypeIntegrationTest is a test THIS PR adds for the primitive class objects, and probe 2 reverted the code under it, so its cmake --build step fails. Not a signal. So the cause is on the runtime side, and the two bytecode-rewriting passes are exonerated -- probe 1 had already shown that turning them off left the Linux and Windows signature completely unchanged at pngs=145. They are restored here, which also answers the P2 review comment on the literal `false`: it was a deliberate, temporary bisect probe, and it is gone. This probe restores everything on the runtime side EXCEPT vm/JavaAPI/src/java/util, leaving ArrayList and IdentityHashMap at the merge base. Green means the cause is the ArrayList iterator or the IdentityHashMap iterator inlining; red means it is in java/lang (the primitive TYPE statics on nine wrapper classes), the primitive class structs in cn1_globals, the String.equals/compareTo fast path, or the pacing edits. Restoring java/lang also puts PrimitiveTypeIntegrationTest back on its own code, so vm-tests should go green again and stop masking the result. Gates D and A byte-identical over 797 files (797 rather than 795 because the ArrayList$1 access bridge exists again at the merge base), negative control still detecting an injected corruption. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.h | 32 +- vm/ByteCodeTranslator/src/cn1_globals.m | 436 +++++++++++++++++- .../codename1/tools/translator/Parser.java | 16 +- vm/ByteCodeTranslator/src/nativeMethods.m | 135 +++++- vm/JavaAPI/src/java/lang/Boolean.java | 6 + vm/JavaAPI/src/java/lang/Byte.java | 2 +- vm/JavaAPI/src/java/lang/Character.java | 2 +- vm/JavaAPI/src/java/lang/Class.java | 110 ++++- vm/JavaAPI/src/java/lang/Double.java | 10 +- vm/JavaAPI/src/java/lang/Float.java | 20 + vm/JavaAPI/src/java/lang/Integer.java | 14 +- vm/JavaAPI/src/java/lang/Long.java | 2 +- vm/JavaAPI/src/java/lang/Short.java | 6 + vm/JavaAPI/src/java/lang/StringBuilder.java | 19 +- vm/JavaAPI/src/java/lang/Void.java | 2 +- 15 files changed, 773 insertions(+), 39 deletions(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index b12ce93136d..6813f4ea164 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -2185,7 +2185,7 @@ static inline JAVA_OBJECT cn1BibopFastAllocNoZero(CODENAME_ONE_THREAD_STATE, int // because bibopCurrent[] is shared across all classes of the same size class). #if !defined(CN1_DISABLE_INLINE_ALLOC) && !defined(CN1_DISABLE_BIBOP) #define CN1_FAST_NEW(X) ({ \ - if(__builtin_expect(!class__##X.initialized, 0)) __STATIC_INITIALIZER_##X(threadStateData); \ + if(__builtin_expect(!__atomic_load_n(&class__##X.initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_##X(threadStateData); \ JAVA_OBJECT __cn1fo = cn1BibopFastAlloc(threadStateData, sizeof(struct obj__##X), &class__##X, CN1_BIBOP_CIDX(sizeof(struct obj__##X))); \ if(__builtin_expect(__cn1fo == (JAVA_OBJECT)0, 0)) __cn1fo = __NEW_##X(threadStateData); \ __cn1fo; }) @@ -2193,7 +2193,7 @@ static inline JAVA_OBJECT cn1BibopFastAllocNoZero(CODENAME_ONE_THREAD_STATE, int // still fully zeroes (calloc) -- correct, just un-elided on the rare page-full // path. #define CN1_FAST_NEW_NOZERO(X) ({ \ - if(__builtin_expect(!class__##X.initialized, 0)) __STATIC_INITIALIZER_##X(threadStateData); \ + if(__builtin_expect(!__atomic_load_n(&class__##X.initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_##X(threadStateData); \ JAVA_OBJECT __cn1fo = cn1BibopFastAllocNoZero(threadStateData, sizeof(struct obj__##X), &class__##X, CN1_BIBOP_CIDX(sizeof(struct obj__##X))); \ if(__builtin_expect(__cn1fo == (JAVA_OBJECT)0, 0)) __cn1fo = __NEW_##X(threadStateData); \ __cn1fo; }) @@ -2827,6 +2827,34 @@ extern struct clazz class_array1__JAVA_DOUBLE; extern struct clazz class_array2__JAVA_DOUBLE; extern struct clazz class_array3__JAVA_DOUBLE; +/** + * The nine scalar primitive class objects -- int.class, Integer.TYPE and friends. + * + * javac lowers a primitive class literal to a read of the boxed type's own TYPE + * field, so `TYPE = int.class` inside Integer's initializer compiles to + * `getstatic TYPE; putstatic TYPE` -- it reads the field it is initializing and + * leaves it null. Every wrapper that declared TYPE that way had a null one, and + * a Map keyed on them collapsed to a single entry, so a lookup for int answered + * with whatever type was stored last. Nothing threw. The wrappers now go through + * java_lang_Class_getPrimitiveClass, which hands back one of these. + * + * classId is CN1_PRIMITIVE_CLASS_ID for all nine: these never take part in an + * instanceof, and instanceofFunction indexes tables by classId, so the callers + * that could reach one (isAssignableFrom, isInstance) test primitiveType first + * rather than indexing with a value no table has a row for. + */ +#define CN1_PRIMITIVE_CLASS_ID (-1) + +extern struct clazz cn1_primitive_class_int; +extern struct clazz cn1_primitive_class_long; +extern struct clazz cn1_primitive_class_short; +extern struct clazz cn1_primitive_class_byte; +extern struct clazz cn1_primitive_class_char; +extern struct clazz cn1_primitive_class_float; +extern struct clazz cn1_primitive_class_double; +extern struct clazz cn1_primitive_class_boolean; +extern struct clazz cn1_primitive_class_void; + extern JAVA_OBJECT newString(CODENAME_ONE_THREAD_STATE, int length, JAVA_CHAR data[]); /** * Like newStringFromCString but DECODES, in the PLATFORM's encoding, instead of diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index c4a93106b1a..e16ca1cc202 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -956,6 +956,54 @@ static void init_gc_thresholds() { //#define DEBUG_GC_OBJECTS_IN_HEAP +/** + * Scalar primitive class objects. See the comment on CN1_PRIMITIVE_CLASS_ID in + * cn1_globals.h for why these exist and why their classId is a sentinel. + * + * baseClass is 0 because int.class.getSuperclass() is null, which + * java_lang_Class_getSuperclass already returns for a null baseClass. isArray is + * false and arrayType is 0: these are the scalar types, not the array classes, + * which already exist as class_arrayN__JAVA_*. + * + * Designated initializers, unlike the positional generated ones beside them, so + * that a future field added to struct clazz cannot silently shift every value. + */ +/* + * __codenameOneParentClsReference is the class OF this object. Every generated + * clazz sets it to class__java_lang_Class, and CN1_CLASS_OF reads it to find the + * vtable when a clazz is used as an ordinary object -- which is what happens the + * moment one becomes a Map key. Leaving it zero segfaults on the first + * hashCode(), well away from anything that names it. + * + * The comment sits outside the macro on purpose: backslash-newline splicing + * happens before comments are removed, so an unbackslashed comment line inside + * the macro would silently end the definition. + */ +#define CN1_DEFINE_PRIMITIVE_CLASS(cname, jname) \ +struct clazz cn1_primitive_class_##cname = { \ + .__codenameOneParentClsReference = &class__java_lang_Class, \ + .classId = CN1_PRIMITIVE_CLASS_ID, \ + .clsName = jname, \ + .isArray = JAVA_FALSE, \ + .dimensions = 0, \ + .arrayType = 0, \ + .primitiveType = JAVA_TRUE, \ + .baseClass = 0, \ + .baseInterfaces = EMPTY_INTERFACES, \ + .baseInterfaceCount = 0, \ + .initialized = JAVA_TRUE \ +} + +CN1_DEFINE_PRIMITIVE_CLASS(int, "int"); +CN1_DEFINE_PRIMITIVE_CLASS(long, "long"); +CN1_DEFINE_PRIMITIVE_CLASS(short, "short"); +CN1_DEFINE_PRIMITIVE_CLASS(byte, "byte"); +CN1_DEFINE_PRIMITIVE_CLASS(char, "char"); +CN1_DEFINE_PRIMITIVE_CLASS(float, "float"); +CN1_DEFINE_PRIMITIVE_CLASS(double, "double"); +CN1_DEFINE_PRIMITIVE_CLASS(boolean, "boolean"); +CN1_DEFINE_PRIMITIVE_CLASS(void, "void"); + struct clazz class_array1__JAVA_BOOLEAN = { DEBUG_GC_INIT 0, 0, 0, 0, 0, 0, 0, cn1_array_1_id_JAVA_BOOLEAN, "boolean[]", JAVA_TRUE, 1, &class__java_lang_Boolean, JAVA_TRUE, &class__java_lang_Object, EMPTY_INTERFACES, 0, 0, 0 }; @@ -1743,6 +1791,15 @@ static void cn1DrainDeadThreadPending() { // walks the page registry and its slots before their definitions. static inline JAVA_OBJECT cn1BibopSlot(CN1BibopPage* p, int i); static CN1BibopPage* _Atomic bibopAllPages; +#ifdef CN1_ALLOC_CENSUS +// Defined far below, beside the BiBOP page structures they read. Declared up here +// because the post-sweep hook that calls them is compiled earlier -- and OUTSIDE the +// CN1_GC_VERIFY block just above, which is off in an ordinary census build. +void cn1HeapAccounting(const char* label); +void cn1AllocCensus(const char* label); +void cn1LiveCensus(const char* label); +#endif + #ifdef CN1_GRACE_AUDIT static void cn1GraceAuditPreSweep(CODENAME_ONE_THREAD_STATE); #endif @@ -4672,6 +4729,14 @@ static void cn1GcReportStaleIndexSkip(void) { void codenameOneGCSweep() { struct ThreadLocalData* threadStateData = getThreadLocalData(); +#ifdef CN1_ALLOC_CENSUS + // BEFORE the sweep on purpose. This is the only point where the four slot + // states are still distinguishable -- the sweep stamps every fresh object with + // the current mark, after which "traced" and "kept by grace" look identical. + if(getenv("CN1_HEAP_REPORT")) { + cn1LiveCensus("pre-sweep"); + } +#endif // THE MARK THIS SWEEP WOULD ACT ON MAY BE INCOMPLETE. cn1GcPageIndexStale says the // page index could not be rebuilt, so every reference into a page registered since // the last successful rebuild failed to resolve and its object was never marked -- @@ -4817,6 +4882,15 @@ void codenameOneGCSweep() { // permanently broken. cn1GcVerifyHeap(threadStateData); #endif +#ifdef CN1_ALLOC_CENSUS + // Same reasoning as the verify hook above: post-sweep is when "live" means + // live. cn1HeapAccounting and cn1AllocCensus were written but never called + // from anywhere, so nothing could answer "what is the footprint made of". + if(getenv("CN1_HEAP_REPORT")) { + cn1HeapAccounting("post-sweep"); + cn1LiveCensus("post-sweep"); + } +#endif } JAVA_BOOLEAN removeObjectFromHeapCollection(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT o) { @@ -5477,6 +5551,10 @@ JAVA_INT java_lang_System_identityHashCode___java_lang_Object_R_int(CODENAME_ONE // Non-static: the inlined bump fast path (cn1_globals.h) reads bibopCurrent[ci]. __thread CN1BibopPage* bibopCurrent[CN1_BIBOP_NUM_CLASSES]; +#ifdef CN1_ALLOC_CENSUS +static void cn1BibopExitReport(void); +#endif + static void cn1BibopDoInit() { int ci = 0; // DIAGNOSTIC KNOB -- CN1_GC_TRIGGER_MB overrides how many uncollected bytes @@ -5508,8 +5586,38 @@ static void cn1BibopDoInit() { atomic_store_explicit(&bibopBypassGeneration[i], 0, memory_order_relaxed); bibopHighSurvivalStreak[i] = 0; } + // Prime the free-memory snapshot the pacing cap is computed from. + // + // Its only other caller is the mark cycle, so until the FIRST collection + // cn1CachedFreeMem was 0 and cn1BibopPacingCap's `fm / 8` evaluated to 0, leaving + // the cap at its floor of trigger * CN1_BIBOP_GC_HARD_CAP_MULTIPLIER = 72MB -- + // during exactly the window where there is least reason to throttle anything, + // since nothing has been collected yet. ProcessBudgetPacingIntegrationTest's + // control arm reports minCapKb=4194304 with this in place and the 72MB floor + // without it. + // + // Priming it matters twice over: the run-ahead bound's own floor is scaled off + // the same reading (see cn1PacingGrowthFloorBytes), so a zero here would arm that + // bound at its absolute 512MB minimum no matter how much memory the host has. + cn1RefreshFreeMemCache(); +#ifdef CN1_ALLOC_CENSUS + if(getenv("CN1_HEAP_REPORT")) { + atexit(cn1BibopExitReport); + } +#endif } +#ifdef CN1_ALLOC_CENSUS +// Registered from cn1BibopDoInit under CN1_HEAP_REPORT. A batch program usually +// ends between collections, so the post-sweep reports alone never show the state +// the process actually died holding. +static void cn1BibopExitReport(void) { + cn1HeapAccounting("exit"); + cn1LiveCensus("exit"); + cn1AllocCensus("exit"); +} +#endif + static void cn1BibopFormatPage(CN1BibopPage* p, int ci) { int slotSize = cn1BibopClassSize[ci]; // slot 0 starts after the page header, rounded up to 16-byte alignment so @@ -6220,6 +6328,39 @@ static inline JAVA_OBJECT cn1BibopSlot(CN1BibopPage* p, int i) { #ifndef CN1_PACING_GROWTH_FLOOR_BYTES #define CN1_PACING_GROWTH_FLOOR_BYTES (512LL*1024*1024) #endif +// Ceiling on how far a mutator may run ahead of a cycle in flight, regardless of +// how much RAM the host has. See the measurement table in cn1BibopPacingCap. +#ifndef CN1_BIBOP_PACING_MAX_RUNAHEAD_BYTES +#define CN1_BIBOP_PACING_MAX_RUNAHEAD_BYTES (1024L*1024*1024) +#endif +// cn1_available_memory answers a flat 100MB on every platform where it cannot +// measure: Linux, Windows, and the non-Apple fallback. That number is not a +// reading, and a bound DERIVED from it is not a bound -- it is a constant that +// happens to look like one. +// +// This matters asymmetrically. cn1PacingGrowthFloorBytes above only ever RAISES +// its floor from fm, so on a placeholder host the absolute floor wins and +// behaviour is bit-for-bit unchanged. The run-ahead bound below only ever LOWERS +// the cap, so scaling it by a placeholder TIGHTENS pacing on exactly the hosts we +// know nothing about. It did: BibopPageFloorIntegrationTest went red on arm64 +// Linux, where fm/8 is 12.5MB, while the same code passed on macOS where fm is +// real. +// +// So the bound applies only where fm is a genuine reading. Returns 0 to mean +// "not measurable here, leave the cap alone". +#ifndef CN1_PACING_PLACEHOLDER_FREE_MEM +#define CN1_PACING_PLACEHOLDER_FREE_MEM (1024L*1024*100) +#endif +static long cn1PacingRunAheadBound(long fm) { + if(fm <= CN1_PACING_PLACEHOLDER_FREE_MEM) { + return 0; + } + long bound = CN1_BIBOP_PACING_MAX_RUNAHEAD_BYTES; + if(bound > fm / 8) { + bound = fm / 8; + } + return bound; +} // How stale a below-floor footprint reading may be before the bound re-probes it. The // probe is task_info on Apple and one /proc read on Linux -- a microsecond or two -- and // it is taken at most once per interval across the whole process, and only when the bound @@ -6393,14 +6534,48 @@ static long long cn1PacingFootprintNow(void) { return fp; } +// The footprint at which the run-ahead bound starts applying, scaled to the memory +// this host actually has. +// +// A fixed 512MB says "this process has grown"; it does not say the machine is under +// any pressure, and the bound exists for pressure. On a host with tens of GB free, a +// process holding a couple of GB is nowhere near runaway, and clamping it there +// parks the mutator against a collector that cannot get under the ceiling: measured +// at 6.7-8.7s versus 1.4s for the same work, to save 2% of peak footprint. +// +// So take the larger of the absolute floor and a quarter of available memory. Two +// properties this has to keep: +// +// - Where cn1_available_memory is the flat 100MB placeholder (Linux, Windows, and +// the non-Apple fallback), fm/4 is 25MB, the absolute floor wins, and behaviour is +// bit-for-bit what it was. Nothing changes on a platform where we cannot measure. +// - It only ever RAISES the floor, so the bound can only engage later than before, +// never earlier. It cannot make a constrained host more permissive than it was. +// +// This is the no-per-process-ceiling path only. Where a ceiling exists -- iOS's dirty +// memory limit, or an explicit process budget -- cn1PacingPark takes the bounded +// branch instead and never reaches cn1BibopPacingCap, so none of this loosens the +// admission control that keeps an app inside its own limit. +static long long cn1PacingGrowthFloorBytes(void) { + long long floor = CN1_PACING_GROWTH_FLOOR_BYTES; + long fm = atomic_load_explicit(&cn1CachedFreeMem, memory_order_relaxed); + if(fm > 0) { + long long scaled = (long long)fm / 4; + if(scaled > floor) { + floor = scaled; + } + } + return floor; +} + static JAVA_BOOLEAN cn1PacingPastGrowthFloor(void) { + long long floor = cn1PacingGrowthFloorBytes(); // Once the cache is over the floor the bound is engaged and a syscall to re-confirm // it buys nothing, so this stays ahead of the probe. - if(atomic_load_explicit(&cn1CachedProcFootprint, memory_order_relaxed) - > CN1_PACING_GROWTH_FLOOR_BYTES) { + if(atomic_load_explicit(&cn1CachedProcFootprint, memory_order_relaxed) > floor) { return JAVA_TRUE; } - return cn1PacingFootprintNow() > CN1_PACING_GROWTH_FLOOR_BYTES; + return cn1PacingFootprintNow() > floor; } static long cn1BibopPacingCap(CODENAME_ONE_THREAD_STATE) { @@ -6460,10 +6635,71 @@ static long cn1BibopPacingCap(CODENAME_ONE_THREAD_STATE) { if(capCeiling < base) { capCeiling = base; } + // FLOOR the clamp at the point where run-ahead stops paying, when the host + // can afford it. + // + // capCeiling is derived from the TRIGGER, and the trigger spends most of a + // run at its 24MB minimum, so this clamp lands at 24*8 = 192MB. Confirmed + // at runtime, not inferred: `[PACING] minCapKb=196608`. That is what + // actually throttles the mutator -- NOT the fm/8 and fm/2 figures above, + // which never bind on a large host. It is also why the diagnostic knob + // CN1_GC_PACING_CAP_MB appears to work miracles: returning early, it + // bypasses this clamp entirely. + // + // MEASURED, 5782-class hellocodenameone translation, min of 3 interleaved + // reps, phys_footprint: + // + // cap in force wall peak + // 192MB 46.3s 9736MB <- this clamp, as it stood + // 1024MB 23.8s 8325MB + // 2048MB 22.9s 12870MB <- 2 more seconds for 4GB + // + // Run-ahead saturates near 1GB: below it the mutator parks waiting on a + // cycle it cannot help finish, and the resulting bigger heap costs kernel + // time faulting pages in, so tightening this clamp lost on BOTH axes. + // + // Kept proportionate rather than absolute: on a host where fm/8 is already + // under the saturation point -- a phone, a container, the flat 100MB + // placeholder off Apple -- the floor follows fm/8 and nothing loosens. + { + long runAhead = cn1PacingRunAheadBound(fm); + if(runAhead > 0 && capCeiling < runAhead) { + capCeiling = runAhead; + } + } if(cap > capCeiling && cn1PacingPastGrowthFloor()) { cap = capCeiling; } } + // FINAL absolute bound on run-ahead. Applied last, after the trigger-derived + // clamp above, because the two failure modes are opposite and BOTH were + // measured on this workload: + // + // - the clamp alone drove cap down to 192MB (trigger 24MB x 8), which parks + // the mutator on a cycle it cannot help finish: 46.3s / 9736MB. + // - flooring the clamp without bounding the top left cap at fm/8 = 4GB (or + // fm/2 = 16GB for a thread flagged high-throughput), so the heap ran to + // 11848MB and the run took 48.0s -- worse on both axes. + // + // Pinning run-ahead near 1GB gives 23.8s / 8325MB. The saturation is real: at + // 2GB the run is 22.9s but the footprint is 12870MB, i.e. 2 more GB per second + // saved. So the useful range is narrow and this is its top. + // + // Proportionate, not absolute: on a host where fm/8 is already below the + // saturation point -- a phone, a container, the flat 100MB placeholder off + // Apple -- this follows fm/8 and nothing is loosened. `base` is still honoured + // so a build with a large static trigger keeps the admission it had. + { + long runAhead = cn1PacingRunAheadBound(fm); + if(runAhead > 0) { + if(cap > runAhead) { + cap = runAhead; + } + if(cap < base) { + cap = base; + } + } + } if(cn1PacingTraceOn()) { long seen = atomic_load_explicit(&cn1PacingMinCap, memory_order_relaxed); while(cap < seen && @@ -6698,6 +6934,31 @@ static void cn1PacingPark(CODENAME_ONE_THREAD_STATE, int which, long long pendin // sleep-until-done park. See cn1GcMutatorAssist. if(!threadStateData->threadBlockedByGC && cn1GcMutatorAssist(threadStateData) > 0) { + // HONOUR A STOP REQUESTED WHILE WE WERE ASSISTING. + // + // The test above is taken BEFORE the assist, and the assist marks a + // batch, so the collector can raise threadBlockedByGC while this + // thread is inside it. Without the check below this path continues + // with threadActive still TRUE and never passes the safepoint wait + // further down, so a thread with marking work available can loop + // here indefinitely: the collector waits out its handshake and then + // force-stops it. + // + // OBSERVED on the iOS simulator, where the app finished its suite + // and then hung without emitting the completion marker: + // [GC] force-stopped thread 3 after 250000us at a safepoint it + // never reached (2 so far) ... (16 so far) + // The hazard predates the run-ahead bound; tightening the cap keeps + // `volume > cap` true for longer, which is what made it reachable. + if(threadStateData->threadBlockedByGC) { + threadStateData->threadActive = JAVA_FALSE; + while(threadStateData->threadBlockedByGC) { + if(!cn1VirtualThreadYieldIfVirtual()) { + usleep((JAVA_INT)(500)); + } + } + threadStateData->threadActive = JAVA_TRUE; + } continue; } threadStateData->threadActive = JAVA_FALSE; @@ -7256,6 +7517,175 @@ void cn1HeapAccounting(const char* label) { fflush(stderr); } +/** + * Prints the LIVE heap by class, biggest first. + * + * The twin of cn1AllocCensus and the one that answers a different question. + * cn1AllocCensus is a census of what was ALLOCATED -- churn, which is what costs + * CPU. This is a census of what is still HERE at the moment the sweep finished, + * which is what costs memory. A class can dominate one and not appear in the + * other: a short-lived iterator allocated a million times retains nothing, and a + * cache allocated once retains everything. + * + * Sizes are what the object OCCUPIES, not what it asked for: a BiBOP object is + * charged its whole size-class slot and a legacy object its whole malloc block, + * so the per-class totals add up to the footprint rather than to a smaller + * idealised number. Rounding waste therefore shows up against the class that + * causes it, which is the class that can be made to stop causing it. + * + * Classes are collected into a local open-addressed table keyed on the clazz + * pointer rather than read out of cn1ClazzSet, which only exists under + * CN1_CONSERVATIVE_GC_ROOTS. + * + * Must run where the marks are meaningful -- the post-sweep hook, the same point + * the GC verifier uses. + */ +#define CN1_LIVE_CENSUS_SLOTS 8192 +// Four states a slot can be in when the SWEEP is about to look at it. Read +// pre-sweep they are distinguishable; read post-sweep they are not, because the +// sweep stamps every fresh object live and that is exactly the population the +// question is about. +#define CN1_LB_TRACED 0 /* mark == currentGcMarkValue: traced live this cycle */ +#define CN1_LB_FRESH 1 /* mark == -1: allocated since the mark, gets one grace */ +#define CN1_LB_AGING 2 /* mark == V-1: not traced, kept one more cycle anyway */ +#define CN1_LB_DEAD 3 /* older: this sweep reclaims it */ +#define CN1_LB_COUNT 4 +struct CN1LiveRow { struct clazz* c; long count; long long bytes; long b[CN1_LB_COUNT]; }; +static struct CN1LiveRow cn1LiveRows[CN1_LIVE_CENSUS_SLOTS]; + +static int cn1LiveBucket(int m) { + // -1 must be tested before the "older than V-1" arm: it is numerically less + // than V-1 for any live epoch, so the ordering is what keeps a fresh object + // out of the reclaimable bucket. + if(m == -1) { + return CN1_LB_FRESH; + } + if(m == currentGcMarkValue) { + return CN1_LB_TRACED; + } + if(m == currentGcMarkValue - 1) { + return CN1_LB_AGING; + } + return CN1_LB_DEAD; +} + +static void cn1LiveTally(struct clazz* c, long long bytes, int bucket) { + if(c == 0) { + return; + } + size_t h = (((uintptr_t)c) >> 4) & (CN1_LIVE_CENSUS_SLOTS - 1); + for(int probe = 0 ; probe < CN1_LIVE_CENSUS_SLOTS ; probe++) { + size_t i = (h + (size_t)probe) & (CN1_LIVE_CENSUS_SLOTS - 1); + if(cn1LiveRows[i].c == 0) { + cn1LiveRows[i].c = c; + } + if(cn1LiveRows[i].c == c) { + cn1LiveRows[i].count++; + cn1LiveRows[i].bytes += bytes; + cn1LiveRows[i].b[bucket]++; + return; + } + } + // Table full: 8192 slots against the ~170 classes a large program allocates, + // so this is unreachable short of a pathological program. Dropping the row is + // still better than looping forever, and the printed total will not match the + // per-class rows, which is the visible signal that it happened. +} + +void cn1LiveCensus(const char* label) { + memset(cn1LiveRows, 0, sizeof(cn1LiveRows)); + long long bibopBytes = 0, legacyBytes = 0; + long bibopObjs = 0, legacyObjs = 0; + long totals[CN1_LB_COUNT]; + for(int i = 0 ; i < CN1_LB_COUNT ; i++) { + totals[i] = 0; + } + + CN1BibopPage* p = atomic_load_explicit(&bibopAllPages, memory_order_acquire); + while(p != 0) { + int n = atomic_load_explicit(&p->bumpIndex, memory_order_acquire); + for(int i = 0 ; i < n ; i++) { + JAVA_OBJECT o = cn1BibopSlot(p, i); + int m = __atomic_load_n(&o->__codenameOneGcMark, __ATOMIC_ACQUIRE); + // Occupied, not "provably reachable": a slot awaiting collection is + // still holding memory, and this census is about what memory is being + // held. A slot on the page free-list is the one that costs nothing -- + // the same test cn1ConservativeResolve uses. (CN1_GC_POISON_MARK is + // deliberately not consulted: it is defined further down, inside the + // verifier's section, and exists only in a CN1_GC_VERIFY build.) + if(m == CN1_BIBOP_FREE_MARK) { + continue; + } + int bucket = cn1LiveBucket(m); + cn1LiveTally(o->__codenameOneParentClsReference, (long long)p->slotSize, bucket); + bibopBytes += (long long)p->slotSize; + bibopObjs++; + totals[bucket]++; + } + p = atomic_load_explicit(&p->nextAll, memory_order_acquire); + } + + int nHeap = currentSizeOfAllObjectsInHeap; + for(int i = 0 ; i < nHeap ; i++) { + JAVA_OBJECT o = allObjectsInHeap[i]; + if(o == JAVA_NULL) { + continue; + } + // An adopted object lives in a BiBOP slot and was already charged by the + // page walk; malloc_size on it would read a block header that is not there. + if(o->__heapPosition == CN1_BIBOP_ADOPTED) { + continue; + } + long long sz = 0; +#if defined(__APPLE__) + sz = (long long)malloc_size((void*)o); +#endif + int lbucket = cn1LiveBucket(o->__codenameOneGcMark); + cn1LiveTally(o->__codenameOneParentClsReference, sz, lbucket); + legacyBytes += sz; + legacyObjs++; + totals[lbucket]++; + } + + // OCCUPIED is what costs memory. The four buckets say WHY each object is still + // occupying a slot, and they call for different fixes: traced means the program + // really is holding it, fresh and aging mean the collector is holding it under + // the grace and aging rules, and dead means this sweep is about to return it. + long occupied = bibopObjs + legacyObjs; + fprintf(stderr, "[LIVE:%s] occupied %ld objects %.2fMB | traced %ld (%.0f%%) " + "fresh %ld (%.0f%%) aging %ld (%.0f%%) dead %ld (%.0f%%) | bibop %.2fMB legacy %.2fMB\n", + label, occupied, (bibopBytes + legacyBytes) / 1048576.0, + totals[CN1_LB_TRACED], 100.0 * totals[CN1_LB_TRACED] / (occupied > 0 ? occupied : 1), + totals[CN1_LB_FRESH], 100.0 * totals[CN1_LB_FRESH] / (occupied > 0 ? occupied : 1), + totals[CN1_LB_AGING], 100.0 * totals[CN1_LB_AGING] / (occupied > 0 ? occupied : 1), + totals[CN1_LB_DEAD], 100.0 * totals[CN1_LB_DEAD] / (occupied > 0 ? occupied : 1), + bibopBytes / 1048576.0, legacyBytes / 1048576.0); + for(int shown = 0 ; shown < 30 ; shown++) { + int best = -1; + for(int i = 0 ; i < CN1_LIVE_CENSUS_SLOTS ; i++) { + if(cn1LiveRows[i].c != 0 && cn1LiveRows[i].bytes > 0 + && (best < 0 || cn1LiveRows[i].bytes > cn1LiveRows[best].bytes)) { + best = i; + } + } + if(best < 0) { + break; + } + long rc = cn1LiveRows[best].count > 0 ? cn1LiveRows[best].count : 1; + fprintf(stderr, "[LIVE:%s] %8.2fMB %9ld objs %4lld B/obj traced %3.0f%% fresh %3.0f%% " + "aging %3.0f%% dead %3.0f%% %s\n", + label, cn1LiveRows[best].bytes / 1048576.0, cn1LiveRows[best].count, + cn1LiveRows[best].bytes / rc, + 100.0 * cn1LiveRows[best].b[CN1_LB_TRACED] / rc, + 100.0 * cn1LiveRows[best].b[CN1_LB_FRESH] / rc, + 100.0 * cn1LiveRows[best].b[CN1_LB_AGING] / rc, + 100.0 * cn1LiveRows[best].b[CN1_LB_DEAD] / rc, + cn1LiveRows[best].c->clsName ? cn1LiveRows[best].c->clsName : "?"); + cn1LiveRows[best].bytes = 0; + } + fflush(stderr); +} + void cn1AllocCensus(const char* label) { struct Row { const char* name; long count; long bytes; }; static struct Row rows[4096]; diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java index 0fcbab9952d..d5f84574329 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java @@ -846,20 +846,8 @@ public static void writeOutput(File outputDirectory) throws Exception { if (BytecodeMethod.optimizerOn) { for (ByteCodeClass fuseCls : classes) { for (BytecodeMethod fuseMtd : fuseCls.getMethods()) { - // BISECT PROBE (PR #5766, not a fix): both passes are off. - // These are the only changes in this PR that REWRITE bytecode, - // and the failure under investigation is heap corruption -- an - // int[] header reading length 0, which then takes an unchecked - // [-1] store in Display.edtLoopImpl. Wrong stack depth or a - // wrong receiver type from a rewrite is the most plausible - // source of that, so turning both off splits the suspect space - // in half: green means the cause is an emitted-code rewrite, - // red means it is in the JavaAPI or the C runtime instead. - // Restore both once the answer is in. - if (false) { - fuseMtd.fuseStringBuilderConcat(); - fuseMtd.lowerIteratorCalls(); - } + fuseMtd.fuseStringBuilderConcat(); + fuseMtd.lowerIteratorCalls(); } } } diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 4e88a1d8ebc..4b7f8d1259b 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -423,6 +423,26 @@ JAVA_BOOLEAN java_lang_String_equals___java_lang_Object_R_boolean(CODENAME_ONE_T // Fast path: both backing arrays are char[] -- byte-equality of UTF-16 code // units == string equality; libc memcmp is the SIMD-optimized comparison on // every target. + // BOTH LATIN-1 -- the overwhelmingly common case, and until now the SLOW one. + // + // The char[] path below already had a memcmp; the compact byte[] path did + // not, so two ASCII strings (every class name, method name and descriptor + // this translator compares) fell into the per-character loop at the bottom, + // which calls cn1StrCharAtRaw TWICE per character. That helper reloads + // `value` and `offset` and branches on the backing array's class pointer + // EVERY time, so the common case paid a branch and two field loads per char + // where a single memcmp would do. + // + // Latin-1 stores each char as its raw 0..255 byte, so memcmp's unsigned byte + // ordering is exactly char ordering; equality is bit-identical. + // + // MEASURED before the fix: java_lang_String_equals was 6.78% of mutator + // self-time on the 5782-class hellocodenameone translation. + if(cn1StrIsLatin1(__cn1ThisObject) && cn1StrIsLatin1(__cn1Arg1)) { + JAVA_ARRAY_BYTE* ta = ((JAVA_ARRAY_BYTE*)((JAVA_ARRAY)t->java_lang_String_value)->data) + t->java_lang_String_offset; + JAVA_ARRAY_BYTE* oa = ((JAVA_ARRAY_BYTE*)((JAVA_ARRAY)o->java_lang_String_value)->data) + o->java_lang_String_offset; + return memcmp(ta, oa, (size_t)t->java_lang_String_count) == 0 ? JAVA_TRUE : JAVA_FALSE; + } if(!cn1StrIsLatin1(__cn1ThisObject) && !cn1StrIsLatin1(__cn1Arg1)) { JAVA_ARRAY_CHAR* oa = ((JAVA_ARRAY_CHAR*)((JAVA_ARRAY)o->java_lang_String_value)->data) + o->java_lang_String_offset; JAVA_ARRAY_CHAR* ta = ((JAVA_ARRAY_CHAR*)((JAVA_ARRAY)t->java_lang_String_value)->data) + t->java_lang_String_offset; @@ -471,8 +491,25 @@ JAVA_INT java_lang_String_compareTo___java_lang_String_R_int(CODENAME_ONE_THREAD } return tc - oc; } - // Coder-aware path: at least one string is Latin-1 (byte[]); compare logical - // chars. Same UTF-16 code-unit ordering, bit-identical to the char[] path. + // BOTH Latin-1: hoist the coder test and the field reloads OUT of the loop. + // cn1StrCharAtRaw re-derives the base pointer and re-tests the backing array's + // class on every character, twice per iteration; with both coders known the + // loop is two raw byte pointers. Ordering is unchanged -- Latin-1 bytes are + // the char values 0..255. + if(cn1StrIsLatin1(__cn1ThisObject) && cn1StrIsLatin1(__cn1Arg1)) { + struct obj__java_lang_String* ts = (struct obj__java_lang_String*)__cn1ThisObject; + struct obj__java_lang_String* os = (struct obj__java_lang_String*)__cn1Arg1; + const JAVA_ARRAY_BYTE* tb = ((JAVA_ARRAY_BYTE*)((JAVA_ARRAY)ts->java_lang_String_value)->data) + ts->java_lang_String_offset; + const JAVA_ARRAY_BYTE* ob = ((JAVA_ARRAY_BYTE*)((JAVA_ARRAY)os->java_lang_String_value)->data) + os->java_lang_String_offset; + for(JAVA_INT k = 0; k < minL; k++) { + int d = (int)(tb[k] & 0xff) - (int)(ob[k] & 0xff); + if(d) { + return d; + } + } + return tc - oc; + } + // Mixed coders: one Latin-1, one UTF-16. Rare; keep the general helper. for(JAVA_INT k = 0; k < minL; k++) { int d = (int)cn1StrCharAtRaw(__cn1ThisObject, k) - (int)cn1StrCharAtRaw(__cn1Arg1, k); if(d) { @@ -1463,16 +1500,6 @@ JAVA_LONG java_lang_Double_doubleToLongBits___double_R_long(CODENAME_ONE_THREAD_ return u.l; } -JAVA_LONG java_lang_Double_doubleToRawLongBits___double_R_long(CODENAME_ONE_THREAD_STATE, JAVA_DOUBLE n1) { - union { - JAVA_DOUBLE d; - JAVA_LONG l; - } u; - - u.d = n1; - return u.l; -} - JAVA_FLOAT java_lang_Float_intBitsToFloat___int_R_float(CODENAME_ONE_THREAD_STATE, JAVA_INT n1) { union { @@ -1973,6 +2000,81 @@ JAVA_OBJECT java_lang_Class_getName___R_java_lang_String(CODENAME_ONE_THREAD_STA return newStringFromCString(threadStateData, clz->clsName); } +/** + * Backs Integer.TYPE and the eight other wrapper TYPE fields. The JDK needs a + * native here for the same reason we do: `TYPE = int.class` cannot initialize the + * field, because javac lowers a primitive class literal to a read of that very + * field (getstatic TYPE; putstatic TYPE), leaving it null. + * + * Takes an int code rather than the JDK's String name deliberately. This runs + * inside the wrapper class initializers, which are among the earliest code in the + * process, and decoding a Java String here would drag in String.getBytes and the + * charset machinery during Integer's own clinit. An int argument allocates + * nothing and initializes nothing. + * + * The codes are an implementation detail shared only with java/lang/Class.java; + * they are matched by CN1_PRIM_* there. + */ +JAVA_OBJECT java_lang_Class_getPrimitiveClass___int_R_java_lang_Class(CODENAME_ONE_THREAD_STATE, JAVA_INT typeCode) { + switch(typeCode) { + case 0: return (JAVA_OBJECT)&cn1_primitive_class_int; + case 1: return (JAVA_OBJECT)&cn1_primitive_class_long; + case 2: return (JAVA_OBJECT)&cn1_primitive_class_short; + case 3: return (JAVA_OBJECT)&cn1_primitive_class_byte; + case 4: return (JAVA_OBJECT)&cn1_primitive_class_char; + case 5: return (JAVA_OBJECT)&cn1_primitive_class_float; + case 6: return (JAVA_OBJECT)&cn1_primitive_class_double; + case 7: return (JAVA_OBJECT)&cn1_primitive_class_boolean; + case 8: return (JAVA_OBJECT)&cn1_primitive_class_void; + } + // Only java/lang/Class.java calls this, always with one of its own constants, + // so this is unreachable short of the two files disagreeing. Returning null + // would restore exactly the silent null TYPE this code exists to remove. + fprintf(stderr, "getPrimitiveClass: unknown primitive type code %d\n", (int)typeCode); + exit(1); + return JAVA_NULL; +} + +/** + * Resources linked into the executable, backing Class.getResourceAsStream. + * + * cn1FindResource has a weak definition here that finds nothing. A target that + * embeds resources emits a strong one (the generated cn1_resources_table.c) and + * overrides it; everywhere else this one stands and getResourceAsStream falls + * through to the filesystem. That keeps every existing target unchanged -- + * getResourceAsStream returned a hard-coded null before this existed, so nothing + * can regress, only start working. + * + * A weak DEFINITION rather than a weak declaration: Mach-O will not link an + * undefined weak symbol without weak_import, while a weak definition is overridable + * on both Mach-O and ELF. + */ +__attribute__((weak)) const unsigned char* cn1FindResource(const char* name, int* lenOut) { + (void)name; + if(lenOut) { + *lenOut = 0; + } + return 0; +} + +JAVA_OBJECT java_lang_Class_cn1EmbeddedResource___java_lang_String_R_byte_1ARRAY(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT name) { + if(name == JAVA_NULL) { + return JAVA_NULL; + } + const char* n = stringToUTF8(threadStateData, name); + if(n == 0) { + return JAVA_NULL; + } + int len = 0; + const unsigned char* data = cn1FindResource(n, &len); + if(data == 0 || len <= 0) { + return JAVA_NULL; + } + JAVA_OBJECT arr = __NEW_ARRAY_JAVA_BYTE(threadStateData, len); + memcpy(((JAVA_ARRAY)arr)->data, data, len); + return arr; +} + JAVA_BOOLEAN java_lang_Class_isArray___R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT cls) { struct clazz* clz = (struct clazz*)cls; return clz->isArray; @@ -1988,6 +2090,12 @@ JAVA_BOOLEAN java_lang_Class_isArray___R_boolean(CODENAME_ONE_THREAD_STATE, JAVA JAVA_BOOLEAN java_lang_Class_isAssignableFrom___java_lang_Class_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT cls, JAVA_OBJECT cls2) { struct clazz* clz1 = (struct clazz*)cls; struct clazz* clz2 = (struct clazz*)cls2; + // A primitive class carries CN1_PRIMITIVE_CLASS_ID, which indexes no row of + // the instanceof tables, so it must never reach instanceofFunction. The JDK + // rule is also simply identity: int is assignable only from int. + if(clz1->primitiveType || clz2->primitiveType) { + return clz1 == clz2 ? JAVA_TRUE : JAVA_FALSE; + } // A.isAssignableFrom(B): target is A, the class under test is B. return instanceofFunction(clz1->classId, clz2->classId); } @@ -1995,6 +2103,9 @@ JAVA_BOOLEAN java_lang_Class_isAssignableFrom___java_lang_Class_R_boolean(CODENA JAVA_BOOLEAN java_lang_Class_isInstance___java_lang_Object_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT cls, JAVA_OBJECT obj) { if(obj == JAVA_NULL) { return JAVA_FALSE; } struct clazz* clz1 = (struct clazz*)cls; + // No object is ever an instance of a primitive class, and its sentinel + // classId indexes no instanceof table row -- see isAssignableFrom above. + if(((struct clazz*)cls)->primitiveType) { return JAVA_FALSE; } struct clazz* clz2 = (struct clazz*)CN1_CLASS_OF(obj); // tag-aware: a tagged Integer has no header // A.isInstance(o): target is A, the class under test is o's class. These were // reversed, so isInstance searched the TARGET's supertype table for the diff --git a/vm/JavaAPI/src/java/lang/Boolean.java b/vm/JavaAPI/src/java/lang/Boolean.java index 043fee9956d..2f56aa21520 100644 --- a/vm/JavaAPI/src/java/lang/Boolean.java +++ b/vm/JavaAPI/src/java/lang/Boolean.java @@ -27,6 +27,12 @@ * Since: JDK1.0, CLDC 1.0 */ public final class Boolean implements Comparable { + + /** + * The class object for the primitive type this class wraps. + */ + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_BOOLEAN); + /** * The Boolean object corresponding to the primitive value false. */ diff --git a/vm/JavaAPI/src/java/lang/Byte.java b/vm/JavaAPI/src/java/lang/Byte.java index 9a7fa9d99e9..b7b5ff186f0 100644 --- a/vm/JavaAPI/src/java/lang/Byte.java +++ b/vm/JavaAPI/src/java/lang/Byte.java @@ -28,7 +28,7 @@ */ public final class Byte extends Number implements Comparable { - public static final Class TYPE = byte.class; + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_BYTE); public static final int SIZE = 8; /** diff --git a/vm/JavaAPI/src/java/lang/Character.java b/vm/JavaAPI/src/java/lang/Character.java index 93ce6f67946..448f24d6a51 100644 --- a/vm/JavaAPI/src/java/lang/Character.java +++ b/vm/JavaAPI/src/java/lang/Character.java @@ -155,7 +155,7 @@ public final class Character implements Comparable{ //public static final int MAX_RADIX = 36; //public static final char MIN_VALUE = '\0'; //public static final char MAX_VALUE = '\uFFFF'; - public static final Class TYPE = char.class; + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_CHAR); public static final byte UNASSIGNED = 0; public static final byte UPPERCASE_LETTER = 1; public static final byte LOWERCASE_LETTER = 2; diff --git a/vm/JavaAPI/src/java/lang/Class.java b/vm/JavaAPI/src/java/lang/Class.java index 00b5f6466ec..ff1f5aa0f3e 100644 --- a/vm/JavaAPI/src/java/lang/Class.java +++ b/vm/JavaAPI/src/java/lang/Class.java @@ -50,6 +50,21 @@ public ClassLoader getClassLoader() { * following code fragment returns the runtime Class descriptor for the * class named java.lang.Thread: Classt= Class.forName("java.lang.Thread") */ + /** + * Returns the Class object for {@code className}. + * + * ParparVM links the whole program ahead of time, so there is no second class + * loader to consult and nothing to defer: both extra arguments are accepted and + * ignored, and the class is resolved exactly as the one-argument form resolves + * it. The overload exists because library bytecode calls it -- ASM's + * ClassWriter.getCommonSuperClass does -- and an absent overload is a link + * error in translated code, not a compile error here. + */ + public static java.lang.Class forName(java.lang.String className, boolean initialize, + ClassLoader loader) throws java.lang.ClassNotFoundException { + return forName(className); + } + public static java.lang.Class forName(java.lang.String className) throws java.lang.ClassNotFoundException { className = className.replace('$', '.'); Class c = forNameImpl(className); @@ -136,9 +151,102 @@ public static java.lang.Class forName(java.lang.String className) throws java.la * class upon which the getResourceAsStream method was called. */ public java.io.InputStream getResourceAsStream(java.lang.String name){ - return null; + if (name == null) { + return null; + } + String absolute = name; + if (!absolute.startsWith("/")) { + // Relative names resolve against this class's package, as the javadoc + // above describes. + String className = getName(); + int lastDot = className.lastIndexOf('.'); + absolute = lastDot < 0 ? "/" + name + : "/" + className.substring(0, lastDot).replace('.', '/') + "/" + name; + } + byte[] embedded = cn1EmbeddedResource(absolute); + if (embedded != null) { + return new java.io.ByteArrayInputStream(embedded); + } + return cn1FileResource(absolute); + } + + /** + * Resources linked into the executable, or null when there are none. + * + * The native side calls a weakly-linked {@code cn1FindResource}, which the + * generated resource table overrides on targets that embed resources. Where + * nothing provides it the weak symbol is null and this returns null, so a target + * that embeds nothing behaves exactly as it did before this existed. + */ + private static native byte[] cn1EmbeddedResource(String name); + + /** + * The filesystem half of {@link #getResourceAsStream}: looks the resource up + * under a search path, so a translated command-line program can read files that + * sit beside it rather than being linked into it. + * + * The path comes from CN1_RESOURCE_PATH, else a "cn1runtime" directory next to + * the executable. Entries are separated the way the platform separates path + * entries. + */ + private static java.io.InputStream cn1FileResource(String absolute) { + String path = System.getenv("CN1_RESOURCE_PATH"); + if (path == null || path.length() == 0) { + return null; + } + String relative = absolute.substring(1); + int from = 0; + while (from <= path.length()) { + int end = path.indexOf(java.io.File.pathSeparatorChar, from); + String root = end < 0 ? path.substring(from) : path.substring(from, end); + if (root.length() > 0) { + java.io.File candidate = new java.io.File(root, relative); + if (candidate.exists()) { + try { + return new java.io.FileInputStream(candidate); + } catch (java.io.IOException err) { + return null; + } + } + } + if (end < 0) { + break; + } + from = end + 1; + } + return null; } + /** + * Type codes for {@link #getPrimitiveClass(int)}. Shared only with + * nativeMethods.m, which switches on the same values. + */ + static final int CN1_PRIM_INT = 0; + static final int CN1_PRIM_LONG = 1; + static final int CN1_PRIM_SHORT = 2; + static final int CN1_PRIM_BYTE = 3; + static final int CN1_PRIM_CHAR = 4; + static final int CN1_PRIM_FLOAT = 5; + static final int CN1_PRIM_DOUBLE = 6; + static final int CN1_PRIM_BOOLEAN = 7; + static final int CN1_PRIM_VOID = 8; + + /** + * Returns the class object for a primitive type, e.g. the one + * {@code int.class} and {@link Integer#TYPE} denote. + * + * The wrapper classes cannot initialize their {@code TYPE} fields with a + * primitive class literal: javac lowers {@code int.class} to a read of + * {@code Integer.TYPE} itself, so {@code TYPE = int.class} compiles to + * {@code getstatic TYPE; putstatic TYPE} and leaves the field null. The JDK + * declares an equivalent native for the same reason. + * + * Takes an int code rather than a name so that it allocates nothing and + * decodes nothing: it runs inside the wrapper class initializers, which are + * among the earliest code in the process. + */ + static native Class getPrimitiveClass(int typeCode); + /** * Determines if this Class object represents an array class. */ diff --git a/vm/JavaAPI/src/java/lang/Double.java b/vm/JavaAPI/src/java/lang/Double.java index 22f161feb5d..05d12f67f31 100644 --- a/vm/JavaAPI/src/java/lang/Double.java +++ b/vm/JavaAPI/src/java/lang/Double.java @@ -29,7 +29,7 @@ */ public final class Double extends Number implements Comparable { - public static final Class TYPE = double.class; + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_DOUBLE); /** * The largest positive finite value of type double. It is equal to the value returned by Double.longBitsToDouble(0x7fefffffffffffffL) * See Also:Constant Field Values @@ -88,6 +88,14 @@ public byte byteValue(){ * If the argument is NaN, the result is 0x7ff8000000000000L. * In all cases, the result is a long integer that, when given to the longBitsToDouble(long) method, will produce a floating-point value equal to the argument to doubleToLongBits. */ + /** + * The raw IEEE 754 bits of {@code value}, without collapsing NaN to the + * canonical NaN. See {@link Float#floatToRawIntBits} for why this delegates. + */ + public static long doubleToRawLongBits(double value) { + return doubleToLongBits(value); + } + public native static long doubleToLongBits(double value); /** diff --git a/vm/JavaAPI/src/java/lang/Float.java b/vm/JavaAPI/src/java/lang/Float.java index 5b257d00f64..93e9d560a81 100644 --- a/vm/JavaAPI/src/java/lang/Float.java +++ b/vm/JavaAPI/src/java/lang/Float.java @@ -28,6 +28,12 @@ * Since: JDK1.0, CLDC 1.1 */ public final class Float extends Number implements Comparable { + + /** + * The class object for the primitive type this class wraps. + */ + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_FLOAT); + /** * The largest positive value of type float. It is equal to the value returned by Float.intBitsToFloat(0x7f7fffff). * See Also:Constant Field Values @@ -113,6 +119,20 @@ public boolean equals(java.lang.Object obj){ * Returns the bit representation of a single-float value. The result is a representation of the floating-point argument according to the IEEE 754 floating-point "single precision" bit layout. Bit 31 (the bit that is selected by the mask 0x80000000) represents the sign of the floating-point number. Bits 30-23 (the bits that are selected by the mask 0x7f800000) represent the exponent. Bits 22-0 (the bits that are selected by the mask 0x007fffff) represent the significand (sometimes called the mantissa) of the floating-point number. If the argument is positive infinity, the result is 0x7f800000. If the argument is negative infinity, the result is 0xff800000. If the argument is NaN, the result is 0x7fc00000. In all cases, the result is an integer that, when given to the * method, will produce a floating-point value equal to the argument to floatToIntBits. */ + /** + * The raw IEEE 754 bits of {@code value}, without collapsing NaN to the + * canonical NaN. + * + * Delegates rather than declaring a second native. ParparVM's floatToIntBits + * is a bare union punt that does not collapse NaN to the canonical NaN -- so it + * is already the raw operation, and the two differ in the spec but not here. A + * separate native would be one more mangled symbol to get wrong, silently, for + * no behavioural difference. + */ + public static int floatToRawIntBits(float value) { + return floatToIntBits(value); + } + public native static int floatToIntBits(float value); /** diff --git a/vm/JavaAPI/src/java/lang/Integer.java b/vm/JavaAPI/src/java/lang/Integer.java index 0bcf391a733..4a1cb1a85d0 100644 --- a/vm/JavaAPI/src/java/lang/Integer.java +++ b/vm/JavaAPI/src/java/lang/Integer.java @@ -29,7 +29,7 @@ */ public final class Integer extends Number implements Comparable { - public static final Class TYPE = int.class; + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_INT); private static final char[] DIGITS = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', @@ -359,6 +359,18 @@ public static int signum(int i) { return (i >> 31) | (-i >>> 31); // Hacker's delight 2-7 } + /** + * Rotates the two's-complement binary representation of {@code i} left by + * {@code distance} bits. + * + * The shift distance is used modulo 32 by the JLS shift rules, which is what + * makes the negation on the right half correct for every distance, including + * zero and multiples of 32. + */ + public static int rotateLeft(int i, int distance) { + return (i << distance) | (i >>> -distance); + } + public static int compare(int f1, int f2) { if (f1 > f2) return 1; diff --git a/vm/JavaAPI/src/java/lang/Long.java b/vm/JavaAPI/src/java/lang/Long.java index 0e938265153..fce50a48abd 100644 --- a/vm/JavaAPI/src/java/lang/Long.java +++ b/vm/JavaAPI/src/java/lang/Long.java @@ -29,7 +29,7 @@ */ public final class Long extends Number implements Comparable { - public static Class TYPE = long.class; + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_LONG); /** * The largest value of type long. diff --git a/vm/JavaAPI/src/java/lang/Short.java b/vm/JavaAPI/src/java/lang/Short.java index 233a1698268..f0800e1f219 100644 --- a/vm/JavaAPI/src/java/lang/Short.java +++ b/vm/JavaAPI/src/java/lang/Short.java @@ -27,6 +27,12 @@ * Since: JDK1.1, CLDC 1.0 */ public final class Short extends Number implements Comparable { + + /** + * The class object for the primitive type this class wraps. + */ + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_SHORT); + /** * The maximum value a Short can have. * See Also:Constant Field Values diff --git a/vm/JavaAPI/src/java/lang/StringBuilder.java b/vm/JavaAPI/src/java/lang/StringBuilder.java index 4c5fd18d310..c031b06fc03 100644 --- a/vm/JavaAPI/src/java/lang/StringBuilder.java +++ b/vm/JavaAPI/src/java/lang/StringBuilder.java @@ -100,7 +100,24 @@ private StringBuilder(char[] data, int offset, int charCount) { } private void enlargeBuffer(int min) { - int newCount = ((value.length >> 1) + value.length) + 2; + // Double, as OpenJDK's AbstractStringBuilder does, rather than the 1.5x + // ((len>>1)+len+2) inherited from Harmony. + // + // Growing to N chars costs sum(capacity) in ABANDONED intermediate + // arrays, and that sum is N*r/(r-1): 3N at r=1.5, 2N at r=2. The + // difference is pure garbage, and on ParparVM garbage is expensive in a + // way it is not on a generational JVM -- the collector is a concurrent + // mark/sweep with no nursery, so a dead intermediate array occupies its + // slot until a later cycle sweeps it. + // + // MEASURED on the 5782-class hellocodenameone translation, where the + // emit phase is StringBuilder-bound: char[] occupancy 1529.84MB and the + // legacy (large-array) heap 1533.43MB before the emit-buffer reuse fix. + // + // The cost is peak overshoot: a buffer can now be up to 2x the chars + // actually needed rather than 1.5x. That is bounded and transient, where + // the reallocation garbage is unbounded in the number of appends. + int newCount = (value.length << 1) + 2; char[] newData = new char[min > newCount ? min : newCount]; System.arraycopy(value, 0, newData, 0, count); value = newData; diff --git a/vm/JavaAPI/src/java/lang/Void.java b/vm/JavaAPI/src/java/lang/Void.java index c1391f982e0..96dbd87a71e 100644 --- a/vm/JavaAPI/src/java/lang/Void.java +++ b/vm/JavaAPI/src/java/lang/Void.java @@ -27,5 +27,5 @@ * @author Shai Almog */ public final class Void { - public static final Class TYPE = Void.class; + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_VOID); } From 25205a899ecd9910f8965097023cdc713682b1db Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Fri, 11 Sep 2026 16:40:29 +0300 Subject: [PATCH 26/66] Bounds-check the ArrayList iterator's array read This is the defect. The specialised ArrayListIterator added in this PR does if (i >= size) throw new NoSuchElementException(); return array[firstIndex + i]; and the size test is only a bounds check while the list's firstIndex + size <= array.length invariant holds. The iterator it replaced could not read out of range at all: it went through get(), which bounds-checks, inside a try that turned IndexOutOfBoundsException into NoSuchElementException. Removing that try was the entire point -- ParparVM has no zero-cost exception tables, so it was a setjmp per element in the hottest loop in the program -- but it also removed the only bounds check on the read, and ParparVM does NOT check an array read in a release build. A recoverable exception became an out-of-bounds read of the heap. OpenJDK's own ArrayList.Itr.next() carries the identical guard, if (i >= elementData.length) throw new ConcurrentModificationException(); so omitting it is the whole bug, and restoring it costs one compare against a hoisted local while the measured win stays. Bisected rather than guessed, over three CI probes: 1. both bytecode-rewriting passes off -- Linux and Windows signature completely unchanged at pngs=145, so the rewrites are exonerated (they are restored). 2. the entire runtime side reverted to the merge base -- Linux native, ParparVM Java Tests (Windows) and Windows cross-build all green, so the cause is in the runtime. 3. the runtime restored EXCEPT vm/JavaAPI/src/java/util -- the Windows cross leg went from 100+ failures to pass=185 fail=3, the three being Media360Panorama, VRStereoScene and VideoIODecodedFrames, which are media/VR tests that produced no output and are a separate question. That isolates java/util, and of the two changes there IdentityHashMap is sound -- it hoists elementData into a local and tests p < len against that same snapshot, which is strictly safer than what it replaced. Why it presented as corruption rather than an exception: the read walks off the end of the backing array into whatever object follows it in the BiBOP page, so the damage surfaces somewhere else entirely. The core dump named an int[] whose header read length 0 -- Display's inputEventStackTmp, which is new int[1000] and can never legitimately be empty -- and the unchecked [-1] store that followed segfaulted in cn1_set_array_element_int. The AIOOBE index varied run to run (69, 89) while the stop point did not, because 145 is simply where AccessibilityTest sits in the run order. 3000 differential fuzz seeds against java.util.ArrayList still clean, so the guard changes nothing for correct single-threaded use. Gates D and A byte-identical over 795 files. Co-Authored-By: Claude Opus 5 (1M context) --- vm/JavaAPI/src/java/util/ArrayList.java | 115 ++++++++++++++++++ vm/JavaAPI/src/java/util/IdentityHashMap.java | 102 ++++++++++++---- 2 files changed, 191 insertions(+), 26 deletions(-) diff --git a/vm/JavaAPI/src/java/util/ArrayList.java b/vm/JavaAPI/src/java/util/ArrayList.java index 8a3c4129a15..4f99090a2a5 100644 --- a/vm/JavaAPI/src/java/util/ArrayList.java +++ b/vm/JavaAPI/src/java/util/ArrayList.java @@ -38,6 +38,27 @@ public class ArrayList extends AbstractList implements List, RandomAcce /** * Constructs a new instance of {@code ArrayList} with ten capacity. */ + // ISOLATION (PR #5766): the lazy default-capacity allocation that used to sit + // here is withdrawn. It replaced the eager new Object[10] with a SHARED static + // zero-length array, which also gave java.util.ArrayList a it had + // never had -- master's only static is a compile-time serialVersionUID, so the + // class previously emitted no static initializer at all. + // + // The suite then began stopping after exactly 145 of 166 screenshots on every + // target except glibc-x64, with ArrayList state corrupt at the point of + // failure: AIOOBE 89 inside pendingIdleSerialCalls.add, then AIOOBE -1, then a + // NullPointerException inside ArrayList.get, which only happens when the + // backing array reference itself is null. + // + // The list logic is NOT at fault: a differential fuzz of this exact source + // against java.util.ArrayList ran 3000 seeds x 200 random operations with no + // divergence, and every access to the corrupted list in Display is inside + // synchronized(lock). The corruption is therefore below Java, which makes the + // new and the process-wide shared array the part worth removing + // before anything subtler is blamed. + // + // The iterator below is the change that carried the measured win (iteration + // 25.5% -> 12.4% of mutator self-time) and is kept. public ArrayList() { this(10); } @@ -322,6 +343,100 @@ public void ensureCapacity(int minimumCapacity) { } } + /** + * Direct-array iterator, overriding AbstractList's generic SimpleListIterator. + * + * The inherited one was the single hottest method in a large translation -- + * 16.45% of mutator self-time on the 5782-class hellocodenameone corpus, more + * than twice the next entry. Three costs per element, none inherent: + * + * - a try/catch around the body, to turn IndexOutOfBoundsException into + * NoSuchElementException. ParparVM has no zero-cost exception tables, so a + * try block is a setjmp -- once per element, in the hottest loop in the + * program. An explicit bounds test costs a compare. + * - size() and get() as VIRTUAL calls on the outer list, with no JIT to + * inline them. + * - the index recomputed as size() - numLeft every iteration instead of + * being carried in a cursor. + * + * MEASURED after: the iteration path fell from 25.5% of mutator self-time to + * 12.4%, ArrayList.get from 7.42% to 0.55%, and _setjmp from 1.61% to zero. + * + * Semantics are unchanged: same ConcurrentModificationException on structural + * modification, same NoSuchElementException past the end, remove() still + * works. Reads array[firstIndex + i] exactly as get(int) does. + * + * Applies to every `for (x : list)` in every translated application whatever + * the loop's static type, because dispatch lands on the concrete ArrayList. + */ + // Package-private, not private: a private inner class whose constructor is + // reached from the outer class makes javac synthesise an access bridge and a + // ArrayList$1 marker type, so every iterator() paid an extra class and an + // aconst_null for the bridge argument. Nothing outside java.util can see it + // either way. + class ArrayListIterator implements Iterator { + private int cursor; + private int lastReturned = -1; + private int expectedModCount = modCount; + + public boolean hasNext() { + return cursor < size; + } + + public E next() { + if (modCount != expectedModCount) { + throw new ConcurrentModificationException(); + } + int i = cursor; + if (i >= size) { + throw new NoSuchElementException(); + } + // The i < size test is only a bounds check while the list's + // firstIndex + size <= array.length invariant holds, so the array + // itself has to be checked too. The iterator this replaced could not + // read out of range: it went through get(), which bounds-checks, inside + // a try that turned IndexOutOfBoundsException into + // NoSuchElementException. Dropping that -- the try was the point, since + // ParparVM has no zero-cost exception tables -- also dropped the only + // bounds check on the read, and ParparVM does NOT check an array read in + // a release build. The result was an out-of-bounds read of the heap + // rather than a recoverable exception, which is how an unrelated int[] + // ended up with a zeroed header and the screenshot suite died 145 tests + // in. OpenJDK's own ArrayList.Itr carries this identical guard + // (`if (i >= elementData.length) throw new ConcurrentModificationException()`); + // omitting it is the whole defect. One compare, and the measured win + // stays. + E[] a = array; + int idx = firstIndex + i; + if (idx < 0 || idx >= a.length) { + throw new ConcurrentModificationException(); + } + cursor = i + 1; + lastReturned = i; + return a[idx]; + } + + public void remove() { + if (lastReturned < 0) { + throw new IllegalStateException(); + } + if (modCount != expectedModCount) { + throw new ConcurrentModificationException(); + } + ArrayList.this.remove(lastReturned); + if (lastReturned < cursor) { + cursor--; + } + lastReturned = -1; + expectedModCount = modCount; + } + } + + @Override + public Iterator iterator() { + return new ArrayListIterator(); + } + @Override public E get(int location) { if (location < 0 || location >= size) { diff --git a/vm/JavaAPI/src/java/util/IdentityHashMap.java b/vm/JavaAPI/src/java/util/IdentityHashMap.java index 4010c21c92a..188a307ac74 100644 --- a/vm/JavaAPI/src/java/util/IdentityHashMap.java +++ b/vm/JavaAPI/src/java/util/IdentityHashMap.java @@ -125,25 +125,52 @@ static class IdentityHashMapIterator implements Iterator { final MapEntry.Type type; + /** + * Which of the three views this iterator serves. + * + * Keys and values come straight out of the table; only entrySet has to + * materialise an Entry, and only there can the caller observe one. The + * generic {@code type} callback cannot express that, because it takes a + * MapEntry -- so serving a key iterator through it allocated an Entry per + * next() purely to read one field back out and drop it. Measured on a + * self-hosting translation of the ParparVM translator: 1,366,140 such + * entries, 43.7MB, all garbage. java.util.HashMap already had separate + * key/value/entry iterators for exactly this reason; this one was missed. + */ + static final int KIND_ENTRY = 0; + static final int KIND_KEY = 1; + static final int KIND_VALUE = 2; + + final int kind; + boolean canRemove = false; IdentityHashMapIterator(MapEntry.Type value, IdentityHashMap hm) { associatedMap = hm; type = value; + kind = KIND_ENTRY; + expectedModCount = hm.modCount; + } + + IdentityHashMapIterator(int iteratorKind, IdentityHashMap hm) { + associatedMap = hm; + type = null; + kind = iteratorKind; expectedModCount = hm.modCount; } public boolean hasNext() { - while (position < associatedMap.elementData.length) { - // if this is an empty spot, go to the next one - if (associatedMap.elementData[position] == null) { - position += 2; - } else { - return true; - } + // elementData hoisted into a local: it was re-loaded from the outer map + // on every comparison AND on every array access, twice per probe step. + Object[] data = associatedMap.elementData; + int p = position; + int len = data.length; + while (p < len && data[p] == null) { + p += 2; } - return false; + position = p; + return p < len; } void checkConcurrentMod() throws ConcurrentModificationException { @@ -152,19 +179,50 @@ void checkConcurrentMod() throws ConcurrentModificationException { } } + @SuppressWarnings("unchecked") public E next() { - checkConcurrentMod(); - if (!hasNext()) { + // The concurrent-modification test and the null-skipping scan are + // INLINED here rather than reached through checkConcurrentMod() and + // hasNext(). + // + // An enhanced-for already pays two interface dispatches per element + // (hasNext then next); routing next() through two more non-inlined + // calls made it four, and ParparVM has no JIT to fold them away. + // MEASURED on the 5782-class hellocodenameone translation: + // IdentityHashMapIterator.next 6.43% of mutator self-time with + // checkConcurrentMod a further 1.84%, second only to the ArrayList + // iterator. + // + // Behaviour is unchanged: same ConcurrentModificationException on a + // structural change, same NoSuchElementException past the end, and + // position still advances past empty slots exactly as hasNext() did. + if (expectedModCount != associatedMap.modCount) { + throw new ConcurrentModificationException(); + } + Object[] data = associatedMap.elementData; + int p = position; + int len = data.length; + while (p < len && data[p] == null) { + p += 2; + } + if (p >= len) { + position = p; throw new NoSuchElementException(); } - IdentityHashMapEntry result = associatedMap - .getEntry(position); - lastPosition = position; - position += 2; - + lastPosition = p; + position = p + 2; canRemove = true; - return type.get(result); + + if (kind == KIND_KEY) { + Object key = associatedMap.elementData[lastPosition]; + return (E) (key == NULL_OBJECT ? null : key); + } + if (kind == KIND_VALUE) { + Object value = associatedMap.elementData[lastPosition + 1]; + return (E) (value == NULL_OBJECT ? null : value); + } + return type.get(associatedMap.getEntry(lastPosition)); } public void remove() { @@ -687,11 +745,7 @@ public boolean remove(Object key) { @Override public Iterator iterator() { return new IdentityHashMapIterator( - new MapEntry.Type() { - public K get(MapEntry entry) { - return entry.key; - } - }, IdentityHashMap.this); + IdentityHashMapIterator.KIND_KEY, IdentityHashMap.this); } }; } @@ -739,11 +793,7 @@ public void clear() { @Override public Iterator iterator() { return new IdentityHashMapIterator( - new MapEntry.Type() { - public V get(MapEntry entry) { - return entry.value; - } - }, IdentityHashMap.this); + IdentityHashMapIterator.KIND_VALUE, IdentityHashMap.this); } @Override From 2700998b6454061d1e754e640714497ab4def8f0 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 12 Sep 2026 07:42:39 +0300 Subject: [PATCH 27/66] Withdraw the GC pacing work and the String natives; keep what is justified The ArrayList bounds guard cleared the mass failure but not all of it, and the residue is real rather than flaky. On the fix commit the Windows cross leg reports pass=185 fail=3 -- the three being Media360Panorama, VRStereoScene and VideoIODecodedFrames, all of which "did not produce output". Master is pass=189 fail=0 with all three matching their stored reference, so they are this PR's doing. macOS improved from pass=34 fail=95 to pass=177 fail=8 and Windows tests from 174/11 to 181/7, so the guard fixed most of it and something smaller is left. Probe 2 had reverted the whole runtime side and was green everywhere, so the residue is in what probe 3 restored: java/lang, cn1_globals.{m,h} and nativeMethods.m. Of those the GC pacing work is the only part that plausibly makes a memory-heavy media test emit nothing at all -- changing when the heap is allowed to grow changes how hard a 360-panorama, a stereo VR scene or a video decode is squeezed, and "no output" is what a starved or timed-out test looks like. So this withdraws it: cn1_globals.m goes from 420 added lines to 73. What is kept is the part that is load-bearing rather than an optimisation: - the nine primitive class objects and their guards, which the self-hosting needs (Integer.TYPE and friends were null on every ParparVM target, which is the latent bug this PR opened with). - the cn1PacingPark safepoint fix, which is an independent correctness bug: the assist path continued with threadActive still TRUE and never reached the safepoint, so the collector force-stopped the thread. It was OBSERVED on the iOS simulator as a suite that finished and then hung without emitting its completion marker. Tightening the cap made it reachable; it did not create it, so the fix stays whether or not the cap comes back. Also withdrawn: the String.equals memcmp path and the compareTo coder hoist. Those were MEASURED at no improvement (java_lang_String_equals 6.78% -> 6.94% of mutator self-time) and were kept only because the old structure read backwards, which is not a reason to carry native string code through a corruption investigation. Recorded plainly because it is a real cost, not a footnote: the pacing work is where the self-hosting speedup came from (91s -> 27.4s on the 5782-class corpus). Withdrawing it gives that back. If this lands green the cap is confirmed as the cause and can be re-introduced with the specific defect fixed rather than wholesale -- note the run-ahead bound was already disabled on placeholder hosts, so Linux and Windows were never scaled by it and cn1PacingGrowthFloorBytes is the remaining candidate there. Gates D and A byte-identical over 795 files, negative control still detecting an injected corruption. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.m | 363 +--------------------- vm/ByteCodeTranslator/src/nativeMethods.m | 41 +-- 2 files changed, 5 insertions(+), 399 deletions(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index e16ca1cc202..960dc157c0b 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -1791,15 +1791,6 @@ static void cn1DrainDeadThreadPending() { // walks the page registry and its slots before their definitions. static inline JAVA_OBJECT cn1BibopSlot(CN1BibopPage* p, int i); static CN1BibopPage* _Atomic bibopAllPages; -#ifdef CN1_ALLOC_CENSUS -// Defined far below, beside the BiBOP page structures they read. Declared up here -// because the post-sweep hook that calls them is compiled earlier -- and OUTSIDE the -// CN1_GC_VERIFY block just above, which is off in an ordinary census build. -void cn1HeapAccounting(const char* label); -void cn1AllocCensus(const char* label); -void cn1LiveCensus(const char* label); -#endif - #ifdef CN1_GRACE_AUDIT static void cn1GraceAuditPreSweep(CODENAME_ONE_THREAD_STATE); #endif @@ -4729,14 +4720,6 @@ static void cn1GcReportStaleIndexSkip(void) { void codenameOneGCSweep() { struct ThreadLocalData* threadStateData = getThreadLocalData(); -#ifdef CN1_ALLOC_CENSUS - // BEFORE the sweep on purpose. This is the only point where the four slot - // states are still distinguishable -- the sweep stamps every fresh object with - // the current mark, after which "traced" and "kept by grace" look identical. - if(getenv("CN1_HEAP_REPORT")) { - cn1LiveCensus("pre-sweep"); - } -#endif // THE MARK THIS SWEEP WOULD ACT ON MAY BE INCOMPLETE. cn1GcPageIndexStale says the // page index could not be rebuilt, so every reference into a page registered since // the last successful rebuild failed to resolve and its object was never marked -- @@ -4882,15 +4865,6 @@ void codenameOneGCSweep() { // permanently broken. cn1GcVerifyHeap(threadStateData); #endif -#ifdef CN1_ALLOC_CENSUS - // Same reasoning as the verify hook above: post-sweep is when "live" means - // live. cn1HeapAccounting and cn1AllocCensus were written but never called - // from anywhere, so nothing could answer "what is the footprint made of". - if(getenv("CN1_HEAP_REPORT")) { - cn1HeapAccounting("post-sweep"); - cn1LiveCensus("post-sweep"); - } -#endif } JAVA_BOOLEAN removeObjectFromHeapCollection(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT o) { @@ -5551,10 +5525,6 @@ JAVA_INT java_lang_System_identityHashCode___java_lang_Object_R_int(CODENAME_ONE // Non-static: the inlined bump fast path (cn1_globals.h) reads bibopCurrent[ci]. __thread CN1BibopPage* bibopCurrent[CN1_BIBOP_NUM_CLASSES]; -#ifdef CN1_ALLOC_CENSUS -static void cn1BibopExitReport(void); -#endif - static void cn1BibopDoInit() { int ci = 0; // DIAGNOSTIC KNOB -- CN1_GC_TRIGGER_MB overrides how many uncollected bytes @@ -5586,38 +5556,8 @@ static void cn1BibopDoInit() { atomic_store_explicit(&bibopBypassGeneration[i], 0, memory_order_relaxed); bibopHighSurvivalStreak[i] = 0; } - // Prime the free-memory snapshot the pacing cap is computed from. - // - // Its only other caller is the mark cycle, so until the FIRST collection - // cn1CachedFreeMem was 0 and cn1BibopPacingCap's `fm / 8` evaluated to 0, leaving - // the cap at its floor of trigger * CN1_BIBOP_GC_HARD_CAP_MULTIPLIER = 72MB -- - // during exactly the window where there is least reason to throttle anything, - // since nothing has been collected yet. ProcessBudgetPacingIntegrationTest's - // control arm reports minCapKb=4194304 with this in place and the 72MB floor - // without it. - // - // Priming it matters twice over: the run-ahead bound's own floor is scaled off - // the same reading (see cn1PacingGrowthFloorBytes), so a zero here would arm that - // bound at its absolute 512MB minimum no matter how much memory the host has. - cn1RefreshFreeMemCache(); -#ifdef CN1_ALLOC_CENSUS - if(getenv("CN1_HEAP_REPORT")) { - atexit(cn1BibopExitReport); - } -#endif } -#ifdef CN1_ALLOC_CENSUS -// Registered from cn1BibopDoInit under CN1_HEAP_REPORT. A batch program usually -// ends between collections, so the post-sweep reports alone never show the state -// the process actually died holding. -static void cn1BibopExitReport(void) { - cn1HeapAccounting("exit"); - cn1LiveCensus("exit"); - cn1AllocCensus("exit"); -} -#endif - static void cn1BibopFormatPage(CN1BibopPage* p, int ci) { int slotSize = cn1BibopClassSize[ci]; // slot 0 starts after the page header, rounded up to 16-byte alignment so @@ -6328,39 +6268,6 @@ static inline JAVA_OBJECT cn1BibopSlot(CN1BibopPage* p, int i) { #ifndef CN1_PACING_GROWTH_FLOOR_BYTES #define CN1_PACING_GROWTH_FLOOR_BYTES (512LL*1024*1024) #endif -// Ceiling on how far a mutator may run ahead of a cycle in flight, regardless of -// how much RAM the host has. See the measurement table in cn1BibopPacingCap. -#ifndef CN1_BIBOP_PACING_MAX_RUNAHEAD_BYTES -#define CN1_BIBOP_PACING_MAX_RUNAHEAD_BYTES (1024L*1024*1024) -#endif -// cn1_available_memory answers a flat 100MB on every platform where it cannot -// measure: Linux, Windows, and the non-Apple fallback. That number is not a -// reading, and a bound DERIVED from it is not a bound -- it is a constant that -// happens to look like one. -// -// This matters asymmetrically. cn1PacingGrowthFloorBytes above only ever RAISES -// its floor from fm, so on a placeholder host the absolute floor wins and -// behaviour is bit-for-bit unchanged. The run-ahead bound below only ever LOWERS -// the cap, so scaling it by a placeholder TIGHTENS pacing on exactly the hosts we -// know nothing about. It did: BibopPageFloorIntegrationTest went red on arm64 -// Linux, where fm/8 is 12.5MB, while the same code passed on macOS where fm is -// real. -// -// So the bound applies only where fm is a genuine reading. Returns 0 to mean -// "not measurable here, leave the cap alone". -#ifndef CN1_PACING_PLACEHOLDER_FREE_MEM -#define CN1_PACING_PLACEHOLDER_FREE_MEM (1024L*1024*100) -#endif -static long cn1PacingRunAheadBound(long fm) { - if(fm <= CN1_PACING_PLACEHOLDER_FREE_MEM) { - return 0; - } - long bound = CN1_BIBOP_PACING_MAX_RUNAHEAD_BYTES; - if(bound > fm / 8) { - bound = fm / 8; - } - return bound; -} // How stale a below-floor footprint reading may be before the bound re-probes it. The // probe is task_info on Apple and one /proc read on Linux -- a microsecond or two -- and // it is taken at most once per interval across the whole process, and only when the bound @@ -6534,48 +6441,14 @@ static long long cn1PacingFootprintNow(void) { return fp; } -// The footprint at which the run-ahead bound starts applying, scaled to the memory -// this host actually has. -// -// A fixed 512MB says "this process has grown"; it does not say the machine is under -// any pressure, and the bound exists for pressure. On a host with tens of GB free, a -// process holding a couple of GB is nowhere near runaway, and clamping it there -// parks the mutator against a collector that cannot get under the ceiling: measured -// at 6.7-8.7s versus 1.4s for the same work, to save 2% of peak footprint. -// -// So take the larger of the absolute floor and a quarter of available memory. Two -// properties this has to keep: -// -// - Where cn1_available_memory is the flat 100MB placeholder (Linux, Windows, and -// the non-Apple fallback), fm/4 is 25MB, the absolute floor wins, and behaviour is -// bit-for-bit what it was. Nothing changes on a platform where we cannot measure. -// - It only ever RAISES the floor, so the bound can only engage later than before, -// never earlier. It cannot make a constrained host more permissive than it was. -// -// This is the no-per-process-ceiling path only. Where a ceiling exists -- iOS's dirty -// memory limit, or an explicit process budget -- cn1PacingPark takes the bounded -// branch instead and never reaches cn1BibopPacingCap, so none of this loosens the -// admission control that keeps an app inside its own limit. -static long long cn1PacingGrowthFloorBytes(void) { - long long floor = CN1_PACING_GROWTH_FLOOR_BYTES; - long fm = atomic_load_explicit(&cn1CachedFreeMem, memory_order_relaxed); - if(fm > 0) { - long long scaled = (long long)fm / 4; - if(scaled > floor) { - floor = scaled; - } - } - return floor; -} - static JAVA_BOOLEAN cn1PacingPastGrowthFloor(void) { - long long floor = cn1PacingGrowthFloorBytes(); // Once the cache is over the floor the bound is engaged and a syscall to re-confirm // it buys nothing, so this stays ahead of the probe. - if(atomic_load_explicit(&cn1CachedProcFootprint, memory_order_relaxed) > floor) { + if(atomic_load_explicit(&cn1CachedProcFootprint, memory_order_relaxed) + > CN1_PACING_GROWTH_FLOOR_BYTES) { return JAVA_TRUE; } - return cn1PacingFootprintNow() > floor; + return cn1PacingFootprintNow() > CN1_PACING_GROWTH_FLOOR_BYTES; } static long cn1BibopPacingCap(CODENAME_ONE_THREAD_STATE) { @@ -6635,71 +6508,10 @@ static long cn1BibopPacingCap(CODENAME_ONE_THREAD_STATE) { if(capCeiling < base) { capCeiling = base; } - // FLOOR the clamp at the point where run-ahead stops paying, when the host - // can afford it. - // - // capCeiling is derived from the TRIGGER, and the trigger spends most of a - // run at its 24MB minimum, so this clamp lands at 24*8 = 192MB. Confirmed - // at runtime, not inferred: `[PACING] minCapKb=196608`. That is what - // actually throttles the mutator -- NOT the fm/8 and fm/2 figures above, - // which never bind on a large host. It is also why the diagnostic knob - // CN1_GC_PACING_CAP_MB appears to work miracles: returning early, it - // bypasses this clamp entirely. - // - // MEASURED, 5782-class hellocodenameone translation, min of 3 interleaved - // reps, phys_footprint: - // - // cap in force wall peak - // 192MB 46.3s 9736MB <- this clamp, as it stood - // 1024MB 23.8s 8325MB - // 2048MB 22.9s 12870MB <- 2 more seconds for 4GB - // - // Run-ahead saturates near 1GB: below it the mutator parks waiting on a - // cycle it cannot help finish, and the resulting bigger heap costs kernel - // time faulting pages in, so tightening this clamp lost on BOTH axes. - // - // Kept proportionate rather than absolute: on a host where fm/8 is already - // under the saturation point -- a phone, a container, the flat 100MB - // placeholder off Apple -- the floor follows fm/8 and nothing loosens. - { - long runAhead = cn1PacingRunAheadBound(fm); - if(runAhead > 0 && capCeiling < runAhead) { - capCeiling = runAhead; - } - } if(cap > capCeiling && cn1PacingPastGrowthFloor()) { cap = capCeiling; } } - // FINAL absolute bound on run-ahead. Applied last, after the trigger-derived - // clamp above, because the two failure modes are opposite and BOTH were - // measured on this workload: - // - // - the clamp alone drove cap down to 192MB (trigger 24MB x 8), which parks - // the mutator on a cycle it cannot help finish: 46.3s / 9736MB. - // - flooring the clamp without bounding the top left cap at fm/8 = 4GB (or - // fm/2 = 16GB for a thread flagged high-throughput), so the heap ran to - // 11848MB and the run took 48.0s -- worse on both axes. - // - // Pinning run-ahead near 1GB gives 23.8s / 8325MB. The saturation is real: at - // 2GB the run is 22.9s but the footprint is 12870MB, i.e. 2 more GB per second - // saved. So the useful range is narrow and this is its top. - // - // Proportionate, not absolute: on a host where fm/8 is already below the - // saturation point -- a phone, a container, the flat 100MB placeholder off - // Apple -- this follows fm/8 and nothing is loosened. `base` is still honoured - // so a build with a large static trigger keeps the admission it had. - { - long runAhead = cn1PacingRunAheadBound(fm); - if(runAhead > 0) { - if(cap > runAhead) { - cap = runAhead; - } - if(cap < base) { - cap = base; - } - } - } if(cn1PacingTraceOn()) { long seen = atomic_load_explicit(&cn1PacingMinCap, memory_order_relaxed); while(cap < seen && @@ -7517,175 +7329,6 @@ void cn1HeapAccounting(const char* label) { fflush(stderr); } -/** - * Prints the LIVE heap by class, biggest first. - * - * The twin of cn1AllocCensus and the one that answers a different question. - * cn1AllocCensus is a census of what was ALLOCATED -- churn, which is what costs - * CPU. This is a census of what is still HERE at the moment the sweep finished, - * which is what costs memory. A class can dominate one and not appear in the - * other: a short-lived iterator allocated a million times retains nothing, and a - * cache allocated once retains everything. - * - * Sizes are what the object OCCUPIES, not what it asked for: a BiBOP object is - * charged its whole size-class slot and a legacy object its whole malloc block, - * so the per-class totals add up to the footprint rather than to a smaller - * idealised number. Rounding waste therefore shows up against the class that - * causes it, which is the class that can be made to stop causing it. - * - * Classes are collected into a local open-addressed table keyed on the clazz - * pointer rather than read out of cn1ClazzSet, which only exists under - * CN1_CONSERVATIVE_GC_ROOTS. - * - * Must run where the marks are meaningful -- the post-sweep hook, the same point - * the GC verifier uses. - */ -#define CN1_LIVE_CENSUS_SLOTS 8192 -// Four states a slot can be in when the SWEEP is about to look at it. Read -// pre-sweep they are distinguishable; read post-sweep they are not, because the -// sweep stamps every fresh object live and that is exactly the population the -// question is about. -#define CN1_LB_TRACED 0 /* mark == currentGcMarkValue: traced live this cycle */ -#define CN1_LB_FRESH 1 /* mark == -1: allocated since the mark, gets one grace */ -#define CN1_LB_AGING 2 /* mark == V-1: not traced, kept one more cycle anyway */ -#define CN1_LB_DEAD 3 /* older: this sweep reclaims it */ -#define CN1_LB_COUNT 4 -struct CN1LiveRow { struct clazz* c; long count; long long bytes; long b[CN1_LB_COUNT]; }; -static struct CN1LiveRow cn1LiveRows[CN1_LIVE_CENSUS_SLOTS]; - -static int cn1LiveBucket(int m) { - // -1 must be tested before the "older than V-1" arm: it is numerically less - // than V-1 for any live epoch, so the ordering is what keeps a fresh object - // out of the reclaimable bucket. - if(m == -1) { - return CN1_LB_FRESH; - } - if(m == currentGcMarkValue) { - return CN1_LB_TRACED; - } - if(m == currentGcMarkValue - 1) { - return CN1_LB_AGING; - } - return CN1_LB_DEAD; -} - -static void cn1LiveTally(struct clazz* c, long long bytes, int bucket) { - if(c == 0) { - return; - } - size_t h = (((uintptr_t)c) >> 4) & (CN1_LIVE_CENSUS_SLOTS - 1); - for(int probe = 0 ; probe < CN1_LIVE_CENSUS_SLOTS ; probe++) { - size_t i = (h + (size_t)probe) & (CN1_LIVE_CENSUS_SLOTS - 1); - if(cn1LiveRows[i].c == 0) { - cn1LiveRows[i].c = c; - } - if(cn1LiveRows[i].c == c) { - cn1LiveRows[i].count++; - cn1LiveRows[i].bytes += bytes; - cn1LiveRows[i].b[bucket]++; - return; - } - } - // Table full: 8192 slots against the ~170 classes a large program allocates, - // so this is unreachable short of a pathological program. Dropping the row is - // still better than looping forever, and the printed total will not match the - // per-class rows, which is the visible signal that it happened. -} - -void cn1LiveCensus(const char* label) { - memset(cn1LiveRows, 0, sizeof(cn1LiveRows)); - long long bibopBytes = 0, legacyBytes = 0; - long bibopObjs = 0, legacyObjs = 0; - long totals[CN1_LB_COUNT]; - for(int i = 0 ; i < CN1_LB_COUNT ; i++) { - totals[i] = 0; - } - - CN1BibopPage* p = atomic_load_explicit(&bibopAllPages, memory_order_acquire); - while(p != 0) { - int n = atomic_load_explicit(&p->bumpIndex, memory_order_acquire); - for(int i = 0 ; i < n ; i++) { - JAVA_OBJECT o = cn1BibopSlot(p, i); - int m = __atomic_load_n(&o->__codenameOneGcMark, __ATOMIC_ACQUIRE); - // Occupied, not "provably reachable": a slot awaiting collection is - // still holding memory, and this census is about what memory is being - // held. A slot on the page free-list is the one that costs nothing -- - // the same test cn1ConservativeResolve uses. (CN1_GC_POISON_MARK is - // deliberately not consulted: it is defined further down, inside the - // verifier's section, and exists only in a CN1_GC_VERIFY build.) - if(m == CN1_BIBOP_FREE_MARK) { - continue; - } - int bucket = cn1LiveBucket(m); - cn1LiveTally(o->__codenameOneParentClsReference, (long long)p->slotSize, bucket); - bibopBytes += (long long)p->slotSize; - bibopObjs++; - totals[bucket]++; - } - p = atomic_load_explicit(&p->nextAll, memory_order_acquire); - } - - int nHeap = currentSizeOfAllObjectsInHeap; - for(int i = 0 ; i < nHeap ; i++) { - JAVA_OBJECT o = allObjectsInHeap[i]; - if(o == JAVA_NULL) { - continue; - } - // An adopted object lives in a BiBOP slot and was already charged by the - // page walk; malloc_size on it would read a block header that is not there. - if(o->__heapPosition == CN1_BIBOP_ADOPTED) { - continue; - } - long long sz = 0; -#if defined(__APPLE__) - sz = (long long)malloc_size((void*)o); -#endif - int lbucket = cn1LiveBucket(o->__codenameOneGcMark); - cn1LiveTally(o->__codenameOneParentClsReference, sz, lbucket); - legacyBytes += sz; - legacyObjs++; - totals[lbucket]++; - } - - // OCCUPIED is what costs memory. The four buckets say WHY each object is still - // occupying a slot, and they call for different fixes: traced means the program - // really is holding it, fresh and aging mean the collector is holding it under - // the grace and aging rules, and dead means this sweep is about to return it. - long occupied = bibopObjs + legacyObjs; - fprintf(stderr, "[LIVE:%s] occupied %ld objects %.2fMB | traced %ld (%.0f%%) " - "fresh %ld (%.0f%%) aging %ld (%.0f%%) dead %ld (%.0f%%) | bibop %.2fMB legacy %.2fMB\n", - label, occupied, (bibopBytes + legacyBytes) / 1048576.0, - totals[CN1_LB_TRACED], 100.0 * totals[CN1_LB_TRACED] / (occupied > 0 ? occupied : 1), - totals[CN1_LB_FRESH], 100.0 * totals[CN1_LB_FRESH] / (occupied > 0 ? occupied : 1), - totals[CN1_LB_AGING], 100.0 * totals[CN1_LB_AGING] / (occupied > 0 ? occupied : 1), - totals[CN1_LB_DEAD], 100.0 * totals[CN1_LB_DEAD] / (occupied > 0 ? occupied : 1), - bibopBytes / 1048576.0, legacyBytes / 1048576.0); - for(int shown = 0 ; shown < 30 ; shown++) { - int best = -1; - for(int i = 0 ; i < CN1_LIVE_CENSUS_SLOTS ; i++) { - if(cn1LiveRows[i].c != 0 && cn1LiveRows[i].bytes > 0 - && (best < 0 || cn1LiveRows[i].bytes > cn1LiveRows[best].bytes)) { - best = i; - } - } - if(best < 0) { - break; - } - long rc = cn1LiveRows[best].count > 0 ? cn1LiveRows[best].count : 1; - fprintf(stderr, "[LIVE:%s] %8.2fMB %9ld objs %4lld B/obj traced %3.0f%% fresh %3.0f%% " - "aging %3.0f%% dead %3.0f%% %s\n", - label, cn1LiveRows[best].bytes / 1048576.0, cn1LiveRows[best].count, - cn1LiveRows[best].bytes / rc, - 100.0 * cn1LiveRows[best].b[CN1_LB_TRACED] / rc, - 100.0 * cn1LiveRows[best].b[CN1_LB_FRESH] / rc, - 100.0 * cn1LiveRows[best].b[CN1_LB_AGING] / rc, - 100.0 * cn1LiveRows[best].b[CN1_LB_DEAD] / rc, - cn1LiveRows[best].c->clsName ? cn1LiveRows[best].c->clsName : "?"); - cn1LiveRows[best].bytes = 0; - } - fflush(stderr); -} - void cn1AllocCensus(const char* label) { struct Row { const char* name; long count; long bytes; }; static struct Row rows[4096]; diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 4b7f8d1259b..da508632c4e 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -423,26 +423,6 @@ JAVA_BOOLEAN java_lang_String_equals___java_lang_Object_R_boolean(CODENAME_ONE_T // Fast path: both backing arrays are char[] -- byte-equality of UTF-16 code // units == string equality; libc memcmp is the SIMD-optimized comparison on // every target. - // BOTH LATIN-1 -- the overwhelmingly common case, and until now the SLOW one. - // - // The char[] path below already had a memcmp; the compact byte[] path did - // not, so two ASCII strings (every class name, method name and descriptor - // this translator compares) fell into the per-character loop at the bottom, - // which calls cn1StrCharAtRaw TWICE per character. That helper reloads - // `value` and `offset` and branches on the backing array's class pointer - // EVERY time, so the common case paid a branch and two field loads per char - // where a single memcmp would do. - // - // Latin-1 stores each char as its raw 0..255 byte, so memcmp's unsigned byte - // ordering is exactly char ordering; equality is bit-identical. - // - // MEASURED before the fix: java_lang_String_equals was 6.78% of mutator - // self-time on the 5782-class hellocodenameone translation. - if(cn1StrIsLatin1(__cn1ThisObject) && cn1StrIsLatin1(__cn1Arg1)) { - JAVA_ARRAY_BYTE* ta = ((JAVA_ARRAY_BYTE*)((JAVA_ARRAY)t->java_lang_String_value)->data) + t->java_lang_String_offset; - JAVA_ARRAY_BYTE* oa = ((JAVA_ARRAY_BYTE*)((JAVA_ARRAY)o->java_lang_String_value)->data) + o->java_lang_String_offset; - return memcmp(ta, oa, (size_t)t->java_lang_String_count) == 0 ? JAVA_TRUE : JAVA_FALSE; - } if(!cn1StrIsLatin1(__cn1ThisObject) && !cn1StrIsLatin1(__cn1Arg1)) { JAVA_ARRAY_CHAR* oa = ((JAVA_ARRAY_CHAR*)((JAVA_ARRAY)o->java_lang_String_value)->data) + o->java_lang_String_offset; JAVA_ARRAY_CHAR* ta = ((JAVA_ARRAY_CHAR*)((JAVA_ARRAY)t->java_lang_String_value)->data) + t->java_lang_String_offset; @@ -491,25 +471,8 @@ JAVA_INT java_lang_String_compareTo___java_lang_String_R_int(CODENAME_ONE_THREAD } return tc - oc; } - // BOTH Latin-1: hoist the coder test and the field reloads OUT of the loop. - // cn1StrCharAtRaw re-derives the base pointer and re-tests the backing array's - // class on every character, twice per iteration; with both coders known the - // loop is two raw byte pointers. Ordering is unchanged -- Latin-1 bytes are - // the char values 0..255. - if(cn1StrIsLatin1(__cn1ThisObject) && cn1StrIsLatin1(__cn1Arg1)) { - struct obj__java_lang_String* ts = (struct obj__java_lang_String*)__cn1ThisObject; - struct obj__java_lang_String* os = (struct obj__java_lang_String*)__cn1Arg1; - const JAVA_ARRAY_BYTE* tb = ((JAVA_ARRAY_BYTE*)((JAVA_ARRAY)ts->java_lang_String_value)->data) + ts->java_lang_String_offset; - const JAVA_ARRAY_BYTE* ob = ((JAVA_ARRAY_BYTE*)((JAVA_ARRAY)os->java_lang_String_value)->data) + os->java_lang_String_offset; - for(JAVA_INT k = 0; k < minL; k++) { - int d = (int)(tb[k] & 0xff) - (int)(ob[k] & 0xff); - if(d) { - return d; - } - } - return tc - oc; - } - // Mixed coders: one Latin-1, one UTF-16. Rare; keep the general helper. + // Coder-aware path: at least one string is Latin-1 (byte[]); compare logical + // chars. Same UTF-16 code-unit ordering, bit-identical to the char[] path. for(JAVA_INT k = 0; k < minL; k++) { int d = (int)cn1StrCharAtRaw(__cn1ThisObject, k) - (int)cn1StrCharAtRaw(__cn1Arg1, k); if(d) { From 84065db2cb00e1ae150b9a547642d1e0d978bc12 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 12 Sep 2026 08:40:23 +0300 Subject: [PATCH 28/66] Restore the pacing cap, drop the StringBuilder growth change Withdrawing the pacing work made the suite WORSE, not better: the Windows cross leg went from pass=185 fail=3 to pass=173 fail=11, and the whole Window-* group went missing again. So the cap was not the cause of the residual failures -- if anything it was holding them off -- and it is restored here. That inverts the reading of the residue. "More GC makes it worse" is not what a corruption bug looks like; it is what MEMORY PRESSURE looks like. And the tests that fail are the ones with the largest buffers: Media360Panorama, VRStereoScene, VideoIODecodedFrames, and under the tighter heap the Window-* dialogs as well. They do not fail an assertion, they produce no output at all. The StringBuilder growth change is the part of this PR that raises peak memory. Going from Harmony's 1.5x ((len>>1)+len+2) to OpenJDK's 2x halves the abandoned intermediate arrays -- N*r/(r-1) is 3N at r=1.5 and 2N at r=2 -- which is why it was worth doing for a translation, where the emit phase is StringBuilder-bound. But it does that by allowing a buffer to sit at up to 2x the chars actually needed rather than 1.5x, and the screenshot suite is evidently close enough to its ceiling that the difference shows. Trading a translator-workload win for screenshot tests that cannot allocate is not a trade worth making, so it goes. Also still withdrawn from the previous commit: the String.equals memcmp path and the compareTo coder hoist, which were measured at no improvement (6.78% -> 6.94%) and so cost nothing to drop. What this leaves of the PR: the self-hosting harness and its CI gate, the primitive class objects (Integer.TYPE and friends were null on every ParparVM target), the concat fusion, the iterator lowering, the ArrayList iterator with its bounds guard, the IdentityHashMap iterator split, the pacing cap, and the cn1PacingPark safepoint fix. Stated rather than buried: this is reasoning from two CI runs, and these suites are not perfectly deterministic. If the media tests still fail the memory-pressure reading is wrong and the next step is to measure the app's peak footprint directly instead of inferring it from pass counts. Gates D and A byte-identical over 795 files. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.m | 363 +++++++++++++++++++- vm/JavaAPI/src/java/lang/StringBuilder.java | 19 +- 2 files changed, 361 insertions(+), 21 deletions(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index 960dc157c0b..e16ca1cc202 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -1791,6 +1791,15 @@ static void cn1DrainDeadThreadPending() { // walks the page registry and its slots before their definitions. static inline JAVA_OBJECT cn1BibopSlot(CN1BibopPage* p, int i); static CN1BibopPage* _Atomic bibopAllPages; +#ifdef CN1_ALLOC_CENSUS +// Defined far below, beside the BiBOP page structures they read. Declared up here +// because the post-sweep hook that calls them is compiled earlier -- and OUTSIDE the +// CN1_GC_VERIFY block just above, which is off in an ordinary census build. +void cn1HeapAccounting(const char* label); +void cn1AllocCensus(const char* label); +void cn1LiveCensus(const char* label); +#endif + #ifdef CN1_GRACE_AUDIT static void cn1GraceAuditPreSweep(CODENAME_ONE_THREAD_STATE); #endif @@ -4720,6 +4729,14 @@ static void cn1GcReportStaleIndexSkip(void) { void codenameOneGCSweep() { struct ThreadLocalData* threadStateData = getThreadLocalData(); +#ifdef CN1_ALLOC_CENSUS + // BEFORE the sweep on purpose. This is the only point where the four slot + // states are still distinguishable -- the sweep stamps every fresh object with + // the current mark, after which "traced" and "kept by grace" look identical. + if(getenv("CN1_HEAP_REPORT")) { + cn1LiveCensus("pre-sweep"); + } +#endif // THE MARK THIS SWEEP WOULD ACT ON MAY BE INCOMPLETE. cn1GcPageIndexStale says the // page index could not be rebuilt, so every reference into a page registered since // the last successful rebuild failed to resolve and its object was never marked -- @@ -4865,6 +4882,15 @@ void codenameOneGCSweep() { // permanently broken. cn1GcVerifyHeap(threadStateData); #endif +#ifdef CN1_ALLOC_CENSUS + // Same reasoning as the verify hook above: post-sweep is when "live" means + // live. cn1HeapAccounting and cn1AllocCensus were written but never called + // from anywhere, so nothing could answer "what is the footprint made of". + if(getenv("CN1_HEAP_REPORT")) { + cn1HeapAccounting("post-sweep"); + cn1LiveCensus("post-sweep"); + } +#endif } JAVA_BOOLEAN removeObjectFromHeapCollection(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT o) { @@ -5525,6 +5551,10 @@ JAVA_INT java_lang_System_identityHashCode___java_lang_Object_R_int(CODENAME_ONE // Non-static: the inlined bump fast path (cn1_globals.h) reads bibopCurrent[ci]. __thread CN1BibopPage* bibopCurrent[CN1_BIBOP_NUM_CLASSES]; +#ifdef CN1_ALLOC_CENSUS +static void cn1BibopExitReport(void); +#endif + static void cn1BibopDoInit() { int ci = 0; // DIAGNOSTIC KNOB -- CN1_GC_TRIGGER_MB overrides how many uncollected bytes @@ -5556,8 +5586,38 @@ static void cn1BibopDoInit() { atomic_store_explicit(&bibopBypassGeneration[i], 0, memory_order_relaxed); bibopHighSurvivalStreak[i] = 0; } + // Prime the free-memory snapshot the pacing cap is computed from. + // + // Its only other caller is the mark cycle, so until the FIRST collection + // cn1CachedFreeMem was 0 and cn1BibopPacingCap's `fm / 8` evaluated to 0, leaving + // the cap at its floor of trigger * CN1_BIBOP_GC_HARD_CAP_MULTIPLIER = 72MB -- + // during exactly the window where there is least reason to throttle anything, + // since nothing has been collected yet. ProcessBudgetPacingIntegrationTest's + // control arm reports minCapKb=4194304 with this in place and the 72MB floor + // without it. + // + // Priming it matters twice over: the run-ahead bound's own floor is scaled off + // the same reading (see cn1PacingGrowthFloorBytes), so a zero here would arm that + // bound at its absolute 512MB minimum no matter how much memory the host has. + cn1RefreshFreeMemCache(); +#ifdef CN1_ALLOC_CENSUS + if(getenv("CN1_HEAP_REPORT")) { + atexit(cn1BibopExitReport); + } +#endif } +#ifdef CN1_ALLOC_CENSUS +// Registered from cn1BibopDoInit under CN1_HEAP_REPORT. A batch program usually +// ends between collections, so the post-sweep reports alone never show the state +// the process actually died holding. +static void cn1BibopExitReport(void) { + cn1HeapAccounting("exit"); + cn1LiveCensus("exit"); + cn1AllocCensus("exit"); +} +#endif + static void cn1BibopFormatPage(CN1BibopPage* p, int ci) { int slotSize = cn1BibopClassSize[ci]; // slot 0 starts after the page header, rounded up to 16-byte alignment so @@ -6268,6 +6328,39 @@ static inline JAVA_OBJECT cn1BibopSlot(CN1BibopPage* p, int i) { #ifndef CN1_PACING_GROWTH_FLOOR_BYTES #define CN1_PACING_GROWTH_FLOOR_BYTES (512LL*1024*1024) #endif +// Ceiling on how far a mutator may run ahead of a cycle in flight, regardless of +// how much RAM the host has. See the measurement table in cn1BibopPacingCap. +#ifndef CN1_BIBOP_PACING_MAX_RUNAHEAD_BYTES +#define CN1_BIBOP_PACING_MAX_RUNAHEAD_BYTES (1024L*1024*1024) +#endif +// cn1_available_memory answers a flat 100MB on every platform where it cannot +// measure: Linux, Windows, and the non-Apple fallback. That number is not a +// reading, and a bound DERIVED from it is not a bound -- it is a constant that +// happens to look like one. +// +// This matters asymmetrically. cn1PacingGrowthFloorBytes above only ever RAISES +// its floor from fm, so on a placeholder host the absolute floor wins and +// behaviour is bit-for-bit unchanged. The run-ahead bound below only ever LOWERS +// the cap, so scaling it by a placeholder TIGHTENS pacing on exactly the hosts we +// know nothing about. It did: BibopPageFloorIntegrationTest went red on arm64 +// Linux, where fm/8 is 12.5MB, while the same code passed on macOS where fm is +// real. +// +// So the bound applies only where fm is a genuine reading. Returns 0 to mean +// "not measurable here, leave the cap alone". +#ifndef CN1_PACING_PLACEHOLDER_FREE_MEM +#define CN1_PACING_PLACEHOLDER_FREE_MEM (1024L*1024*100) +#endif +static long cn1PacingRunAheadBound(long fm) { + if(fm <= CN1_PACING_PLACEHOLDER_FREE_MEM) { + return 0; + } + long bound = CN1_BIBOP_PACING_MAX_RUNAHEAD_BYTES; + if(bound > fm / 8) { + bound = fm / 8; + } + return bound; +} // How stale a below-floor footprint reading may be before the bound re-probes it. The // probe is task_info on Apple and one /proc read on Linux -- a microsecond or two -- and // it is taken at most once per interval across the whole process, and only when the bound @@ -6441,14 +6534,48 @@ static long long cn1PacingFootprintNow(void) { return fp; } +// The footprint at which the run-ahead bound starts applying, scaled to the memory +// this host actually has. +// +// A fixed 512MB says "this process has grown"; it does not say the machine is under +// any pressure, and the bound exists for pressure. On a host with tens of GB free, a +// process holding a couple of GB is nowhere near runaway, and clamping it there +// parks the mutator against a collector that cannot get under the ceiling: measured +// at 6.7-8.7s versus 1.4s for the same work, to save 2% of peak footprint. +// +// So take the larger of the absolute floor and a quarter of available memory. Two +// properties this has to keep: +// +// - Where cn1_available_memory is the flat 100MB placeholder (Linux, Windows, and +// the non-Apple fallback), fm/4 is 25MB, the absolute floor wins, and behaviour is +// bit-for-bit what it was. Nothing changes on a platform where we cannot measure. +// - It only ever RAISES the floor, so the bound can only engage later than before, +// never earlier. It cannot make a constrained host more permissive than it was. +// +// This is the no-per-process-ceiling path only. Where a ceiling exists -- iOS's dirty +// memory limit, or an explicit process budget -- cn1PacingPark takes the bounded +// branch instead and never reaches cn1BibopPacingCap, so none of this loosens the +// admission control that keeps an app inside its own limit. +static long long cn1PacingGrowthFloorBytes(void) { + long long floor = CN1_PACING_GROWTH_FLOOR_BYTES; + long fm = atomic_load_explicit(&cn1CachedFreeMem, memory_order_relaxed); + if(fm > 0) { + long long scaled = (long long)fm / 4; + if(scaled > floor) { + floor = scaled; + } + } + return floor; +} + static JAVA_BOOLEAN cn1PacingPastGrowthFloor(void) { + long long floor = cn1PacingGrowthFloorBytes(); // Once the cache is over the floor the bound is engaged and a syscall to re-confirm // it buys nothing, so this stays ahead of the probe. - if(atomic_load_explicit(&cn1CachedProcFootprint, memory_order_relaxed) - > CN1_PACING_GROWTH_FLOOR_BYTES) { + if(atomic_load_explicit(&cn1CachedProcFootprint, memory_order_relaxed) > floor) { return JAVA_TRUE; } - return cn1PacingFootprintNow() > CN1_PACING_GROWTH_FLOOR_BYTES; + return cn1PacingFootprintNow() > floor; } static long cn1BibopPacingCap(CODENAME_ONE_THREAD_STATE) { @@ -6508,10 +6635,71 @@ static long cn1BibopPacingCap(CODENAME_ONE_THREAD_STATE) { if(capCeiling < base) { capCeiling = base; } + // FLOOR the clamp at the point where run-ahead stops paying, when the host + // can afford it. + // + // capCeiling is derived from the TRIGGER, and the trigger spends most of a + // run at its 24MB minimum, so this clamp lands at 24*8 = 192MB. Confirmed + // at runtime, not inferred: `[PACING] minCapKb=196608`. That is what + // actually throttles the mutator -- NOT the fm/8 and fm/2 figures above, + // which never bind on a large host. It is also why the diagnostic knob + // CN1_GC_PACING_CAP_MB appears to work miracles: returning early, it + // bypasses this clamp entirely. + // + // MEASURED, 5782-class hellocodenameone translation, min of 3 interleaved + // reps, phys_footprint: + // + // cap in force wall peak + // 192MB 46.3s 9736MB <- this clamp, as it stood + // 1024MB 23.8s 8325MB + // 2048MB 22.9s 12870MB <- 2 more seconds for 4GB + // + // Run-ahead saturates near 1GB: below it the mutator parks waiting on a + // cycle it cannot help finish, and the resulting bigger heap costs kernel + // time faulting pages in, so tightening this clamp lost on BOTH axes. + // + // Kept proportionate rather than absolute: on a host where fm/8 is already + // under the saturation point -- a phone, a container, the flat 100MB + // placeholder off Apple -- the floor follows fm/8 and nothing loosens. + { + long runAhead = cn1PacingRunAheadBound(fm); + if(runAhead > 0 && capCeiling < runAhead) { + capCeiling = runAhead; + } + } if(cap > capCeiling && cn1PacingPastGrowthFloor()) { cap = capCeiling; } } + // FINAL absolute bound on run-ahead. Applied last, after the trigger-derived + // clamp above, because the two failure modes are opposite and BOTH were + // measured on this workload: + // + // - the clamp alone drove cap down to 192MB (trigger 24MB x 8), which parks + // the mutator on a cycle it cannot help finish: 46.3s / 9736MB. + // - flooring the clamp without bounding the top left cap at fm/8 = 4GB (or + // fm/2 = 16GB for a thread flagged high-throughput), so the heap ran to + // 11848MB and the run took 48.0s -- worse on both axes. + // + // Pinning run-ahead near 1GB gives 23.8s / 8325MB. The saturation is real: at + // 2GB the run is 22.9s but the footprint is 12870MB, i.e. 2 more GB per second + // saved. So the useful range is narrow and this is its top. + // + // Proportionate, not absolute: on a host where fm/8 is already below the + // saturation point -- a phone, a container, the flat 100MB placeholder off + // Apple -- this follows fm/8 and nothing is loosened. `base` is still honoured + // so a build with a large static trigger keeps the admission it had. + { + long runAhead = cn1PacingRunAheadBound(fm); + if(runAhead > 0) { + if(cap > runAhead) { + cap = runAhead; + } + if(cap < base) { + cap = base; + } + } + } if(cn1PacingTraceOn()) { long seen = atomic_load_explicit(&cn1PacingMinCap, memory_order_relaxed); while(cap < seen && @@ -7329,6 +7517,175 @@ void cn1HeapAccounting(const char* label) { fflush(stderr); } +/** + * Prints the LIVE heap by class, biggest first. + * + * The twin of cn1AllocCensus and the one that answers a different question. + * cn1AllocCensus is a census of what was ALLOCATED -- churn, which is what costs + * CPU. This is a census of what is still HERE at the moment the sweep finished, + * which is what costs memory. A class can dominate one and not appear in the + * other: a short-lived iterator allocated a million times retains nothing, and a + * cache allocated once retains everything. + * + * Sizes are what the object OCCUPIES, not what it asked for: a BiBOP object is + * charged its whole size-class slot and a legacy object its whole malloc block, + * so the per-class totals add up to the footprint rather than to a smaller + * idealised number. Rounding waste therefore shows up against the class that + * causes it, which is the class that can be made to stop causing it. + * + * Classes are collected into a local open-addressed table keyed on the clazz + * pointer rather than read out of cn1ClazzSet, which only exists under + * CN1_CONSERVATIVE_GC_ROOTS. + * + * Must run where the marks are meaningful -- the post-sweep hook, the same point + * the GC verifier uses. + */ +#define CN1_LIVE_CENSUS_SLOTS 8192 +// Four states a slot can be in when the SWEEP is about to look at it. Read +// pre-sweep they are distinguishable; read post-sweep they are not, because the +// sweep stamps every fresh object live and that is exactly the population the +// question is about. +#define CN1_LB_TRACED 0 /* mark == currentGcMarkValue: traced live this cycle */ +#define CN1_LB_FRESH 1 /* mark == -1: allocated since the mark, gets one grace */ +#define CN1_LB_AGING 2 /* mark == V-1: not traced, kept one more cycle anyway */ +#define CN1_LB_DEAD 3 /* older: this sweep reclaims it */ +#define CN1_LB_COUNT 4 +struct CN1LiveRow { struct clazz* c; long count; long long bytes; long b[CN1_LB_COUNT]; }; +static struct CN1LiveRow cn1LiveRows[CN1_LIVE_CENSUS_SLOTS]; + +static int cn1LiveBucket(int m) { + // -1 must be tested before the "older than V-1" arm: it is numerically less + // than V-1 for any live epoch, so the ordering is what keeps a fresh object + // out of the reclaimable bucket. + if(m == -1) { + return CN1_LB_FRESH; + } + if(m == currentGcMarkValue) { + return CN1_LB_TRACED; + } + if(m == currentGcMarkValue - 1) { + return CN1_LB_AGING; + } + return CN1_LB_DEAD; +} + +static void cn1LiveTally(struct clazz* c, long long bytes, int bucket) { + if(c == 0) { + return; + } + size_t h = (((uintptr_t)c) >> 4) & (CN1_LIVE_CENSUS_SLOTS - 1); + for(int probe = 0 ; probe < CN1_LIVE_CENSUS_SLOTS ; probe++) { + size_t i = (h + (size_t)probe) & (CN1_LIVE_CENSUS_SLOTS - 1); + if(cn1LiveRows[i].c == 0) { + cn1LiveRows[i].c = c; + } + if(cn1LiveRows[i].c == c) { + cn1LiveRows[i].count++; + cn1LiveRows[i].bytes += bytes; + cn1LiveRows[i].b[bucket]++; + return; + } + } + // Table full: 8192 slots against the ~170 classes a large program allocates, + // so this is unreachable short of a pathological program. Dropping the row is + // still better than looping forever, and the printed total will not match the + // per-class rows, which is the visible signal that it happened. +} + +void cn1LiveCensus(const char* label) { + memset(cn1LiveRows, 0, sizeof(cn1LiveRows)); + long long bibopBytes = 0, legacyBytes = 0; + long bibopObjs = 0, legacyObjs = 0; + long totals[CN1_LB_COUNT]; + for(int i = 0 ; i < CN1_LB_COUNT ; i++) { + totals[i] = 0; + } + + CN1BibopPage* p = atomic_load_explicit(&bibopAllPages, memory_order_acquire); + while(p != 0) { + int n = atomic_load_explicit(&p->bumpIndex, memory_order_acquire); + for(int i = 0 ; i < n ; i++) { + JAVA_OBJECT o = cn1BibopSlot(p, i); + int m = __atomic_load_n(&o->__codenameOneGcMark, __ATOMIC_ACQUIRE); + // Occupied, not "provably reachable": a slot awaiting collection is + // still holding memory, and this census is about what memory is being + // held. A slot on the page free-list is the one that costs nothing -- + // the same test cn1ConservativeResolve uses. (CN1_GC_POISON_MARK is + // deliberately not consulted: it is defined further down, inside the + // verifier's section, and exists only in a CN1_GC_VERIFY build.) + if(m == CN1_BIBOP_FREE_MARK) { + continue; + } + int bucket = cn1LiveBucket(m); + cn1LiveTally(o->__codenameOneParentClsReference, (long long)p->slotSize, bucket); + bibopBytes += (long long)p->slotSize; + bibopObjs++; + totals[bucket]++; + } + p = atomic_load_explicit(&p->nextAll, memory_order_acquire); + } + + int nHeap = currentSizeOfAllObjectsInHeap; + for(int i = 0 ; i < nHeap ; i++) { + JAVA_OBJECT o = allObjectsInHeap[i]; + if(o == JAVA_NULL) { + continue; + } + // An adopted object lives in a BiBOP slot and was already charged by the + // page walk; malloc_size on it would read a block header that is not there. + if(o->__heapPosition == CN1_BIBOP_ADOPTED) { + continue; + } + long long sz = 0; +#if defined(__APPLE__) + sz = (long long)malloc_size((void*)o); +#endif + int lbucket = cn1LiveBucket(o->__codenameOneGcMark); + cn1LiveTally(o->__codenameOneParentClsReference, sz, lbucket); + legacyBytes += sz; + legacyObjs++; + totals[lbucket]++; + } + + // OCCUPIED is what costs memory. The four buckets say WHY each object is still + // occupying a slot, and they call for different fixes: traced means the program + // really is holding it, fresh and aging mean the collector is holding it under + // the grace and aging rules, and dead means this sweep is about to return it. + long occupied = bibopObjs + legacyObjs; + fprintf(stderr, "[LIVE:%s] occupied %ld objects %.2fMB | traced %ld (%.0f%%) " + "fresh %ld (%.0f%%) aging %ld (%.0f%%) dead %ld (%.0f%%) | bibop %.2fMB legacy %.2fMB\n", + label, occupied, (bibopBytes + legacyBytes) / 1048576.0, + totals[CN1_LB_TRACED], 100.0 * totals[CN1_LB_TRACED] / (occupied > 0 ? occupied : 1), + totals[CN1_LB_FRESH], 100.0 * totals[CN1_LB_FRESH] / (occupied > 0 ? occupied : 1), + totals[CN1_LB_AGING], 100.0 * totals[CN1_LB_AGING] / (occupied > 0 ? occupied : 1), + totals[CN1_LB_DEAD], 100.0 * totals[CN1_LB_DEAD] / (occupied > 0 ? occupied : 1), + bibopBytes / 1048576.0, legacyBytes / 1048576.0); + for(int shown = 0 ; shown < 30 ; shown++) { + int best = -1; + for(int i = 0 ; i < CN1_LIVE_CENSUS_SLOTS ; i++) { + if(cn1LiveRows[i].c != 0 && cn1LiveRows[i].bytes > 0 + && (best < 0 || cn1LiveRows[i].bytes > cn1LiveRows[best].bytes)) { + best = i; + } + } + if(best < 0) { + break; + } + long rc = cn1LiveRows[best].count > 0 ? cn1LiveRows[best].count : 1; + fprintf(stderr, "[LIVE:%s] %8.2fMB %9ld objs %4lld B/obj traced %3.0f%% fresh %3.0f%% " + "aging %3.0f%% dead %3.0f%% %s\n", + label, cn1LiveRows[best].bytes / 1048576.0, cn1LiveRows[best].count, + cn1LiveRows[best].bytes / rc, + 100.0 * cn1LiveRows[best].b[CN1_LB_TRACED] / rc, + 100.0 * cn1LiveRows[best].b[CN1_LB_FRESH] / rc, + 100.0 * cn1LiveRows[best].b[CN1_LB_AGING] / rc, + 100.0 * cn1LiveRows[best].b[CN1_LB_DEAD] / rc, + cn1LiveRows[best].c->clsName ? cn1LiveRows[best].c->clsName : "?"); + cn1LiveRows[best].bytes = 0; + } + fflush(stderr); +} + void cn1AllocCensus(const char* label) { struct Row { const char* name; long count; long bytes; }; static struct Row rows[4096]; diff --git a/vm/JavaAPI/src/java/lang/StringBuilder.java b/vm/JavaAPI/src/java/lang/StringBuilder.java index c031b06fc03..4c5fd18d310 100644 --- a/vm/JavaAPI/src/java/lang/StringBuilder.java +++ b/vm/JavaAPI/src/java/lang/StringBuilder.java @@ -100,24 +100,7 @@ private StringBuilder(char[] data, int offset, int charCount) { } private void enlargeBuffer(int min) { - // Double, as OpenJDK's AbstractStringBuilder does, rather than the 1.5x - // ((len>>1)+len+2) inherited from Harmony. - // - // Growing to N chars costs sum(capacity) in ABANDONED intermediate - // arrays, and that sum is N*r/(r-1): 3N at r=1.5, 2N at r=2. The - // difference is pure garbage, and on ParparVM garbage is expensive in a - // way it is not on a generational JVM -- the collector is a concurrent - // mark/sweep with no nursery, so a dead intermediate array occupies its - // slot until a later cycle sweeps it. - // - // MEASURED on the 5782-class hellocodenameone translation, where the - // emit phase is StringBuilder-bound: char[] occupancy 1529.84MB and the - // legacy (large-array) heap 1533.43MB before the emit-buffer reuse fix. - // - // The cost is peak overshoot: a buffer can now be up to 2x the chars - // actually needed rather than 1.5x. That is bounded and transient, where - // the reallocation garbage is unbounded in the number of appends. - int newCount = (value.length << 1) + 2; + int newCount = ((value.length >> 1) + value.length) + 2; char[] newData = new char[min > newCount ? min : newCount]; System.arraycopy(value, 0, newData, 0, count); value = newData; From c81f41de91fd45ff190eec50346b688dbf75faa5 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 12 Sep 2026 09:28:18 +0300 Subject: [PATCH 29/66] Re-apply the self-hosting rewrites on top of master's refactor The merge in the previous commit resolved ByteCodeTranslator.java to master's side wholesale, which was right for master's refactor (the copyVendoredResource/copyRuntimeResource helpers and the source manifest) but silently dropped every self-hosting rewrite this PR had made in that file. The self-hosted build then failed with 13 errors, all of them JavaAPI gaps, and master's new code had added more uses of the same APIs. Re-applied against master's version: - String.split -> Util.splitLiteral. JavaAPI declares no regex at all; BytecodeComplianceMojo rewrites split/replaceAll/matches to JdkApiRewriteHelper precisely because it does not. - OutputStreamWriter(.., Charset) -> the charset-name overload. - StringBuilder.indexOf/replace -> through toString(). - Path.relativize -> an absolute-prefix strip. The file always sits under the root here because it came from a walk of it. - File.list(FilenameFilter) -> the plain listing, filtered in a loop. - SourceManifest, which master added on the core path, was writing through java.nio.file and a Charset; both are now java.io. The two stubs had also fallen behind master: JavascriptSuspensionAnalysis gained an outputDirectory parameter and JavascriptReachability gained resetExportedFacts(). Worth stating because it is the point of the gate: nothing here was caught by review or by a normal build. Master added a java.nio.file dependency to a class every translation reaches, and the only thing that noticed was compiling the translator against JavaAPI. That is exactly the drift the self-hosting workflow exists to catch, and it has now caught it once before landing. Gates D and A byte-identical over 798 files -- three more than before the merge, which is master's SourceManifest and its .h/.c pair entering the corpus. Co-Authored-By: Claude Opus 5 (1M context) --- .../tools/translator/ByteCodeTranslator.java | 47 +++++++++++++++---- .../tools/translator/SourceManifest.java | 2 +- .../translator/JavascriptReachability.java | 5 ++ .../JavascriptSuspensionAnalysis.java | 2 +- 4 files changed, 44 insertions(+), 12 deletions(-) diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java index 82e293e437f..c6686aca697 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java @@ -431,7 +431,7 @@ public static void main(String[] args) throws Exception { // Unrecognized output type falls back to the plain copy-through default handler recognizedOutputType = false; } - String[] sourceDirectories = args[1].split(";"); + String[] sourceDirectories = Util.splitLiteral(args[1], ';'); File[] sources = new File[sourceDirectories.length]; for(int iter = 0 ; iter < sourceDirectories.length ; iter++) { sources[iter] = new File(sourceDirectories[iter]); @@ -776,7 +776,19 @@ private static void collectResources(File root, File dir, java.util.LinkedHashMa || ext.equals("mm") || ext.equals("rc")) { continue; } - String rel = root.toPath().relativize(f.toPath()).toString().replace('\\', '/'); + // Relative path by absolute-prefix strip rather than Path.relativize: + // JavaAPI has no java.nio.file, and the translator compiles against it + // when it translates itself. f is always under root here -- it came from + // a walk of root -- so the prefix always matches. + String rootAbs = root.getAbsolutePath(); + String fileAbs = f.getAbsolutePath(); + String rel = fileAbs.startsWith(rootAbs) + ? fileAbs.substring(rootAbs.length()) + : fileAbs; + while (rel.startsWith(File.separator) || rel.startsWith("/")) { + rel = rel.substring(1); + } + rel = rel.replace('\\', '/'); String key = "/" + rel; if (!out.containsKey(key)) { out.put(key, f); @@ -910,8 +922,20 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File File projectPbx = new File(xcproj, "project.pbxproj"); copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template.xcodeproj/project.pbxproj"), new FileOutputStream(projectPbx)); - String[] sourceFiles = srcRoot.list((pathname, string) -> - string.endsWith(".bundle") || string.endsWith(".xcdatamodeld") || !pathname.isHidden() && !string.startsWith(".") && !"Images.xcassets".equals(string)); + // File.list(FilenameFilter) is not in JavaAPI; filter the plain listing. + String[] allNames = srcRoot.list(); + java.util.List keptNames = new java.util.ArrayList(); + if (allNames != null) { + for (String string : allNames) { + File pathname = new File(srcRoot, string); + if (string.endsWith(".bundle") || string.endsWith(".xcdatamodeld") + || !pathname.isHidden() && !string.startsWith(".") + && !"Images.xcassets".equals(string)) { + keptNames.add(string); + } + } + } + String[] sourceFiles = keptNames.toArray(new String[keptNames.size()]); StringBuilder fileOneEntry = new StringBuilder(); StringBuilder fileTwoEntry = new StringBuilder(); @@ -927,7 +951,7 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File List includeFrameworks = new ArrayList<>(); Set optionalFrameworks = new HashSet<>(); - for (String optionalFramework : Util.getProperty("optional.frameworks", "").split(";")) { + for (String optionalFramework : Util.splitLiteral(Util.getProperty("optional.frameworks", ""), ';')) { optionalFramework = optionalFramework.trim(); if (!optionalFramework.isEmpty()) { optionalFrameworks.add(optionalFramework); @@ -997,7 +1021,7 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File includeFrameworks.add("libz.dylib"); includeFrameworks.add("AVKit.framework"); if(!addFrameworks.equalsIgnoreCase("none")) { - includeFrameworks.addAll(Arrays.asList(addFrameworks.split(";"))); + includeFrameworks.addAll(Arrays.asList(Util.splitLiteral(addFrameworks, ';'))); } int currentValue = 0xF63EAAA; @@ -1167,7 +1191,7 @@ private static void writeCmakeProject(File projectRoot, File srcRoot, String app boolean windows = "windows".equalsIgnoreCase(appType); boolean linux = "linux".equalsIgnoreCase(appType); boolean executable = windows || linux; - try (Writer writer = new OutputStreamWriter(new FileOutputStream(cmakeLists), StandardCharsets.UTF_8)) { + try (Writer writer = new OutputStreamWriter(new FileOutputStream(cmakeLists), "UTF-8")) { writer.append("cmake_minimum_required(VERSION 3.10)\n"); // The native Windows port mixes the translated C runtime with a C++ // layer for the COM APIs that have no C binding (DirectWrite), so the @@ -1594,9 +1618,12 @@ private static void replaceInFile(File sourceFile, String... values) throws IOEx String target = values[iter]; String replacement = values[iter + 1]; int index = 0; - while ((index = str.indexOf(target, index)) >= 0) { + while ((index = str.toString().indexOf(target, index)) >= 0) { int targetSize = target.length(); - str.replace(index, index + targetSize, replacement); + String replaced = str.toString().substring(0, index) + replacement + + str.toString().substring(index + targetSize); + str.setLength(0); + str.append(replaced); index += replacement.length(); totchanges++; } @@ -1608,7 +1635,7 @@ private static void replaceInFile(File sourceFile, String... values) throws IOEx if(verbose) { System.out.println("Rewrite " + sourceFile + " with " + totchanges + " changes"); } - try(Writer fios = new OutputStreamWriter(new FileOutputStream(sourceFile), StandardCharsets.UTF_8)) { + try(Writer fios = new OutputStreamWriter(new FileOutputStream(sourceFile), "UTF-8")) { fios.write(str.toString()); } } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/SourceManifest.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/SourceManifest.java index 0287489273f..b4fb7cba6bd 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/SourceManifest.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/SourceManifest.java @@ -262,7 +262,7 @@ public void write(File projectRoot) throws IOException { // java.io rather than java.nio.file: the translator compiles against JavaAPI // when it translates itself, and JavaAPI has no java.nio.file. See // vm/selfhost. - try (Writer w = new OutputStreamWriter(new FileOutputStream(out), UTF8)) { + try (Writer w = new OutputStreamWriter(new FileOutputStream(out), "UTF-8")) { w.write("# Provenance of every file in the generated project's source directory.\n"); w.write("# Written by the ParparVM translator; consumed by\n"); w.write("# scripts/check-native-warnings.py to decide who owns a compiler warning.\n"); diff --git a/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptReachability.java b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptReachability.java index bde73904500..a6c9bec6385 100644 --- a/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptReachability.java +++ b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptReachability.java @@ -39,6 +39,11 @@ final class JavascriptReachability { private JavascriptReachability() { } + /// Stub: the JavaScript target is excluded from the self-hosted build, so the + /// per-application fact cache it clears does not exist here. + static void resetExportedFacts() { + } + static int run(List classes, List classPool, String[] nativeSources) { throw new UnsupportedOperationException("JavaScript target not built into this translator"); diff --git a/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptSuspensionAnalysis.java b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptSuspensionAnalysis.java index a8db88412cf..4d1c96eced1 100644 --- a/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptSuspensionAnalysis.java +++ b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptSuspensionAnalysis.java @@ -39,7 +39,7 @@ final class JavascriptSuspensionAnalysis { private JavascriptSuspensionAnalysis() { } - static int run(List classes) { + static int run(List classes, java.io.File outputDirectory) { throw new UnsupportedOperationException("JavaScript target not built into this translator"); } } From f00db7e6fb5336138c93ec8387f9e949a82c5d60 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 12 Sep 2026 09:49:19 +0300 Subject: [PATCH 30/66] Fix the iterator lowering's lifetime bug, and withdraw the concat fusion Two P1 review findings, both correct, both about rewrites this PR added. ITERATOR LOWERING -- fixed. countStoresTo(slot) == 1 was taken to mean the slot holds one value, but a PARAMETER reaches its slot without an ASTORE. A method that takes an Iterator and later reuses that slot for this loop's iterator therefore has two values in it, and retypeIteratorUses() scanned the whole method and rewrote the parameter's interface calls to the concrete type as well. On this VM that dispatches methods which read the wrong object layout, unchecked. Two changes: reject any slot that could hold an incoming argument (firstNonParameterSlot(), counting long and double as two slots per the JVM numbering the instruction stream uses), and rewrite only uses that FOLLOW the store. CONCAT FUSION -- withdrawn rather than patched again. The finding is that matching on owner alone mistakes calls on a different builder for calls on the allocated one, so consume(new StringBuilder(), existing.append(a).append(b).toString()) fuses the wrong chain and hands consume() the wrong arguments. That is a miscompile, and fixing it properly needs the receiver tracked through the operand stack, which this translator has no infrastructure for -- the existing bail list is opcode-shaped, and no amount of widening it distinguishes two builders of the same type. This is the THIRD correctness defect out of that one pass: maxStack under-reservation (C stack overflow), running after the cull (calls emitted into deleted methods), and now receiver identity. A fourth attempt under time pressure is not a good trade against a measured win on one workload, so it comes out and can return on its own with real stack tracking behind it. Probe 1 established it is not what the suite is failing on, so removing it is scope control, not a fix. Note during verification: one self-hosted run segfaulted at virtual_java_lang_Comparable_compareTo <- NULL receiver java_util_TreeMap_containsKey ByteCodeClass.updateAllDependencies Parser.writeOutput which is the known open intermittent crash in that path, first seen 2026-09-10 and unattributed. It did not recur in 5 consecutive runs on the same binary, but 0/5 has roughly a 44% chance of missing a 15% event, so that is NOT evidence it is unrelated to anything here -- it is only evidence it is still rare. Recorded rather than closed. Gates D and A byte-identical over the corpus, negative control still detecting an injected corruption. Co-Authored-By: Claude Opus 5 (1M context) --- .../tools/translator/BytecodeMethod.java | 251 ++---------------- .../codename1/tools/translator/Parser.java | 9 +- 2 files changed, 33 insertions(+), 227 deletions(-) diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java index d680d232a69..bb6469c6528 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java @@ -2532,6 +2532,23 @@ private int prevExecutable(int from) { return -1; } + /// The first local slot that cannot hold an incoming argument. + /// + /// Parameters occupy locals WITHOUT an ASTORE, so a slot-write count of one + /// does not mean the slot holds one value over the method's lifetime -- an + /// Iterator parameter in that slot is a second, earlier value. Long and double + /// take two slots each, per the JVM numbering the instruction stream uses. + /// + /// @return the lowest slot index that is definitely not a parameter + private int firstNonParameterSlot() { + int slots = isStatic() ? 0 : 1; + for (ByteCodeMethodArg arg : arguments) { + char q = arg.getQualifier(); + slots += (q == 'l' || q == 'd') ? 2 : 1; + } + return slots; + } + private int countStoresTo(int slot) { int n = 0; for (Instruction ins : instructions) { @@ -2600,19 +2617,27 @@ public void lowerIteratorCalls() { continue; } int slot = ((VarOp) store).getIndex(); - if (countStoresTo(slot) != 1) { + // Exactly one ASTORE is not enough on its own: a parameter reaches its + // slot without one, so a method that takes an Iterator and later reuses + // that slot for this loop's iterator has TWO values in it. Rewriting the + // parameter's calls to the concrete type would dispatch methods that + // read the wrong object layout -- unchecked, on this VM. + if (slot < firstNonParameterSlot() || countStoresTo(slot) != 1) { continue; } - retypeIteratorUses(slot, itType); + retypeIteratorUses(slot, itType, st); } } - private void retypeIteratorUses(int slot, String itType) { + /// @param storeIdx index of the ASTORE that put the concrete iterator in the + /// slot; only uses AFTER it are rewritten, since anything + /// earlier cannot be reading the value this store wrote + private void retypeIteratorUses(int slot, String itType, int storeIdx) { ByteCodeClass itClass = Parser.getClassObject(Util.mangle(itType)); if (itClass == null) { return; } - for (int i = 0; i < instructions.size(); i++) { + for (int i = storeIdx + 1; i < instructions.size(); i++) { Instruction ins = instructions.get(i); if (!(ins instanceof Invoke) || ins.getOpcode() != Opcodes.INVOKEINTERFACE) { continue; @@ -4443,224 +4468,6 @@ private void removeRepeatedCheckcasts() { } } - /** - * Route the javac string-concatenation idiom to the SAME fused path that - * invokedynamic concat already uses. - * - * `a + b` compiles two different ways depending on the source/target level. - * JDK 9+ emits `invokedynamic makeConcat(WithConstants)`, which - * Parser.visitInvokeDynamicInsn already rewrites to String.cn1ConcatN when - * every part is String-typed -- two allocations and no conversion, against - * the StringBuilder's four plus a byte->char decode per append and a - * char->byte re-encode in toString (StringBuilder is char[]-backed while - * Strings are compact byte[]). - * - * Anything compiled at source/target 8 emits the StringBuilder idiom - * directly instead, and reached NONE of that. That is not a corner: the - * Codename One core, every port and every cn1lib are built that way, so the - * fallback was what nearly all linked code paid, no matter which JDK built - * the application on top. MEASURED on the 5782-class hellocodenameone - * corpus: 3058 StringBuilder-idiom sites against 499 invokedynamic ones. - * - * The rewrite is a deletion, because the stack discipline already lines up: - * - * NEW/DUP/ -> [sb] - * -> [sb, a] - * append -> [sb] (consumes sb and a, returns sb) - * -> [sb, b] - * append -> [sb] - * toString -> [String] - * - * Drop the NEW, the DUP, the constructor and every append, and what is left - * is `` leaving exactly [a, b] -- the argument shape - * cn1Concat2 wants. Only the terminating toString is replaced, by the static - * call. No new runtime: cn1Concat2..5 and their cn1FusedConcatN natives are - * the ones the invokedynamic path has been using. - * - * Conservative on purpose; every bail-out below is a case where a naive - * deletion would change behaviour: - * - only all-String append chains, because cn1ConcatN takes Strings. An - * append(int) renders digits straight into the builder, and routing it - * here would mean materialising an intermediate String, which is not - * obviously cheaper. Those chains are left alone. - * - only 2..5 parts, matching the cn1ConcatN arity that exists. - * - a control-flow join inside the chain ends it (srNextRealNoJoin, and - * the explicit isJumpTarget check): control could enter mid-chain, so - * the builder would not be the one this NEW created. - * - a nested `new StringBuilder` inside the chain ends it, so the inner - * concat of `"a" + (x + y)` is not mistaken for the outer one. The inner - * site is rewritten on its own, and the outer becomes eligible on a - * later pass -- hence the fixpoint loop in the caller. - * - any other StringBuilder method (charAt, reverse, ...), or a store or - * return of the builder, ends it: the builder escapes the chain. - * - * @return true when at least one chain was rewritten - */ - private boolean fuseStringBuilderConcatOnce() { - final String SB = "java/lang/StringBuilder"; - final String APPEND_STR = "(Ljava/lang/String;)Ljava/lang/StringBuilder;"; - for (int i = 0; i < instructions.size(); i++) { - Instruction in = instructions.get(i); - if (!(in instanceof TypeInstruction) || in.getOpcode() != Opcodes.NEW - || !SB.equals(((TypeInstruction) in).getTypeName())) { - continue; - } - int iDup = srNextRealNoJoin(i + 1); - if (iDup < 0 || instructions.get(iDup).getOpcode() != Opcodes.DUP) { - continue; - } - int iInit = srNextRealNoJoin(iDup + 1); - if (iInit < 0) { - continue; - } - Instruction initIns = instructions.get(iInit); - if (!(initIns instanceof Invoke) || initIns.getOpcode() != Opcodes.INVOKESPECIAL) { - continue; - } - Invoke init = (Invoke) initIns; - if (!SB.equals(init.getOwner()) || !"".equals(init.getName()) - || !"()V".equals(init.getDesc())) { - continue; - } - - java.util.List appends = new java.util.ArrayList(); - int toStringIdx = -1; - boolean ok = true; - for (int j = iInit + 1; j < instructions.size(); j++) { - Instruction c = instructions.get(j); - if (c instanceof LabelInstruction) { - if (LabelInstruction.isJumpTarget(((LabelInstruction) c).getLabel())) { - ok = false; - } - if (!ok) { - break; - } - continue; - } - if (c instanceof LineNumber || c instanceof LocalVariable) { - continue; - } - if (c instanceof Jump) { - ok = false; - break; - } - if (c instanceof TypeInstruction && c.getOpcode() == Opcodes.NEW - && SB.equals(((TypeInstruction) c).getTypeName())) { - ok = false; - break; - } - int op = c.getOpcode(); - // Any opcode that can MOVE OR DISCARD the builder reference ends the - // chain, not just the ones that store it somewhere. - // - // The matcher recognises appends by owner, not by tracking which - // object is on the stack, so without this it accepts - // new StringBuilder(); POP; return existing.append(a).append(b).toString(); - // -- valid bytecode -- and mistakes the appends on `existing` for - // appends on the builder it just allocated. Deleting the allocation - // and the appends would then leave the POP behind: an operand-stack - // underflow, and a concat of the wrong operands. - // - // The whole DUP/POP/SWAP family is refused rather than reasoned - // about. This costs coverage on chains whose argument expressions - // happen to contain one, which is the right trade: a missed fusion - // is slower, a wrong one is memory corruption. The pattern's own DUP - // sits before the scan window and is unaffected. - if (op == Opcodes.POP || op == Opcodes.POP2 || op == Opcodes.SWAP - || op == Opcodes.DUP || op == Opcodes.DUP_X1 || op == Opcodes.DUP_X2 - || op == Opcodes.DUP2 || op == Opcodes.DUP2_X1 || op == Opcodes.DUP2_X2) { - ok = false; - break; - } - if (op == Opcodes.ASTORE || op == Opcodes.PUTFIELD || op == Opcodes.PUTSTATIC - || op == Opcodes.AASTORE || op == Opcodes.ARETURN) { - ok = false; - break; - } - if (c instanceof Invoke) { - Invoke ci = (Invoke) c; - if (SB.equals(ci.getOwner())) { - if ("append".equals(ci.getName()) && APPEND_STR.equals(ci.getDesc())) { - appends.add(Integer.valueOf(j)); - continue; - } - if ("toString".equals(ci.getName()) && "()Ljava/lang/String;".equals(ci.getDesc())) { - toStringIdx = j; - break; - } - ok = false; - break; - } - } - } - int n = appends.size(); - if (!ok || toStringIdx < 0 || n < 2 || n > 5) { - continue; - } - - StringBuilder sig = new StringBuilder("("); - for (int k = 0; k < n; k++) { - sig.append("Ljava/lang/String;"); - } - sig.append(")Ljava/lang/String;"); - Invoke fused = new Invoke(Opcodes.INVOKESTATIC, "java/lang/String", - "cn1Concat" + n, sig.toString(), false); - instructions.set(toStringIdx, fused); - // Register it exactly as addInstruction() would. Setting the list entry - // alone leaves the new call with no owning method, no class dependency - // and -- the one that bites -- NO EDGE IN THE DEPENDENCY GRAPH, so the - // unused-method cull cannot see that String.cn1ConcatN is now called. - fused.setMethod(this); - fused.addDependencies(dependentClasses); - if (dependencyGraph != null) { - String fusedUses = fused.getMethodUsed(); - if (fusedUses != null) { - dependencyGraph.recordMethodCall(this, fusedUses); - } - } - for (int k = n - 1; k >= 0; k--) { - instructions.remove(appends.get(k).intValue()); - } - instructions.remove(iInit); - instructions.remove(iDup); - instructions.remove(i); - // GROW the frame; do not clamp it. - // - // maxStack sizes the emitted C stack array (DEFINE_METHOD_STACK), so a - // value that is too small writes PAST that array. The failure is silent - // and arrives far away: the first symptom here was a SIGSEGV in - // java_io_File_getParentFile, called from File.mkdirs, nowhere near any - // concat. - // - // While evaluating the LAST part the builder held one persistent slot - // (sb) under that part's own working set; the fused form instead holds - // n-1 finished parts under it. So the requirement rises by n-2 over - // whatever the chain needed before. An earlier `if (maxStack < n + 1)` - // was a no-op for every real method, because maxStack is essentially - // always already larger than 6. - // - // Adding n (rather than the n-2 strictly implied) buys a slot of margin - // for a couple of pointers per frame, which is the right trade against a - // memory-corrupting underestimate. - maxStack += n; - cn1ConcatFused++; - return true; - } - return false; - } - - /** Count of chains rewritten by {@link #fuseStringBuilderConcatOnce}, for reporting. */ - static int cn1ConcatFused; - - void fuseStringBuilderConcat() { - // Fixpoint: rewriting an inner concat makes the outer one all-String and - // free of the nested NEW that had disqualified it. Bounded so a bug here - // cannot hang a build. - int guard = 0; - while (guard++ < 64 && fuseStringBuilderConcatOnce()) { - // keep going - } - } boolean optimize() { // FUSED OBJECTS, constructor side: rewrite each planned diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java index 1a934ebd4b9..7dae4f551ca 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java @@ -844,14 +844,13 @@ public static void writeOutput(File outputDirectory) throws Exception { // runs later -- inside BytecodeMethod.optimize(), which happens during // generateCCode -- inserts calls to methods the cull has already deleted, // and a deleted method is emitted as `return 0;`. That is not a build - // error: cn1ConcatN silently answered null, java.io.File got a null path, - // and the translator died in File.getParentFile with a SIGSEGV nowhere - // near a concat. Running here, the references exist before anything is - // eliminated. See BytecodeMethod.fuseStringBuilderConcat. + // error: the rewritten call silently answered null, java.io.File got a + // null path, and the translator died in File.getParentFile with a SIGSEGV + // nowhere near the rewrite. Running here, the references exist before + // anything is eliminated. See BytecodeMethod.lowerIteratorCalls. if (BytecodeMethod.optimizerOn) { for (ByteCodeClass fuseCls : classes) { for (BytecodeMethod fuseMtd : fuseCls.getMethods()) { - fuseMtd.fuseStringBuilderConcat(); fuseMtd.lowerIteratorCalls(); } } From 8275e5bd410b027c6abba2790a3541c60bda531a Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 12 Sep 2026 10:04:01 +0300 Subject: [PATCH 31/66] Review round: primitive-class reflection, empty resources, duplicate cores Six review findings, all in surface this PR adds. P1, re-applied after the merge lost it: handleAppleOutput copied the port's java_io_File.m to srcRoot/java_io_File.m, and Parser.writeOutput then emitted the RETAINED java.io.File class over the top of it, so File.exists() linked against a missing existsImpl. The clean target already dodged this by emitting java_io_File_runtime.c; the Apple path now does the same with java_io_File_runtime.m. This was fixed once in a57b3ecd and reverted by resolving ByteCodeTranslator.java to master's side during the merge -- the second thing that resolution quietly undid. The primitive class objects made three reflective paths reachable that were previously unreachable, because Integer.TYPE and friends were null: - newInstance() on a primitive jumped through a zero newInstanceFp. The native calls that pointer unconditionally, so this was a jump to address zero rather than an InstantiationException. Guarded in Java, where newInstanceImpl's only caller is. - Array.newInstance(int.class, n) was rejected outright, because the descriptors left arrayClass zero. Now linked to class_array1__JAVA_*, with void deliberately left at zero since void[] does not exist. AND the allocator's element width had to move with it: it hardcoded sizeof(JAVA_OBJECT), which was correct while only reference component types could reach it. Setting arrayClass without that would have allocated int[] at 8 bytes per element -- corruption, not an exception. Both halves land together. - getClassLoader() returned the system loader and toString() returned "int class". Primitives must report a null loader and their bare name; reflection code tests both. Empty embedded resources: a zero LENGTH was treated as "not found", so getResourceAsStream returned null for a resource that exists and is legitimately empty. Only a null pointer means absent now. And one of my own: the Linux job copied every core and the unstripped binary into the screenshot artifact, while master's "Package core dump for offline autopsy" step already ships the ELF plus .debug and zstd-compresses each core into its own artifact. That duplicated multi-gigabyte uploads on exactly the runs that crashed -- the runs whose screenshots and backtrace most need to fit in the disk and upload budget. Removed. Not implemented, and worth surfacing rather than burying: Class.forName's initialize flag is still ignored. Honouring it needs an init function pointer in struct clazz that does not exist, and the overload's only caller -- ASM's ClassWriter.getCommonSuperClass -- passes false. The javadoc now states the consequence instead of just the fact: a class you go on to touch initializes normally at the first NEW/GETSTATIC/ INVOKESTATIC, and what does not work is forName purely for a registration side effect, which obfuscation rules out on this platform anyway. Gates D and A byte-identical over 798 files, PrimitiveTypeIntegrationTest green. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/linux-build-run.yml | 15 ++++----- vm/ByteCodeTranslator/src/cn1_globals.m | 28 ++++++++++------ .../tools/translator/ByteCodeTranslator.java | 15 ++++++++- vm/ByteCodeTranslator/src/nativeMethods.m | 25 +++++++++++++-- vm/JavaAPI/src/java/lang/Class.java | 32 +++++++++++++++++-- 5 files changed, 91 insertions(+), 24 deletions(-) diff --git a/.github/workflows/linux-build-run.yml b/.github/workflows/linux-build-run.yml index 4216a5e1677..ada9d9920dd 100644 --- a/.github/workflows/linux-build-run.yml +++ b/.github/workflows/linux-build-run.yml @@ -338,15 +338,12 @@ jobs: # frames (e.g. a stack overflow's recursion / the caller that ran CN1 on a small # native stack). The suite binary is not stripped, so addr2line resolves them. elf="$(/usr/bin/find /tmp -maxdepth 4 -name LinuxHelloMain -type f 2>/dev/null | head -1)" - # Ship the unstripped binary with the artifact: a core or a raw backtrace - # is addresses and nothing else once the runner is gone. - if [ -n "$elf" ]; then - cp "$elf" "$(dirname "$CN1_APP_LOG_TEE")/LinuxHelloMain" 2>/dev/null || true - fi - for core in /tmp/cn1-cores/core.*; do - [ -f "$core" ] || continue - cp "$core" "$(dirname "$CN1_APP_LOG_TEE")/" 2>/dev/null || true - done + # The binary and the cores are NOT copied here. "Package core dump for + # offline autopsy" below already ships the unstripped ELF (plus .debug) + # and zstd-compresses each core into its own artifact. Copying them into + # the screenshot directory as well would upload multi-gigabyte cores twice + # on exactly the runs that crashed -- the runs whose screenshots and + # backtrace most need to survive the disk and upload budget. if [ -n "$elf" ] && [ -f "$CN1_APP_LOG_TEE" ]; then { echo "=== addr2line of CN1 backtrace addresses (from app log) ===" diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index c101ffd053d..cda232c68b0 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -979,7 +979,7 @@ static void init_gc_thresholds() { * happens before comments are removed, so an unbackslashed comment line inside * the macro would silently end the definition. */ -#define CN1_DEFINE_PRIMITIVE_CLASS(cname, jname) \ +#define CN1_DEFINE_PRIMITIVE_CLASS_ARR(cname, jname, arrCls) \ struct clazz cn1_primitive_class_##cname = { \ .__codenameOneParentClsReference = &class__java_lang_Class, \ .classId = CN1_PRIMITIVE_CLASS_ID, \ @@ -987,6 +987,7 @@ static void init_gc_thresholds() { .isArray = JAVA_FALSE, \ .dimensions = 0, \ .arrayType = 0, \ + .arrayClass = arrCls, \ .primitiveType = JAVA_TRUE, \ .baseClass = 0, \ .baseInterfaces = EMPTY_INTERFACES, \ @@ -994,14 +995,23 @@ static void init_gc_thresholds() { .initialized = JAVA_TRUE \ } -CN1_DEFINE_PRIMITIVE_CLASS(int, "int"); -CN1_DEFINE_PRIMITIVE_CLASS(long, "long"); -CN1_DEFINE_PRIMITIVE_CLASS(short, "short"); -CN1_DEFINE_PRIMITIVE_CLASS(byte, "byte"); -CN1_DEFINE_PRIMITIVE_CLASS(char, "char"); -CN1_DEFINE_PRIMITIVE_CLASS(float, "float"); -CN1_DEFINE_PRIMITIVE_CLASS(double, "double"); -CN1_DEFINE_PRIMITIVE_CLASS(boolean, "boolean"); +/* + * arrayClass is what java.lang.reflect.Array.newInstance resolves int.class to + * before allocating; left zero the reflective allocator rejects every primitive + * array outright. void is the one that stays zero on purpose -- void[] does not + * exist, so Array.newInstance(void.class, n) must keep throwing. + */ +#define CN1_DEFINE_PRIMITIVE_CLASS(cname, jname) \ + CN1_DEFINE_PRIMITIVE_CLASS_ARR(cname, jname, 0) + +CN1_DEFINE_PRIMITIVE_CLASS_ARR(int, "int", &class_array1__JAVA_INT); +CN1_DEFINE_PRIMITIVE_CLASS_ARR(long, "long", &class_array1__JAVA_LONG); +CN1_DEFINE_PRIMITIVE_CLASS_ARR(short, "short", &class_array1__JAVA_SHORT); +CN1_DEFINE_PRIMITIVE_CLASS_ARR(byte, "byte", &class_array1__JAVA_BYTE); +CN1_DEFINE_PRIMITIVE_CLASS_ARR(char, "char", &class_array1__JAVA_CHAR); +CN1_DEFINE_PRIMITIVE_CLASS_ARR(float, "float", &class_array1__JAVA_FLOAT); +CN1_DEFINE_PRIMITIVE_CLASS_ARR(double, "double", &class_array1__JAVA_DOUBLE); +CN1_DEFINE_PRIMITIVE_CLASS_ARR(boolean, "boolean", &class_array1__JAVA_BOOLEAN); CN1_DEFINE_PRIMITIVE_CLASS(void, "void"); struct clazz class_array1__JAVA_BOOLEAN = { diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java index c6686aca697..6c7baf4c87d 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java @@ -887,7 +887,20 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File } copyRuntimeResource(srcRoot, "cn1_globals.m"); copyRuntimeResource(srcRoot, "nativeMethods.m"); - copyRuntimeResource(srcRoot, "java_io_File.m"); + // java_io_File_RUNTIME.m, not java_io_File.m. When the application retains + // java.io.File -- which the filesystem fallback makes ordinary -- Parser + // .writeOutput emits the translated class to java_io_File.m and overwrites + // the port's hand-written native that was copied here first. The generated + // File.exists() then has no existsImpl to link against. + // + // OBSERVED as `Undefined symbols: _java_io_File_existsImpl ... referenced + // from _java_io_File_exists___R_boolean in java_io_File.o` on the iOS legs. + // The clean target already avoids the same collision by emitting + // java_io_File_runtime.c; this is that fix for the Apple path. The name only + // has to differ from the generated one -- the compiler globs the directory, + // and NativeSignatureVerifier reads the RESOURCE "/java_io_File.m" rather + // than the emitted filename. + copyRuntimeResource(srcRoot, "java_io_File.m", "java_io_File_runtime.m"); if (Util.getProperty("USE_RPMALLOC", "false").equals("true")) { copyRuntimeResource(srcRoot, "malloc.c"); diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index da508632c4e..736cc309fe4 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -610,7 +610,25 @@ JAVA_OBJECT java_lang_reflect_Array_newInstanceImpl___java_lang_Class_int_R_java throwException(threadStateData, ex); return NULL; } - JAVA_OBJECT out = allocArray(CN1_THREAD_STATE_PASS_ARG len, clz->arrayClass, sizeof(JAVA_OBJECT), 1); + // Element WIDTH, not sizeof(JAVA_OBJECT). This allocator only ever saw + // reference component types before, because a primitive class object did not + // exist to pass in -- Integer.TYPE and friends were null. Now that they do, + // allocating an int[] at 8 bytes per element would size the block off the end + // of what the array header declares, so the width has to come from the + // component type. + int cn1ElemSize = (int)sizeof(JAVA_OBJECT); + if (clz->primitiveType) { + if (clz == &cn1_primitive_class_boolean || clz == &cn1_primitive_class_byte) { + cn1ElemSize = 1; + } else if (clz == &cn1_primitive_class_char || clz == &cn1_primitive_class_short) { + cn1ElemSize = 2; + } else if (clz == &cn1_primitive_class_int || clz == &cn1_primitive_class_float) { + cn1ElemSize = 4; + } else if (clz == &cn1_primitive_class_long || clz == &cn1_primitive_class_double) { + cn1ElemSize = 8; + } + } + JAVA_OBJECT out = allocArray(CN1_THREAD_STATE_PASS_ARG len, clz->arrayClass, cn1ElemSize, 1); finishedNativeAllocations(); return out; } @@ -2030,7 +2048,10 @@ JAVA_OBJECT java_lang_Class_cn1EmbeddedResource___java_lang_String_R_byte_1ARRAY } int len = 0; const unsigned char* data = cn1FindResource(n, &len); - if(data == 0 || len <= 0) { + // A NULL pointer means "not found". A zero LENGTH does not -- an embedded + // resource is allowed to be empty, and getResourceAsStream must hand back an + // empty stream for one rather than null, which callers read as absent. + if(data == 0 || len < 0) { return JAVA_NULL; } JAVA_OBJECT arr = __NEW_ARRAY_JAVA_BYTE(threadStateData, len); diff --git a/vm/JavaAPI/src/java/lang/Class.java b/vm/JavaAPI/src/java/lang/Class.java index ff1f5aa0f3e..45921e6d648 100644 --- a/vm/JavaAPI/src/java/lang/Class.java +++ b/vm/JavaAPI/src/java/lang/Class.java @@ -40,6 +40,11 @@ public final class Class implements java.lang.reflect.Type { public ClassLoader getClassLoader() { + if (isPrimitive()) { + // A primitive class is bootstrap-defined and must report null, which is + // what reflection code tests to tell such a type from a loaded one. + return null; + } return ClassLoader.getSystemClassLoader(); } @@ -54,11 +59,21 @@ public ClassLoader getClassLoader() { * Returns the Class object for {@code className}. * * ParparVM links the whole program ahead of time, so there is no second class - * loader to consult and nothing to defer: both extra arguments are accepted and - * ignored, and the class is resolved exactly as the one-argument form resolves - * it. The overload exists because library bytecode calls it -- ASM's + * loader to consult: both extra arguments are accepted and ignored, and the + * class is resolved exactly as the one-argument form resolves it. The overload + * exists because library bytecode calls it -- ASM's * ClassWriter.getCommonSuperClass does -- and an absent overload is a link * error in translated code, not a compile error here. + * + * <p>What {@code initialize == true} does NOT do here: it does not run the + * named class's static initializer. ParparVM runs one on first use -- the + * generated code calls the class's static initializer at every NEW, GETSTATIC + * and INVOKESTATIC -- so any code that goes on to TOUCH the class sees its + * statics initialized as normal. What does not work is using forName purely for + * a registration side effect and never referencing the class again, the + * JDBC-driver idiom. That pattern cannot work on this platform for a second + * reason anyway: obfuscation rewrites class names, so a name looked up as a + * string does not survive a release build. */ public static java.lang.Class forName(java.lang.String className, boolean initialize, ClassLoader loader) throws java.lang.ClassNotFoundException { @@ -301,6 +316,12 @@ private static java.io.InputStream cn1FileResource(String absolute) { * Creates a new instance of a class. */ public java.lang.Object newInstance() throws java.lang.InstantiationException, java.lang.IllegalAccessException { + if (isPrimitive()) { + // A primitive descriptor has no constructor, and its newInstanceFp is + // zero -- the native calls that pointer unconditionally, so letting one + // through jumps to address zero instead of throwing. + throw new InstantiationException(); + } Object o = newInstanceImpl(); if(o == null) { throw new InstantiationException(); @@ -319,6 +340,11 @@ public java.lang.Object newInstance() throws java.lang.InstantiationException, j * returns "void". */ public java.lang.String toString() { + if (isPrimitive()) { + // "int", not "int class" -- java.lang.Class documents the primitive form + // as the name alone. + return getName(); + } return getName() + " class"; } From ea47102471214d326ccea2e0a8f561ff6d684269 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 12 Sep 2026 10:18:48 +0300 Subject: [PATCH 32/66] Restore readFileAsString, and make replaceInFile single-pass vm-tests failed on BytecodeInstructionIntegrationTest with NoSuchMethod readFileAsString(File). Another casualty of resolving ByteCodeTranslator.java to master's side: master has readFileAsStringBuilder returning a StringBuilder, this PR had changed it to return a String, and the test that reflects on it came through the merge from this branch while the method it names did not. Restored to the String form, because the reason for it still holds -- the translator compiles against ParparVM's JavaAPI to translate itself, and StringBuilder there has neither indexOf nor replace. And fixed the way I had bridged that gap earlier in the merge, which a review comment correctly called out. Doing indexOf on str.toString() and then rebuilding the buffer with substring + concat for every match copies the ENTIRE file twice per occurrence. That is the opposite of what this method is for: it exists to avoid the memory spike that made large Xcode project.pbxproj rewrites fail with OutOfMemoryError, and the Xcode file-list placeholders are exactly the large, many-match case. It now walks each target once, appending into a single output buffer, so a target costs one traversal regardless of how often it matches. BytecodeInstructionIntegrationTest#readFileAsStringReadsContent and #replaceInFileModifiesContent both green; gates D and A byte-identical. Co-Authored-By: Claude Opus 5 (1M context) --- .../tools/translator/ByteCodeTranslator.java | 42 ++++++++++++------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java index 6c7baf4c87d..998e9353d1f 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java @@ -1606,12 +1606,12 @@ private static String getFileType(String s) { // to be mutated. Also, expire the temporary byte[] buffer so it can // be collected. // - private static StringBuilder readFileAsStringBuilder(File sourceFile) throws IOException + private static String readFileAsString(File sourceFile) throws IOException { try(DataInputStream dis = new DataInputStream(new FileInputStream(sourceFile))) { byte[] data = new byte[(int) sourceFile.length()]; dis.readFully(data); - return new StringBuilder(new String(data, StandardCharsets.UTF_8)); + return new String(data, StandardCharsets.UTF_8); } } // @@ -1622,24 +1622,38 @@ private static StringBuilder readFileAsStringBuilder(File sourceFile) throws IOE // process for large projects. // private static void replaceInFile(File sourceFile, String... values) throws IOException { - StringBuilder str = readFileAsStringBuilder(sourceFile); + // A String rather than a StringBuilder because the translator has to compile + // against ParparVM's own JavaAPI in order to translate itself, and + // StringBuilder there has neither indexOf nor replace. + // + // One pass per target, appending into a fresh builder. The obvious + // translation of the old in-place edit -- indexOf on str.toString(), then + // substring/concat the whole buffer back together per match -- copies the + // ENTIRE file twice for every occurrence, which is the opposite of this + // method's purpose: it exists to avoid the memory spike that made large + // Xcode project.pbxproj rewrites fail with OutOfMemoryError. Each target + // now costs one traversal and one output buffer regardless of how many + // times it matches. + String str = readFileAsString(sourceFile); int totchanges = 0; - // perform the mutations on stringbuilder, which ought to implement - // these operations efficiently. for (int iter = 0; iter < values.length; iter += 2) { String target = values[iter]; String replacement = values[iter + 1]; - int index = 0; - while ((index = str.toString().indexOf(target, index)) >= 0) { - int targetSize = target.length(); - String replaced = str.toString().substring(0, index) + replacement - + str.toString().substring(index + targetSize); - str.setLength(0); - str.append(replaced); - index += replacement.length(); + int index = str.indexOf(target); + if (index < 0) { + continue; + } + StringBuilder out = new StringBuilder(str.length() + 64); + int from = 0; + while (index >= 0) { + out.append(str, from, index).append(replacement); + from = index + target.length(); totchanges++; + index = str.indexOf(target, from); } + out.append(str, from, str.length()); + str = out.toString(); } // @@ -1649,7 +1663,7 @@ private static void replaceInFile(File sourceFile, String... values) throws IOEx System.out.println("Rewrite " + sourceFile + " with " + totchanges + " changes"); } try(Writer fios = new OutputStreamWriter(new FileOutputStream(sourceFile), "UTF-8")) { - fios.write(str.toString()); + fios.write(str); } } From 43331747d5e195e672e1d3e9e560af984b3454b5 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 12 Sep 2026 10:51:11 +0300 Subject: [PATCH 33/66] Install gdb from an absolute path: it has never been installed at all The suite step declares working-directory: vm, so bash scripts/ci/apt-get-install.sh gdb resolved to vm/scripts/ci/apt-get-install.sh and bash answered "No such file or directory". The `|| echo "WARNING: gdb install failed"` then swallowed it into a warning nobody read. The retry.sh call twenty lines below in the same step already used "$GITHUB_WORKSPACE/scripts/ci/...". So gdb has never been present on ANY Linux leg, and every hang-stacks.txt and crash-stacks.txt this job has produced was empty for that reason alone -- not because the process was healthy, and not because ptrace was restricted. That covers the four stalls the surrounding comment mentions and the entire investigation on this branch, during which the x64 leg dumped a 12.3GB core and packaged it faithfully while nothing on the machine could symbolise it. What the run that found this did establish, from the harness rather than from gdb: CN1SS:HARNESS: suite process exited early, exitValue=139 pngs=145 139 is 128+11, so x64 SIGSEGVs at AccessibilityTest. arm64 does not crash -- it HANGS, 12 minutes of silence in VideoIODecodedFramesScreenshotTest at pngs=162, which is one of the media tests that has been reported as "produced no output" all along. Two different failure modes, which is worth knowing before treating a pass count as one number to optimise. Also worth recording against my own earlier reasoning: the ArrayList bounds guard did fix the corruption. The Windows app log for this commit contains zero exceptions, where before it had AIOOBE 69, AIOOBE -1 and an NPE inside ArrayList.get, and AccessibilityTest completes there instead of throwing. The remaining failures are a separate defect, and the memory-pressure theory I offered for them is dead: this commit restores the pacing cap AND reverts the StringBuilder growth, and scored worse than the commit that had neither. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/linux-build-run.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linux-build-run.yml b/.github/workflows/linux-build-run.yml index ada9d9920dd..17011790410 100644 --- a/.github/workflows/linux-build-run.yml +++ b/.github/workflows/linux-build-run.yml @@ -283,7 +283,15 @@ jobs: # hang-stacks.txt with nothing but sample headers -- which is how four # occurrences of the suite stall ended up with no evidence at all. Still # best-effort: a runner without gdb must not fail the suite, it must say so. - bash scripts/ci/apt-get-install.sh gdb || echo "WARNING: gdb install failed" + # $GITHUB_WORKSPACE, not a relative path: this step runs with + # working-directory: vm, so "scripts/ci/..." resolved to vm/scripts/ci and + # bash answered "No such file or directory". gdb was therefore NEVER + # installed on any Linux leg, and every hang-stacks.txt and crash-stacks.txt + # this job has ever produced was empty for that reason alone -- including + # the ones collected while chasing a SIGSEGV that dumped a 12GB core nobody + # could symbolise. The retry.sh call twenty lines below already had this + # right. + bash "$GITHUB_WORKSPACE/scripts/ci/apt-get-install.sh" gdb || echo "WARNING: gdb install failed" if command -v gdb >/dev/null 2>&1; then echo "gdb available: $(gdb --version | head -1)" else From 1b2cbfbb02b655354acfbde38755201ec31ad3df Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 12 Sep 2026 12:47:47 +0300 Subject: [PATCH 34/66] Guard class use on completion, not on "initialisation started" The inline guards emitted at allocation and stack-allocation sites tested the WRONG FLAG: if(!__atomic_load_n(&class__X.initialized, __ATOMIC_ACQUIRE)) __STATIC_INITIALIZER_X(threadStateData); class__X.initialized means STARTED, not completed. The JLS requires a class whose initialiser re-enters itself to proceed rather than deadlock, so __STATIC_INITIALIZER_X sets that flag BEFORE calling __CLINIT__ -- it is the recursion guard for the check above the monitor. A second thread whose inline guard observed it therefore SKIPPED the initialiser entirely, never took the class monitor that would have made it wait, and went on to use a class whose static initialiser was still running on another thread. Its statics read as whatever they were before __CLINIT__ wrote them: null. The release store on that flag does not help and the comment claiming it did was wrong, which is the part of this that is mine. A release publishes writes that happened BEFORE it; every write __CLINIT__ makes happens after. Releasing on "started" cannot publish them. __X_LOADED__ is the completion flag -- stored with release after __CLINIT__ returns -- but it was `static`, so a guard in another translation unit could not name it and used the visible flag instead. It is now emitted non-static with an extern in the class header, and both guards (TypeInstruction, FusedConstructor) test it. Verified in the emitted C: 816 guards on the completion flag, zero left on .initialized. This matches the failures better than anything else examined. Uninitialised statics give a null where a field is required, which is what virtual_java_lang_Comparable_compareTo <- NULL receiver java_util_TreeMap_containsKey ByteCodeClass.updateAllDependencies is, and what a TreeSet with a null backing map is; both are recorded as open and unattributed. It is also consistent with a window that widens with thread count and class-initialiser cost, and with master being green while carrying the same guard -- this PR gives nine wrapper classes a TYPE static whose initialiser now calls a native, which is real work inside exactly that window. Stated plainly: this is a mechanism that fits, not a confirmed fix for the x64 SIGSEGV. That crash is a store into Display's event stack where cn1_set_array_element_int bounds-checked ->length successfully and then faulted through ->data, so the two fields disagreed -- a corrupted or dangling header. The post-mortem now dumps those arrays through the Display object, which frame 1 keeps as __cn1ThisObject even at -O3, so the next occurrence says which field is wrong instead of leaving it to inference. Ruled out by direct experiment on the way here, so they are not re-litigated: the shared EMIT_BUFFER emits byte-identical C (diffed a full translation against a fresh-StringBuilder build -- only the embedded output path differs), and Util.splitLiteral matches String.split on every edge case plus 200k fuzz inputs. Gates D and A byte-identical over 798 files; gauntlet green. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/linux-build-run.yml | 21 ++++++++++++++- .../tools/translator/ByteCodeClass.java | 27 ++++++++++++++----- .../bytecodes/FusedConstructor.java | 4 +-- .../translator/bytecodes/TypeInstruction.java | 4 +-- 4 files changed, 45 insertions(+), 11 deletions(-) diff --git a/.github/workflows/linux-build-run.yml b/.github/workflows/linux-build-run.yml index 17011790410..f9f01773538 100644 --- a/.github/workflows/linux-build-run.yml +++ b/.github/workflows/linux-build-run.yml @@ -332,11 +332,30 @@ jobs: mkdir -p "$(dirname "$CN1_APP_LOG_TEE")" { echo "=== post-mortem of $core (elf=$elf) ===" + # The event-stack arrays are dumped explicitly because the crash this + # keeps producing is a store into one of them from + # Display.edtLoopImpl, and -O3 optimises the array temporary out of + # the frame -- "array=" is all `info locals` gives. + # The Display object itself survives as frame 1's __cn1ThisObject, so + # the arrays are reachable THROUGH it, and what matters is whether + # each header's length and data agree: cn1_set_array_element_int + # bounds-checks against ->length and then stores through ->data, so a + # fault there means those two fields disagree, which is a corrupted or + # dangling header rather than a bad index. Harmless noise when the + # faulting frame is something else -- gdb just prints an error. gdb "$elf" "$core" -batch -ex 'set pagination off' \ -ex 'thread apply all bt' \ -ex 'thread 1' -ex 'bt full' \ - -ex 'frame 0' -ex 'info args' \ + -ex 'frame 0' -ex 'info args' -ex 'info registers' \ -ex 'frame 1' -ex 'info locals' -ex 'info args' \ + -ex 'set $d = (struct obj__com_codename1_ui_Display*)__cn1ThisObject' \ + -ex 'p $d->com_codename1_ui_Display_inputEventStackTmp' \ + -ex 'p *(JAVA_ARRAY)$d->com_codename1_ui_Display_inputEventStackTmp' \ + -ex 'p *(JAVA_ARRAY)$d->com_codename1_ui_Display_inputEventStack' \ + -ex 'p *(JAVA_ARRAY)$d->com_codename1_ui_Display_pointerMetaStackTmp' \ + -ex 'p *(JAVA_ARRAY)$d->com_codename1_ui_Display_pointerMetaStack' \ + -ex 'p $d->com_codename1_ui_Display_inputEventStackPointerTmp' \ + -ex 'p $d->com_codename1_ui_Display_inputEventStackPointer' \ -ex 'p gcMarkWorklistTop' -ex 'p currentGcMarkValue' \ 2>&1 } >> "$(dirname "$CN1_APP_LOG_TEE")/crash-stacks.txt" || true diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java index c481f473e1a..03b54b1d160 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java @@ -1549,7 +1549,15 @@ public String generateCCode(List allClasses) { } // insert static initializer - b.append("static int __").append(clsName).append("_LOADED__=0;\n"); + // NOT static: the inline guards emitted at allocation and static-access + // sites live in OTHER translation units and have to test COMPLETION. They + // used to test class__X.initialized instead, which is the wrong flag -- + // that one is the JLS recursion guard and is deliberately set BEFORE + // __CLINIT__ runs, so a thread observing it could skip the initialiser + // while another thread was still inside the class initialiser, and then + // read statics that had not been written yet. Releasing on "started" + // cannot publish writes that happen after it. + b.append("int __").append(clsName).append("_LOADED__=0;\n"); b.append("void __STATIC_INITIALIZER_"); b.append(clsName); // ACQUIRE, not a plain load. This is the fast path of a double-checked @@ -1658,11 +1666,13 @@ public String generateCCode(List allClasses) { } b.append(" __atomic_store_n(&class__"); b.append(clsName); - // RELEASE store, matching the one on __X_LOADED__ above. Readers outside - // this monitor (the inline guards emitted at allocation and static-access - // sites) do a plain-or-acquire load of this flag and SKIP the call - // entirely when it is set, so the monitor's own release is not enough on - // its own -- the guard never takes the monitor. + // This flag means STARTED, not completed: the JLS requires a class whose + // initialiser re-enters itself to proceed rather than deadlock, so it has + // to be set before __CLINIT__ runs, and the check above the monitor is + // that recursion guard. Nothing outside this function may treat it as + // "safe to use the class" -- the inline guards test __X_LOADED__, which is + // stored after __CLINIT__ returns. The release here is still wanted for + // the vtable and classToInterfaceMap rows written just above. b.append(".initialized, JAVA_TRUE, __ATOMIC_RELEASE);\n"); // init static fields and invoke the static initializer code block if(clInitMethod != null) { @@ -2048,6 +2058,11 @@ public String generateCHeader() { b.append("extern void __STATIC_INITIALIZER_"); b.append(clsName); b.append("(CODENAME_ONE_THREAD_STATE);\n"); + // The COMPLETION flag, for the inline guards. Set with a release store + // after __CLINIT__ returns; see the note where it is defined. + b.append("extern int __"); + b.append(clsName); + b.append("_LOADED__;\n"); b.append("extern void __FINALIZER_"); b.append(clsName); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedConstructor.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedConstructor.java index 8157fb040e3..8f9e0dae048 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedConstructor.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedConstructor.java @@ -555,8 +555,8 @@ public void appendFusedAlloc(StringBuilder b, String cType, String[] lenExprs, int recvSlot, int survSlot) { b.append(" { /* FUSED construction of ").append(cType).append(" */\n"); // ACQUIRE; see the note in TypeInstruction. - b.append(" if(__builtin_expect(!__atomic_load_n(&class__").append(cType) - .append(".initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_").append(cType).append("(threadStateData);\n"); + b.append(" if(__builtin_expect(!__atomic_load_n(&__").append(cType) + .append("_LOADED__, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_").append(cType).append("(threadStateData);\n"); for (int i = 0; i < children.size(); i++) { b.append(" int __fLen").append(i).append(" = ").append(lenExprs[i]).append(";\n"); } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java index feb8f2095aa..66bbbb76f0a 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java @@ -270,9 +270,9 @@ public void appendInstruction(StringBuilder b, List l) { // in ByteCodeClass. A plain load here let a thread see the flag // set while the vtable / classToInterfaceMap rows it describes // were still invisible. - b.append("if(__builtin_expect(!__atomic_load_n(&class__"); + b.append("if(__builtin_expect(!__atomic_load_n(&__"); b.append(type); - b.append(".initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_"); + b.append("_LOADED__, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_"); b.append(type); b.append("(threadStateData); memset(&__cn1stk_"); b.append(stackAllocId); From 670fb6c66387101f179ea72e745e584fba7ba2e2 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:13:54 +0300 Subject: [PATCH 35/66] Bind Class.getResourceAsStream to the generated port resource tables On a native Linux or Windows build, SomeClass.class.getResourceAsStream ("/theme.res") returned null for a resource the build had just embedded. The native calls cn1FindResource, whose only definition was the weak null-returning fallback in nativeMethods; the generated tables define cn1LinuxFindResource and cn1WinFindResourceId, which nothing on this path called. The resources were staged, assembled into .rodata or the PE resource section, linked, and never consulted -- the port's own Implementation.getResourceAsStream used the tables, but a direct Class.getResourceAsStream did not. Each generated table now emits a strong cn1FindResource that overrides the weak one. Linux delegates straight through, the signature being identical. Windows cannot: its table maps a name to an RCDATA id, so the id still has to be resolved to bytes through the image's resource section, which the override now does -- following the port's existing loader, including its note that RT_RCDATA expands to the NARROW MAKEINTRESOURCE while FindResourceW wants LPCWSTR, which MSVC only warns about (C4133) and clang-cl, the path this target actually cross-compiles through, rejects outright. The bytes stay mapped for the life of the process, so returning the locked pointer is sound. Verified by emitting both tables and reading them rather than assuming: the Linux one delegates and compiles clean under -fsyntax-only, the Windows one carries #include and the full resolve. The appType gate is args[7], so this needs `clean` as the output type with `linux`/`windows` as the app type -- passing the target as args[0] silently produces no table at all, which is how this first looked like it had not been emitted. Gates D and A byte-identical. Co-Authored-By: Claude Opus 5 (1M context) --- .../tools/translator/ByteCodeTranslator.java | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java index 998e9353d1f..c8d902e0d7e 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java @@ -623,7 +623,8 @@ private static void embedWindowsResources(File[] sources, File srcRoot) throws I StringBuilder table = new StringBuilder(); table.append("/* Auto-generated by the ParparVM windows target: maps a classpath\n"); table.append(" * resource path to the RCDATA id embedded in the executable. */\n"); - table.append("#include \n\n"); + table.append("#include \n"); + table.append("#include \n\n"); table.append("typedef struct { const char* name; int id; } CN1ResourceEntry;\n\n"); table.append("static const CN1ResourceEntry cn1ResourceTable[] = {\n"); @@ -655,6 +656,36 @@ private static void embedWindowsResources(File[] sources, File srcRoot) throws I table.append(" if (strcmp(cn1ResourceTable[i].name, name) == 0) { return cn1ResourceTable[i].id; }\n"); table.append(" }\n"); table.append(" return 0;\n"); + table.append("}\n\n"); + // Class.getResourceAsStream calls cn1FindResource, which has only a weak + // null-returning definition in nativeMethods -- the id table alone is not + // enough, because the id still has to be resolved to bytes. The image's + // resources stay mapped for the life of the process, so handing back the + // locked pointer is safe. + table.append("/* Strong override of the weak cn1FindResource in nativeMethods. */\n"); + table.append("const unsigned char* cn1FindResource(const char* name, int* lenOut) {\n"); + table.append(" int id;\n"); + table.append(" HMODULE module;\n"); + table.append(" HRSRC info;\n"); + table.append(" DWORD size;\n"); + table.append(" HGLOBAL loaded;\n"); + table.append(" void* data;\n"); + table.append(" if (lenOut) { *lenOut = 0; }\n"); + table.append(" id = cn1WinFindResourceId(name);\n"); + table.append(" if (id == 0) { return 0; }\n"); + table.append(" module = GetModuleHandleW(NULL);\n"); + // RT_RCDATA expands to the NARROW MAKEINTRESOURCE; FindResourceW wants + // LPCWSTR. MSVC only warns (C4133) but clang-cl -- the cross-compile path + // this target actually uses -- errors, so spell the wide form explicitly. + table.append(" info = FindResourceW(module, MAKEINTRESOURCEW(id), (LPCWSTR) RT_RCDATA);\n"); + table.append(" if (info == NULL) { return 0; }\n"); + table.append(" size = SizeofResource(module, info);\n"); + table.append(" loaded = LoadResource(module, info);\n"); + table.append(" if (loaded == NULL) { return 0; }\n"); + table.append(" data = LockResource(loaded);\n"); + table.append(" if (data == NULL) { return 0; }\n"); + table.append(" if (lenOut) { *lenOut = (int) size; }\n"); + table.append(" return (const unsigned char*) data;\n"); table.append("}\n"); sourceManifest.recordGenerated("cn1_resources_table.c"); Util.writeBytes(new File(srcRoot, "cn1_resources_table.c"), @@ -745,6 +776,14 @@ private static void embedLinuxResources(File[] sources, File srcRoot) throws IOE table.append(" }\n"); table.append(" if (lenOut) { *lenOut = 0; }\n"); table.append(" return 0;\n"); + table.append("}\n\n"); + // Class.getResourceAsStream calls cn1FindResource, which has only a weak + // null-returning definition in nativeMethods. Without this strong override + // that weak one stands and every embedded resource reads as absent on this + // target -- the table is built, linked, and never consulted. + table.append("/* Strong override of the weak cn1FindResource in nativeMethods. */\n"); + table.append("const unsigned char* cn1FindResource(const char* name, int* lenOut) {\n"); + table.append(" return cn1LinuxFindResource(name, lenOut);\n"); table.append("}\n"); sourceManifest.recordGenerated("cn1_resources_table.c"); Util.writeBytes(new File(srcRoot, "cn1_resources_table.c"), From 73a49fc3e2b151c71300c94dc7c4e4451a343b7d Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:54:57 +0300 Subject: [PATCH 36/66] PROBE: run the Linux suite with the mutator assist disabled Not a fix. The post-mortem on 1b2cbfb finally named the fault, and it is not a bad index: Display.inputEventStackTmp holds a DANGLING reference. $2 inputEventStackTmp parentCls=0x264d5420 gcMark=642602016 length=642602016 dimensions=0 primitiveSize=39 data=0x264d54f0 $3 inputEventStack parentCls=&class_array1__JAVA_INT gcMark=127 length=1000 dimensions=1 primitiveSize=4 length is byte-identical to gcMark, so those offsets are being read out of some other struct, and data sits 0xd0 bytes after parentCls, which is the shape of a FUSED object carrying its payload inline. An int[1000] reachable from a live Display field was reclaimed and a fused allocation now owns its slot. Both *Tmp fields are like this; both non-Tmp fields beside them are intact int[1000]. That one fact explains everything this branch has been failing on: the SIGSEGV is a store through the stale data pointer, the NPEs are reads of a field whose object is gone, and the stalls are the same corruption landing somewhere that waits instead of faulting. It also explains why the affected test moves between runs and why two Windows legs scored 184/4 and 175/12 on identical code. Write barriers are NOT what changed here -- this PR's only edit to Field.java is swapping an inline replace() for Util.mangle, and nothing in the translator touches barrier emission. So this is a latent collector bug that the PR's timing changes expose. The mutator assist is the obvious thing to test first: it is the part of the collector that runs Java mark functions on a MUTATOR's own thread, this PR's safepoint fix sits in that exact path, and the comment on the park directly below it warns that letting a mutator run under a scan in progress "loses reachable objects" -- which is the observed fault stated in advance. It ships with CN1_GC_NO_MUTATOR_ASSIST precisely so it can be A/B'd in one binary rather than two builds, so this costs a run and no rebuild. Green means the assist is where objects are lost and the fix belongs there. Red means the assist is exonerated and the next instrument is CN1_GC_VERIFY over this workload, which aborts at the cycle that dropped the object and names holder, victim and mark call site. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/linux-build-run.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/linux-build-run.yml b/.github/workflows/linux-build-run.yml index f9f01773538..02db5de79ad 100644 --- a/.github/workflows/linux-build-run.yml +++ b/.github/workflows/linux-build-run.yml @@ -270,6 +270,19 @@ jobs: # itself is still stripped; only the companion grows, and it is uploaded beside # the core so the crash can be autopsied off the runner. CN1_LINUX_FULL_DEBUG: 'true' + # PROBE, not a setting to keep. The post-mortem on 1b2cbfb showed + # Display.inputEventStackTmp holding a DANGLING reference: its header read + # length == gcMark == 642602016 with data 0xd0 bytes after parentCls -- the + # shape of a fused object occupying a slot an int[1000] used to own, while + # inputEventStack beside it was still a valid int[1000]. An object reachable + # from a live field had been swept. + # + # The mutator assist is the part of the collector that runs Java mark + # functions on a MUTATOR's own thread, and it ships with this A/B switch so + # it can be tested against the sleep it replaced in one binary. If the suite + # stops losing objects with the assist off, the assist is where they are + # lost. Remove this line once the answer is in either way. + CN1_GC_NO_MUTATOR_ASSIST: '1' # Build the native ELF (and the demo) against an old glibc for portability. CN1_CC: /usr/local/bin/cn1-zig-cc # After the suite runs, the capture test relinks the same objects into a @@ -497,6 +510,7 @@ jobs: -e CN1_SHOT_OUTPUT_DIR=/cn1/artifacts/linux-port/raw-musl \ -e CN1_APP_LOG_TEE=/cn1/artifacts/linux-port/raw-musl/app-output.log \ -e CN1_REQUIRE_SUITE=true \ + -e CN1_GC_NO_MUTATOR_ASSIST=1 \ -e LIBGL_ALWAYS_SOFTWARE=1 \ docker.io/library/alpine:3.20 sh -ec ' sed -i "s|^#\(.*/community\)|\1|" /etc/apk/repositories From a9959873162b8321a0d93f5cc0e4a6c5e510f065 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 12 Sep 2026 14:26:48 +0300 Subject: [PATCH 37/66] Add a diagnostic define hook to the generated Linux CMake project Empty by default, so an unset knob changes nothing for anybody. It exists because the collector ships its own heap-integrity verifier -- CN1_GC_VERIFY, which asserts after every sweep that no object the sweep KEPT references memory it RECLAIMED, and aborts naming holder class, victim class and the field's mark call site -- and there was no way to turn it on for a generated project short of hand-editing the emitted CMakeLists. That gap is what made the current fault expensive. A dangling Display.inputEventStackTmp had to be reconstructed from a 12GB core and a hand-written gdb dump of four fields, when the verifier would have named it at the cycle that dropped it. The harness passes it through CN1_LINUX_EXTRA_DEFINES, mirroring CN1_LINUX_FULL_DEBUG beside it. Checked rather than assumed, since a misplaced target_compile_definitions would break the Linux build outright: the hook lands at line 25 of the emitted project, add_executable is at line 9, and with the variable unset the if() is false and no define is emitted. Note the CLEAN target uses a different CMake writer and gets neither this hook nor the debug one -- testing with `clean` as the app type silently proves nothing, which is the same trap the resource table set earlier. Gates D and A byte-identical. Co-Authored-By: Claude Opus 5 (1M context) --- .../tools/translator/ByteCodeTranslator.java | 12 ++++++++++++ .../translator/CleanTargetLinuxIntegrationTest.java | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java index c8d902e0d7e..66a54aa61da 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java @@ -1544,6 +1544,18 @@ private static void writeLinuxLinkSet(Writer writer) throws IOException { // binary, where the companion exists to turn an address back into a Java method. // It is the wrong one for a CI build whose whole job is to be autopsied, so the // level is a cache variable: unset it and nothing changes for anybody. + // A diagnostic-only define hook, empty by default so nothing changes for + // anybody who does not ask. The reason it exists: the collector's own + // heap-integrity verifier (-DCN1_GC_VERIFY) is the designed detector for + // "the sweep reclaimed something a retained object still references", and + // there was no way to turn it on for a generated project without editing + // the emitted CMakeLists by hand. Chasing a dangling field reference + // through core dumps is what made that gap expensive. + writer.append("set(CN1_EXTRA_DEFINES \"\" CACHE STRING\n"); + writer.append(" \"Extra preprocessor defines for diagnostic builds, semicolon separated (e.g. CN1_GC_VERIFY)\")\n"); + writer.append("if(CN1_EXTRA_DEFINES)\n"); + writer.append(" target_compile_definitions(${PROJECT_NAME} PRIVATE ${CN1_EXTRA_DEFINES})\n"); + writer.append("endif()\n"); writer.append("set(CN1_DEBUG_INFO_LEVEL \"1\" CACHE STRING\n"); writer.append(" \"DWARF level for the .debug companion: 1 = lines + function names (lean, the default), 3 = full variable and type information (autopsyable)\")\n"); writer.append("target_compile_options(${PROJECT_NAME} PRIVATE -g${CN1_DEBUG_INFO_LEVEL} -fno-asynchronous-unwind-tables -fno-unwind-tables)\n"); diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java index 6b6d8ba29f5..8e33ef2507c 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java @@ -287,6 +287,12 @@ static Path buildHelloCodenameOneElf(String launcherSource) throws Exception { if (Boolean.parseBoolean(System.getenv("CN1_LINUX_FULL_DEBUG"))) { configure.add("-DCN1_DEBUG_INFO_LEVEL=3"); } + // Diagnostic defines, e.g. CN1_GC_VERIFY for the collector's heap-integrity + // checker. Unset it and the build is exactly what it was. + String extraDefines = System.getenv("CN1_LINUX_EXTRA_DEFINES"); + if (extraDefines != null && !extraDefines.trim().isEmpty()) { + configure.add("-DCN1_EXTRA_DEFINES=" + extraDefines.trim()); + } CleanTargetIntegrationTest.runCommand(configure, cmakeRoot); CleanTargetIntegrationTest.runCommand(Arrays.asList("cmake", "--build", buildDir.toString()), cmakeRoot); Path elf = buildDir.resolve("LinuxHelloMain"); From 3c32a10939981f4c4e251b3c466793126a3826c1 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 12 Sep 2026 14:32:07 +0300 Subject: [PATCH 38/66] Guard the Windows resource override behind _WIN32 My regression, caught by CleanTargetIntegrationTest. The strong cn1FindResource added for the windows app type pulled in unconditionally, and the windows APP TYPE is compiled on a Linux host by generatesRunnableExecutableForWindowsAppType -- all five compiler configurations failed with cn1_resources_table.c:4:10: fatal error: 'windows.h' file not found The include and the override now sit behind _WIN32. The name -> id table stays unguarded because it is plain C with string.h, and off Windows the weak null-returning cn1FindResource stands, which is the correct answer there: no PE resource section exists to read. Worth naming the verification mistake, because it was avoidable. I emitted both tables and read them, and syntax-checked the LINUX one with clang -fsyntax-only -- but only eyeballed the Windows one, on the reasoning that it could not be compiled here without windows.h. That reasoning was exactly backwards: being uncompilable on this host IS the bug, and the check that would have found it was the cheap one I skipped. Both tables now compile under -fsyntax-only on a non-Windows host, and the five-configuration test passes locally. Co-Authored-By: Claude Opus 5 (1M context) --- .../tools/translator/ByteCodeTranslator.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java index 66a54aa61da..d3be7c0a6b7 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java @@ -624,7 +624,13 @@ private static void embedWindowsResources(File[] sources, File srcRoot) throws I table.append("/* Auto-generated by the ParparVM windows target: maps a classpath\n"); table.append(" * resource path to the RCDATA id embedded in the executable. */\n"); table.append("#include \n"); - table.append("#include \n\n"); + // Behind _WIN32: the windows APP TYPE is compiled on a Linux host by + // CleanTargetIntegrationTest#generatesRunnableExecutableForWindowsAppType, + // where windows.h does not exist. The id table below is plain C and stays + // unguarded; only the resource-resolving override needs the platform. + table.append("#if defined(_WIN32)\n"); + table.append("#include \n"); + table.append("#endif\n\n"); table.append("typedef struct { const char* name; int id; } CN1ResourceEntry;\n\n"); table.append("static const CN1ResourceEntry cn1ResourceTable[] = {\n"); @@ -662,7 +668,10 @@ private static void embedWindowsResources(File[] sources, File srcRoot) throws I // enough, because the id still has to be resolved to bytes. The image's // resources stay mapped for the life of the process, so handing back the // locked pointer is safe. - table.append("/* Strong override of the weak cn1FindResource in nativeMethods. */\n"); + table.append("#if defined(_WIN32)\n"); + table.append("/* Strong override of the weak cn1FindResource in nativeMethods.\n"); + table.append(" * Off Windows the weak null-returning one stands, which is correct:\n"); + table.append(" * there is no PE resource section to read. */\n"); table.append("const unsigned char* cn1FindResource(const char* name, int* lenOut) {\n"); table.append(" int id;\n"); table.append(" HMODULE module;\n"); @@ -687,6 +696,7 @@ private static void embedWindowsResources(File[] sources, File srcRoot) throws I table.append(" if (lenOut) { *lenOut = (int) size; }\n"); table.append(" return (const unsigned char*) data;\n"); table.append("}\n"); + table.append("#endif\n"); sourceManifest.recordGenerated("cn1_resources_table.c"); Util.writeBytes(new File(srcRoot, "cn1_resources_table.c"), table.toString().getBytes(StandardCharsets.UTF_8)); From 7a4b086217be367cf0d33448f5a363784c3b7f2f Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 12 Sep 2026 14:52:47 +0300 Subject: [PATCH 39/66] The mutator assist is exonerated; turn the heap verifier on instead The probe answered, and the answer is no. With CN1_GC_NO_MUTATOR_ASSIST=1 confirmed in the job environment, the x64 suite still died the same way: exitValue=139 suite never emitted CN1SS:SUITE:FINISHED; pngs=146; stopped in WindowLayoutTest 139 is 128+11, so objects are still being lost with the assist switched off entirely. It is not where they go. The probe is removed from both Linux legs rather than left lying around. In its place, CN1_GC_VERIFY on the x64 leg through the define hook added alongside. The verifier asserts after every sweep that no object the sweep KEPT references memory it RECLAIMED, and aborts at the cycle that broke it, naming holder class, victim class and the field's mark call site. That is precisely the fault -- Display.inputEventStackTmp holding a reclaimed int[1000] whose slot a fused object had taken -- reported at the moment it happens instead of reconstructed from a 12GB corpse. x64 only, keyed off the matrix arch: the verifier walks the heap after every sweep, and x64 is the leg that reproduces the SIGSEGV, so running it on arm64 as well would double the cost for nothing. The job's timeout is already 90 minutes. Also worth recording against the stop point moving again -- pngs=146, WindowLayoutTest this time, against 145/AccessibilityTest and 162/VideoIODecodedFrames before. The victim varies because the corruption is a dangling reference, not a broken test; which test happens to touch the reclaimed slot first is timing. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/linux-build-run.yml | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/.github/workflows/linux-build-run.yml b/.github/workflows/linux-build-run.yml index 02db5de79ad..665675d8617 100644 --- a/.github/workflows/linux-build-run.yml +++ b/.github/workflows/linux-build-run.yml @@ -270,19 +270,18 @@ jobs: # itself is still stripped; only the companion grows, and it is uploaded beside # the core so the crash can be autopsied off the runner. CN1_LINUX_FULL_DEBUG: 'true' - # PROBE, not a setting to keep. The post-mortem on 1b2cbfb showed - # Display.inputEventStackTmp holding a DANGLING reference: its header read - # length == gcMark == 642602016 with data 0xd0 bytes after parentCls -- the - # shape of a fused object occupying a slot an int[1000] used to own, while - # inputEventStack beside it was still a valid int[1000]. An object reachable - # from a live field had been swept. + # The collector's heap-integrity verifier, on the x64 leg only. # - # The mutator assist is the part of the collector that runs Java mark - # functions on a MUTATOR's own thread, and it ships with this A/B switch so - # it can be tested against the sleep it replaced in one binary. If the suite - # stops losing objects with the assist off, the assist is where they are - # lost. Remove this line once the answer is in either way. - CN1_GC_NO_MUTATOR_ASSIST: '1' + # It asserts after every sweep that no object the sweep KEPT references + # memory it RECLAIMED, and aborts at the cycle that broke it naming holder + # class, victim class and the field's mark call site. That is exactly the + # fault here: Display.inputEventStackTmp was found holding a reclaimed + # int[1000] whose slot a fused object had taken over. + # + # x64 only, via the matrix arch, because the verifier walks the heap after + # every sweep and doubling that across the matrix buys nothing -- x64 is + # the leg that reproduces the SIGSEGV. + CN1_LINUX_EXTRA_DEFINES: ${{ matrix.arch == 'x64' && 'CN1_GC_VERIFY' || '' }} # Build the native ELF (and the demo) against an old glibc for portability. CN1_CC: /usr/local/bin/cn1-zig-cc # After the suite runs, the capture test relinks the same objects into a @@ -510,7 +509,6 @@ jobs: -e CN1_SHOT_OUTPUT_DIR=/cn1/artifacts/linux-port/raw-musl \ -e CN1_APP_LOG_TEE=/cn1/artifacts/linux-port/raw-musl/app-output.log \ -e CN1_REQUIRE_SUITE=true \ - -e CN1_GC_NO_MUTATOR_ASSIST=1 \ -e LIBGL_ALWAYS_SOFTWARE=1 \ docker.io/library/alpine:3.20 sh -ec ' sed -i "s|^#\(.*/community\)|\1|" /etc/apk/repositories From 848326814edd804067990882a26467a94a4b2096 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 12 Sep 2026 15:55:06 +0300 Subject: [PATCH 40/66] Make the verifier run prove it ran The first CN1_GC_VERIFY attempt measured nothing and looked clean. The app log contained no [GC-VERIFY] line at all, and I could not tell whether that meant the verifier was compiled in and found no violation, or that the define never reached the compiler -- because the verifier prints a clean epoch ONLY when CN1_GC_VERIFY_LOG is set. Silent otherwise; it speaks on its own only when it aborts. Two changes so the next run is unambiguous: - CN1_GC_VERIFY_LOG=1, so every clean epoch prints "[GC-VERIFY] epoch=N holders=.. refs=.. clean (...)". Absence of those lines now means the verifier is absent, not that the heap is sound. - the harness prints the cmake configure command, so the job log shows whether -DCN1_EXTRA_DEFINES=CN1_GC_VERIFY was actually passed. Both changes exist because a check that cannot fail is not a check, and this is the third time in this investigation that an empty log read as a pass: gdb was never installed on any Linux leg (a relative path under working-directory: vm), hang-stacks.txt held only sample headers, and now this. What the run did show, which is a new face of the same fault rather than a new one: java.lang.IndexOutOfBoundsException at java_util_ArrayList.get:443 at com_codename1_ui_Display.paintTransitionAnimation:1339 at com_codename1_ui_Display.edtLoopImpl:1833 and it stopped at pngs=160 in WindowDialogTest, where the previous run stopped at 146 in WindowLayoutTest and the one before at 145 in AccessibilityTest. A dangling reference does not pick a victim. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/linux-build-run.yml | 6 ++++++ .../tools/translator/CleanTargetLinuxIntegrationTest.java | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/linux-build-run.yml b/.github/workflows/linux-build-run.yml index 665675d8617..253583c6fc3 100644 --- a/.github/workflows/linux-build-run.yml +++ b/.github/workflows/linux-build-run.yml @@ -282,6 +282,12 @@ jobs: # every sweep and doubling that across the matrix buys nothing -- x64 is # the leg that reproduces the SIGSEGV. CN1_LINUX_EXTRA_DEFINES: ${{ matrix.arch == 'x64' && 'CN1_GC_VERIFY' || '' }} + # REQUIRED to tell "verifier clean" from "verifier absent". Without it the + # verifier prints nothing on a clean epoch and only speaks when it aborts, + # so a silent log is indistinguishable from a build where the define never + # reached the compiler -- which is exactly how the first attempt at this + # read as "no violations found" when it had in fact measured nothing. + CN1_GC_VERIFY_LOG: '1' # Build the native ELF (and the demo) against an old glibc for portability. CN1_CC: /usr/local/bin/cn1-zig-cc # After the suite runs, the capture test relinks the same objects into a diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java index 8e33ef2507c..f2f50047645 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java @@ -293,6 +293,10 @@ static Path buildHelloCodenameOneElf(String launcherSource) throws Exception { if (extraDefines != null && !extraDefines.trim().isEmpty()) { configure.add("-DCN1_EXTRA_DEFINES=" + extraDefines.trim()); } + // Printed so a diagnostic build proves itself from the job log. A define + // that silently fails to reach the compiler leaves a clean-looking run that + // measured nothing, which is worse than no diagnostic at all. + System.out.println("CN1SS:HARNESS: cmake configure: " + String.join(" ", configure)); CleanTargetIntegrationTest.runCommand(configure, cmakeRoot); CleanTargetIntegrationTest.runCommand(Arrays.asList("cmake", "--build", buildDir.toString()), cmakeRoot); Path elf = buildDir.resolve("LinuxHelloMain"); From a86b7cdddca8a933c7c741404918213f618911fd Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 12 Sep 2026 16:59:01 +0300 Subject: [PATCH 41/66] Report whether the generated project declares the diagnostic hook The verifier still did not run, and the log now proves the define is not the missing piece: the harness printed cmake configure: ... -DCN1_DEBUG_INFO_LEVEL=3 -DCN1_EXTRA_DEFINES=CN1_GC_VERIFY so -DCN1_EXTRA_DEFINES reached cmake, CN1_GC_VERIFY_LOG=1 was set, and there is still not one [GC-VERIFY] line. cn1GcVerifyHeap is called directly after the sweep under #ifdef CN1_GC_VERIFY, so silence means the macro is not defined in the compiled binary. Passing -DCN1_EXTRA_DEFINES to a CMakeLists that never declares the variable is accepted by cmake without a murmur and compiles nothing extra, which in a log looks exactly like a diagnostic that ran and found nothing. So the harness now prints whether the project it is about to configure actually declares the hook, alongside CN1_DEBUG_INFO_LEVEL as a control -- that one is known to work, since the crash dumps this investigation has been reading carry full types and locals. If the next run says "declares CN1_EXTRA_DEFINES: false" while the control says true, the hook is not reaching the writer this build uses and the emitter is at fault. If it says true, the define is arriving and being compiled, and the absence of output is the verifier's own. Third instance of the same trap in this investigation, which is why it gets a printed answer rather than another round of inference: gdb was never installed on any Linux leg, hang-stacks.txt held only sample headers, and CN1_GC_VERIFY prints nothing on a clean epoch unless CN1_GC_VERIFY_LOG is set. Co-Authored-By: Claude Opus 5 (1M context) --- .../CleanTargetLinuxIntegrationTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java index f2f50047645..92d088b053e 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java @@ -297,6 +297,21 @@ static Path buildHelloCodenameOneElf(String launcherSource) throws Exception { // that silently fails to reach the compiler leaves a clean-looking run that // measured nothing, which is worse than no diagnostic at all. System.out.println("CN1SS:HARNESS: cmake configure: " + String.join(" ", configure)); + // And whether the project being configured actually HAS the hook those + // defines hang on. Passing -DCN1_EXTRA_DEFINES to a CMakeLists that never + // declares it is silently accepted by cmake and compiles nothing extra, + // which is indistinguishable in the log from a diagnostic that ran and + // found nothing. + try { + String cml = new String(Files.readAllBytes(cmakeRoot.resolve("CMakeLists.txt")), + StandardCharsets.UTF_8); + System.out.println("CN1SS:HARNESS: CMakeLists declares CN1_EXTRA_DEFINES: " + + cml.contains("CN1_EXTRA_DEFINES") + + "; declares CN1_DEBUG_INFO_LEVEL: " + + cml.contains("CN1_DEBUG_INFO_LEVEL")); + } catch (IOException readFailed) { + System.out.println("CN1SS:HARNESS: could not read the generated CMakeLists: " + readFailed); + } CleanTargetIntegrationTest.runCommand(configure, cmakeRoot); CleanTargetIntegrationTest.runCommand(Arrays.asList("cmake", "--build", buildDir.toString()), cmakeRoot); Path elf = buildDir.resolve("LinuxHelloMain"); From 7eaab0b7a65420b8feb8388ae7dd33644adb4b15 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:27:41 +0300 Subject: [PATCH 42/66] Probe the built ELF for the verifier instead of trusting the chain Every link in the chain now reports success and the verifier still does not run. From the last job log: cmake configure: ... -DCN1_EXTRA_DEFINES=CN1_GC_VERIFY CMakeLists declares CN1_EXTRA_DEFINES: true; declares CN1_DEBUG_INFO_LEVEL: true [GC-VERIFY] lines: 0 with CN1_GC_VERIFY_LOG=1 set. cn1GcVerifyHeap has no early return -- it registers an atexit summary on its first pass and sets its own active flag -- so if the macro were defined there would be output no matter what the heap looked like. The hook mechanism itself is not at fault: an isolated CMake project with the identical set(... CACHE STRING) / if() / target_compile_definitions prints VERIFY_ON with -DCN1_EXTRA_DEFINES=CN1_GC_VERIFY and VERIFY_OFF without it. The emitted project globs *.c into a single executable, so cn1_globals.c is in the target the definitions attach to. And the suite is not running a prebuilt image: CN1_PREBUILT_EXE is unset, "Running prebuilt suite ELF" never appears, and configure ran exactly once. So the remaining question is not answerable by reading any more of the chain: it is whether the BINARY carries the code. The harness now says so directly, by looking for the "[GC-VERIFY]" literal from cn1GcVerifyHeap's own reporting in the linked image. Fourth time in this investigation that a green-looking link was doing nothing, which is the reason for asking the artifact rather than the process: gdb was never installed, hang-stacks.txt held only headers, the verifier is silent on clean epochs without CN1_GC_VERIFY_LOG, and now this. Co-Authored-By: Claude Opus 5 (1M context) --- .../CleanTargetLinuxIntegrationTest.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java index 92d088b053e..a1e9a3d0a60 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java @@ -315,6 +315,20 @@ static Path buildHelloCodenameOneElf(String launcherSource) throws Exception { CleanTargetIntegrationTest.runCommand(configure, cmakeRoot); CleanTargetIntegrationTest.runCommand(Arrays.asList("cmake", "--build", buildDir.toString()), cmakeRoot); Path elf = buildDir.resolve("LinuxHelloMain"); + // Does the BINARY carry the diagnostic? Every link in the chain above can + // report success while the define fails to reach the compiler, and the only + // unambiguous answer is whether the code it guards is in the executable. + // "[GC-VERIFY]" is a literal in cn1GcVerifyHeap's reporting, so its presence + // in the image means CN1_GC_VERIFY was compiled in. + try { + byte[] image = Files.readAllBytes(elf); + String needle = "[GC-VERIFY]"; + boolean present = new String(image, StandardCharsets.ISO_8859_1).contains(needle); + System.out.println("CN1SS:HARNESS: built ELF contains " + needle + ": " + present + + " (" + image.length + " bytes)"); + } catch (IOException probeFailed) { + System.out.println("CN1SS:HARNESS: could not probe the built ELF: " + probeFailed); + } assertTrue(Files.exists(elf), "native ELF should be produced: " + elf); return elf; } From ef5b6f944a24f5f121bdbb40a0d0212cec253741 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:35:43 +0300 Subject: [PATCH 43/66] The GC is exonerated: 2213 clean epochs. Look at page recycling instead CN1_GC_VERIFY finally ran -- the ELF probe confirmed the code was linked in ("built ELF contains [GC-VERIFY]: true"), and the output was in the APP log rather than the job log, which is where I had been grepping. [GC-VERIFY] epoch=2 ... clean (ok=269189 unknown=2 free=0 stale=0 ...) ... [GC-VERIFY] epoch=2214 ... clean (ok=179384 unknown=12 free=0 ...) 2213 epochs, zero VIOLATIONS, and the suite still failed at pngs=160. The sweep is not reclaiming anything a survivor still references, so the premise this investigation ran on -- "a live Display field holds a RECLAIMED int[1000]" -- is WRONG. I read a fused-object-shaped pointer in a dangling field and concluded the object had been freed; it had not. Second measurement, taken because the other half of that premise was that something wrote the field wrongly: translating CN1 core with master's translator and with this branch's and diffing com_codename1_ui_Display.c gives 14 differing lines, ALL of them this PR's class-init guards (acquire/release on __X_LOADED__ and the completion-flag test). The rest is identical once the determinism fixes are normalised -- local declaration order and catch-label naming, which master derives from identityHashCode and this branch numbers sequentially. So the code that writes that field is master's, unchanged. What survives both measurements is a pointer that looks valid and addresses the wrong object. A BiBOP page retired and reformatted for a different size class does exactly that, and the sweep would not call it reclaimed -- which is why the verifier is silent about it. So the x64 leg now builds with CN1_BIBOP_VALIDATE instead. It checks on every fast allocation that bibopCurrent[ci] is owned, matches its size class, and that the bumped slot lies inside the page, and aborts at the allocation that breaks it with page, class index and slot bounds. Its own comment names "the intermittent x64 cn1BibopFastAlloc crash"; this is an intermittent x64 crash. It also costs a few compares per allocation rather than a heap walk per sweep, and deliberately runs in a non-ASan build so ASan's layout changes cannot mask it. CN1_GC_VERIFY_LOG goes away with the verifier, and the ELF probe now keys off whichever define the run asked for so it stays honest. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/linux-build-run.yml | 33 ++++++++++--------- .../CleanTargetLinuxIntegrationTest.java | 6 +++- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/.github/workflows/linux-build-run.yml b/.github/workflows/linux-build-run.yml index 253583c6fc3..8e9ce624ce7 100644 --- a/.github/workflows/linux-build-run.yml +++ b/.github/workflows/linux-build-run.yml @@ -270,24 +270,25 @@ jobs: # itself is still stripped; only the companion grows, and it is uploaded beside # the core so the crash can be autopsied off the runner. CN1_LINUX_FULL_DEBUG: 'true' - # The collector's heap-integrity verifier, on the x64 leg only. + # BiBOP page validation, on the x64 leg only. # - # It asserts after every sweep that no object the sweep KEPT references - # memory it RECLAIMED, and aborts at the cycle that broke it naming holder - # class, victim class and the field's mark call site. That is exactly the - # fault here: Display.inputEventStackTmp was found holding a reclaimed - # int[1000] whose slot a fused object had taken over. + # CN1_GC_VERIFY already ran here and came back CLEAN over 2213 epochs, so + # the sweep is not reclaiming anything a survivor still references -- the + # premise this investigation had been working from is wrong. And the + # generated Display.c is semantically identical to master's (14 lines + # differ, all of them this PR's class-init guards), so the code that + # writes the field is not at fault either. # - # x64 only, via the matrix arch, because the verifier walks the heap after - # every sweep and doubling that across the matrix buys nothing -- x64 is - # the leg that reproduces the SIGSEGV. - CN1_LINUX_EXTRA_DEFINES: ${{ matrix.arch == 'x64' && 'CN1_GC_VERIFY' || '' }} - # REQUIRED to tell "verifier clean" from "verifier absent". Without it the - # verifier prints nothing on a clean epoch and only speaks when it aborts, - # so a silent log is indistinguishable from a build where the define never - # reached the compiler -- which is exactly how the first attempt at this - # read as "no violations found" when it had in fact measured nothing. - CN1_GC_VERIFY_LOG: '1' + # What remains is a pointer that LOOKS valid but addresses the wrong + # object, which is what a retired-and-reformatted page produces and what + # the verifier would not call reclaimed. This guard checks on every fast + # allocation that bibopCurrent[ci] is owned, matches its size class, and + # that the bumped slot lies inside the page -- and aborts at the + # allocation that breaks it. Its comment names "the intermittent x64 + # cn1BibopFastAlloc crash"; this is an intermittent x64 crash. + # + # x64 only via the matrix arch: it is the leg that SIGSEGVs. + CN1_LINUX_EXTRA_DEFINES: ${{ matrix.arch == 'x64' && 'CN1_BIBOP_VALIDATE' || '' }} # Build the native ELF (and the demo) against an old glibc for portability. CN1_CC: /usr/local/bin/cn1-zig-cc # After the suite runs, the capture test relinks the same objects into a diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java index a1e9a3d0a60..451d4471036 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java @@ -322,7 +322,11 @@ static Path buildHelloCodenameOneElf(String launcherSource) throws Exception { // in the image means CN1_GC_VERIFY was compiled in. try { byte[] image = Files.readAllBytes(elf); - String needle = "[GC-VERIFY]"; + // Keyed off whatever diagnostic this run asked for, so the probe stays + // honest when the define changes. + String want = System.getenv("CN1_LINUX_EXTRA_DEFINES"); + String needle = want != null && want.contains("CN1_BIBOP_VALIDATE") + ? "CN1BIBOP FASTALLOC CORRUPT" : "[GC-VERIFY]"; boolean present = new String(image, StandardCharsets.ISO_8859_1).contains(needle); System.out.println("CN1SS:HARNESS: built ELF contains " + needle + ": " + present + " (" + image.length + " bytes)"); From 2f6748b5e6569c92ba9a883546b0b0dbb08d1c0d Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 12 Sep 2026 19:35:06 +0300 Subject: [PATCH 44/66] PROBE: revert java/util, the last delta the collector work did not clear Not a fix. Everything else has now been eliminated by measurement rather than argument: - CN1_GC_VERIFY: 2213 epochs, zero violations. The sweep reclaims nothing a survivor references. - CN1_BIBOP_VALIDATE: compiled in (ELF probe confirms the abort string is linked) and never fired, while the same run still exited 139. Page ownership, size class and slot bounds are intact at every fast allocation. - CN1_GC_NO_MUTATOR_ASSIST: crash unchanged with the assist off. - SATB insertion and deletion barriers are emitted unconditionally on every object field store; mark functions trace every object field; gcCurrentlyMaturing and gcMarkLocalBuf are thread-local. - com_codename1_ui_Display.c generated by this branch differs from master's by 14 lines, all of them this PR's class-init guards. The code that writes the corrupt field is master's. So the bad pointer is not produced by the collector and not by the translated Display. What remains untested is vm/JavaAPI/src/java/util -- the specialised ArrayList iterator and the IdentityHashMap key/value/entry split -- which this reverts to the merge base. Reading them did not find the bug: the ArrayList iterator carries the bounds guard OpenJDK's own Itr has, and IdentityHashMap's keySet() iterator is constructed KIND_KEY while values() is KIND_VALUE, with the key read at elementData[p] and the value at elementData[p+1]. A differential fuzz would settle it locally, but extracting IdentityHashMap from JavaAPI drags in JavaAPI's own AbstractMap (package-private keySet) and transitively more, so the cheaper decisive test is this one. Green on the x64 leg -- specifically, no exitValue=139 -- means the fault is in one of those two classes and I will bisect between them. Red means java/util is clear too, and the remaining candidates are the primitive class objects and the getResourceAsStream natives. The ArrayList iterator is the PR's main measured win (iteration 25.5% -> 12.4% of mutator self-time), so this is a diagnostic state, not a proposal. Co-Authored-By: Claude Opus 5 (1M context) --- vm/JavaAPI/src/java/util/ArrayList.java | 115 ------------------ vm/JavaAPI/src/java/util/IdentityHashMap.java | 102 ++++------------ 2 files changed, 26 insertions(+), 191 deletions(-) diff --git a/vm/JavaAPI/src/java/util/ArrayList.java b/vm/JavaAPI/src/java/util/ArrayList.java index 4f99090a2a5..8a3c4129a15 100644 --- a/vm/JavaAPI/src/java/util/ArrayList.java +++ b/vm/JavaAPI/src/java/util/ArrayList.java @@ -38,27 +38,6 @@ public class ArrayList extends AbstractList implements List, RandomAcce /** * Constructs a new instance of {@code ArrayList} with ten capacity. */ - // ISOLATION (PR #5766): the lazy default-capacity allocation that used to sit - // here is withdrawn. It replaced the eager new Object[10] with a SHARED static - // zero-length array, which also gave java.util.ArrayList a it had - // never had -- master's only static is a compile-time serialVersionUID, so the - // class previously emitted no static initializer at all. - // - // The suite then began stopping after exactly 145 of 166 screenshots on every - // target except glibc-x64, with ArrayList state corrupt at the point of - // failure: AIOOBE 89 inside pendingIdleSerialCalls.add, then AIOOBE -1, then a - // NullPointerException inside ArrayList.get, which only happens when the - // backing array reference itself is null. - // - // The list logic is NOT at fault: a differential fuzz of this exact source - // against java.util.ArrayList ran 3000 seeds x 200 random operations with no - // divergence, and every access to the corrupted list in Display is inside - // synchronized(lock). The corruption is therefore below Java, which makes the - // new and the process-wide shared array the part worth removing - // before anything subtler is blamed. - // - // The iterator below is the change that carried the measured win (iteration - // 25.5% -> 12.4% of mutator self-time) and is kept. public ArrayList() { this(10); } @@ -343,100 +322,6 @@ public void ensureCapacity(int minimumCapacity) { } } - /** - * Direct-array iterator, overriding AbstractList's generic SimpleListIterator. - * - * The inherited one was the single hottest method in a large translation -- - * 16.45% of mutator self-time on the 5782-class hellocodenameone corpus, more - * than twice the next entry. Three costs per element, none inherent: - * - * - a try/catch around the body, to turn IndexOutOfBoundsException into - * NoSuchElementException. ParparVM has no zero-cost exception tables, so a - * try block is a setjmp -- once per element, in the hottest loop in the - * program. An explicit bounds test costs a compare. - * - size() and get() as VIRTUAL calls on the outer list, with no JIT to - * inline them. - * - the index recomputed as size() - numLeft every iteration instead of - * being carried in a cursor. - * - * MEASURED after: the iteration path fell from 25.5% of mutator self-time to - * 12.4%, ArrayList.get from 7.42% to 0.55%, and _setjmp from 1.61% to zero. - * - * Semantics are unchanged: same ConcurrentModificationException on structural - * modification, same NoSuchElementException past the end, remove() still - * works. Reads array[firstIndex + i] exactly as get(int) does. - * - * Applies to every `for (x : list)` in every translated application whatever - * the loop's static type, because dispatch lands on the concrete ArrayList. - */ - // Package-private, not private: a private inner class whose constructor is - // reached from the outer class makes javac synthesise an access bridge and a - // ArrayList$1 marker type, so every iterator() paid an extra class and an - // aconst_null for the bridge argument. Nothing outside java.util can see it - // either way. - class ArrayListIterator implements Iterator { - private int cursor; - private int lastReturned = -1; - private int expectedModCount = modCount; - - public boolean hasNext() { - return cursor < size; - } - - public E next() { - if (modCount != expectedModCount) { - throw new ConcurrentModificationException(); - } - int i = cursor; - if (i >= size) { - throw new NoSuchElementException(); - } - // The i < size test is only a bounds check while the list's - // firstIndex + size <= array.length invariant holds, so the array - // itself has to be checked too. The iterator this replaced could not - // read out of range: it went through get(), which bounds-checks, inside - // a try that turned IndexOutOfBoundsException into - // NoSuchElementException. Dropping that -- the try was the point, since - // ParparVM has no zero-cost exception tables -- also dropped the only - // bounds check on the read, and ParparVM does NOT check an array read in - // a release build. The result was an out-of-bounds read of the heap - // rather than a recoverable exception, which is how an unrelated int[] - // ended up with a zeroed header and the screenshot suite died 145 tests - // in. OpenJDK's own ArrayList.Itr carries this identical guard - // (`if (i >= elementData.length) throw new ConcurrentModificationException()`); - // omitting it is the whole defect. One compare, and the measured win - // stays. - E[] a = array; - int idx = firstIndex + i; - if (idx < 0 || idx >= a.length) { - throw new ConcurrentModificationException(); - } - cursor = i + 1; - lastReturned = i; - return a[idx]; - } - - public void remove() { - if (lastReturned < 0) { - throw new IllegalStateException(); - } - if (modCount != expectedModCount) { - throw new ConcurrentModificationException(); - } - ArrayList.this.remove(lastReturned); - if (lastReturned < cursor) { - cursor--; - } - lastReturned = -1; - expectedModCount = modCount; - } - } - - @Override - public Iterator iterator() { - return new ArrayListIterator(); - } - @Override public E get(int location) { if (location < 0 || location >= size) { diff --git a/vm/JavaAPI/src/java/util/IdentityHashMap.java b/vm/JavaAPI/src/java/util/IdentityHashMap.java index 188a307ac74..4010c21c92a 100644 --- a/vm/JavaAPI/src/java/util/IdentityHashMap.java +++ b/vm/JavaAPI/src/java/util/IdentityHashMap.java @@ -125,52 +125,25 @@ static class IdentityHashMapIterator implements Iterator { final MapEntry.Type type; - /** - * Which of the three views this iterator serves. - * - * Keys and values come straight out of the table; only entrySet has to - * materialise an Entry, and only there can the caller observe one. The - * generic {@code type} callback cannot express that, because it takes a - * MapEntry -- so serving a key iterator through it allocated an Entry per - * next() purely to read one field back out and drop it. Measured on a - * self-hosting translation of the ParparVM translator: 1,366,140 such - * entries, 43.7MB, all garbage. java.util.HashMap already had separate - * key/value/entry iterators for exactly this reason; this one was missed. - */ - static final int KIND_ENTRY = 0; - static final int KIND_KEY = 1; - static final int KIND_VALUE = 2; - - final int kind; - boolean canRemove = false; IdentityHashMapIterator(MapEntry.Type value, IdentityHashMap hm) { associatedMap = hm; type = value; - kind = KIND_ENTRY; - expectedModCount = hm.modCount; - } - - IdentityHashMapIterator(int iteratorKind, IdentityHashMap hm) { - associatedMap = hm; - type = null; - kind = iteratorKind; expectedModCount = hm.modCount; } public boolean hasNext() { - // elementData hoisted into a local: it was re-loaded from the outer map - // on every comparison AND on every array access, twice per probe step. - Object[] data = associatedMap.elementData; - int p = position; - int len = data.length; - while (p < len && data[p] == null) { - p += 2; + while (position < associatedMap.elementData.length) { + // if this is an empty spot, go to the next one + if (associatedMap.elementData[position] == null) { + position += 2; + } else { + return true; + } } - position = p; - return p < len; + return false; } void checkConcurrentMod() throws ConcurrentModificationException { @@ -179,50 +152,19 @@ void checkConcurrentMod() throws ConcurrentModificationException { } } - @SuppressWarnings("unchecked") public E next() { - // The concurrent-modification test and the null-skipping scan are - // INLINED here rather than reached through checkConcurrentMod() and - // hasNext(). - // - // An enhanced-for already pays two interface dispatches per element - // (hasNext then next); routing next() through two more non-inlined - // calls made it four, and ParparVM has no JIT to fold them away. - // MEASURED on the 5782-class hellocodenameone translation: - // IdentityHashMapIterator.next 6.43% of mutator self-time with - // checkConcurrentMod a further 1.84%, second only to the ArrayList - // iterator. - // - // Behaviour is unchanged: same ConcurrentModificationException on a - // structural change, same NoSuchElementException past the end, and - // position still advances past empty slots exactly as hasNext() did. - if (expectedModCount != associatedMap.modCount) { - throw new ConcurrentModificationException(); - } - Object[] data = associatedMap.elementData; - int p = position; - int len = data.length; - while (p < len && data[p] == null) { - p += 2; - } - if (p >= len) { - position = p; + checkConcurrentMod(); + if (!hasNext()) { throw new NoSuchElementException(); } - lastPosition = p; - position = p + 2; - canRemove = true; + IdentityHashMapEntry result = associatedMap + .getEntry(position); + lastPosition = position; + position += 2; - if (kind == KIND_KEY) { - Object key = associatedMap.elementData[lastPosition]; - return (E) (key == NULL_OBJECT ? null : key); - } - if (kind == KIND_VALUE) { - Object value = associatedMap.elementData[lastPosition + 1]; - return (E) (value == NULL_OBJECT ? null : value); - } - return type.get(associatedMap.getEntry(lastPosition)); + canRemove = true; + return type.get(result); } public void remove() { @@ -745,7 +687,11 @@ public boolean remove(Object key) { @Override public Iterator iterator() { return new IdentityHashMapIterator( - IdentityHashMapIterator.KIND_KEY, IdentityHashMap.this); + new MapEntry.Type() { + public K get(MapEntry entry) { + return entry.key; + } + }, IdentityHashMap.this); } }; } @@ -793,7 +739,11 @@ public void clear() { @Override public Iterator iterator() { return new IdentityHashMapIterator( - IdentityHashMapIterator.KIND_VALUE, IdentityHashMap.this); + new MapEntry.Type() { + public V get(MapEntry entry) { + return entry.value; + } + }, IdentityHashMap.this); } @Override From 628c0da762945fbbd13c82b63d5ba40fddee975f Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 12 Sep 2026 20:01:07 +0300 Subject: [PATCH 45/66] PROBE: withdraw the safepoint park; restore java/util, which is clear java/util is exonerated. With ArrayList and IdentityHashMap reverted to the merge base the x64 suite still exited 139, stopping at pngs=132 in RealOsmVectorScreenshotTest -- so neither the specialised ArrayList iterator nor the IdentityHashMap key/value split produces the bad pointer. Both are restored here; the iterator is this PR's main measured win and there is no reason to carry it reverted. That leaves very little. Eliminated by measurement so far, none of it by argument: sweep reclaiming live objects CN1_GC_VERIFY, 2213 epochs, 0 violations BiBOP page recycling CN1_BIBOP_VALIDATE linked in, never fired mutator assist switch verified in env, crash unchanged translated Display 14 lines vs master, all class-init guards java/util reverted, crash unchanged SATB barriers / mark functions present and correct on every object store class statics in the marker gcMarkObject rejects them before deref, by design and by comment This probe withdraws the cn1PacingPark safepoint park -- the last change this branch makes to GC/thread interaction. It was added for a real, observed iOS failure (a suite that finished and then hung while the collector force-stopped a thread "at a safepoint it never reached"), and it matches the established park pattern in the same file exactly, so I do not expect it to be the cause. But it is the only remaining edit in the area where a wild write could plausibly originate, and it is cheap to test. Green means the park is at fault and needs rewriting rather than reverting -- the iOS hang it fixes is real. Red means the C runtime is clear too, and what remains is the primitive class objects and the getResourceAsStream natives, which I intend to bisect locally against the self-hosted translator rather than with more full suite runs: it reproduces a crash in the same family (a NULL receiver reaching TreeMap/Comparable through updateAllDependencies) and costs nothing per iteration. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.m | 29 +---- vm/JavaAPI/src/java/util/ArrayList.java | 115 ++++++++++++++++++ vm/JavaAPI/src/java/util/IdentityHashMap.java | 102 ++++++++++++---- 3 files changed, 195 insertions(+), 51 deletions(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index cda232c68b0..23672634fcb 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -6952,31 +6952,10 @@ static void cn1PacingPark(CODENAME_ONE_THREAD_STATE, int which, long long pendin // sleep-until-done park. See cn1GcMutatorAssist. if(!threadStateData->threadBlockedByGC && cn1GcMutatorAssist(threadStateData) > 0) { - // HONOUR A STOP REQUESTED WHILE WE WERE ASSISTING. - // - // The test above is taken BEFORE the assist, and the assist marks a - // batch, so the collector can raise threadBlockedByGC while this - // thread is inside it. Without the check below this path continues - // with threadActive still TRUE and never passes the safepoint wait - // further down, so a thread with marking work available can loop - // here indefinitely: the collector waits out its handshake and then - // force-stops it. - // - // OBSERVED on the iOS simulator, where the app finished its suite - // and then hung without emitting the completion marker: - // [GC] force-stopped thread 3 after 250000us at a safepoint it - // never reached (2 so far) ... (16 so far) - // The hazard predates the run-ahead bound; tightening the cap keeps - // `volume > cap` true for longer, which is what made it reachable. - if(threadStateData->threadBlockedByGC) { - threadStateData->threadActive = JAVA_FALSE; - while(threadStateData->threadBlockedByGC) { - if(!cn1VirtualThreadYieldIfVirtual()) { - usleep((JAVA_INT)(500)); - } - } - threadStateData->threadActive = JAVA_TRUE; - } + // PROBE: the safepoint park added by this PR is withdrawn here. It + // is the last change this branch makes to GC/thread interaction, and + // everything else has been eliminated by measurement. Restore it + // (with the iOS force-stop rationale) once the answer is in. continue; } threadStateData->threadActive = JAVA_FALSE; diff --git a/vm/JavaAPI/src/java/util/ArrayList.java b/vm/JavaAPI/src/java/util/ArrayList.java index 8a3c4129a15..4f99090a2a5 100644 --- a/vm/JavaAPI/src/java/util/ArrayList.java +++ b/vm/JavaAPI/src/java/util/ArrayList.java @@ -38,6 +38,27 @@ public class ArrayList extends AbstractList implements List, RandomAcce /** * Constructs a new instance of {@code ArrayList} with ten capacity. */ + // ISOLATION (PR #5766): the lazy default-capacity allocation that used to sit + // here is withdrawn. It replaced the eager new Object[10] with a SHARED static + // zero-length array, which also gave java.util.ArrayList a it had + // never had -- master's only static is a compile-time serialVersionUID, so the + // class previously emitted no static initializer at all. + // + // The suite then began stopping after exactly 145 of 166 screenshots on every + // target except glibc-x64, with ArrayList state corrupt at the point of + // failure: AIOOBE 89 inside pendingIdleSerialCalls.add, then AIOOBE -1, then a + // NullPointerException inside ArrayList.get, which only happens when the + // backing array reference itself is null. + // + // The list logic is NOT at fault: a differential fuzz of this exact source + // against java.util.ArrayList ran 3000 seeds x 200 random operations with no + // divergence, and every access to the corrupted list in Display is inside + // synchronized(lock). The corruption is therefore below Java, which makes the + // new and the process-wide shared array the part worth removing + // before anything subtler is blamed. + // + // The iterator below is the change that carried the measured win (iteration + // 25.5% -> 12.4% of mutator self-time) and is kept. public ArrayList() { this(10); } @@ -322,6 +343,100 @@ public void ensureCapacity(int minimumCapacity) { } } + /** + * Direct-array iterator, overriding AbstractList's generic SimpleListIterator. + * + * The inherited one was the single hottest method in a large translation -- + * 16.45% of mutator self-time on the 5782-class hellocodenameone corpus, more + * than twice the next entry. Three costs per element, none inherent: + * + * - a try/catch around the body, to turn IndexOutOfBoundsException into + * NoSuchElementException. ParparVM has no zero-cost exception tables, so a + * try block is a setjmp -- once per element, in the hottest loop in the + * program. An explicit bounds test costs a compare. + * - size() and get() as VIRTUAL calls on the outer list, with no JIT to + * inline them. + * - the index recomputed as size() - numLeft every iteration instead of + * being carried in a cursor. + * + * MEASURED after: the iteration path fell from 25.5% of mutator self-time to + * 12.4%, ArrayList.get from 7.42% to 0.55%, and _setjmp from 1.61% to zero. + * + * Semantics are unchanged: same ConcurrentModificationException on structural + * modification, same NoSuchElementException past the end, remove() still + * works. Reads array[firstIndex + i] exactly as get(int) does. + * + * Applies to every `for (x : list)` in every translated application whatever + * the loop's static type, because dispatch lands on the concrete ArrayList. + */ + // Package-private, not private: a private inner class whose constructor is + // reached from the outer class makes javac synthesise an access bridge and a + // ArrayList$1 marker type, so every iterator() paid an extra class and an + // aconst_null for the bridge argument. Nothing outside java.util can see it + // either way. + class ArrayListIterator implements Iterator { + private int cursor; + private int lastReturned = -1; + private int expectedModCount = modCount; + + public boolean hasNext() { + return cursor < size; + } + + public E next() { + if (modCount != expectedModCount) { + throw new ConcurrentModificationException(); + } + int i = cursor; + if (i >= size) { + throw new NoSuchElementException(); + } + // The i < size test is only a bounds check while the list's + // firstIndex + size <= array.length invariant holds, so the array + // itself has to be checked too. The iterator this replaced could not + // read out of range: it went through get(), which bounds-checks, inside + // a try that turned IndexOutOfBoundsException into + // NoSuchElementException. Dropping that -- the try was the point, since + // ParparVM has no zero-cost exception tables -- also dropped the only + // bounds check on the read, and ParparVM does NOT check an array read in + // a release build. The result was an out-of-bounds read of the heap + // rather than a recoverable exception, which is how an unrelated int[] + // ended up with a zeroed header and the screenshot suite died 145 tests + // in. OpenJDK's own ArrayList.Itr carries this identical guard + // (`if (i >= elementData.length) throw new ConcurrentModificationException()`); + // omitting it is the whole defect. One compare, and the measured win + // stays. + E[] a = array; + int idx = firstIndex + i; + if (idx < 0 || idx >= a.length) { + throw new ConcurrentModificationException(); + } + cursor = i + 1; + lastReturned = i; + return a[idx]; + } + + public void remove() { + if (lastReturned < 0) { + throw new IllegalStateException(); + } + if (modCount != expectedModCount) { + throw new ConcurrentModificationException(); + } + ArrayList.this.remove(lastReturned); + if (lastReturned < cursor) { + cursor--; + } + lastReturned = -1; + expectedModCount = modCount; + } + } + + @Override + public Iterator iterator() { + return new ArrayListIterator(); + } + @Override public E get(int location) { if (location < 0 || location >= size) { diff --git a/vm/JavaAPI/src/java/util/IdentityHashMap.java b/vm/JavaAPI/src/java/util/IdentityHashMap.java index 4010c21c92a..188a307ac74 100644 --- a/vm/JavaAPI/src/java/util/IdentityHashMap.java +++ b/vm/JavaAPI/src/java/util/IdentityHashMap.java @@ -125,25 +125,52 @@ static class IdentityHashMapIterator implements Iterator { final MapEntry.Type type; + /** + * Which of the three views this iterator serves. + * + * Keys and values come straight out of the table; only entrySet has to + * materialise an Entry, and only there can the caller observe one. The + * generic {@code type} callback cannot express that, because it takes a + * MapEntry -- so serving a key iterator through it allocated an Entry per + * next() purely to read one field back out and drop it. Measured on a + * self-hosting translation of the ParparVM translator: 1,366,140 such + * entries, 43.7MB, all garbage. java.util.HashMap already had separate + * key/value/entry iterators for exactly this reason; this one was missed. + */ + static final int KIND_ENTRY = 0; + static final int KIND_KEY = 1; + static final int KIND_VALUE = 2; + + final int kind; + boolean canRemove = false; IdentityHashMapIterator(MapEntry.Type value, IdentityHashMap hm) { associatedMap = hm; type = value; + kind = KIND_ENTRY; + expectedModCount = hm.modCount; + } + + IdentityHashMapIterator(int iteratorKind, IdentityHashMap hm) { + associatedMap = hm; + type = null; + kind = iteratorKind; expectedModCount = hm.modCount; } public boolean hasNext() { - while (position < associatedMap.elementData.length) { - // if this is an empty spot, go to the next one - if (associatedMap.elementData[position] == null) { - position += 2; - } else { - return true; - } + // elementData hoisted into a local: it was re-loaded from the outer map + // on every comparison AND on every array access, twice per probe step. + Object[] data = associatedMap.elementData; + int p = position; + int len = data.length; + while (p < len && data[p] == null) { + p += 2; } - return false; + position = p; + return p < len; } void checkConcurrentMod() throws ConcurrentModificationException { @@ -152,19 +179,50 @@ void checkConcurrentMod() throws ConcurrentModificationException { } } + @SuppressWarnings("unchecked") public E next() { - checkConcurrentMod(); - if (!hasNext()) { + // The concurrent-modification test and the null-skipping scan are + // INLINED here rather than reached through checkConcurrentMod() and + // hasNext(). + // + // An enhanced-for already pays two interface dispatches per element + // (hasNext then next); routing next() through two more non-inlined + // calls made it four, and ParparVM has no JIT to fold them away. + // MEASURED on the 5782-class hellocodenameone translation: + // IdentityHashMapIterator.next 6.43% of mutator self-time with + // checkConcurrentMod a further 1.84%, second only to the ArrayList + // iterator. + // + // Behaviour is unchanged: same ConcurrentModificationException on a + // structural change, same NoSuchElementException past the end, and + // position still advances past empty slots exactly as hasNext() did. + if (expectedModCount != associatedMap.modCount) { + throw new ConcurrentModificationException(); + } + Object[] data = associatedMap.elementData; + int p = position; + int len = data.length; + while (p < len && data[p] == null) { + p += 2; + } + if (p >= len) { + position = p; throw new NoSuchElementException(); } - IdentityHashMapEntry result = associatedMap - .getEntry(position); - lastPosition = position; - position += 2; - + lastPosition = p; + position = p + 2; canRemove = true; - return type.get(result); + + if (kind == KIND_KEY) { + Object key = associatedMap.elementData[lastPosition]; + return (E) (key == NULL_OBJECT ? null : key); + } + if (kind == KIND_VALUE) { + Object value = associatedMap.elementData[lastPosition + 1]; + return (E) (value == NULL_OBJECT ? null : value); + } + return type.get(associatedMap.getEntry(lastPosition)); } public void remove() { @@ -687,11 +745,7 @@ public boolean remove(Object key) { @Override public Iterator iterator() { return new IdentityHashMapIterator( - new MapEntry.Type() { - public K get(MapEntry entry) { - return entry.key; - } - }, IdentityHashMap.this); + IdentityHashMapIterator.KIND_KEY, IdentityHashMap.this); } }; } @@ -739,11 +793,7 @@ public void clear() { @Override public Iterator iterator() { return new IdentityHashMapIterator( - new MapEntry.Type() { - public V get(MapEntry entry) { - return entry.value; - } - }, IdentityHashMap.this); + IdentityHashMapIterator.KIND_VALUE, IdentityHashMap.this); } @Override From c66da379d452d9a656afe5fdc560b3b0e94c0af9 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 12 Sep 2026 21:03:20 +0300 Subject: [PATCH 46/66] PROBE: stop handing out the primitive class objects The safepoint park is exonerated -- withdrawing it left x64 exiting 139 at pngs=155 in WindowEditingTest -- so it is restored here with the iOS force-stop fix intact. That leaves two candidates, and they separate cleanly. Reverting the nine wrapper classes' TYPE statics stops the static `struct clazz` primitive class objects from ever being handed to Java, without touching Class.java and therefore without disturbing the getResourceAsStream work. The natives and the structs stay defined and simply go unused. Red means the primitive class objects are clear and the remaining candidate is the resource path. Green means they are the cause, and the fix belongs in how they are handed out rather than in reverting them -- Integer.TYPE being null on every ParparVM target is the latent bug this PR opened with, and Util's ctype map collapses to two entries without it. Also recorded: the self-hosted TreeMap/Comparable NULL crash, open and unattributed since 2026-09-10, now looks FIXED. 40 consecutive self-hosted translations, zero crashes, against a ~15% historical rate -- 0.85^40 is 0.0015, so unlike the 0/5 and 0/6 checks earlier in this investigation that number means something. The class-init completion-flag fix explains it exactly: a thread that skipped an initialiser still running elsewhere read statics as null, which is a null backing map in a TreeSet and a null receiver in Comparable.compareTo, both reached from updateAllDependencies. The same 40 runs mean the local harness cannot bisect what is left -- it no longer reproduces anything -- so the remaining candidates have to be settled in CI, one Linux leg at a time. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.m | 29 +++++++++++++++++++++---- vm/JavaAPI/src/java/lang/Boolean.java | 6 ----- vm/JavaAPI/src/java/lang/Byte.java | 2 +- vm/JavaAPI/src/java/lang/Character.java | 2 +- vm/JavaAPI/src/java/lang/Double.java | 10 +-------- vm/JavaAPI/src/java/lang/Float.java | 20 ----------------- vm/JavaAPI/src/java/lang/Integer.java | 14 +----------- vm/JavaAPI/src/java/lang/Long.java | 2 +- vm/JavaAPI/src/java/lang/Short.java | 6 ----- vm/JavaAPI/src/java/lang/Void.java | 2 +- 10 files changed, 31 insertions(+), 62 deletions(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index 23672634fcb..cda232c68b0 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -6952,10 +6952,31 @@ static void cn1PacingPark(CODENAME_ONE_THREAD_STATE, int which, long long pendin // sleep-until-done park. See cn1GcMutatorAssist. if(!threadStateData->threadBlockedByGC && cn1GcMutatorAssist(threadStateData) > 0) { - // PROBE: the safepoint park added by this PR is withdrawn here. It - // is the last change this branch makes to GC/thread interaction, and - // everything else has been eliminated by measurement. Restore it - // (with the iOS force-stop rationale) once the answer is in. + // HONOUR A STOP REQUESTED WHILE WE WERE ASSISTING. + // + // The test above is taken BEFORE the assist, and the assist marks a + // batch, so the collector can raise threadBlockedByGC while this + // thread is inside it. Without the check below this path continues + // with threadActive still TRUE and never passes the safepoint wait + // further down, so a thread with marking work available can loop + // here indefinitely: the collector waits out its handshake and then + // force-stops it. + // + // OBSERVED on the iOS simulator, where the app finished its suite + // and then hung without emitting the completion marker: + // [GC] force-stopped thread 3 after 250000us at a safepoint it + // never reached (2 so far) ... (16 so far) + // The hazard predates the run-ahead bound; tightening the cap keeps + // `volume > cap` true for longer, which is what made it reachable. + if(threadStateData->threadBlockedByGC) { + threadStateData->threadActive = JAVA_FALSE; + while(threadStateData->threadBlockedByGC) { + if(!cn1VirtualThreadYieldIfVirtual()) { + usleep((JAVA_INT)(500)); + } + } + threadStateData->threadActive = JAVA_TRUE; + } continue; } threadStateData->threadActive = JAVA_FALSE; diff --git a/vm/JavaAPI/src/java/lang/Boolean.java b/vm/JavaAPI/src/java/lang/Boolean.java index 2f56aa21520..043fee9956d 100644 --- a/vm/JavaAPI/src/java/lang/Boolean.java +++ b/vm/JavaAPI/src/java/lang/Boolean.java @@ -27,12 +27,6 @@ * Since: JDK1.0, CLDC 1.0 */ public final class Boolean implements Comparable { - - /** - * The class object for the primitive type this class wraps. - */ - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_BOOLEAN); - /** * The Boolean object corresponding to the primitive value false. */ diff --git a/vm/JavaAPI/src/java/lang/Byte.java b/vm/JavaAPI/src/java/lang/Byte.java index b7b5ff186f0..9a7fa9d99e9 100644 --- a/vm/JavaAPI/src/java/lang/Byte.java +++ b/vm/JavaAPI/src/java/lang/Byte.java @@ -28,7 +28,7 @@ */ public final class Byte extends Number implements Comparable { - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_BYTE); + public static final Class TYPE = byte.class; public static final int SIZE = 8; /** diff --git a/vm/JavaAPI/src/java/lang/Character.java b/vm/JavaAPI/src/java/lang/Character.java index 448f24d6a51..93ce6f67946 100644 --- a/vm/JavaAPI/src/java/lang/Character.java +++ b/vm/JavaAPI/src/java/lang/Character.java @@ -155,7 +155,7 @@ public final class Character implements Comparable{ //public static final int MAX_RADIX = 36; //public static final char MIN_VALUE = '\0'; //public static final char MAX_VALUE = '\uFFFF'; - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_CHAR); + public static final Class TYPE = char.class; public static final byte UNASSIGNED = 0; public static final byte UPPERCASE_LETTER = 1; public static final byte LOWERCASE_LETTER = 2; diff --git a/vm/JavaAPI/src/java/lang/Double.java b/vm/JavaAPI/src/java/lang/Double.java index 05d12f67f31..22f161feb5d 100644 --- a/vm/JavaAPI/src/java/lang/Double.java +++ b/vm/JavaAPI/src/java/lang/Double.java @@ -29,7 +29,7 @@ */ public final class Double extends Number implements Comparable { - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_DOUBLE); + public static final Class TYPE = double.class; /** * The largest positive finite value of type double. It is equal to the value returned by Double.longBitsToDouble(0x7fefffffffffffffL) * See Also:Constant Field Values @@ -88,14 +88,6 @@ public byte byteValue(){ * If the argument is NaN, the result is 0x7ff8000000000000L. * In all cases, the result is a long integer that, when given to the longBitsToDouble(long) method, will produce a floating-point value equal to the argument to doubleToLongBits. */ - /** - * The raw IEEE 754 bits of {@code value}, without collapsing NaN to the - * canonical NaN. See {@link Float#floatToRawIntBits} for why this delegates. - */ - public static long doubleToRawLongBits(double value) { - return doubleToLongBits(value); - } - public native static long doubleToLongBits(double value); /** diff --git a/vm/JavaAPI/src/java/lang/Float.java b/vm/JavaAPI/src/java/lang/Float.java index 93e9d560a81..5b257d00f64 100644 --- a/vm/JavaAPI/src/java/lang/Float.java +++ b/vm/JavaAPI/src/java/lang/Float.java @@ -28,12 +28,6 @@ * Since: JDK1.0, CLDC 1.1 */ public final class Float extends Number implements Comparable { - - /** - * The class object for the primitive type this class wraps. - */ - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_FLOAT); - /** * The largest positive value of type float. It is equal to the value returned by Float.intBitsToFloat(0x7f7fffff). * See Also:Constant Field Values @@ -119,20 +113,6 @@ public boolean equals(java.lang.Object obj){ * Returns the bit representation of a single-float value. The result is a representation of the floating-point argument according to the IEEE 754 floating-point "single precision" bit layout. Bit 31 (the bit that is selected by the mask 0x80000000) represents the sign of the floating-point number. Bits 30-23 (the bits that are selected by the mask 0x7f800000) represent the exponent. Bits 22-0 (the bits that are selected by the mask 0x007fffff) represent the significand (sometimes called the mantissa) of the floating-point number. If the argument is positive infinity, the result is 0x7f800000. If the argument is negative infinity, the result is 0xff800000. If the argument is NaN, the result is 0x7fc00000. In all cases, the result is an integer that, when given to the * method, will produce a floating-point value equal to the argument to floatToIntBits. */ - /** - * The raw IEEE 754 bits of {@code value}, without collapsing NaN to the - * canonical NaN. - * - * Delegates rather than declaring a second native. ParparVM's floatToIntBits - * is a bare union punt that does not collapse NaN to the canonical NaN -- so it - * is already the raw operation, and the two differ in the spec but not here. A - * separate native would be one more mangled symbol to get wrong, silently, for - * no behavioural difference. - */ - public static int floatToRawIntBits(float value) { - return floatToIntBits(value); - } - public native static int floatToIntBits(float value); /** diff --git a/vm/JavaAPI/src/java/lang/Integer.java b/vm/JavaAPI/src/java/lang/Integer.java index 4a1cb1a85d0..0bcf391a733 100644 --- a/vm/JavaAPI/src/java/lang/Integer.java +++ b/vm/JavaAPI/src/java/lang/Integer.java @@ -29,7 +29,7 @@ */ public final class Integer extends Number implements Comparable { - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_INT); + public static final Class TYPE = int.class; private static final char[] DIGITS = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', @@ -359,18 +359,6 @@ public static int signum(int i) { return (i >> 31) | (-i >>> 31); // Hacker's delight 2-7 } - /** - * Rotates the two's-complement binary representation of {@code i} left by - * {@code distance} bits. - * - * The shift distance is used modulo 32 by the JLS shift rules, which is what - * makes the negation on the right half correct for every distance, including - * zero and multiples of 32. - */ - public static int rotateLeft(int i, int distance) { - return (i << distance) | (i >>> -distance); - } - public static int compare(int f1, int f2) { if (f1 > f2) return 1; diff --git a/vm/JavaAPI/src/java/lang/Long.java b/vm/JavaAPI/src/java/lang/Long.java index fce50a48abd..0e938265153 100644 --- a/vm/JavaAPI/src/java/lang/Long.java +++ b/vm/JavaAPI/src/java/lang/Long.java @@ -29,7 +29,7 @@ */ public final class Long extends Number implements Comparable { - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_LONG); + public static Class TYPE = long.class; /** * The largest value of type long. diff --git a/vm/JavaAPI/src/java/lang/Short.java b/vm/JavaAPI/src/java/lang/Short.java index f0800e1f219..233a1698268 100644 --- a/vm/JavaAPI/src/java/lang/Short.java +++ b/vm/JavaAPI/src/java/lang/Short.java @@ -27,12 +27,6 @@ * Since: JDK1.1, CLDC 1.0 */ public final class Short extends Number implements Comparable { - - /** - * The class object for the primitive type this class wraps. - */ - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_SHORT); - /** * The maximum value a Short can have. * See Also:Constant Field Values diff --git a/vm/JavaAPI/src/java/lang/Void.java b/vm/JavaAPI/src/java/lang/Void.java index 96dbd87a71e..c1391f982e0 100644 --- a/vm/JavaAPI/src/java/lang/Void.java +++ b/vm/JavaAPI/src/java/lang/Void.java @@ -27,5 +27,5 @@ * @author Shai Almog */ public final class Void { - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_VOID); + public static final Class TYPE = Void.class; } From ed057bebdcad7a7cea67ce4d2cbe74db4620f936 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 12 Sep 2026 21:38:43 +0300 Subject: [PATCH 47/66] Initialise java.lang.Class before handing out a primitive class object FOUND IT. Reverting the nine wrapper TYPE statics made all three Linux suite legs pass -- x64, arm64 and musl -- with SUITE:FINISHED and pass=188 fail=1 not-run=0, the one failure being a ValidatorLightweightPicker screenshot difference rather than a crash. Every previous run on this branch had at least one leg exiting 139. So the primitive class objects are what the crash was about. The bug is that they were handed out without initialising the class they claim to be instances of. getPrimitiveClass returns (JAVA_OBJECT)&cn1_primitive_class_int and friends -- static struct clazz whose __codenameOneParentClsReference is &class__java_lang_Class. Every virtual call on Integer.TYPE therefore dispatches through class__java_lang_Class.vtable, and that vtable is malloc'd by java.lang.Class's own static initialiser. Nothing on this path ran it: the callers are the wrapper classes' clinits, any of which can be the first class a program touches, so the vtable is whatever a static initialises to -- zero. The initialiser is idempotent and returns on its completion flag, so calling it here costs one acquire load afterwards. How the space was closed, all of it by measurement rather than argument: sweep reclaiming live objects CN1_GC_VERIFY, 2213 epochs, 0 violations BiBOP page recycling CN1_BIBOP_VALIDATE linked in, never fired mutator assist switch verified in env, crash unchanged translated Display 14 lines vs master, all class-init guards java/util reverted, crash unchanged cn1PacingPark safepoint park withdrawn, crash unchanged getResourceAsStream excluded by history: cn1FindResource was the weak null stub on Linux until 670fb6c, and the crash long predates it primitive class objects reverted -> all three legs green Everything exonerated above is restored, including the safepoint park and the ArrayList iterator. Stated honestly: the mechanism above is a real defect and its removal is what the evidence points at, but I have not proved it is the exact instruction that produced the observed bad pointer -- the fault address was not the low-address read a null vtable call gives directly. If the crash survives this, the remaining possibility inside the same unit is the CN1_PRIMITIVE_CLASS_ID sentinel of -1 reaching something that indexes by class id. Gates D and A byte-identical over 798 files. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/nativeMethods.m | 15 +++++++++++++++ vm/JavaAPI/src/java/lang/Boolean.java | 6 ++++++ vm/JavaAPI/src/java/lang/Byte.java | 2 +- vm/JavaAPI/src/java/lang/Character.java | 2 +- vm/JavaAPI/src/java/lang/Double.java | 10 +++++++++- vm/JavaAPI/src/java/lang/Float.java | 20 ++++++++++++++++++++ vm/JavaAPI/src/java/lang/Integer.java | 14 +++++++++++++- vm/JavaAPI/src/java/lang/Long.java | 2 +- vm/JavaAPI/src/java/lang/Short.java | 6 ++++++ vm/JavaAPI/src/java/lang/Void.java | 2 +- 10 files changed, 73 insertions(+), 6 deletions(-) diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 736cc309fe4..4f11a41e199 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -1997,6 +1997,21 @@ JAVA_OBJECT java_lang_Class_getName___R_java_lang_String(CODENAME_ONE_THREAD_STA * they are matched by CN1_PRIM_* there. */ JAVA_OBJECT java_lang_Class_getPrimitiveClass___int_R_java_lang_Class(CODENAME_ONE_THREAD_STATE, JAVA_INT typeCode) { + // java.lang.Class MUST be initialised before one of these is handed out. + // + // The nine descriptors are static struct clazz, and they are returned as + // java.lang.Class OBJECTS: each carries + // __codenameOneParentClsReference = &class__java_lang_Class, so every virtual + // call on Integer.TYPE and friends -- hashCode, equals, toString, getClass -- + // dispatches through class__java_lang_Class.vtable. That vtable is malloc'd by + // java.lang.Class's own static initialiser, and NOTHING on this path ran it: + // the callers are the wrapper classes' clinits (Integer, Boolean, ...), any of + // which can be the first class the program touches. Until it runs, the vtable + // is the zero a static initialises to. + // + // The initialiser is idempotent and returns on its completion flag, so this + // costs one acquire load once the class is up. + __STATIC_INITIALIZER_java_lang_Class(threadStateData); switch(typeCode) { case 0: return (JAVA_OBJECT)&cn1_primitive_class_int; case 1: return (JAVA_OBJECT)&cn1_primitive_class_long; diff --git a/vm/JavaAPI/src/java/lang/Boolean.java b/vm/JavaAPI/src/java/lang/Boolean.java index 043fee9956d..2f56aa21520 100644 --- a/vm/JavaAPI/src/java/lang/Boolean.java +++ b/vm/JavaAPI/src/java/lang/Boolean.java @@ -27,6 +27,12 @@ * Since: JDK1.0, CLDC 1.0 */ public final class Boolean implements Comparable { + + /** + * The class object for the primitive type this class wraps. + */ + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_BOOLEAN); + /** * The Boolean object corresponding to the primitive value false. */ diff --git a/vm/JavaAPI/src/java/lang/Byte.java b/vm/JavaAPI/src/java/lang/Byte.java index 9a7fa9d99e9..b7b5ff186f0 100644 --- a/vm/JavaAPI/src/java/lang/Byte.java +++ b/vm/JavaAPI/src/java/lang/Byte.java @@ -28,7 +28,7 @@ */ public final class Byte extends Number implements Comparable { - public static final Class TYPE = byte.class; + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_BYTE); public static final int SIZE = 8; /** diff --git a/vm/JavaAPI/src/java/lang/Character.java b/vm/JavaAPI/src/java/lang/Character.java index 93ce6f67946..448f24d6a51 100644 --- a/vm/JavaAPI/src/java/lang/Character.java +++ b/vm/JavaAPI/src/java/lang/Character.java @@ -155,7 +155,7 @@ public final class Character implements Comparable{ //public static final int MAX_RADIX = 36; //public static final char MIN_VALUE = '\0'; //public static final char MAX_VALUE = '\uFFFF'; - public static final Class TYPE = char.class; + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_CHAR); public static final byte UNASSIGNED = 0; public static final byte UPPERCASE_LETTER = 1; public static final byte LOWERCASE_LETTER = 2; diff --git a/vm/JavaAPI/src/java/lang/Double.java b/vm/JavaAPI/src/java/lang/Double.java index 22f161feb5d..05d12f67f31 100644 --- a/vm/JavaAPI/src/java/lang/Double.java +++ b/vm/JavaAPI/src/java/lang/Double.java @@ -29,7 +29,7 @@ */ public final class Double extends Number implements Comparable { - public static final Class TYPE = double.class; + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_DOUBLE); /** * The largest positive finite value of type double. It is equal to the value returned by Double.longBitsToDouble(0x7fefffffffffffffL) * See Also:Constant Field Values @@ -88,6 +88,14 @@ public byte byteValue(){ * If the argument is NaN, the result is 0x7ff8000000000000L. * In all cases, the result is a long integer that, when given to the longBitsToDouble(long) method, will produce a floating-point value equal to the argument to doubleToLongBits. */ + /** + * The raw IEEE 754 bits of {@code value}, without collapsing NaN to the + * canonical NaN. See {@link Float#floatToRawIntBits} for why this delegates. + */ + public static long doubleToRawLongBits(double value) { + return doubleToLongBits(value); + } + public native static long doubleToLongBits(double value); /** diff --git a/vm/JavaAPI/src/java/lang/Float.java b/vm/JavaAPI/src/java/lang/Float.java index 5b257d00f64..93e9d560a81 100644 --- a/vm/JavaAPI/src/java/lang/Float.java +++ b/vm/JavaAPI/src/java/lang/Float.java @@ -28,6 +28,12 @@ * Since: JDK1.0, CLDC 1.1 */ public final class Float extends Number implements Comparable { + + /** + * The class object for the primitive type this class wraps. + */ + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_FLOAT); + /** * The largest positive value of type float. It is equal to the value returned by Float.intBitsToFloat(0x7f7fffff). * See Also:Constant Field Values @@ -113,6 +119,20 @@ public boolean equals(java.lang.Object obj){ * Returns the bit representation of a single-float value. The result is a representation of the floating-point argument according to the IEEE 754 floating-point "single precision" bit layout. Bit 31 (the bit that is selected by the mask 0x80000000) represents the sign of the floating-point number. Bits 30-23 (the bits that are selected by the mask 0x7f800000) represent the exponent. Bits 22-0 (the bits that are selected by the mask 0x007fffff) represent the significand (sometimes called the mantissa) of the floating-point number. If the argument is positive infinity, the result is 0x7f800000. If the argument is negative infinity, the result is 0xff800000. If the argument is NaN, the result is 0x7fc00000. In all cases, the result is an integer that, when given to the * method, will produce a floating-point value equal to the argument to floatToIntBits. */ + /** + * The raw IEEE 754 bits of {@code value}, without collapsing NaN to the + * canonical NaN. + * + * Delegates rather than declaring a second native. ParparVM's floatToIntBits + * is a bare union punt that does not collapse NaN to the canonical NaN -- so it + * is already the raw operation, and the two differ in the spec but not here. A + * separate native would be one more mangled symbol to get wrong, silently, for + * no behavioural difference. + */ + public static int floatToRawIntBits(float value) { + return floatToIntBits(value); + } + public native static int floatToIntBits(float value); /** diff --git a/vm/JavaAPI/src/java/lang/Integer.java b/vm/JavaAPI/src/java/lang/Integer.java index 0bcf391a733..4a1cb1a85d0 100644 --- a/vm/JavaAPI/src/java/lang/Integer.java +++ b/vm/JavaAPI/src/java/lang/Integer.java @@ -29,7 +29,7 @@ */ public final class Integer extends Number implements Comparable { - public static final Class TYPE = int.class; + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_INT); private static final char[] DIGITS = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', @@ -359,6 +359,18 @@ public static int signum(int i) { return (i >> 31) | (-i >>> 31); // Hacker's delight 2-7 } + /** + * Rotates the two's-complement binary representation of {@code i} left by + * {@code distance} bits. + * + * The shift distance is used modulo 32 by the JLS shift rules, which is what + * makes the negation on the right half correct for every distance, including + * zero and multiples of 32. + */ + public static int rotateLeft(int i, int distance) { + return (i << distance) | (i >>> -distance); + } + public static int compare(int f1, int f2) { if (f1 > f2) return 1; diff --git a/vm/JavaAPI/src/java/lang/Long.java b/vm/JavaAPI/src/java/lang/Long.java index 0e938265153..fce50a48abd 100644 --- a/vm/JavaAPI/src/java/lang/Long.java +++ b/vm/JavaAPI/src/java/lang/Long.java @@ -29,7 +29,7 @@ */ public final class Long extends Number implements Comparable { - public static Class TYPE = long.class; + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_LONG); /** * The largest value of type long. diff --git a/vm/JavaAPI/src/java/lang/Short.java b/vm/JavaAPI/src/java/lang/Short.java index 233a1698268..f0800e1f219 100644 --- a/vm/JavaAPI/src/java/lang/Short.java +++ b/vm/JavaAPI/src/java/lang/Short.java @@ -27,6 +27,12 @@ * Since: JDK1.1, CLDC 1.0 */ public final class Short extends Number implements Comparable { + + /** + * The class object for the primitive type this class wraps. + */ + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_SHORT); + /** * The maximum value a Short can have. * See Also:Constant Field Values diff --git a/vm/JavaAPI/src/java/lang/Void.java b/vm/JavaAPI/src/java/lang/Void.java index c1391f982e0..96dbd87a71e 100644 --- a/vm/JavaAPI/src/java/lang/Void.java +++ b/vm/JavaAPI/src/java/lang/Void.java @@ -27,5 +27,5 @@ * @author Shai Almog */ public final class Void { - public static final Class TYPE = Void.class; + public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_VOID); } From 1c6dc85cfd4b577ec98c843e06a9810563dbd54a Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 12 Sep 2026 22:38:38 +0300 Subject: [PATCH 48/66] Register the primitive class descriptors in the GC's clazz registry The java.lang.Class initialisation fix was necessary but not sufficient: x64 still exited 139 with it in place, while reverting the nine TYPE statics still makes all three Linux legs pass. So something else about handing out these descriptors is the problem, and there is exactly one structural difference left between them and every other clazz in the process. struct clazz carries a TRAILING cn1ClazzRegistered flag. Its comment: "Set once by cn1GcRegisterClazz when the first object of this class is allocated; the conservative GC's mark guard then recognises the clazz ADDRESS as genuine via an exact registry instead of a distance heuristic." Every generated class__X is registered by CN1_CLAZZ_REGISTER from every allocation entry point, so by construction the collector has seen every clazz address that can reach it. These nine allocate nothing. They are static, and getPrimitiveClass hands their addresses straight to Java -- so they were the ONLY clazz addresses in the process reaching Java while permanently unregistered, in a collector whose guard is documented to depend on that registry being exact. They are now registered on first hand-out, which is the same moment the invariant fires for every other class. The macro tests the flag first, so it costs one predictable load afterwards. Extracted cn1PrimitiveClassFor so the code-to-descriptor mapping is not written twice. Being explicit about confidence: this is the last structural difference I can find, and it is a real invariant violation, but I have not yet reproduced the mechanism end to end. If x64 still exits 139 with this in, I will stop attacking it from the VM side and propose lifting the primitive class objects out of this PR entirely -- they are the first commit and the rest of the branch does not depend on them except through Util's ctype map, which matters only when the translator is self-hosted. Gates D and A byte-identical. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/nativeMethods.m | 30 +++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 4f11a41e199..60cab21f523 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -1996,6 +1996,22 @@ JAVA_OBJECT java_lang_Class_getName___R_java_lang_String(CODENAME_ONE_THREAD_STA * The codes are an implementation detail shared only with java/lang/Class.java; * they are matched by CN1_PRIM_* there. */ +/** The descriptor for a primitive type code, or 0 for an unknown code. */ +static struct clazz* cn1PrimitiveClassFor(JAVA_INT typeCode) { + switch(typeCode) { + case 0: return &cn1_primitive_class_int; + case 1: return &cn1_primitive_class_long; + case 2: return &cn1_primitive_class_short; + case 3: return &cn1_primitive_class_byte; + case 4: return &cn1_primitive_class_char; + case 5: return &cn1_primitive_class_float; + case 6: return &cn1_primitive_class_double; + case 7: return &cn1_primitive_class_boolean; + case 8: return &cn1_primitive_class_void; + } + return 0; +} + JAVA_OBJECT java_lang_Class_getPrimitiveClass___int_R_java_lang_Class(CODENAME_ONE_THREAD_STATE, JAVA_INT typeCode) { // java.lang.Class MUST be initialised before one of these is handed out. // @@ -2012,6 +2028,20 @@ JAVA_OBJECT java_lang_Class_getPrimitiveClass___int_R_java_lang_Class(CODENAME_O // The initialiser is idempotent and returns on its completion flag, so this // costs one acquire load once the class is up. __STATIC_INITIALIZER_java_lang_Class(threadStateData); + // And REGISTER the descriptor in the GC's exact clazz registry. + // + // Every generated class__X is registered by CN1_CLAZZ_REGISTER on the first + // allocation of one of its instances, from every allocation entry point, so + // by construction the collector has seen every clazz address that can reach + // it. These nine allocate nothing -- they are static and handed out directly + // -- so they were the only clazz addresses in the process that reached Java + // while permanently unregistered, and the mark guard is documented to + // recognise a genuine clazz address "via an exact registry instead of a + // distance heuristic". + // + // The macro is idempotent and tests the trailing cn1ClazzRegistered flag + // first, so this is one predictable load after the first call. + CN1_CLAZZ_REGISTER(cn1PrimitiveClassFor(typeCode)); switch(typeCode) { case 0: return (JAVA_OBJECT)&cn1_primitive_class_int; case 1: return (JAVA_OBJECT)&cn1_primitive_class_long; From f1da54314be6245d702fa68470b8f3174fa59521 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:50:14 +0300 Subject: [PATCH 49/66] Withdraw the primitive class objects; key the C-type tables on an enum The nine primitive class objects are the one part of this branch I could not make safe, and they are now out of it. Reverting them (c66da37) was the only change that made all three Linux legs pass, reproducibly; keeping them, x64 exited 139 in the suite every time, stopping in a different test each run. Two fixes were tried on top and neither moved it: ed057be initialised java.lang.Class before handing a descriptor out, so the vtable a virtual call on Integer.TYPE dispatches through is no longer the zero a static initialises to. Necessary, not sufficient. 1c6dc85 registered the nine in the GC's exact clazz registry, on the grounds that every other class__X is registered from its first allocation and these allocate nothing. Also did not move it. What I can state is what I ruled out, all by measurement rather than argument: the sweep does not reclaim live objects (2213 clean CN1_GC_VERIFY epochs, zero violations), BiBOP page recycling is not it (guard linked, never fired), nor mutator assist, the safepoint park, java/util, or the translated Display.c. Every route by which a descriptor can reach the collector -- SATB drain, the exact static-root scan in markStatics, the conservative scan, the reflection natives, instanceofFunction -- is guarded, and I verified each one. So I know WHAT causes the crash and not WHY, and those are different things. Shipping a feature on the second of them is not something I am willing to do. Nothing is lost by removing it, because the dependency was never real. Util kept two HashMap keyed on Integer.TYPE and its eight siblings, and NOTHING ever reflected on those objects: every use was an identity comparison against one of the nine constants. Class was standing in for an enum. PrimitiveType is that enum, carrying the C type, the Java keyword and the JVM descriptor character, and it is strictly better than what it replaces: - It works on a ParparVM target, which X.TYPE does not. javac lowers the primitive class literal in `TYPE = int.class` to a read of the field being initialized, so the wrapper's own clinit stores the null straight back, and three of the nine wrappers never declared the field at all. Keyed on those, both maps collapsed to one entry and getCType answered the same C type for every primitive -- valid C, every type wrong, nothing thrown. - ByteCodeMethodArg.hashCode was returning Class's identity hash, which varies between runs of one JVM. It now returns ordinal(), explicitly: Enum.hashCode is an identity hash on OpenJDK and the ordinal in ParparVM's java.lang.Enum, so hashing on it would make the two runtimes disagree on iteration order and the self-hosting gate would report that as a VM divergence. Short.TYPE, Float.TYPE and Boolean.TYPE stay, declared as an explicit null, because org.objectweb.asm.Type reads all three and ASM is a jar we cannot edit -- the one case where JavaAPI grows to meet a dependency instead of the dependency being removed. The other six keep master's declarations byte for byte. Kept out of the revert, because they are correct independently of the feature: the primitiveType guards in isAssignableFrom/isInstance (instanceofFunction indexes classInstanceOf[] by classId and has no guard against a negative one, so a primitive must never reach it), and the isInstance argument-order fix -- the two were reversed, which made every Class.isInstance in a native build answer false for every subclass. verify-output-neutral.sh is the gate this change needed and Gate A structurally cannot provide: gate A compares the JVM translator against the native one, so a refactor lands on both sides at once and it stays green while every emitted signature changes. This runs the JVM translator alone, before and after, over the same corpus. The refactor is byte-identical over 798 files, the descriptor collapse over 800, and the comparator was watched failing on a single flipped byte first -- a comparator nobody has seen fail is not a comparator. Gates D and A both pass on the result, with the negative control. --- vm/ByteCodeTranslator/src/cn1_globals.h | 28 --- vm/ByteCodeTranslator/src/cn1_globals.m | 58 ----- .../tools/translator/ByteCodeField.java | 47 ++-- .../tools/translator/ByteCodeMethodArg.java | 22 +- .../tools/translator/BytecodeMethod.java | 46 ++-- .../translator/JavascriptMethodGenerator.java | 18 +- .../tools/translator/PrimitiveType.java | 90 ++++++++ .../com/codename1/tools/translator/Util.java | 50 +--- vm/ByteCodeTranslator/src/nativeMethods.m | 85 +------ vm/JavaAPI/src/java/lang/Boolean.java | 18 +- vm/JavaAPI/src/java/lang/Byte.java | 2 +- vm/JavaAPI/src/java/lang/Character.java | 2 +- vm/JavaAPI/src/java/lang/Class.java | 30 --- vm/JavaAPI/src/java/lang/Double.java | 2 +- vm/JavaAPI/src/java/lang/Float.java | 18 +- vm/JavaAPI/src/java/lang/Integer.java | 2 +- vm/JavaAPI/src/java/lang/Long.java | 2 +- vm/JavaAPI/src/java/lang/Short.java | 18 +- vm/JavaAPI/src/java/lang/Void.java | 2 +- vm/selfhost/verify-output-neutral.sh | 59 +++++ .../PrimitiveTypeIntegrationTest.java | 215 ------------------ .../tools/translator/PrimitiveTypeApp.java | 97 -------- 22 files changed, 277 insertions(+), 634 deletions(-) create mode 100644 vm/ByteCodeTranslator/src/com/codename1/tools/translator/PrimitiveType.java create mode 100755 vm/selfhost/verify-output-neutral.sh delete mode 100644 vm/tests/src/test/java/com/codename1/tools/translator/PrimitiveTypeIntegrationTest.java delete mode 100644 vm/tests/src/test/resources/com/codename1/tools/translator/PrimitiveTypeApp.java diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index 9d9f97c05c9..1baad97e679 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -2860,34 +2860,6 @@ extern struct clazz class_array1__JAVA_DOUBLE; extern struct clazz class_array2__JAVA_DOUBLE; extern struct clazz class_array3__JAVA_DOUBLE; -/** - * The nine scalar primitive class objects -- int.class, Integer.TYPE and friends. - * - * javac lowers a primitive class literal to a read of the boxed type's own TYPE - * field, so `TYPE = int.class` inside Integer's initializer compiles to - * `getstatic TYPE; putstatic TYPE` -- it reads the field it is initializing and - * leaves it null. Every wrapper that declared TYPE that way had a null one, and - * a Map keyed on them collapsed to a single entry, so a lookup for int answered - * with whatever type was stored last. Nothing threw. The wrappers now go through - * java_lang_Class_getPrimitiveClass, which hands back one of these. - * - * classId is CN1_PRIMITIVE_CLASS_ID for all nine: these never take part in an - * instanceof, and instanceofFunction indexes tables by classId, so the callers - * that could reach one (isAssignableFrom, isInstance) test primitiveType first - * rather than indexing with a value no table has a row for. - */ -#define CN1_PRIMITIVE_CLASS_ID (-1) - -extern struct clazz cn1_primitive_class_int; -extern struct clazz cn1_primitive_class_long; -extern struct clazz cn1_primitive_class_short; -extern struct clazz cn1_primitive_class_byte; -extern struct clazz cn1_primitive_class_char; -extern struct clazz cn1_primitive_class_float; -extern struct clazz cn1_primitive_class_double; -extern struct clazz cn1_primitive_class_boolean; -extern struct clazz cn1_primitive_class_void; - extern JAVA_OBJECT newString(CODENAME_ONE_THREAD_STATE, int length, JAVA_CHAR data[]); /** * Like newStringFromCString but DECODES, in the PLATFORM's encoding, instead of diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index cda232c68b0..21b762c1732 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -956,64 +956,6 @@ static void init_gc_thresholds() { //#define DEBUG_GC_OBJECTS_IN_HEAP -/** - * Scalar primitive class objects. See the comment on CN1_PRIMITIVE_CLASS_ID in - * cn1_globals.h for why these exist and why their classId is a sentinel. - * - * baseClass is 0 because int.class.getSuperclass() is null, which - * java_lang_Class_getSuperclass already returns for a null baseClass. isArray is - * false and arrayType is 0: these are the scalar types, not the array classes, - * which already exist as class_arrayN__JAVA_*. - * - * Designated initializers, unlike the positional generated ones beside them, so - * that a future field added to struct clazz cannot silently shift every value. - */ -/* - * __codenameOneParentClsReference is the class OF this object. Every generated - * clazz sets it to class__java_lang_Class, and CN1_CLASS_OF reads it to find the - * vtable when a clazz is used as an ordinary object -- which is what happens the - * moment one becomes a Map key. Leaving it zero segfaults on the first - * hashCode(), well away from anything that names it. - * - * The comment sits outside the macro on purpose: backslash-newline splicing - * happens before comments are removed, so an unbackslashed comment line inside - * the macro would silently end the definition. - */ -#define CN1_DEFINE_PRIMITIVE_CLASS_ARR(cname, jname, arrCls) \ -struct clazz cn1_primitive_class_##cname = { \ - .__codenameOneParentClsReference = &class__java_lang_Class, \ - .classId = CN1_PRIMITIVE_CLASS_ID, \ - .clsName = jname, \ - .isArray = JAVA_FALSE, \ - .dimensions = 0, \ - .arrayType = 0, \ - .arrayClass = arrCls, \ - .primitiveType = JAVA_TRUE, \ - .baseClass = 0, \ - .baseInterfaces = EMPTY_INTERFACES, \ - .baseInterfaceCount = 0, \ - .initialized = JAVA_TRUE \ -} - -/* - * arrayClass is what java.lang.reflect.Array.newInstance resolves int.class to - * before allocating; left zero the reflective allocator rejects every primitive - * array outright. void is the one that stays zero on purpose -- void[] does not - * exist, so Array.newInstance(void.class, n) must keep throwing. - */ -#define CN1_DEFINE_PRIMITIVE_CLASS(cname, jname) \ - CN1_DEFINE_PRIMITIVE_CLASS_ARR(cname, jname, 0) - -CN1_DEFINE_PRIMITIVE_CLASS_ARR(int, "int", &class_array1__JAVA_INT); -CN1_DEFINE_PRIMITIVE_CLASS_ARR(long, "long", &class_array1__JAVA_LONG); -CN1_DEFINE_PRIMITIVE_CLASS_ARR(short, "short", &class_array1__JAVA_SHORT); -CN1_DEFINE_PRIMITIVE_CLASS_ARR(byte, "byte", &class_array1__JAVA_BYTE); -CN1_DEFINE_PRIMITIVE_CLASS_ARR(char, "char", &class_array1__JAVA_CHAR); -CN1_DEFINE_PRIMITIVE_CLASS_ARR(float, "float", &class_array1__JAVA_FLOAT); -CN1_DEFINE_PRIMITIVE_CLASS_ARR(double, "double", &class_array1__JAVA_DOUBLE); -CN1_DEFINE_PRIMITIVE_CLASS_ARR(boolean, "boolean", &class_array1__JAVA_BOOLEAN); -CN1_DEFINE_PRIMITIVE_CLASS(void, "void"); - struct clazz class_array1__JAVA_BOOLEAN = { DEBUG_GC_INIT 0, 0, 0, 0, 0, 0, 0, cn1_array_1_id_JAVA_BOOLEAN, "boolean[]", JAVA_TRUE, 1, &class__java_lang_Boolean, JAVA_TRUE, &class__java_lang_Object, EMPTY_INTERFACES, 0, 0, 0 }; diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeField.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeField.java index 8eac3b6d495..3b31238b03c 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeField.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeField.java @@ -41,7 +41,7 @@ public class ByteCodeField { private int arrayDimensions; private String type; - private Class primitiveType; + private PrimitiveType primitiveType; private boolean finalField; private Object value; private boolean privateField; @@ -81,28 +81,28 @@ public ByteCodeField(String clsName, int access, String name, String desc, Strin type = objectType; break; case 'I': - primitiveType = Integer.TYPE; + primitiveType = PrimitiveType.INT; break; case 'J': - primitiveType = Long.TYPE; + primitiveType = PrimitiveType.LONG; break; case 'B': - primitiveType = Byte.TYPE; + primitiveType = PrimitiveType.BYTE; break; case 'S': - primitiveType = Short.TYPE; + primitiveType = PrimitiveType.SHORT; break; case 'F': - primitiveType = Float.TYPE; + primitiveType = PrimitiveType.FLOAT; break; case 'D': - primitiveType = Double.TYPE; + primitiveType = PrimitiveType.DOUBLE; break; case 'Z': - primitiveType = Boolean.TYPE; + primitiveType = PrimitiveType.BOOLEAN; break; case 'C': - primitiveType = Character.TYPE; + primitiveType = PrimitiveType.CHAR; break; } } @@ -211,31 +211,10 @@ public String getRuntimeDescriptor() { if (primitiveType == null) { return type; } - if (primitiveType == Integer.TYPE) { - return "I"; - } - if (primitiveType == Long.TYPE) { - return "J"; - } - if (primitiveType == Byte.TYPE) { - return "B"; - } - if (primitiveType == Short.TYPE) { - return "S"; - } - if (primitiveType == Float.TYPE) { - return "F"; - } - if (primitiveType == Double.TYPE) { - return "D"; - } - if (primitiveType == Boolean.TYPE) { - return "Z"; - } - if (primitiveType == Character.TYPE) { - return "C"; - } - return null; + // A field is never void, so PrimitiveType.VOID's "V" is unreachable here; + // the chain this replaces returned null for it, which no caller handled + // either. + return primitiveType.getDescriptor(); } public boolean isPrivate() { diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeMethodArg.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeMethodArg.java index e956745d0cb..b2b961f107e 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeMethodArg.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeMethodArg.java @@ -33,14 +33,14 @@ public class ByteCodeMethodArg { private final int arrayDimensions; private String type; - private Class primitiveType; + private PrimitiveType primitiveType; public ByteCodeMethodArg(String type, int dim) { this.type = type.replace('/', '_').replace('$', '_'); arrayDimensions = dim; } - public ByteCodeMethodArg(Class type, int dim) { + public ByteCodeMethodArg(PrimitiveType type, int dim) { this.primitiveType = type; arrayDimensions = dim; } @@ -49,13 +49,13 @@ public char getQualifier() { if(type != null || arrayDimensions > 0) { return 'o'; } - if(primitiveType == Long.TYPE) { + if(primitiveType == PrimitiveType.LONG) { return 'l'; } - if(primitiveType == Double.TYPE) { + if(primitiveType == PrimitiveType.DOUBLE) { return 'd'; } - if(primitiveType == Float.TYPE) { + if(primitiveType == PrimitiveType.FLOAT) { return 'f'; } return 'i'; @@ -93,7 +93,11 @@ public int hashCode() { if(type != null) { return type.hashCode(); } - return primitiveType.hashCode(); + // ordinal(), not hashCode(): Enum.hashCode is an identity hash on OpenJDK + // and the ordinal in ParparVM's java.lang.Enum, so hashing on it would make + // a hash container of these args iterate in a different order under the + // self-hosted translator than under the JVM-hosted one. + return primitiveType.ordinal(); } @Override @@ -121,11 +125,11 @@ public boolean equals(Object obj) { } public boolean isVoid() { - return primitiveType == Void.TYPE; + return primitiveType == PrimitiveType.VOID; } public boolean isDoubleOrLong() { - return (primitiveType == Double.TYPE || primitiveType == Long.TYPE) && arrayDimensions == 0; + return (primitiveType == PrimitiveType.DOUBLE || primitiveType == PrimitiveType.LONG) && arrayDimensions == 0; } /** @@ -139,7 +143,7 @@ public String getTypeName() { return type; } - public Class getPrimitiveType() { + public PrimitiveType getPrimitiveType() { return primitiveType; } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java index bb6469c6528..e9e533d0174 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java @@ -786,16 +786,16 @@ public BytecodeMethod(String clsName, int access, String name, String desc, Stri if(methodName.equals("")) { methodName = "__INIT__"; constructor = true; - returnType = new ByteCodeMethodArg(Void.TYPE, 0); + returnType = new ByteCodeMethodArg(PrimitiveType.VOID, 0); } else { if(methodName.equals("")) { methodName = "__CLINIT__"; - returnType = new ByteCodeMethodArg(Void.TYPE, 0); + returnType = new ByteCodeMethodArg(PrimitiveType.VOID, 0); staticMethod = true; } else { String retType = desc.substring(pos + 1); if(retType.equals("V")) { - returnType = new ByteCodeMethodArg(Void.TYPE, 0); + returnType = new ByteCodeMethodArg(PrimitiveType.VOID, 0); } else { int dim = 0; while(retType.startsWith("[")) { @@ -818,28 +818,28 @@ public BytecodeMethod(String clsName, int access, String name, String desc, Stri returnType = new ByteCodeMethodArg(objectType, dim); break; case 'I': - returnType = new ByteCodeMethodArg(Integer.TYPE, dim); + returnType = new ByteCodeMethodArg(PrimitiveType.INT, dim); break; case 'J': - returnType = new ByteCodeMethodArg(Long.TYPE, dim); + returnType = new ByteCodeMethodArg(PrimitiveType.LONG, dim); break; case 'B': - returnType = new ByteCodeMethodArg(Byte.TYPE, dim); + returnType = new ByteCodeMethodArg(PrimitiveType.BYTE, dim); break; case 'S': - returnType = new ByteCodeMethodArg(Short.TYPE, dim); + returnType = new ByteCodeMethodArg(PrimitiveType.SHORT, dim); break; case 'F': - returnType = new ByteCodeMethodArg(Float.TYPE, dim); + returnType = new ByteCodeMethodArg(PrimitiveType.FLOAT, dim); break; case 'D': - returnType = new ByteCodeMethodArg(Double.TYPE, dim); + returnType = new ByteCodeMethodArg(PrimitiveType.DOUBLE, dim); break; case 'Z': - returnType = new ByteCodeMethodArg(Boolean.TYPE, dim); + returnType = new ByteCodeMethodArg(PrimitiveType.BOOLEAN, dim); break; case 'C': - returnType = new ByteCodeMethodArg(Character.TYPE, dim); + returnType = new ByteCodeMethodArg(PrimitiveType.CHAR, dim); break; } } @@ -869,28 +869,28 @@ public BytecodeMethod(String clsName, int access, String name, String desc, Stri arguments.add(new ByteCodeMethodArg(objectType, currentArrayDim)); break; case 'I': - arguments.add(new ByteCodeMethodArg(Integer.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.INT, currentArrayDim)); break; case 'J': - arguments.add(new ByteCodeMethodArg(Long.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.LONG, currentArrayDim)); break; case 'B': - arguments.add(new ByteCodeMethodArg(Byte.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.BYTE, currentArrayDim)); break; case 'S': - arguments.add(new ByteCodeMethodArg(Short.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.SHORT, currentArrayDim)); break; case 'F': - arguments.add(new ByteCodeMethodArg(Float.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.FLOAT, currentArrayDim)); break; case 'D': - arguments.add(new ByteCodeMethodArg(Double.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.DOUBLE, currentArrayDim)); break; case 'Z': - arguments.add(new ByteCodeMethodArg(Boolean.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.BOOLEAN, currentArrayDim)); break; case 'C': - arguments.add(new ByteCodeMethodArg(Character.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.CHAR, currentArrayDim)); break; } currentArrayDim = 0; @@ -2915,10 +2915,10 @@ public void appendOnDeviceDebugInvokeThunk(String declaringClsName, StringBuilde * is more specific (e.g. boolean / byte / short / char). */ private String returnTypeChar() { - if (returnType.getPrimitiveType() == Boolean.TYPE) return "Z"; - if (returnType.getPrimitiveType() == Byte.TYPE) return "B"; - if (returnType.getPrimitiveType() == Short.TYPE) return "S"; - if (returnType.getPrimitiveType() == Character.TYPE) return "C"; + if (returnType.getPrimitiveType() == PrimitiveType.BOOLEAN) return "Z"; + if (returnType.getPrimitiveType() == PrimitiveType.BYTE) return "B"; + if (returnType.getPrimitiveType() == PrimitiveType.SHORT) return "S"; + if (returnType.getPrimitiveType() == PrimitiveType.CHAR) return "C"; return "I"; } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptMethodGenerator.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptMethodGenerator.java index fed2a486679..23f6bb7e83c 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptMethodGenerator.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptMethodGenerator.java @@ -5838,22 +5838,22 @@ private static void appendJsBodyMethod(StringBuilder out, ByteCodeClass cls, Byt if (returnTypeName != null) { jsReturnType = JavascriptNameUtil.sanitizeClassName(returnTypeName); } else { - Class primitiveType = returnType.getPrimitiveType(); - if (primitiveType == Integer.TYPE) { + PrimitiveType primitiveType = returnType.getPrimitiveType(); + if (primitiveType == PrimitiveType.INT) { jsReturnType = "int"; - } else if (primitiveType == Long.TYPE) { + } else if (primitiveType == PrimitiveType.LONG) { jsReturnType = "long"; - } else if (primitiveType == Double.TYPE) { + } else if (primitiveType == PrimitiveType.DOUBLE) { jsReturnType = "double"; - } else if (primitiveType == Float.TYPE) { + } else if (primitiveType == PrimitiveType.FLOAT) { jsReturnType = "float"; - } else if (primitiveType == Boolean.TYPE) { + } else if (primitiveType == PrimitiveType.BOOLEAN) { jsReturnType = "boolean"; - } else if (primitiveType == Byte.TYPE) { + } else if (primitiveType == PrimitiveType.BYTE) { jsReturnType = "byte"; - } else if (primitiveType == Short.TYPE) { + } else if (primitiveType == PrimitiveType.SHORT) { jsReturnType = "short"; - } else if (primitiveType == Character.TYPE) { + } else if (primitiveType == PrimitiveType.CHAR) { jsReturnType = "char"; } else { jsReturnType = "java_lang_Object"; diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/PrimitiveType.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/PrimitiveType.java new file mode 100644 index 00000000000..1b06b7e0d04 --- /dev/null +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/PrimitiveType.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +/** + * The nine primitive types, as a token the translator can compare and hash. + * + *

This used to be {@code java.lang.Class}, holding {@code Integer.TYPE} and its + * eight siblings. Nothing ever reflected on those objects: every use was an + * identity comparison against one of the nine constants, or a lookup in a + * {@code HashMap} keyed on them. {@code Class} was standing in for + * an enum, and it carried two problems that an enum does not. + * + *

The first is that {@code X.TYPE} does not exist on a ParparVM target. javac + * lowers the primitive class literal in {@code Integer.TYPE = int.class} to a read + * of the field being initialized, so the wrapper's own {@code } stores + * null into it; three of the nine wrappers do not declare the field at all. Keyed + * on those, both maps collapsed to a single entry and {@code getCType} answered the + * same C type for every primitive -- valid C, every type wrong, nothing thrown. + * That made the maps unusable in a self-hosted translator, which is what forced + * this change. + * + *

The second is ordering. {@code Class} has no {@code hashCode} of its own, so + * {@code ByteCodeMethodArg.hashCode} was returning an identity hash, which varies + * between runs of one JVM. Anything that iterated a hash container of those keys + * and wrote the result would emit a different file each time. + * + *

Note for the same reason that {@link #ordinal()} is used explicitly wherever a + * hash is needed rather than calling {@code hashCode()} on a constant here: + * {@code Enum.hashCode} is an identity hash on OpenJDK and the ordinal in + * ParparVM's {@code java.lang.Enum}, so relying on it would make the JVM-hosted and + * self-hosted translators disagree on hash order -- a difference the self-hosting + * gate would report as a VM divergence. + */ +public enum PrimitiveType { + INT("JAVA_INT", "int", "I"), + LONG("JAVA_LONG", "long", "J"), + SHORT("JAVA_SHORT", "short", "S"), + BYTE("JAVA_BYTE", "byte", "B"), + DOUBLE("JAVA_DOUBLE", "double", "D"), + FLOAT("JAVA_FLOAT", "float", "F"), + BOOLEAN("JAVA_BOOLEAN", "boolean", "Z"), + CHAR("JAVA_CHAR", "char", "C"), + VOID("JAVA_VOID", "void", "V"); + + private final String cType; + private final String sigType; + private final String descriptor; + + private PrimitiveType(String cType, String sigType, String descriptor) { + this.cType = cType; + this.sigType = sigType; + this.descriptor = descriptor; + } + + /** The C type the generated code uses for this primitive, e.g. JAVA_INT. */ + public String getCType() { + return cType; + } + + /** The Java keyword, as it appears in a mangled C method name. */ + public String getSigType() { + return sigType; + } + + /** The JVM field descriptor character, e.g. I for int. */ + public String getDescriptor() { + return descriptor; + } +} diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Util.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Util.java index 1f05d960e38..2644dc9a35b 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Util.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Util.java @@ -29,9 +29,7 @@ import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.objectweb.asm.Opcodes; /** @@ -40,36 +38,12 @@ */ public class Util { - private static final Map ctypeMap = new HashMap(); - private static final Map sigTypeMap = new HashMap(); - - static { - ctypeMap.put(Integer.TYPE, "JAVA_INT"); - ctypeMap.put(Long.TYPE, "JAVA_LONG"); - ctypeMap.put(Short.TYPE, "JAVA_SHORT"); - ctypeMap.put(Byte.TYPE, "JAVA_BYTE"); - ctypeMap.put(Double.TYPE, "JAVA_DOUBLE"); - ctypeMap.put(Float.TYPE, "JAVA_FLOAT"); - ctypeMap.put(Boolean.TYPE, "JAVA_BOOLEAN"); - ctypeMap.put(Character.TYPE, "JAVA_CHAR"); - ctypeMap.put(Void.TYPE, "JAVA_VOID"); - sigTypeMap.put(Integer.TYPE, "int"); - sigTypeMap.put(Long.TYPE, "long"); - sigTypeMap.put(Short.TYPE, "short"); - sigTypeMap.put(Byte.TYPE, "byte"); - sigTypeMap.put(Double.TYPE, "double"); - sigTypeMap.put(Float.TYPE, "float"); - sigTypeMap.put(Boolean.TYPE, "boolean"); - sigTypeMap.put(Character.TYPE, "char"); - sigTypeMap.put(Void.TYPE, "void"); + public static String getCType(PrimitiveType type) { + return type == null ? null : type.getCType(); } - public static String getCType(Class cls) { - return ctypeMap.get(cls); - } - - public static String getSigType(Class cls) { - return sigTypeMap.get(cls); + public static String getSigType(PrimitiveType type) { + return type == null ? null : type.getSigType(); } public static List getMethodArgs(String methodDesc) { @@ -98,28 +72,28 @@ public static List getMethodArgs(String methodDesc) { arguments.add(new ByteCodeMethodArg(objectType, currentArrayDim)); break; case 'I': - arguments.add(new ByteCodeMethodArg(Integer.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.INT, currentArrayDim)); break; case 'J': - arguments.add(new ByteCodeMethodArg(Long.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.LONG, currentArrayDim)); break; case 'B': - arguments.add(new ByteCodeMethodArg(Byte.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.BYTE, currentArrayDim)); break; case 'S': - arguments.add(new ByteCodeMethodArg(Short.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.SHORT, currentArrayDim)); break; case 'F': - arguments.add(new ByteCodeMethodArg(Float.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.FLOAT, currentArrayDim)); break; case 'D': - arguments.add(new ByteCodeMethodArg(Double.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.DOUBLE, currentArrayDim)); break; case 'Z': - arguments.add(new ByteCodeMethodArg(Boolean.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.BOOLEAN, currentArrayDim)); break; case 'C': - arguments.add(new ByteCodeMethodArg(Character.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.CHAR, currentArrayDim)); break; } currentArrayDim = 0; diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 60cab21f523..42a9d2d4d3c 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -610,25 +610,7 @@ JAVA_OBJECT java_lang_reflect_Array_newInstanceImpl___java_lang_Class_int_R_java throwException(threadStateData, ex); return NULL; } - // Element WIDTH, not sizeof(JAVA_OBJECT). This allocator only ever saw - // reference component types before, because a primitive class object did not - // exist to pass in -- Integer.TYPE and friends were null. Now that they do, - // allocating an int[] at 8 bytes per element would size the block off the end - // of what the array header declares, so the width has to come from the - // component type. - int cn1ElemSize = (int)sizeof(JAVA_OBJECT); - if (clz->primitiveType) { - if (clz == &cn1_primitive_class_boolean || clz == &cn1_primitive_class_byte) { - cn1ElemSize = 1; - } else if (clz == &cn1_primitive_class_char || clz == &cn1_primitive_class_short) { - cn1ElemSize = 2; - } else if (clz == &cn1_primitive_class_int || clz == &cn1_primitive_class_float) { - cn1ElemSize = 4; - } else if (clz == &cn1_primitive_class_long || clz == &cn1_primitive_class_double) { - cn1ElemSize = 8; - } - } - JAVA_OBJECT out = allocArray(CN1_THREAD_STATE_PASS_ARG len, clz->arrayClass, cn1ElemSize, 1); + JAVA_OBJECT out = allocArray(CN1_THREAD_STATE_PASS_ARG len, clz->arrayClass, sizeof(JAVA_OBJECT), 1); finishedNativeAllocations(); return out; } @@ -1996,71 +1978,6 @@ JAVA_OBJECT java_lang_Class_getName___R_java_lang_String(CODENAME_ONE_THREAD_STA * The codes are an implementation detail shared only with java/lang/Class.java; * they are matched by CN1_PRIM_* there. */ -/** The descriptor for a primitive type code, or 0 for an unknown code. */ -static struct clazz* cn1PrimitiveClassFor(JAVA_INT typeCode) { - switch(typeCode) { - case 0: return &cn1_primitive_class_int; - case 1: return &cn1_primitive_class_long; - case 2: return &cn1_primitive_class_short; - case 3: return &cn1_primitive_class_byte; - case 4: return &cn1_primitive_class_char; - case 5: return &cn1_primitive_class_float; - case 6: return &cn1_primitive_class_double; - case 7: return &cn1_primitive_class_boolean; - case 8: return &cn1_primitive_class_void; - } - return 0; -} - -JAVA_OBJECT java_lang_Class_getPrimitiveClass___int_R_java_lang_Class(CODENAME_ONE_THREAD_STATE, JAVA_INT typeCode) { - // java.lang.Class MUST be initialised before one of these is handed out. - // - // The nine descriptors are static struct clazz, and they are returned as - // java.lang.Class OBJECTS: each carries - // __codenameOneParentClsReference = &class__java_lang_Class, so every virtual - // call on Integer.TYPE and friends -- hashCode, equals, toString, getClass -- - // dispatches through class__java_lang_Class.vtable. That vtable is malloc'd by - // java.lang.Class's own static initialiser, and NOTHING on this path ran it: - // the callers are the wrapper classes' clinits (Integer, Boolean, ...), any of - // which can be the first class the program touches. Until it runs, the vtable - // is the zero a static initialises to. - // - // The initialiser is idempotent and returns on its completion flag, so this - // costs one acquire load once the class is up. - __STATIC_INITIALIZER_java_lang_Class(threadStateData); - // And REGISTER the descriptor in the GC's exact clazz registry. - // - // Every generated class__X is registered by CN1_CLAZZ_REGISTER on the first - // allocation of one of its instances, from every allocation entry point, so - // by construction the collector has seen every clazz address that can reach - // it. These nine allocate nothing -- they are static and handed out directly - // -- so they were the only clazz addresses in the process that reached Java - // while permanently unregistered, and the mark guard is documented to - // recognise a genuine clazz address "via an exact registry instead of a - // distance heuristic". - // - // The macro is idempotent and tests the trailing cn1ClazzRegistered flag - // first, so this is one predictable load after the first call. - CN1_CLAZZ_REGISTER(cn1PrimitiveClassFor(typeCode)); - switch(typeCode) { - case 0: return (JAVA_OBJECT)&cn1_primitive_class_int; - case 1: return (JAVA_OBJECT)&cn1_primitive_class_long; - case 2: return (JAVA_OBJECT)&cn1_primitive_class_short; - case 3: return (JAVA_OBJECT)&cn1_primitive_class_byte; - case 4: return (JAVA_OBJECT)&cn1_primitive_class_char; - case 5: return (JAVA_OBJECT)&cn1_primitive_class_float; - case 6: return (JAVA_OBJECT)&cn1_primitive_class_double; - case 7: return (JAVA_OBJECT)&cn1_primitive_class_boolean; - case 8: return (JAVA_OBJECT)&cn1_primitive_class_void; - } - // Only java/lang/Class.java calls this, always with one of its own constants, - // so this is unreachable short of the two files disagreeing. Returning null - // would restore exactly the silent null TYPE this code exists to remove. - fprintf(stderr, "getPrimitiveClass: unknown primitive type code %d\n", (int)typeCode); - exit(1); - return JAVA_NULL; -} - /** * Resources linked into the executable, backing Class.getResourceAsStream. * diff --git a/vm/JavaAPI/src/java/lang/Boolean.java b/vm/JavaAPI/src/java/lang/Boolean.java index 2f56aa21520..62c36722b01 100644 --- a/vm/JavaAPI/src/java/lang/Boolean.java +++ b/vm/JavaAPI/src/java/lang/Boolean.java @@ -30,8 +30,24 @@ public final class Boolean implements Comparable { /** * The class object for the primitive type this class wraps. + * + * Null on every ParparVM target, and declared only because ASM's compiled + * bytecode reads it: org.objectweb.asm.Type compares against Short.TYPE, + * Float.TYPE and Boolean.TYPE, so the self-hosted translator does not link + * without the three fields existing. ASM is a jar we cannot edit, which is + * the one case where JavaAPI grows to meet a dependency rather than the + * dependency being removed. + * + * It cannot be given a real value here. javac lowers a primitive class + * literal to a read of the boxed type's own TYPE field, so the obvious + * initializer compiles to "getstatic TYPE; putstatic TYPE" -- it reads the + * field it is initializing and stores the null straight back. The six + * wrappers that already declare TYPE are null for exactly that reason. + * Giving all nine real values needs VM-side primitive class objects; that + * work is not part of this change, and nothing in the translator depends on + * it now that the C-type tables are keyed on the PrimitiveType enum. */ - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_BOOLEAN); + public static final Class TYPE = null; /** * The Boolean object corresponding to the primitive value false. diff --git a/vm/JavaAPI/src/java/lang/Byte.java b/vm/JavaAPI/src/java/lang/Byte.java index b7b5ff186f0..9a7fa9d99e9 100644 --- a/vm/JavaAPI/src/java/lang/Byte.java +++ b/vm/JavaAPI/src/java/lang/Byte.java @@ -28,7 +28,7 @@ */ public final class Byte extends Number implements Comparable { - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_BYTE); + public static final Class TYPE = byte.class; public static final int SIZE = 8; /** diff --git a/vm/JavaAPI/src/java/lang/Character.java b/vm/JavaAPI/src/java/lang/Character.java index 448f24d6a51..93ce6f67946 100644 --- a/vm/JavaAPI/src/java/lang/Character.java +++ b/vm/JavaAPI/src/java/lang/Character.java @@ -155,7 +155,7 @@ public final class Character implements Comparable{ //public static final int MAX_RADIX = 36; //public static final char MIN_VALUE = '\0'; //public static final char MAX_VALUE = '\uFFFF'; - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_CHAR); + public static final Class TYPE = char.class; public static final byte UNASSIGNED = 0; public static final byte UPPERCASE_LETTER = 1; public static final byte LOWERCASE_LETTER = 2; diff --git a/vm/JavaAPI/src/java/lang/Class.java b/vm/JavaAPI/src/java/lang/Class.java index 45921e6d648..775b93717d7 100644 --- a/vm/JavaAPI/src/java/lang/Class.java +++ b/vm/JavaAPI/src/java/lang/Class.java @@ -232,36 +232,6 @@ private static java.io.InputStream cn1FileResource(String absolute) { return null; } - /** - * Type codes for {@link #getPrimitiveClass(int)}. Shared only with - * nativeMethods.m, which switches on the same values. - */ - static final int CN1_PRIM_INT = 0; - static final int CN1_PRIM_LONG = 1; - static final int CN1_PRIM_SHORT = 2; - static final int CN1_PRIM_BYTE = 3; - static final int CN1_PRIM_CHAR = 4; - static final int CN1_PRIM_FLOAT = 5; - static final int CN1_PRIM_DOUBLE = 6; - static final int CN1_PRIM_BOOLEAN = 7; - static final int CN1_PRIM_VOID = 8; - - /** - * Returns the class object for a primitive type, e.g. the one - * {@code int.class} and {@link Integer#TYPE} denote. - * - * The wrapper classes cannot initialize their {@code TYPE} fields with a - * primitive class literal: javac lowers {@code int.class} to a read of - * {@code Integer.TYPE} itself, so {@code TYPE = int.class} compiles to - * {@code getstatic TYPE; putstatic TYPE} and leaves the field null. The JDK - * declares an equivalent native for the same reason. - * - * Takes an int code rather than a name so that it allocates nothing and - * decodes nothing: it runs inside the wrapper class initializers, which are - * among the earliest code in the process. - */ - static native Class getPrimitiveClass(int typeCode); - /** * Determines if this Class object represents an array class. */ diff --git a/vm/JavaAPI/src/java/lang/Double.java b/vm/JavaAPI/src/java/lang/Double.java index 05d12f67f31..79a3e648abe 100644 --- a/vm/JavaAPI/src/java/lang/Double.java +++ b/vm/JavaAPI/src/java/lang/Double.java @@ -29,7 +29,7 @@ */ public final class Double extends Number implements Comparable { - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_DOUBLE); + public static final Class TYPE = double.class; /** * The largest positive finite value of type double. It is equal to the value returned by Double.longBitsToDouble(0x7fefffffffffffffL) * See Also:Constant Field Values diff --git a/vm/JavaAPI/src/java/lang/Float.java b/vm/JavaAPI/src/java/lang/Float.java index 93e9d560a81..fc940da7ff7 100644 --- a/vm/JavaAPI/src/java/lang/Float.java +++ b/vm/JavaAPI/src/java/lang/Float.java @@ -31,8 +31,24 @@ public final class Float extends Number implements Comparable { /** * The class object for the primitive type this class wraps. + * + * Null on every ParparVM target, and declared only because ASM's compiled + * bytecode reads it: org.objectweb.asm.Type compares against Short.TYPE, + * Float.TYPE and Boolean.TYPE, so the self-hosted translator does not link + * without the three fields existing. ASM is a jar we cannot edit, which is + * the one case where JavaAPI grows to meet a dependency rather than the + * dependency being removed. + * + * It cannot be given a real value here. javac lowers a primitive class + * literal to a read of the boxed type's own TYPE field, so the obvious + * initializer compiles to "getstatic TYPE; putstatic TYPE" -- it reads the + * field it is initializing and stores the null straight back. The six + * wrappers that already declare TYPE are null for exactly that reason. + * Giving all nine real values needs VM-side primitive class objects; that + * work is not part of this change, and nothing in the translator depends on + * it now that the C-type tables are keyed on the PrimitiveType enum. */ - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_FLOAT); + public static final Class TYPE = null; /** * The largest positive value of type float. It is equal to the value returned by Float.intBitsToFloat(0x7f7fffff). diff --git a/vm/JavaAPI/src/java/lang/Integer.java b/vm/JavaAPI/src/java/lang/Integer.java index 4a1cb1a85d0..387648fe877 100644 --- a/vm/JavaAPI/src/java/lang/Integer.java +++ b/vm/JavaAPI/src/java/lang/Integer.java @@ -29,7 +29,7 @@ */ public final class Integer extends Number implements Comparable { - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_INT); + public static final Class TYPE = int.class; private static final char[] DIGITS = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', diff --git a/vm/JavaAPI/src/java/lang/Long.java b/vm/JavaAPI/src/java/lang/Long.java index fce50a48abd..0e938265153 100644 --- a/vm/JavaAPI/src/java/lang/Long.java +++ b/vm/JavaAPI/src/java/lang/Long.java @@ -29,7 +29,7 @@ */ public final class Long extends Number implements Comparable { - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_LONG); + public static Class TYPE = long.class; /** * The largest value of type long. diff --git a/vm/JavaAPI/src/java/lang/Short.java b/vm/JavaAPI/src/java/lang/Short.java index f0800e1f219..16610a821a9 100644 --- a/vm/JavaAPI/src/java/lang/Short.java +++ b/vm/JavaAPI/src/java/lang/Short.java @@ -30,8 +30,24 @@ public final class Short extends Number implements Comparable { /** * The class object for the primitive type this class wraps. + * + * Null on every ParparVM target, and declared only because ASM's compiled + * bytecode reads it: org.objectweb.asm.Type compares against Short.TYPE, + * Float.TYPE and Boolean.TYPE, so the self-hosted translator does not link + * without the three fields existing. ASM is a jar we cannot edit, which is + * the one case where JavaAPI grows to meet a dependency rather than the + * dependency being removed. + * + * It cannot be given a real value here. javac lowers a primitive class + * literal to a read of the boxed type's own TYPE field, so the obvious + * initializer compiles to "getstatic TYPE; putstatic TYPE" -- it reads the + * field it is initializing and stores the null straight back. The six + * wrappers that already declare TYPE are null for exactly that reason. + * Giving all nine real values needs VM-side primitive class objects; that + * work is not part of this change, and nothing in the translator depends on + * it now that the C-type tables are keyed on the PrimitiveType enum. */ - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_SHORT); + public static final Class TYPE = null; /** * The maximum value a Short can have. diff --git a/vm/JavaAPI/src/java/lang/Void.java b/vm/JavaAPI/src/java/lang/Void.java index 96dbd87a71e..c1391f982e0 100644 --- a/vm/JavaAPI/src/java/lang/Void.java +++ b/vm/JavaAPI/src/java/lang/Void.java @@ -27,5 +27,5 @@ * @author Shai Almog */ public final class Void { - public static final Class TYPE = Class.getPrimitiveClass(Class.CN1_PRIM_VOID); + public static final Class TYPE = Void.class; } diff --git a/vm/selfhost/verify-output-neutral.sh b/vm/selfhost/verify-output-neutral.sh new file mode 100755 index 00000000000..eec550dce53 --- /dev/null +++ b/vm/selfhost/verify-output-neutral.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Proves a translator source change does not alter the emitted C. +# +# verify-output-neutral.sh capture # run the JVM translator, save the tree +# verify-output-neutral.sh compare # diff two captured trees +# +# Gate A (in verify-selfhost.sh) compares the JVM translator against the native one +# and CANNOT see this: a refactor lands on both sides at once, so both move together +# and the gate stays green while every emitted signature changes. This runs the JVM +# translator alone, before and after, over the same corpus. +# +# Same fixed output path and constructed environment as verify-selfhost.sh, and for +# the same reasons: the generated CMakeLists embeds srcRoot.getAbsolutePath(), and +# the translator reads its knobs from getenv. +set -e +cd "$(dirname "$0")" +REPO="$(cd ../.. && pwd)" +J8="${JDK_8_HOME:?set JDK_8_HOME to a working JDK 8}" +W="$REPO/vm/selfhost/target/neutral" +OUT="$W/out" + +case "${1:?usage: capture | compare }" in +capture) + TAG="${2:?}" + TR="$REPO/vm/ByteCodeTranslator/target/classes" + ASM="$(cat "$REPO/vm/ByteCodeTranslator/target/selfhost-asm-classpath.txt")" + JAPI="$REPO/vm/selfhost/target/javaapi-classes" + # Same staleness guard verify-selfhost.sh carries: a source newer than its class + # would capture the OLD translator under the NEW tag and report a real change as + # neutral -- the exact failure this script exists to prevent. + newest="$(find "$REPO/vm/ByteCodeTranslator/src" -name '*.java' -newer "$TR" -print -quit 2>/dev/null || true)" + [ -z "$newest" ] || { echo "STALE: $TR older than $newest -- run mvn package first" >&2; exit 1; } + rm -rf "$W/$TAG-tree" "$OUT"; mkdir -p "$OUT" + ( cd "$W" && env -i PATH=/usr/bin:/bin HOME="$HOME" TMPDIR=/tmp LC_ALL=C \ + CN1_RESOURCE_PATH="$REPO/vm/ByteCodeTranslator/src" \ + "$J8/bin/java" -cp "$TR:$ASM" com.codename1.tools.translator.ByteCodeTranslator \ + clean "$JAPI;$REPO/vm/selfhost/target/asm-classes;$REPO/vm/selfhost/target/classes" \ + "$OUT" com_codename1_tools_translator_ByteCodeTranslator \ + com.codename1.tools.translator com_codename1_tools_translator_ByteCodeTranslator \ + 1.0 clean none ) > "$W/$TAG.log" 2>&1 \ + || { echo "capture $TAG FAILED"; tail -20 "$W/$TAG.log"; exit 1; } + mv "$OUT" "$W/$TAG-tree" + n=$(find "$W/$TAG-tree" -type f | wc -l | tr -d ' ') + [ "$n" -gt 10 ] || { echo "VACUOUS: only $n files"; exit 1; } + echo "captured $TAG: $n files" + ;; +compare) + A="$W/${2:?}-tree"; B="$W/${3:?}-tree" + for d in "$A" "$B"; do [ -d "$d" ] || { echo "no $d"; exit 1; }; done + na=$(find "$A" -type f | wc -l | tr -d ' ') + if diff -rq "$A" "$B" > "$W/neutral.txt" 2>&1; then + echo "OUTPUT-NEUTRAL: PASS -- $na files byte-identical" + else + echo "OUTPUT-NEUTRAL: FAIL -- $(grep -c . "$W/neutral.txt") differing paths" + head -20 "$W/neutral.txt"; exit 1 + fi + ;; +*) echo "usage: capture | compare " >&2; exit 1 ;; +esac diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/PrimitiveTypeIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/PrimitiveTypeIntegrationTest.java deleted file mode 100644 index c73f2e967d3..00000000000 --- a/vm/tests/src/test/java/com/codename1/tools/translator/PrimitiveTypeIntegrationTest.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ -package com.codename1.tools.translator; - -import org.junit.jupiter.api.Test; - -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - -/** - * Pins the nine primitive class objects -- {@code Integer.TYPE} and friends -- against - * the JVM. - * - *

These were all null on ParparVM until the primitive {@code struct clazz} objects - * existed. javac lowers a primitive class literal to a read of the boxed type's own - * {@code TYPE} field, so {@code TYPE = int.class} compiled to - * {@code getstatic TYPE; putstatic TYPE} and left the field null. Nothing threw: a - * {@code Map} keyed on them collapsed to a single entry and answered every lookup with - * whatever had been stored last, which is exactly how the translator's own - * primitive-to-C-type maps in {@code Util} would have typed every primitive alike.

- * - *

Comparing against a real JVM rather than a hard-coded expectation is deliberate -- - * the failure mode here was self-consistent and silent, so only an independent - * reference catches it.

- */ -class PrimitiveTypeIntegrationTest { - - @Test - void primitiveClassObjectsMatchTheJvm() throws Exception { - Parser.cleanup(); - - Path sourceDir = Files.createTempDirectory("primitive-type-sources"); - Path classesDir = Files.createTempDirectory("primitive-type-classes"); - Path javaApiDir = Files.createTempDirectory("primitive-type-java-api"); - - Path source = sourceDir.resolve("PrimitiveTypeApp.java"); - Files.write(source, loadAppSource().getBytes(StandardCharsets.UTF_8)); - - CompilerHelper.CompilerConfig config = selectCompiler(); - if (config == null) { - fail("No compatible compiler available for the primitive type integration test"); - } - assertTrue(CompilerHelper.isJavaApiCompatible(config), - "JDK " + config.jdkVersion + " must target matching bytecode level for JavaAPI"); - - CompilerHelper.compileJavaAPI(javaApiDir, config); - - List compileArgs = new ArrayList<>(); - compileArgs.add("-source"); - compileArgs.add(config.targetVersion); - compileArgs.add("-target"); - compileArgs.add(config.targetVersion); - if (CompilerHelper.useClasspath(config)) { - compileArgs.add("-classpath"); - compileArgs.add(javaApiDir.toString()); - } else { - compileArgs.add("-bootclasspath"); - compileArgs.add(javaApiDir.toString()); - compileArgs.add("-Xlint:-options"); - } - compileArgs.add("-d"); - compileArgs.add(classesDir.toString()); - compileArgs.add(source.toString()); - - assertEquals(0, CompilerHelper.compile(config.jdkHome, compileArgs), - "PrimitiveTypeApp should compile against the JavaAPI"); - - Map expected = parseCases(runJavaMain(config, classesDir, javaApiDir)); - assertFalse(expected.isEmpty(), "JVM run should emit cases"); - - CompilerHelper.copyDirectory(javaApiDir, classesDir); - - Path outputDir = Files.createTempDirectory("primitive-type-output"); - CleanTargetIntegrationTest.runTranslator(classesDir, outputDir, "PrimitiveTypeApp"); - - Path distDir = outputDir.resolve("dist"); - Path cmakeLists = distDir.resolve("CMakeLists.txt"); - assertTrue(Files.exists(cmakeLists), "Translator should emit a CMake project"); - CleanTargetIntegrationTest.replaceLibraryWithExecutableTarget(cmakeLists, "PrimitiveTypeApp-src"); - - Path buildDir = distDir.resolve("build"); - Files.createDirectories(buildDir); - CleanTargetIntegrationTest.runCommand(Arrays.asList( - "cmake", - "-S", distDir.toString(), - "-B", buildDir.toString(), - "-DCMAKE_C_COMPILER=clang", - "-DCMAKE_OBJC_COMPILER=clang" - ), distDir); - CleanTargetIntegrationTest.runCommand(Arrays.asList("cmake", "--build", buildDir.toString()), distDir); - - Path executable = buildDir.resolve("PrimitiveTypeApp"); - String parparOutput = CleanTargetIntegrationTest.runCommand( - Arrays.asList(executable.toString()), buildDir); - assertTrue(parparOutput.contains("DONE"), - "ParparVM run should complete. Output: " + parparOutput); - - Map actual = parseCases(parparOutput); - assertEquals(expected.keySet(), actual.keySet(), "ParparVM should emit the same cases"); - - List differences = new ArrayList<>(); - for (Map.Entry entry : expected.entrySet()) { - if (!entry.getValue().equals(actual.get(entry.getKey()))) { - differences.add(entry.getKey() - + "\n jvm : " + entry.getValue() - + "\n parparvm: " + actual.get(entry.getKey())); - } - } - assertTrue(differences.isEmpty(), - "Primitive class objects diverged from the JVM:\n" + String.join("\n", differences)); - - // Stated explicitly so a regression names the original symptom rather than - // showing up only as a generic diff. - assertEquals("int", actual.get("name.int"), "Integer.TYPE must be the int class"); - assertEquals("void", actual.get("name.void"), "Void.TYPE must be void, not java.lang.Void"); - assertEquals("9", actual.get("distinctIdentities"), - "the nine primitive class objects must be distinct"); - assertEquals("9", actual.get("mapSize"), - "a Map keyed on the nine must hold nine entries, not collapse onto null"); - assertEquals("0", actual.get("lookupFailures"), - "each primitive class must look up its own value"); - } - - private Map parseCases(String output) { - Map cases = new LinkedHashMap<>(); - for (String line : output.split("\\R")) { - if (!line.startsWith("CASE|")) { - continue; - } - String body = line.substring("CASE|".length()); - int separator = body.indexOf('|'); - assertTrue(separator > 0, "Malformed case line: " + line); - cases.put(body.substring(0, separator), body.substring(separator + 1)); - } - return cases; - } - - private String loadAppSource() throws Exception { - java.io.InputStream in = PrimitiveTypeIntegrationTest.class - .getResourceAsStream("/com/codename1/tools/translator/PrimitiveTypeApp.java"); - assertNotNull(in, "PrimitiveTypeApp.java test resource should exist"); - try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) { - return reader.lines().collect(Collectors.joining("\n")) + "\n"; - } - } - - private String runJavaMain(CompilerHelper.CompilerConfig config, Path classesDir, Path javaApiDir) - throws Exception { - String javaExe = config.jdkHome.resolve("bin").resolve(CompilerHelper.executableName("java")).toString(); - ProcessBuilder pb = new ProcessBuilder( - javaExe, - "-cp", - classesDir + System.getProperty("path.separator") + javaApiDir, - "PrimitiveTypeApp" - ); - pb.redirectErrorStream(true); - - Process process = pb.start(); - String output; - try (BufferedReader reader = new BufferedReader( - new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8))) { - output = reader.lines().collect(Collectors.joining("\n")); - } - assertEquals(0, process.waitFor(), "JVM run should exit cleanly. Output: " + output); - return output; - } - - private CompilerHelper.CompilerConfig selectCompiler() { - String[] preferredTargets = {"11", "17", "21", "25", "1.8"}; - for (String target : preferredTargets) { - List configs = CompilerHelper.getAvailableCompilers(target); - for (CompilerHelper.CompilerConfig config : configs) { - if (CompilerHelper.isJavaApiCompatible(config)) { - return config; - } - } - } - return null; - } -} diff --git a/vm/tests/src/test/resources/com/codename1/tools/translator/PrimitiveTypeApp.java b/vm/tests/src/test/resources/com/codename1/tools/translator/PrimitiveTypeApp.java deleted file mode 100644 index 5d61c920ed9..00000000000 --- a/vm/tests/src/test/resources/com/codename1/tools/translator/PrimitiveTypeApp.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ -import java.util.HashMap; -import java.util.Map; - -/** - * Emits the identity and behaviour of the nine primitive class objects so the JVM - * and ParparVM runs can be compared line for line. - * - * These were all null on ParparVM before the primitive class objects existed: - * javac lowers a primitive class literal to a read of the boxed type's own TYPE - * field, so `TYPE = int.class` compiled to `getstatic TYPE; putstatic TYPE`. - * Nothing threw -- a Map keyed on them simply collapsed to one entry and answered - * every lookup with whatever was stored last. - */ -public class PrimitiveTypeApp { - public static void main(String[] args) { - Class[] types = { - Integer.TYPE, Long.TYPE, Short.TYPE, Byte.TYPE, Character.TYPE, - Float.TYPE, Double.TYPE, Boolean.TYPE, Void.TYPE - }; - String[] names = { - "int", "long", "short", "byte", "char", - "float", "double", "boolean", "void" - }; - - for (int i = 0; i < types.length; i++) { - System.out.println("CASE|name." + names[i] + "|" - + (types[i] == null ? "" : types[i].getName())); - } - - // Distinct identities. Any two collapsing is the failure that made the - // translator's own primitive-to-C-type maps answer wrongly. - int distinct = 0; - for (int i = 0; i < types.length; i++) { - boolean unique = true; - for (int j = 0; j < i; j++) { - if (types[i] == types[j]) { - unique = false; - } - } - if (unique) { - distinct++; - } - } - System.out.println("CASE|distinctIdentities|" + distinct); - - // The shape the translator's Util actually uses. - Map byType = new HashMap(); - for (int i = 0; i < types.length; i++) { - byType.put(types[i], names[i]); - } - System.out.println("CASE|mapSize|" + byType.size()); - - int lookupFailures = 0; - for (int i = 0; i < types.length; i++) { - if (!names[i].equals(byType.get(types[i]))) { - lookupFailures++; - } - } - System.out.println("CASE|lookupFailures|" + lookupFailures); - - // isPrimitive, and the two natives that index instanceof tables by classId - // and so must special-case a primitive class rather than look it up. - System.out.println("CASE|isPrimitive.int|" + Integer.TYPE.isPrimitive()); - System.out.println("CASE|isPrimitive.boxed|" + Integer.class.isPrimitive()); - System.out.println("CASE|isArray.int|" + Integer.TYPE.isArray()); - System.out.println("CASE|assignable.self|" + Integer.TYPE.isAssignableFrom(Integer.TYPE)); - System.out.println("CASE|assignable.cross|" + Integer.TYPE.isAssignableFrom(Long.TYPE)); - System.out.println("CASE|assignable.boxed|" + Integer.TYPE.isAssignableFrom(Integer.class)); - System.out.println("CASE|isInstance.boxed|" + Integer.TYPE.isInstance(Integer.valueOf(1))); - System.out.println("CASE|isInstance.string|" + Integer.TYPE.isInstance("x")); - System.out.println("CASE|boxedNotPrimitive|" + (Integer.TYPE == Integer.class)); - - System.out.println("DONE"); - } -} From c851171f9e69d24d823f34d65d210dcb4a6d8e70 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 13 Sep 2026 07:05:39 +0300 Subject: [PATCH 50/66] Review round: resource lookup hardening, CI paths, and the JS getenv throw Addresses the review threads that survive the primitive-class withdrawal. Three of them were bugs in code this branch added, and I had verified none of them: Windows search paths were split on ':'. cn1FileResource used File.pathSeparatorChar, which this class library hard-codes to ':' rather than deriving per platform, so a native Windows build given "C:\res;D:\other" split after the drive letter and every entry was nonsense -- including the single-entry case, which has no ';' to disambiguate it. Both separators are now accepted, and a ':' directly after a single-letter entry followed by a slash is read as a DOS drive prefix instead. Checked against the five shapes that matter (POSIX multi, Windows multi, Windows single, single POSIX, forward slashes). A resource name was treated as a path expression. "../../etc/passwd" resolved straight out of the filesystem; any ".." segment is now refused, because these names frequently come from data. getResourceAsStream threw on JavaScript. It consults CN1_RESOURCE_PATH, and System.getenv is on the unsupported-native path there, which emits `throw new Error("environment variables are not available...")`. So a JavaScript application asking for a resource got an exception where before this branch it got null. The runtime now binds getenv to return null, which is both the safe answer and the correct one -- a browser has no environment, so the variable genuinely is not set. The nested-class finding is real and is NOT fixed here, deliberately. ParparVM builds the runtime class name as clsName.replace('_', '.') in ByteCodeClass, starting from the MANGLED name, so a nested class's '$' arrives as '.' and so does any '_' in a class's own name: Outer$Inner reports "a.b.Outer.Inner" and the derived package is "a.b.Outer". That is a defect in getName() itself, present on master and independent of resources, and correcting it changes a core API's output for every translated application -- it wants its own change and its own testing, not a ride on this one. The consequence here is a MISS (a directory named after a class does not exist, so the lookup returns null exactly as it did before the method was implemented), never a wrong file. Walking shorter prefixes would paper over it and would turn that miss into a confidently wrong hit wherever a package really is named like a class. The call site says all of this. CI, two findings of the same shape as the gdb one fixed in 43331747d5 -- a relative path under a `working-directory:` that resolves to nothing: Both retry.sh invocations in the self-host workflow ran with working-directory: vm, so they resolved to vm/scripts/ci/retry.sh, which does not exist. Anchored to $GITHUB_WORKSPACE. build-selfhost.sh rebuilt the translator only when its classes were MISSING, never when they were STALE, so re-running it after a source edit silently self-hosted the previous build -- and the binary was then compared against a JVM side built from the new sources, which reports the intended change as a VM divergence. It now checks for newer sources and cleans, the same guard verify-selfhost.sh carries; confirmed it flips to "would rebuild" on touch. linux-build-run.yml goes back to master exactly. It had accumulated 165 lines of crash-hunt scaffolding -- core dumps, gdb post-mortems, CN1_BIBOP_VALIDATE on the x64 leg -- and with the crash withdrawn none of it has a subject. That also closes the three threads on the file, including the one asking for a leg on the production mutator-assist path: there is no probe left to displace it. The CleanTargetLinuxIntegrationTest keeps the two inert env hooks (they do nothing unset, and CN1_EXTRA_DEFINES is a real capability of the generated CMakeLists) and loses the ELF-literal probing that only meant something mid-investigation. The verifier entry-point thread is answered by not changing anything: no doc or script names NativeSignatureVerifier as a main class -- check-native-signatures.sh already invokes the Cli -- and adding a delegating main would rebuild the core-to-Cli edge the split exists to remove. Gates D and A pass on the result, 800 files byte-identical, with the negative control. --- .github/workflows/linux-build-run.yml | 170 +----------------- .github/workflows/parparvm-selfhost.yml | 6 +- .../translator/JavascriptNativeRegistry.java | 2 +- .../src/javascript/parparvm_runtime.js | 31 ++-- vm/ByteCodeTranslator/src/nativeMethods.m | 15 -- vm/JavaAPI/src/java/lang/Class.java | 79 +++++++- vm/selfhost/build-selfhost.sh | 19 +- .../CleanTargetLinuxIntegrationTest.java | 33 ---- 8 files changed, 116 insertions(+), 239 deletions(-) diff --git a/.github/workflows/linux-build-run.yml b/.github/workflows/linux-build-run.yml index 8e9ce624ce7..3a87ad3b1c6 100644 --- a/.github/workflows/linux-build-run.yml +++ b/.github/workflows/linux-build-run.yml @@ -263,32 +263,6 @@ jobs: # The Windows pipeline passes a real boolean, which is why its gate # works. 'true' arms it here. CN1_REQUIRE_SUITE: 'true' - # Full DWARF into the .debug companion. The generated project defaults to - # -g1 -- lines and function names only -- so a core from this suite decodes to a - # backtrace and nothing else: every frame answers "No locals". That is what left - # the intermittent SIGSEGV here undiagnosed across four occurrences. The binary - # itself is still stripped; only the companion grows, and it is uploaded beside - # the core so the crash can be autopsied off the runner. - CN1_LINUX_FULL_DEBUG: 'true' - # BiBOP page validation, on the x64 leg only. - # - # CN1_GC_VERIFY already ran here and came back CLEAN over 2213 epochs, so - # the sweep is not reclaiming anything a survivor still references -- the - # premise this investigation had been working from is wrong. And the - # generated Display.c is semantically identical to master's (14 lines - # differ, all of them this PR's class-init guards), so the code that - # writes the field is not at fault either. - # - # What remains is a pointer that LOOKS valid but addresses the wrong - # object, which is what a retired-and-reformatted page produces and what - # the verifier would not call reclaimed. This guard checks on every fast - # allocation that bibopCurrent[ci] is owned, matches its size class, and - # that the bumped slot lies inside the page -- and aborts at the - # allocation that breaks it. Its comment names "the intermittent x64 - # cn1BibopFastAlloc crash"; this is an intermittent x64 crash. - # - # x64 only via the matrix arch: it is the leg that SIGSEGVs. - CN1_LINUX_EXTRA_DEFINES: ${{ matrix.arch == 'x64' && 'CN1_BIBOP_VALIDATE' || '' }} # Build the native ELF (and the demo) against an old glibc for portability. CN1_CC: /usr/local/bin/cn1-zig-cc # After the suite runs, the capture test relinks the same objects into a @@ -297,25 +271,7 @@ jobs: LIBGL_ALWAYS_SOFTWARE: '1' run: | set -e - # Not silenced, and the outcome is stated. The hang dump attaches gdb from - # inside the test JVM, so an install that quietly failed here produced a - # hang-stacks.txt with nothing but sample headers -- which is how four - # occurrences of the suite stall ended up with no evidence at all. Still - # best-effort: a runner without gdb must not fail the suite, it must say so. - # $GITHUB_WORKSPACE, not a relative path: this step runs with - # working-directory: vm, so "scripts/ci/..." resolved to vm/scripts/ci and - # bash answered "No such file or directory". gdb was therefore NEVER - # installed on any Linux leg, and every hang-stacks.txt and crash-stacks.txt - # this job has ever produced was empty for that reason alone -- including - # the ones collected while chasing a SIGSEGV that dumped a 12GB core nobody - # could symbolise. The retry.sh call twenty lines below already had this - # right. - bash "$GITHUB_WORKSPACE/scripts/ci/apt-get-install.sh" gdb || echo "WARNING: gdb install failed" - if command -v gdb >/dev/null 2>&1; then - echo "gdb available: $(gdb --version | head -1)" - else - echo "WARNING: gdb is NOT on PATH -- a hang in this job will capture no thread stacks." - fi + bash scripts/ci/apt-get-install.sh gdb >/dev/null 2>&1 || true Xvfb :99 -screen 0 1200x1600x24 >/tmp/xvfb-run.log 2>&1 & export DISPLAY=:99 sleep 2 @@ -351,30 +307,11 @@ jobs: mkdir -p "$(dirname "$CN1_APP_LOG_TEE")" { echo "=== post-mortem of $core (elf=$elf) ===" - # The event-stack arrays are dumped explicitly because the crash this - # keeps producing is a store into one of them from - # Display.edtLoopImpl, and -O3 optimises the array temporary out of - # the frame -- "array=" is all `info locals` gives. - # The Display object itself survives as frame 1's __cn1ThisObject, so - # the arrays are reachable THROUGH it, and what matters is whether - # each header's length and data agree: cn1_set_array_element_int - # bounds-checks against ->length and then stores through ->data, so a - # fault there means those two fields disagree, which is a corrupted or - # dangling header rather than a bad index. Harmless noise when the - # faulting frame is something else -- gdb just prints an error. gdb "$elf" "$core" -batch -ex 'set pagination off' \ -ex 'thread apply all bt' \ -ex 'thread 1' -ex 'bt full' \ - -ex 'frame 0' -ex 'info args' -ex 'info registers' \ + -ex 'frame 0' -ex 'info args' \ -ex 'frame 1' -ex 'info locals' -ex 'info args' \ - -ex 'set $d = (struct obj__com_codename1_ui_Display*)__cn1ThisObject' \ - -ex 'p $d->com_codename1_ui_Display_inputEventStackTmp' \ - -ex 'p *(JAVA_ARRAY)$d->com_codename1_ui_Display_inputEventStackTmp' \ - -ex 'p *(JAVA_ARRAY)$d->com_codename1_ui_Display_inputEventStack' \ - -ex 'p *(JAVA_ARRAY)$d->com_codename1_ui_Display_pointerMetaStackTmp' \ - -ex 'p *(JAVA_ARRAY)$d->com_codename1_ui_Display_pointerMetaStack' \ - -ex 'p $d->com_codename1_ui_Display_inputEventStackPointerTmp' \ - -ex 'p $d->com_codename1_ui_Display_inputEventStackPointer' \ -ex 'p gcMarkWorklistTop' -ex 'p currentGcMarkValue' \ 2>&1 } >> "$(dirname "$CN1_APP_LOG_TEE")/crash-stacks.txt" || true @@ -384,12 +321,6 @@ jobs: # frames (e.g. a stack overflow's recursion / the caller that ran CN1 on a small # native stack). The suite binary is not stripped, so addr2line resolves them. elf="$(/usr/bin/find /tmp -maxdepth 4 -name LinuxHelloMain -type f 2>/dev/null | head -1)" - # The binary and the cores are NOT copied here. "Package core dump for - # offline autopsy" below already ships the unstripped ELF (plus .debug) - # and zstd-compresses each core into its own artifact. Copying them into - # the screenshot directory as well would upload multi-gigabyte cores twice - # on exactly the runs that crashed -- the runs whose screenshots and - # backtrace most need to survive the disk and upload budget. if [ -n "$elf" ] && [ -f "$CN1_APP_LOG_TEE" ]; then { echo "=== addr2line of CN1 backtrace addresses (from app log) ===" @@ -409,63 +340,6 @@ jobs: if-no-files-found: warn retention-days: 14 - # The core itself, with everything needed to decode it off the runner: the - # stripped ELF the core refers to and its .debug companion. crash-stacks.txt is - # only ever the backtrace gdb could produce in-place; a real autopsy -- printing - # the object a faulting pointer came from, the slot beside it, the register that - # held it -- needs the core in hand. Cores of this app run to hundreds of MB, so - # they are compressed, capped, and collected ONLY when one exists, which means - # only when the suite actually crashed. - - name: Package core dump for offline autopsy - if: always() - run: | - set -u - shopt -s nullglob - cores=(/tmp/cn1-cores/core.*) - if [ ${#cores[@]} -eq 0 ]; then - echo "no core dumped -- nothing to package" - exit 0 - fi - out="${GITHUB_WORKSPACE}/artifacts/linux-port/core" - mkdir -p "$out" - elf="$(/usr/bin/find /tmp -maxdepth 4 -name LinuxHelloMain -type f 2>/dev/null | head -1)" - if [ -n "$elf" ]; then - cp "$elf" "$out/" || true - [ -f "$elf.debug" ] && cp "$elf.debug" "$out/" || true - fi - for c in "${cores[@]}"; do - sz=$(stat -c%s "$c" 2>/dev/null || echo 0) - echo "core $c is $((sz/1024/1024))MB" - # zstd where available (far faster on a GB-scale core), else gzip. - if command -v zstd >/dev/null 2>&1; then - zstd -q -T0 -3 -o "$out/$(basename "$c").zst" "$c" || true - else - gzip -c "$c" > "$out/$(basename "$c").gz" || true - fi - done - echo "--- packaged ---"; ls -la "$out" - cat > "$out/README.txt" <<'TXT' - Decode this core off the runner: - - zstd -d core.LinuxHelloMain.*.zst # or gunzip for the .gz - gdb ./LinuxHelloMain ./core.LinuxHelloMain.* - - LinuxHelloMain.debug must sit beside the binary -- it is linked by - --add-gnu-debuglink and carries all the DWARF, because the shipped binary is - stripped. Built with CN1_LINUX_FULL_DEBUG so variables and types are present: - "info locals", "p *someObject" and "info registers" all work, which they do - not on a default -g1 build. - TXT - - - name: Upload core dump - if: always() - uses: actions/upload-artifact@v4 - with: - name: linux-core-${{ matrix.arch }} - path: artifacts/linux-port/core - if-no-files-found: ignore - retention-days: 14 - # A runnable, self-contained windowed demo binary for this arch (opens a # GTK window showing a Form -- not the headless suite). Download and run it # on a Linux desktop to smoke-test the native port on real hardware. @@ -497,21 +371,9 @@ jobs: name: linux-suite-classes - name: Unpack suite classes run: tar xzf suite-classes.tgz - # The musl leg had no crash wiring at all: when the suite died mid-run the - # only artifact was a short screenshot directory, which is indistinguishable - # from a hang. core_pattern is global to the host kernel rather than per - # namespace, but the kernel writes the file in the CRASHING process's mount - # namespace -- so an absolute path under /cn1 lands inside the container, and - # /cn1 is the bind-mounted workspace, which means the core is uploaded with - # everything else. - - name: Enable core dumps for the container - run: | - mkdir -p artifacts/linux-port/raw-musl - sudo sysctl -w kernel.core_pattern='/cn1/artifacts/linux-port/raw-musl/core.%e.%p' || true - - name: Translate + build + run the suite in Alpine (musl, end-to-end) run: | - docker run --rm --ulimit core=-1 \ + docker run --rm \ -v "$GITHUB_WORKSPACE":/cn1 -w /cn1 \ -e CN1_SHOT_OUTPUT_DIR=/cn1/artifacts/linux-port/raw-musl \ -e CN1_APP_LOG_TEE=/cn1/artifacts/linux-port/raw-musl/app-output.log \ @@ -519,7 +381,7 @@ jobs: -e LIBGL_ALWAYS_SOFTWARE=1 \ docker.io/library/alpine:3.20 sh -ec ' sed -i "s|^#\(.*/community\)|\1|" /etc/apk/repositories - apk add --no-cache build-base cmake samurai pkgconf bash git openjdk8 openjdk17 maven gdb \ + apk add --no-cache build-base cmake samurai pkgconf bash git openjdk8 openjdk17 maven \ gtk+3.0-dev cairo-dev pango-dev gdk-pixbuf-dev glib-dev fontconfig-dev freetype-dev \ curl-dev openssl-dev libepoxy-dev mesa-dev mesa-gles mesa-egl mesa-gbm mesa-dri-gallium \ webkit2gtk-4.1-dev gstreamer-dev gst-plugins-base-dev \ @@ -538,31 +400,9 @@ jobs: # core + Linux port (musl JDK8); the default cc on Alpine already links musl. cd /cn1/maven && mvn -B -pl linux -am -DskipTests -Dmaven.javadoc.skip=true -Plocal-dev-javase install cd /cn1/vm && mvn -B clean package -pl JavaAPI -am -DskipTests - ulimit -c unlimited || true - rc=0 mvn -B test -pl tests -am \ -Dtest=CleanTargetLinuxIntegrationTest#capturesHelloSuiteOverWebSocketLinux \ - -Dsurefire.failIfNoSpecifiedTests=false || rc=$? - # Keep the UNSTRIPPED binary next to any core: without it a core names - # addresses and nothing else, and the container is gone by the time the - # artifact is looked at. - out=/cn1/artifacts/linux-port/raw-musl - mkdir -p "$out" - elf="$(find /tmp -maxdepth 4 -name LinuxHelloMain -type f 2>/dev/null | head -1)" - if [ -n "$elf" ]; then cp "$elf" "$out/LinuxHelloMain" || true; fi - for core in "$out"/core.*; do - [ -f "$core" ] || continue - { - echo "=== post-mortem of $core (elf=$elf) ===" - gdb "$elf" "$core" -batch -ex "set pagination off" \ - -ex "thread apply all bt" -ex "thread 1" -ex "bt full" 2>&1 - } >> "$out/crash-stacks.txt" || true - done - if [ ! -f "$out/crash-stacks.txt" ]; then - echo "no core file was written -- the suite stopped without a fatal signal" \ - > "$out/crash-stacks.txt" - fi - exit $rc + -Dsurefire.failIfNoSpecifiedTests=false ' - name: Upload musl screenshots if: always() diff --git a/.github/workflows/parparvm-selfhost.yml b/.github/workflows/parparvm-selfhost.yml index 0ca9db49e4d..7ba516ab3c5 100644 --- a/.github/workflows/parparvm-selfhost.yml +++ b/.github/workflows/parparvm-selfhost.yml @@ -72,12 +72,14 @@ jobs: # The translator has to exist as classes before it can translate itself. - name: Build the translator - run: scripts/ci/retry.sh mvn -q -B -pl ByteCodeTranslator -am package -DskipTests + run: >- + "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -q -B + -pl ByteCodeTranslator -am package -DskipTests working-directory: vm - name: Resolve the ASM classpath run: >- - scripts/ci/retry.sh mvn -q -B -pl ByteCodeTranslator + "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -q -B -pl ByteCodeTranslator dependency:build-classpath -Dmdep.outputFile=target/selfhost-asm-classpath.txt working-directory: vm diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptNativeRegistry.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptNativeRegistry.java index d1faac4c9f0..2c822e46757 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptNativeRegistry.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptNativeRegistry.java @@ -40,7 +40,6 @@ enum NativeCategory { "cn1_java_lang_Class_forNameImpl_java_lang_String_R_java_lang_Class", "cn1_java_lang_Class_getComponentType_R_java_lang_Class", "cn1_java_lang_Class_getNameImpl_R_java_lang_String", - "cn1_java_lang_Class_getPrimitiveClass_int_R_java_lang_Class", "cn1_java_lang_Class_getName_R_java_lang_String", "cn1_java_lang_Class_getSuperclass_R_java_lang_Class", "cn1_java_lang_Class_hashCode_R_int", @@ -129,6 +128,7 @@ enum NativeCategory { "cn1_java_lang_System_currentTimeMillis_R_long", "cn1_java_lang_System_exit_int", "cn1_java_lang_System_gcLight", + "cn1_java_lang_System_getenvImpl_java_lang_String_R_java_lang_String", "cn1_java_lang_System_gcMarkSweep", "cn1_java_lang_System_identityHashCode_java_lang_Object_R_int", "cn1_java_lang_Integer_cn1Value_R_int", diff --git a/vm/ByteCodeTranslator/src/javascript/parparvm_runtime.js b/vm/ByteCodeTranslator/src/javascript/parparvm_runtime.js index 69330508f58..ba1ecec5e9f 100644 --- a/vm/ByteCodeTranslator/src/javascript/parparvm_runtime.js +++ b/vm/ByteCodeTranslator/src/javascript/parparvm_runtime.js @@ -78,11 +78,7 @@ const PRIMITIVE_INFO = { JAVA_BYTE: { javaName: "byte", descriptor: "B" }, JAVA_SHORT: { javaName: "short", descriptor: "S" }, JAVA_INT: { javaName: "int", descriptor: "I" }, - JAVA_LONG: { javaName: "long", descriptor: "J" }, - // void is a primitive class too -- Void.TYPE is one, and unlike the other - // eight it is not reachable through a primitive class literal, so nothing - // needed it here until getPrimitiveClass did. - JAVA_VOID: { javaName: "void", descriptor: "V" } + JAVA_LONG: { javaName: "long", descriptor: "J" } }; const jsObjectWrappers = typeof WeakMap === "function" ? new WeakMap() : null; const externalIdentityMap = typeof WeakMap === "function" ? new WeakMap() : null; @@ -5759,22 +5755,15 @@ bindNative(["cn1_java_lang_Class_getComponentType_R_java_lang_Class"], function( } return classObjectForName(def.componentClass); }); -// Backs the wrapper classes' TYPE fields. The JavaAPI cannot initialize them with -// a primitive class literal: javac lowers `int.class` to a read of Integer.TYPE -// itself, so `TYPE = int.class` compiles to `getstatic TYPE; putstatic TYPE` and -// leaves the field null. The codes match Class.CN1_PRIM_* in the JavaAPI and the -// switch in nativeMethods.m. -// -// _primClass covers the same ground for a primitive class literal appearing in -// ordinary code; this is the path taken by the wrapper clinits themselves. -bindNative(["cn1_java_lang_Class_getPrimitiveClass_int_R_java_lang_Class"], function(typeCode) { - const names = ["JAVA_INT", "JAVA_LONG", "JAVA_SHORT", "JAVA_BYTE", "JAVA_CHAR", - "JAVA_FLOAT", "JAVA_DOUBLE", "JAVA_BOOLEAN", "JAVA_VOID"]; - const name = names[typeCode | 0]; - if (!name) { - throw new Error("getPrimitiveClass: unknown primitive type code " + typeCode); - } - return classObjectForName(name); +// A browser has no process environment, so getenv ANSWERS null rather than +// failing. Without this the symbol falls through to the unsupported-native path, +// which emits `throw new Error("environment variables are not available...")` -- +// and Class.getResourceAsStream consults CN1_RESOURCE_PATH, so a JavaScript +// application asking for a resource got an exception where it previously got +// null. Returning null is both the safe answer and the correct one: the variable +// genuinely is not set. +bindNative(["cn1_java_lang_System_getenvImpl_java_lang_String_R_java_lang_String"], function(name) { + return null; }); bindNative(["cn1_java_lang_Class_isPrimitive_R_boolean"], function(__cn1ThisObject) { return __cn1ThisObject.__classDef && __cn1ThisObject.__classDef.isPrimitive ? 1 : 0; }); bindNative(["cn1_java_lang_reflect_Array_newInstanceImpl_java_lang_Class_int_R_java_lang_Object"], function(componentClass, length) { diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 42a9d2d4d3c..9fac29f3a01 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -1963,21 +1963,6 @@ JAVA_OBJECT java_lang_Class_getName___R_java_lang_String(CODENAME_ONE_THREAD_STA return newStringFromCString(threadStateData, clz->clsName); } -/** - * Backs Integer.TYPE and the eight other wrapper TYPE fields. The JDK needs a - * native here for the same reason we do: `TYPE = int.class` cannot initialize the - * field, because javac lowers a primitive class literal to a read of that very - * field (getstatic TYPE; putstatic TYPE), leaving it null. - * - * Takes an int code rather than the JDK's String name deliberately. This runs - * inside the wrapper class initializers, which are among the earliest code in the - * process, and decoding a Java String here would drag in String.getBytes and the - * charset machinery during Integer's own clinit. An int argument allocates - * nothing and initializes nothing. - * - * The codes are an implementation detail shared only with java/lang/Class.java; - * they are matched by CN1_PRIM_* there. - */ /** * Resources linked into the executable, backing Class.getResourceAsStream. * diff --git a/vm/JavaAPI/src/java/lang/Class.java b/vm/JavaAPI/src/java/lang/Class.java index 775b93717d7..e691fa890b8 100644 --- a/vm/JavaAPI/src/java/lang/Class.java +++ b/vm/JavaAPI/src/java/lang/Class.java @@ -173,6 +173,24 @@ public java.io.InputStream getResourceAsStream(java.lang.String name){ if (!absolute.startsWith("/")) { // Relative names resolve against this class's package, as the javadoc // above describes. + // + // KNOWN LIMITATION, for a NESTED class only. getName() cannot be told + // apart from a package here, because ParparVM builds the runtime class + // name as clsName.replace('_', '.') in ByteCodeClass -- it starts from + // the MANGLED name, so the '$' that separates a nested class from its + // outer one arrives as a '.', and so does any '_' in a class's own + // name. Outer$Inner therefore reports "a.b.Outer.Inner" where the JDK + // reports "a.b.Outer$Inner", and the package derived below is + // "a.b.Outer" rather than "a.b". + // + // The consequence is a MISS, not a wrong file: the derived path is a + // directory named after a class, which a resource tree does not have, + // so the lookup returns null exactly as it did before this method was + // implemented. It is deliberately not patched up by walking shorter + // prefixes -- a package really can be named like a class, and that + // would turn today's miss into a confidently wrong hit. The fix + // belongs in the name the VM reports, which is a change to getName() + // for every translated application and wants its own testing. String className = getName(); int lastDot = className.lastIndexOf('.'); absolute = lastDot < 0 ? "/" + name @@ -210,9 +228,16 @@ private static java.io.InputStream cn1FileResource(String absolute) { return null; } String relative = absolute.substring(1); + // A resource name is not a path expression. Refusing any ".." segment keeps + // a lookup inside the search root it was found under; without it a name + // like "../../etc/passwd" reads straight out of the filesystem, and the + // caller is usually passing a name that came from data. + if (relative.length() == 0 || cn1EscapesRoot(relative)) { + return null; + } int from = 0; while (from <= path.length()) { - int end = path.indexOf(java.io.File.pathSeparatorChar, from); + int end = cn1PathEntryEnd(path, from); String root = end < 0 ? path.substring(from) : path.substring(from, end); if (root.length() > 0) { java.io.File candidate = new java.io.File(root, relative); @@ -231,6 +256,58 @@ private static java.io.InputStream cn1FileResource(String absolute) { } return null; } + + /** True when any segment of a resource-relative path is "..". */ + private static boolean cn1EscapesRoot(String relative) { + int from = 0; + while (from <= relative.length()) { + int slash = relative.indexOf('/', from); + String segment = slash < 0 ? relative.substring(from) : relative.substring(from, slash); + if (segment.equals("..")) { + return true; + } + if (slash < 0) { + return false; + } + from = slash + 1; + } + return false; + } + + /** + * The index that ends the search-path entry starting at {@code from}, or -1 for + * the last one. + * + * This cannot use {@code File.pathSeparatorChar}, which is a hard-coded ':' in + * this class library rather than a platform value -- on a native Windows build + * that splits "C:\\res;D:\\res" after the drive letter and every entry is + * nonsense. Both separators are therefore accepted, and a ':' is not a + * separator when it sits directly after a single-letter entry and is followed + * by a slash, which is exactly a DOS drive prefix and never a POSIX path. + */ + private static int cn1PathEntryEnd(String path, int from) { + for (int i = from; i < path.length(); i++) { + char c = path.charAt(i); + if (c == ';') { + return i; + } + if (c == ':') { + boolean driveLetter = i == from + 1 + && i + 1 < path.length() + && (path.charAt(i + 1) == '\\' || path.charAt(i + 1) == '/') + && cn1IsLetter(path.charAt(from)); + if (!driveLetter) { + return i; + } + } + } + return -1; + } + + /** ASCII letter test; Character.isLetter is locale-aware and not wanted here. */ + private static boolean cn1IsLetter(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); + } /** * Determines if this Class object represents an array class. diff --git a/vm/selfhost/build-selfhost.sh b/vm/selfhost/build-selfhost.sh index c3e8533af17..301ac24b42d 100755 --- a/vm/selfhost/build-selfhost.sh +++ b/vm/selfhost/build-selfhost.sh @@ -27,8 +27,25 @@ mkdir -p "$OUT" # 1. translator classes + ASM classpath, built once by maven and then cached. TRANSLATOR="$REPO/vm/ByteCodeTranslator/target/classes" +# Rebuild when the classes are MISSING or STALE. Testing only for existence meant +# that re-running this after editing a translator source silently self-hosted the +# previous build, and the resulting binary was then compared against a JVM side +# built from the new sources -- which reports the intended change as a VM +# divergence. verify-selfhost.sh carries the same guard for the same reason, and +# maven's own incremental check is not enough on its own: it answered "Nothing to +# compile - all classes are up to date" for a source three hours newer than its +# class. +needs_build=0 if [ ! -f "$TRANSLATOR/com/codename1/tools/translator/ByteCodeTranslator.class" ]; then - (cd "$REPO/vm" && mvn -q -B -pl ByteCodeTranslator -am package -DskipTests) + needs_build=1 +elif [ -n "$(find "$REPO/vm/ByteCodeTranslator/src" -name '*.java' -newer "$TRANSLATOR" -print -quit 2>/dev/null)" ]; then + echo "translator sources are newer than $TRANSLATOR -- rebuilding" + needs_build=1 +fi +if [ "$needs_build" = 1 ]; then + # `clean` because the incremental check cannot be trusted here; it also removes + # selfhost-asm-classpath.txt, which the next block regenerates. + (cd "$REPO/vm" && mvn -q -B -pl ByteCodeTranslator -am clean package -DskipTests) fi ASM_CP_FILE="$REPO/vm/ByteCodeTranslator/target/selfhost-asm-classpath.txt" if [ ! -f "$ASM_CP_FILE" ]; then diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java index 451d4471036..f2f50047645 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java @@ -297,42 +297,9 @@ static Path buildHelloCodenameOneElf(String launcherSource) throws Exception { // that silently fails to reach the compiler leaves a clean-looking run that // measured nothing, which is worse than no diagnostic at all. System.out.println("CN1SS:HARNESS: cmake configure: " + String.join(" ", configure)); - // And whether the project being configured actually HAS the hook those - // defines hang on. Passing -DCN1_EXTRA_DEFINES to a CMakeLists that never - // declares it is silently accepted by cmake and compiles nothing extra, - // which is indistinguishable in the log from a diagnostic that ran and - // found nothing. - try { - String cml = new String(Files.readAllBytes(cmakeRoot.resolve("CMakeLists.txt")), - StandardCharsets.UTF_8); - System.out.println("CN1SS:HARNESS: CMakeLists declares CN1_EXTRA_DEFINES: " - + cml.contains("CN1_EXTRA_DEFINES") - + "; declares CN1_DEBUG_INFO_LEVEL: " - + cml.contains("CN1_DEBUG_INFO_LEVEL")); - } catch (IOException readFailed) { - System.out.println("CN1SS:HARNESS: could not read the generated CMakeLists: " + readFailed); - } CleanTargetIntegrationTest.runCommand(configure, cmakeRoot); CleanTargetIntegrationTest.runCommand(Arrays.asList("cmake", "--build", buildDir.toString()), cmakeRoot); Path elf = buildDir.resolve("LinuxHelloMain"); - // Does the BINARY carry the diagnostic? Every link in the chain above can - // report success while the define fails to reach the compiler, and the only - // unambiguous answer is whether the code it guards is in the executable. - // "[GC-VERIFY]" is a literal in cn1GcVerifyHeap's reporting, so its presence - // in the image means CN1_GC_VERIFY was compiled in. - try { - byte[] image = Files.readAllBytes(elf); - // Keyed off whatever diagnostic this run asked for, so the probe stays - // honest when the define changes. - String want = System.getenv("CN1_LINUX_EXTRA_DEFINES"); - String needle = want != null && want.contains("CN1_BIBOP_VALIDATE") - ? "CN1BIBOP FASTALLOC CORRUPT" : "[GC-VERIFY]"; - boolean present = new String(image, StandardCharsets.ISO_8859_1).contains(needle); - System.out.println("CN1SS:HARNESS: built ELF contains " + needle + ": " + present - + " (" + image.length + " bytes)"); - } catch (IOException probeFailed) { - System.out.println("CN1SS:HARNESS: could not probe the built ELF: " + probeFailed); - } assertTrue(Files.exists(elf), "native ELF should be produced: " + elf); return elf; } From 23a939944522bf69865ec91a4895b054f4a71714 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 13 Sep 2026 07:16:23 +0300 Subject: [PATCH 51/66] Count a backslash as a separator in the resource traversal check The ".." guard added in the previous commit split on '/' only, which leaves the traversal open on the one platform whose separator is the other one: nothing stops a caller passing "..\..\etc\passwd", and File("root", that) escapes on Windows exactly as the '/' spelling does. Resource names are '/'-separated by specification, but a guard that trusts the specification is not a guard. Checked against ten shapes, including the two that must NOT be refused -- "a/..b/c", where ".." is a prefix of a real segment rather than a segment, and "..." -- since an over-eager contains("..") would reject both. --- vm/JavaAPI/src/java/lang/Class.java | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/vm/JavaAPI/src/java/lang/Class.java b/vm/JavaAPI/src/java/lang/Class.java index e691fa890b8..e4dff54dccb 100644 --- a/vm/JavaAPI/src/java/lang/Class.java +++ b/vm/JavaAPI/src/java/lang/Class.java @@ -257,19 +257,26 @@ private static java.io.InputStream cn1FileResource(String absolute) { return null; } - /** True when any segment of a resource-relative path is "..". */ + /** + * True when any segment of a resource-relative path is "..". + * + * A backslash counts as a separator as well as '/'. Resource names are + * '/'-separated by specification, but nothing stops a caller passing a Windows + * path, and there File("root", "..\\..\\x") escapes exactly as the '/' form + * does -- checking only '/' would leave the traversal open on the one platform + * whose separator it is. + */ private static boolean cn1EscapesRoot(String relative) { int from = 0; - while (from <= relative.length()) { - int slash = relative.indexOf('/', from); - String segment = slash < 0 ? relative.substring(from) : relative.substring(from, slash); - if (segment.equals("..")) { - return true; + for (int i = 0; i <= relative.length(); i++) { + boolean atEnd = i == relative.length(); + if (!atEnd && relative.charAt(i) != '/' && relative.charAt(i) != '\\') { + continue; } - if (slash < 0) { - return false; + if (relative.substring(from, i).equals("..")) { + return true; } - from = slash + 1; + from = i + 1; } return false; } From df53b707d100640fa6001737f20556d839fb3561 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 13 Sep 2026 07:26:51 +0300 Subject: [PATCH 52/66] Withdraw the pacing growth-floor scaling; it disarmed the bound CI enforces GC suite (1 marker, arm64) failed on the previous push: GcOverflowSpiralIntegration Test peaked at 2159916KB against a 2GB limit, with a live set of a few hundred bytes. The test and the workflow are both master's, unmodified by this branch, and master passes them, so the regression is this branch's. This branch had changed the footprint at which pacing starts applying from a flat 512MB to max(512MB, fm/4). The test pins the free-memory reading at 32GB, so the floor became 8GB: the growth test never fired, pacing never engaged, and the peak was bounded only by the 1GB run-ahead allowance. The shape of the failure is the tell -- the FOUR-marker arms passed on both arches and only the one-marker arm failed, which is what a bound that holds only while the collector is fast looks like. A slow collector is the case the bound exists for. Only the arming point moves back. The capCeiling raise is the other half of the same idea and STAYS, because it is measured to help where it matters: on a 5782-class translation, master's 192MB clamp peaked HIGHER than the 1GB one (9736MB against 8325MB) and took twice as long (46.3s against 23.8s). Withdrawing that as well would have made the workload this branch exists to speed up both slower and larger. I was about to, and the measurement table above the clamp is what stopped me. Not re-tuned to sit just under the threshold: the margin would be a few percent on a shared runner, which is a flake rather than a fix. The scaling wants its own change, with the measurement that justifies it and a decision about what the enforced bound should be. HONESTY ABOUT THE VERIFICATION, because a number here is easy to misread: an A/B of this function on an uncontended arm64 Mac measured 107904KB with the constant against 109792KB with the scaling -- identical. Neither arm reaches even the 512MB floor, so the clamp is never armed in EITHER and the value under test does not participate. I first read the single "with fix" figure as confirmation; it was not, and the A/B is what caught it. This change is reasoned from the CI failure and is a revert to master's own constant. The CI leg is the only thing that can actually test it, and the code now says so. Gates D and A pass, 800 files byte-identical, with the negative control. Local vm suite: 567 tests, 0 failures. --- vm/ByteCodeTranslator/src/cn1_globals.m | 66 +++++++++++++------------ 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index 21b762c1732..aabf6dc5850 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -6494,38 +6494,42 @@ static long long cn1PacingFootprintNow(void) { return fp; } -// The footprint at which the run-ahead bound starts applying, scaled to the memory -// this host actually has. -// -// A fixed 512MB says "this process has grown"; it does not say the machine is under -// any pressure, and the bound exists for pressure. On a host with tens of GB free, a -// process holding a couple of GB is nowhere near runaway, and clamping it there -// parks the mutator against a collector that cannot get under the ceiling: measured -// at 6.7-8.7s versus 1.4s for the same work, to save 2% of peak footprint. -// -// So take the larger of the absolute floor and a quarter of available memory. Two -// properties this has to keep: -// -// - Where cn1_available_memory is the flat 100MB placeholder (Linux, Windows, and -// the non-Apple fallback), fm/4 is 25MB, the absolute floor wins, and behaviour is -// bit-for-bit what it was. Nothing changes on a platform where we cannot measure. -// - It only ever RAISES the floor, so the bound can only engage later than before, -// never earlier. It cannot make a constrained host more permissive than it was. -// -// This is the no-per-process-ceiling path only. Where a ceiling exists -- iOS's dirty -// memory limit, or an explicit process budget -- cn1PacingPark takes the bounded -// branch instead and never reaches cn1BibopPacingCap, so none of this loosens the -// admission control that keeps an app inside its own limit. +// The footprint at which the run-ahead bound starts applying. +// +// This SCALED with available memory for a while -- max(512MB, fm/4) -- on the +// reasoning that a fixed 512MB says "this process has grown" and not "the machine +// is under pressure", and that on a host with tens of GB free, parking the mutator +// at 512MB costs real time (measured 6.7-8.7s against 1.4s for the same work) to +// save about 2% of peak footprint. That reasoning still looks right, and the +// scaling is still withdrawn, because it broke an invariant master enforces. +// +// GcOverflowSpiralIntegrationTest pins the free-memory reading at 32GB and requires +// the peak to stay under 2GB with no process ceiling. Scaled, the floor became +// fm/4 = 8GB, so the growth test never fired, pacing never engaged, and the peak +// was bounded only by the run-ahead allowance: 2159916KB on the ONE-marker arm, +// where cycles are longest and the mutator reaches the full allowance every time. +// The four-marker arms passed, which is what a bound that only holds when the +// collector is fast looks like -- and the slow collector is the case the bound is +// for. +// +// It is not re-tuned to sit just under the threshold here. The margin would be a +// few percent on a shared runner, which is a flake rather than a fix. It belongs in +// its own change, with the measurement that justifies it and a decision about what +// the enforced bound should be. +// +// NOT REPRODUCIBLE ON macOS, which is worth knowing before trying: an A/B of this +// function on an uncontended arm64 Mac measured 107904KB with the constant against +// 109792KB with the scaling. Neither arm reaches even the 512MB floor, so the clamp +// is never armed in EITHER and the value under test does not participate. Only the +// CI leg drives the footprint into the gigabytes where the floor decides anything, +// so a local pass here is not evidence about this change. +// +// What is NOT withdrawn is the capCeiling raise below, which is the other half of +// the same idea and is measured to help on a real translation: at the 192MB clamp +// the same workload peaked HIGHER (9736MB against 8325MB) and took twice as long +// (46.3s against 23.8s). Only the arming point moves back. static long long cn1PacingGrowthFloorBytes(void) { - long long floor = CN1_PACING_GROWTH_FLOOR_BYTES; - long fm = atomic_load_explicit(&cn1CachedFreeMem, memory_order_relaxed); - if(fm > 0) { - long long scaled = (long long)fm / 4; - if(scaled > floor) { - floor = scaled; - } - } - return floor; + return CN1_PACING_GROWTH_FLOOR_BYTES; } static JAVA_BOOLEAN cn1PacingPastGrowthFloor(void) { From 66d247a9691d9874c0a2a1ebd99357ed17e5f53f Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 13 Sep 2026 07:36:02 +0300 Subject: [PATCH 53/66] Withdraw the pacing run-ahead bound too; restore master's cap exactly The previous commit withdrew half of this branch's pacing work and CI answered with the other half of the problem. GcOverflowSpiral went green -- peakKb=456216 against the 2159916KB that failed -- and BibopPageFloorIntegrationTest went red in the same job: after dropping a 261492KB live set the footprint only fell to 225396KB against a 143820KB budget, so the pages were not handed back. That test passed on the push before, with the scaling in. So the two halves pull in opposite directions and each reds a test master passes: scaling in GcOverflowSpiral 2159916KB against a 2GB limit (clamp never armed, floor was fm/4 = 8GB against the pinned 32GB reading) scaling out BibopPageFloor does not release (capCeiling raised 192MB -> 1GB, so the mutator runs far enough ahead that pages are not returned) Two red tests traded back and forth is the signal to stop tuning, so the whole optimization is out rather than the half I could argue for. cn1BibopPacingCap now diffs IDENTICAL to master, and cn1PacingRunAheadBound and its two defines are gone. cn1PacingGrowthFloorBytes stays as a function returning master's constant, because the record of what was tried belongs somewhere a future attempt will find it. I argued in the last commit for KEEPING the capCeiling raise, on a measurement that is still true: on a 5782-class translation the 192MB clamp peaked higher (9736MB against 8325MB) and took twice as long (46.3s against 23.8s). It is still not worth a test master passes. That speedup is one of the two things this branch set out to buy and it is now deferred, which is a real cost and not a tidy outcome -- but it needs an environment that reproduces BOTH failures, and this machine reproduces neither: an A/B measured 107904KB against 109792KB, identical, because neither arm reaches even the 512MB floor. Verified here: gates D and A pass, 800 files byte-identical with the negative control, and both GC tests pass locally (peakKb=111392; the floor probe releases). Local passes are weak evidence for the pacing question specifically -- see the comment -- but they do rule out a build or link break. --- vm/ByteCodeTranslator/src/cn1_globals.m | 116 +++++++----------------- 1 file changed, 32 insertions(+), 84 deletions(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index aabf6dc5850..4bd83e9ec75 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -6288,39 +6288,8 @@ static inline JAVA_OBJECT cn1BibopSlot(CN1BibopPage* p, int i) { #ifndef CN1_PACING_GROWTH_FLOOR_BYTES #define CN1_PACING_GROWTH_FLOOR_BYTES (512LL*1024*1024) #endif -// Ceiling on how far a mutator may run ahead of a cycle in flight, regardless of -// how much RAM the host has. See the measurement table in cn1BibopPacingCap. -#ifndef CN1_BIBOP_PACING_MAX_RUNAHEAD_BYTES -#define CN1_BIBOP_PACING_MAX_RUNAHEAD_BYTES (1024L*1024*1024) -#endif -// cn1_available_memory answers a flat 100MB on every platform where it cannot -// measure: Linux, Windows, and the non-Apple fallback. That number is not a -// reading, and a bound DERIVED from it is not a bound -- it is a constant that -// happens to look like one. -// -// This matters asymmetrically. cn1PacingGrowthFloorBytes above only ever RAISES -// its floor from fm, so on a placeholder host the absolute floor wins and -// behaviour is bit-for-bit unchanged. The run-ahead bound below only ever LOWERS -// the cap, so scaling it by a placeholder TIGHTENS pacing on exactly the hosts we -// know nothing about. It did: BibopPageFloorIntegrationTest went red on arm64 -// Linux, where fm/8 is 12.5MB, while the same code passed on macOS where fm is -// real. -// -// So the bound applies only where fm is a genuine reading. Returns 0 to mean -// "not measurable here, leave the cap alone". -#ifndef CN1_PACING_PLACEHOLDER_FREE_MEM -#define CN1_PACING_PLACEHOLDER_FREE_MEM (1024L*1024*100) -#endif -static long cn1PacingRunAheadBound(long fm) { - if(fm <= CN1_PACING_PLACEHOLDER_FREE_MEM) { - return 0; - } - long bound = CN1_BIBOP_PACING_MAX_RUNAHEAD_BYTES; - if(bound > fm / 8) { - bound = fm / 8; - } - return bound; -} +// The run-ahead bound that stood here is withdrawn; see cn1PacingGrowthFloorBytes +// below for the whole story. Pacing is master's again. // How stale a below-floor footprint reading may be before the bound re-probes it. The // probe is task_info on Apple and one /proc read on Linux -- a microsecond or two -- and // it is taken at most once per interval across the whole process, and only when the bound @@ -6494,40 +6463,36 @@ static long long cn1PacingFootprintNow(void) { return fp; } -// The footprint at which the run-ahead bound starts applying. -// -// This SCALED with available memory for a while -- max(512MB, fm/4) -- on the -// reasoning that a fixed 512MB says "this process has grown" and not "the machine -// is under pressure", and that on a host with tens of GB free, parking the mutator -// at 512MB costs real time (measured 6.7-8.7s against 1.4s for the same work) to -// save about 2% of peak footprint. That reasoning still looks right, and the -// scaling is still withdrawn, because it broke an invariant master enforces. -// -// GcOverflowSpiralIntegrationTest pins the free-memory reading at 32GB and requires -// the peak to stay under 2GB with no process ceiling. Scaled, the floor became -// fm/4 = 8GB, so the growth test never fired, pacing never engaged, and the peak -// was bounded only by the run-ahead allowance: 2159916KB on the ONE-marker arm, -// where cycles are longest and the mutator reaches the full allowance every time. -// The four-marker arms passed, which is what a bound that only holds when the -// collector is fast looks like -- and the slow collector is the case the bound is -// for. -// -// It is not re-tuned to sit just under the threshold here. The margin would be a -// few percent on a shared runner, which is a flake rather than a fix. It belongs in -// its own change, with the measurement that justifies it and a decision about what -// the enforced bound should be. -// -// NOT REPRODUCIBLE ON macOS, which is worth knowing before trying: an A/B of this -// function on an uncontended arm64 Mac measured 107904KB with the constant against -// 109792KB with the scaling. Neither arm reaches even the 512MB floor, so the clamp -// is never armed in EITHER and the value under test does not participate. Only the -// CI leg drives the footprint into the gigabytes where the floor decides anything, -// so a local pass here is not evidence about this change. -// -// What is NOT withdrawn is the capCeiling raise below, which is the other half of -// the same idea and is measured to help on a real translation: at the 192MB clamp -// the same workload peaked HIGHER (9736MB against 8325MB) and took twice as long -// (46.3s against 23.8s). Only the arming point moves back. +// The footprint at which the pacing clamp starts applying. Master's constant. +// +// This branch tried to make pacing less eager on a host with memory to spare, in +// two halves, and BOTH are withdrawn. The idea was that a fixed 512MB says "this +// process has grown" and not "the machine is under pressure", and there was a real +// measurement behind it: on a 5782-class translation, a 192MB clamp peaked HIGHER +// than a 1GB one (9736MB against 8325MB) and took twice as long (46.3s against +// 23.8s). The halves were a growth floor of max(512MB, fm/4), and a capCeiling +// raised to a 1GB run-ahead bound. +// +// They are withdrawn because each one reds a test master passes, and the two tests +// pull in OPPOSITE directions -- which is the signal to stop tuning, not to keep +// going: +// +// scaling in GcOverflowSpiral peaked 2159916KB against a 2GB limit. The floor +// became fm/4 = 8GB against the test's pinned 32GB reading, so the +// clamp never armed at all. Only the ONE-marker arm failed; the +// four-marker arms passed, which is what a bound that holds only +// while the collector is fast looks like. +// scaling out GcOverflowSpiral passes (456216KB), and BibopPageFloor fails +// instead: after dropping a 261492KB live set the footprint only +// fell to 225396KB against a 143820KB budget, i.e. the pages were +// not handed back. +// +// Master passes both with the code below and no run-ahead bound, so that is what +// this is. The speedup is worth having and wants its own change -- with an +// environment that reproduces both failures, which is the part missing here: an +// A/B on an uncontended arm64 Mac measured 107904KB against 109792KB, identical, +// because neither arm reaches even the 512MB floor and the value under test never +// participates. A local pass says nothing about any of this. static long long cn1PacingGrowthFloorBytes(void) { return CN1_PACING_GROWTH_FLOOR_BYTES; } @@ -6625,12 +6590,6 @@ static long cn1BibopPacingCap(CODENAME_ONE_THREAD_STATE) { // Kept proportionate rather than absolute: on a host where fm/8 is already // under the saturation point -- a phone, a container, the flat 100MB // placeholder off Apple -- the floor follows fm/8 and nothing loosens. - { - long runAhead = cn1PacingRunAheadBound(fm); - if(runAhead > 0 && capCeiling < runAhead) { - capCeiling = runAhead; - } - } if(cap > capCeiling && cn1PacingPastGrowthFloor()) { cap = capCeiling; } @@ -6653,17 +6612,6 @@ static long cn1BibopPacingCap(CODENAME_ONE_THREAD_STATE) { // saturation point -- a phone, a container, the flat 100MB placeholder off // Apple -- this follows fm/8 and nothing is loosened. `base` is still honoured // so a build with a large static trigger keeps the admission it had. - { - long runAhead = cn1PacingRunAheadBound(fm); - if(runAhead > 0) { - if(cap > runAhead) { - cap = runAhead; - } - if(cap < base) { - cap = base; - } - } - } if(cn1PacingTraceOn()) { long seen = atomic_load_explicit(&cn1PacingMinCap, memory_order_relaxed); while(cap < seen && From 8d2a8f092256ea496fbf2444cb86c081a3e6d542 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 13 Sep 2026 08:06:33 +0300 Subject: [PATCH 54/66] Stop reading embedded resources from getResourceAsStream; it moved app rendering Three screenshot legs -- Windows, the Windows cross-compile, and the third comparison -- reported ValidatorLightweightPicker changed. Decoding both PNGs, the content is IDENTICAL and shifted up about four pixels: everything above the field stack and everything below it is byte-identical, and every differing row matches an actual row at a negative offset. A layout shift, not a rendering fault. A caller that had been falling back on a built-in default now had a file. The cause is a claim I made in this method's own javadoc and did not check: "That keeps every existing target unchanged -- getResourceAsStream returned a hard-coded null before this existed, so nothing can regress, only start working." On master this method is `return null` on EVERY ParparVM target -- the whole resource machinery is this branch's -- so no application has ever received anything from it and every caller has always taken its not-found path. "Only start working" assumed each of those paths was strictly worse than having the resource. Three ports disagreed. A method that previously always failed cannot be given results without moving behaviour, and that is the opposite of "nothing can regress". What self-hosting actually needs is the FILESYSTEM tier, and that tier is opt-in: it answers only when CN1_RESOURCE_PATH names a search root, which the self-hosted translator sets and no application does. So only the embedded tier could return non-null to an app, and only the embedded tier is withdrawn. An application now sees exactly what master gives it, and the translator still finds the C runtime it copies into its output. Removed as a chain rather than left inert, since a native with no Java caller is what NativeSignatureVerifier reports as an ORPHAN: the cn1EmbeddedResource native and its declaration, the weak cn1FindResource, and the strong overrides the Windows and Linux embedders generated. The id table is still built and linked, and a note where each override used to be says that wiring those two halves together is the whole of the future change. Letting applications read their own embedded resources is a good feature; it wants a change where the screenshot baselines it moves are the point rather than the fallout. Also restores .github/workflows/linux-build-run.yml to master EXACTLY. I had reverted it against the pre-merge merge-base, which silently deleted content master added in #5750 (CN1_LINUX_FULL_DEBUG and the gdb install) -- reverting to a base that has since moved, the same shape as the rerere hazard. Gates D and A pass, 798 files byte-identical with the negative control. The file count drops from 800 because ByteArrayInputStream is no longer reachable and the cull drops it, which is the correct consequence. --- .github/workflows/linux-build-run.yml | 76 ++++++++++++++++++- .../tools/translator/ByteCodeTranslator.java | 52 +++---------- vm/ByteCodeTranslator/src/nativeMethods.m | 42 ---------- vm/JavaAPI/src/java/lang/Class.java | 36 +++++---- 4 files changed, 108 insertions(+), 98 deletions(-) diff --git a/.github/workflows/linux-build-run.yml b/.github/workflows/linux-build-run.yml index 3a87ad3b1c6..37c133253fb 100644 --- a/.github/workflows/linux-build-run.yml +++ b/.github/workflows/linux-build-run.yml @@ -263,6 +263,13 @@ jobs: # The Windows pipeline passes a real boolean, which is why its gate # works. 'true' arms it here. CN1_REQUIRE_SUITE: 'true' + # Full DWARF into the .debug companion. The generated project defaults to + # -g1 -- lines and function names only -- so a core from this suite decodes to a + # backtrace and nothing else: every frame answers "No locals". That is what left + # the intermittent SIGSEGV here undiagnosed across four occurrences. The binary + # itself is still stripped; only the companion grows, and it is uploaded beside + # the core so the crash can be autopsied off the runner. + CN1_LINUX_FULL_DEBUG: 'true' # Build the native ELF (and the demo) against an old glibc for portability. CN1_CC: /usr/local/bin/cn1-zig-cc # After the suite runs, the capture test relinks the same objects into a @@ -271,7 +278,17 @@ jobs: LIBGL_ALWAYS_SOFTWARE: '1' run: | set -e - bash scripts/ci/apt-get-install.sh gdb >/dev/null 2>&1 || true + # Not silenced, and the outcome is stated. The hang dump attaches gdb from + # inside the test JVM, so an install that quietly failed here produced a + # hang-stacks.txt with nothing but sample headers -- which is how four + # occurrences of the suite stall ended up with no evidence at all. Still + # best-effort: a runner without gdb must not fail the suite, it must say so. + bash scripts/ci/apt-get-install.sh gdb || echo "WARNING: gdb install failed" + if command -v gdb >/dev/null 2>&1; then + echo "gdb available: $(gdb --version | head -1)" + else + echo "WARNING: gdb is NOT on PATH -- a hang in this job will capture no thread stacks." + fi Xvfb :99 -screen 0 1200x1600x24 >/tmp/xvfb-run.log 2>&1 & export DISPLAY=:99 sleep 2 @@ -340,6 +357,63 @@ jobs: if-no-files-found: warn retention-days: 14 + # The core itself, with everything needed to decode it off the runner: the + # stripped ELF the core refers to and its .debug companion. crash-stacks.txt is + # only ever the backtrace gdb could produce in-place; a real autopsy -- printing + # the object a faulting pointer came from, the slot beside it, the register that + # held it -- needs the core in hand. Cores of this app run to hundreds of MB, so + # they are compressed, capped, and collected ONLY when one exists, which means + # only when the suite actually crashed. + - name: Package core dump for offline autopsy + if: always() + run: | + set -u + shopt -s nullglob + cores=(/tmp/cn1-cores/core.*) + if [ ${#cores[@]} -eq 0 ]; then + echo "no core dumped -- nothing to package" + exit 0 + fi + out="${GITHUB_WORKSPACE}/artifacts/linux-port/core" + mkdir -p "$out" + elf="$(/usr/bin/find /tmp -maxdepth 4 -name LinuxHelloMain -type f 2>/dev/null | head -1)" + if [ -n "$elf" ]; then + cp "$elf" "$out/" || true + [ -f "$elf.debug" ] && cp "$elf.debug" "$out/" || true + fi + for c in "${cores[@]}"; do + sz=$(stat -c%s "$c" 2>/dev/null || echo 0) + echo "core $c is $((sz/1024/1024))MB" + # zstd where available (far faster on a GB-scale core), else gzip. + if command -v zstd >/dev/null 2>&1; then + zstd -q -T0 -3 -o "$out/$(basename "$c").zst" "$c" || true + else + gzip -c "$c" > "$out/$(basename "$c").gz" || true + fi + done + echo "--- packaged ---"; ls -la "$out" + cat > "$out/README.txt" <<'TXT' + Decode this core off the runner: + + zstd -d core.LinuxHelloMain.*.zst # or gunzip for the .gz + gdb ./LinuxHelloMain ./core.LinuxHelloMain.* + + LinuxHelloMain.debug must sit beside the binary -- it is linked by + --add-gnu-debuglink and carries all the DWARF, because the shipped binary is + stripped. Built with CN1_LINUX_FULL_DEBUG so variables and types are present: + "info locals", "p *someObject" and "info registers" all work, which they do + not on a default -g1 build. + TXT + + - name: Upload core dump + if: always() + uses: actions/upload-artifact@v4 + with: + name: linux-core-${{ matrix.arch }} + path: artifacts/linux-port/core + if-no-files-found: ignore + retention-days: 14 + # A runnable, self-contained windowed demo binary for this arch (opens a # GTK window showing a Form -- not the headless suite). Download and run it # on a Linux desktop to smoke-test the native port on real hardware. diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java index d3be7c0a6b7..5b6e22b1648 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java @@ -663,40 +663,11 @@ private static void embedWindowsResources(File[] sources, File srcRoot) throws I table.append(" }\n"); table.append(" return 0;\n"); table.append("}\n\n"); - // Class.getResourceAsStream calls cn1FindResource, which has only a weak - // null-returning definition in nativeMethods -- the id table alone is not - // enough, because the id still has to be resolved to bytes. The image's - // resources stay mapped for the life of the process, so handing back the - // locked pointer is safe. - table.append("#if defined(_WIN32)\n"); - table.append("/* Strong override of the weak cn1FindResource in nativeMethods.\n"); - table.append(" * Off Windows the weak null-returning one stands, which is correct:\n"); - table.append(" * there is no PE resource section to read. */\n"); - table.append("const unsigned char* cn1FindResource(const char* name, int* lenOut) {\n"); - table.append(" int id;\n"); - table.append(" HMODULE module;\n"); - table.append(" HRSRC info;\n"); - table.append(" DWORD size;\n"); - table.append(" HGLOBAL loaded;\n"); - table.append(" void* data;\n"); - table.append(" if (lenOut) { *lenOut = 0; }\n"); - table.append(" id = cn1WinFindResourceId(name);\n"); - table.append(" if (id == 0) { return 0; }\n"); - table.append(" module = GetModuleHandleW(NULL);\n"); - // RT_RCDATA expands to the NARROW MAKEINTRESOURCE; FindResourceW wants - // LPCWSTR. MSVC only warns (C4133) but clang-cl -- the cross-compile path - // this target actually uses -- errors, so spell the wide form explicitly. - table.append(" info = FindResourceW(module, MAKEINTRESOURCEW(id), (LPCWSTR) RT_RCDATA);\n"); - table.append(" if (info == NULL) { return 0; }\n"); - table.append(" size = SizeofResource(module, info);\n"); - table.append(" loaded = LoadResource(module, info);\n"); - table.append(" if (loaded == NULL) { return 0; }\n"); - table.append(" data = LockResource(loaded);\n"); - table.append(" if (data == NULL) { return 0; }\n"); - table.append(" if (lenOut) { *lenOut = (int) size; }\n"); - table.append(" return (const unsigned char*) data;\n"); - table.append("}\n"); - table.append("#endif\n"); + // No cn1FindResource override is emitted. The id table above is built and + // linked, and nothing reads it: Class.getResourceAsStream deliberately does + // not consult embedded resources, because doing so changed how shipping + // applications render (see the comment there). Wiring these together is the + // whole of that future change. sourceManifest.recordGenerated("cn1_resources_table.c"); Util.writeBytes(new File(srcRoot, "cn1_resources_table.c"), table.toString().getBytes(StandardCharsets.UTF_8)); @@ -787,14 +758,11 @@ private static void embedLinuxResources(File[] sources, File srcRoot) throws IOE table.append(" if (lenOut) { *lenOut = 0; }\n"); table.append(" return 0;\n"); table.append("}\n\n"); - // Class.getResourceAsStream calls cn1FindResource, which has only a weak - // null-returning definition in nativeMethods. Without this strong override - // that weak one stands and every embedded resource reads as absent on this - // target -- the table is built, linked, and never consulted. - table.append("/* Strong override of the weak cn1FindResource in nativeMethods. */\n"); - table.append("const unsigned char* cn1FindResource(const char* name, int* lenOut) {\n"); - table.append(" return cn1LinuxFindResource(name, lenOut);\n"); - table.append("}\n"); + // No cn1FindResource override is emitted. The id table above is built and + // linked, and nothing reads it: Class.getResourceAsStream deliberately does + // not consult embedded resources, because doing so changed how shipping + // applications render (see the comment there). Wiring these together is the + // whole of that future change. sourceManifest.recordGenerated("cn1_resources_table.c"); Util.writeBytes(new File(srcRoot, "cn1_resources_table.c"), table.toString().getBytes(StandardCharsets.UTF_8)); diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 9fac29f3a01..a7c9c2965d1 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -1963,48 +1963,6 @@ JAVA_OBJECT java_lang_Class_getName___R_java_lang_String(CODENAME_ONE_THREAD_STA return newStringFromCString(threadStateData, clz->clsName); } -/** - * Resources linked into the executable, backing Class.getResourceAsStream. - * - * cn1FindResource has a weak definition here that finds nothing. A target that - * embeds resources emits a strong one (the generated cn1_resources_table.c) and - * overrides it; everywhere else this one stands and getResourceAsStream falls - * through to the filesystem. That keeps every existing target unchanged -- - * getResourceAsStream returned a hard-coded null before this existed, so nothing - * can regress, only start working. - * - * A weak DEFINITION rather than a weak declaration: Mach-O will not link an - * undefined weak symbol without weak_import, while a weak definition is overridable - * on both Mach-O and ELF. - */ -__attribute__((weak)) const unsigned char* cn1FindResource(const char* name, int* lenOut) { - (void)name; - if(lenOut) { - *lenOut = 0; - } - return 0; -} - -JAVA_OBJECT java_lang_Class_cn1EmbeddedResource___java_lang_String_R_byte_1ARRAY(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT name) { - if(name == JAVA_NULL) { - return JAVA_NULL; - } - const char* n = stringToUTF8(threadStateData, name); - if(n == 0) { - return JAVA_NULL; - } - int len = 0; - const unsigned char* data = cn1FindResource(n, &len); - // A NULL pointer means "not found". A zero LENGTH does not -- an embedded - // resource is allowed to be empty, and getResourceAsStream must hand back an - // empty stream for one rather than null, which callers read as absent. - if(data == 0 || len < 0) { - return JAVA_NULL; - } - JAVA_OBJECT arr = __NEW_ARRAY_JAVA_BYTE(threadStateData, len); - memcpy(((JAVA_ARRAY)arr)->data, data, len); - return arr; -} JAVA_BOOLEAN java_lang_Class_isArray___R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT cls) { struct clazz* clz = (struct clazz*)cls; diff --git a/vm/JavaAPI/src/java/lang/Class.java b/vm/JavaAPI/src/java/lang/Class.java index e4dff54dccb..0e4112103f0 100644 --- a/vm/JavaAPI/src/java/lang/Class.java +++ b/vm/JavaAPI/src/java/lang/Class.java @@ -196,22 +196,32 @@ public java.io.InputStream getResourceAsStream(java.lang.String name){ absolute = lastDot < 0 ? "/" + name : "/" + className.substring(0, lastDot).replace('.', '/') + "/" + name; } - byte[] embedded = cn1EmbeddedResource(absolute); - if (embedded != null) { - return new java.io.ByteArrayInputStream(embedded); - } + // Resources linked INTO the executable are deliberately not consulted here. + // They were, and it changed how shipping applications render. + // + // On master this method is `return null` on every ParparVM target, so no + // application has ever received anything from it, and every caller has + // always taken its not-found path. Reading the embedded table handed some + // of those callers a resource for the first time: the Windows, Linux and + // cross-compiled screenshot legs all began reporting ValidatorLightweight + // Picker as changed, a four-pixel layout shift with identical content -- a + // caller that had been falling back on a built-in default now had a file. + // The javadoc this replaces claimed "nothing can regress, only start + // working", which assumed every not-found path was strictly worse than the + // resource. That assumption was wrong, and three ports disagreed with it. + // + // The filesystem tier below stays, because it is OPT-IN: it answers only + // when CN1_RESOURCE_PATH names a search root, which no application sets and + // the self-hosted translator does. So an application sees exactly what it + // saw on master -- null -- and the translator can still find the C runtime + // it has to copy into its output. + // + // Letting applications read their own embedded resources is a good feature + // and wants its own change, where the screenshot baselines it moves can be + // reviewed as the point of the change rather than as fallout from one. return cn1FileResource(absolute); } - /** - * Resources linked into the executable, or null when there are none. - * - * The native side calls a weakly-linked {@code cn1FindResource}, which the - * generated resource table overrides on targets that embed resources. Where - * nothing provides it the weak symbol is null and this returns null, so a target - * that embeds nothing behaves exactly as it did before this existed. - */ - private static native byte[] cn1EmbeddedResource(String name); /** * The filesystem half of {@link #getResourceAsStream}: looks the resource up From 52f3d4606d8f80c6fcbde6c6ff1923fa2c62b4b5 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 13 Sep 2026 08:34:09 +0300 Subject: [PATCH 55/66] Correct the record: withdrawing the embedded tier did not fix the screenshot The previous commit said the embedded-resource tier caused the Validator LightweightPicker screenshot difference. It did not. The difference persists on 8d2a8f09 with that tier removed, so the claim in that message and in the comment it left behind was wrong, and the cause is still open. The tier stays withdrawn on its own merits rather than that one: handing every caller a resource where master hands it null is a behaviour change for every shipping application, and it is a separate feature from self-hosting, which needs only the opt-in filesystem tier. But that is a scope argument, not evidence, and the comment now says so. Ruled out so far, each by looking rather than reasoning: the Windows resource table generated byte-identical to master's apart from an unused #include; the .rc file is identical IdentityHashMap only MigLayout and SensorSession use it, and this form uses neither translator output diffed master's translator against this branch's over an identical corpus: every difference is an intentional fix -- deterministic labels in place of identity-hash ones, deterministic local-variable declaration order, and the class-init guard moving to __X_LOADED__ with acquire/release. Re-entrant class init is still correctly guarded, because the initializer re-checks .initialized inside a reentrant monitor. Still open: the ArrayList specialised iterator, and the iterator lowering's devirtualization. Both are reachable from layout code and neither is ruled out. Worth recording that the master-versus-branch translator comparison is a gate this branch never had: verify-output-neutral.sh only ever compared the branch against ITSELF, so it could not see a difference from master. That comparison is what ruled the translator out here, and it should exist as a script rather than as something reconstructed by hand each time. --- vm/JavaAPI/src/java/lang/Class.java | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/vm/JavaAPI/src/java/lang/Class.java b/vm/JavaAPI/src/java/lang/Class.java index 0e4112103f0..29ff61e1807 100644 --- a/vm/JavaAPI/src/java/lang/Class.java +++ b/vm/JavaAPI/src/java/lang/Class.java @@ -197,18 +197,21 @@ public java.io.InputStream getResourceAsStream(java.lang.String name){ : "/" + className.substring(0, lastDot).replace('.', '/') + "/" + name; } // Resources linked INTO the executable are deliberately not consulted here. - // They were, and it changed how shipping applications render. // - // On master this method is `return null` on every ParparVM target, so no - // application has ever received anything from it, and every caller has - // always taken its not-found path. Reading the embedded table handed some - // of those callers a resource for the first time: the Windows, Linux and - // cross-compiled screenshot legs all began reporting ValidatorLightweight - // Picker as changed, a four-pixel layout shift with identical content -- a - // caller that had been falling back on a built-in default now had a file. - // The javadoc this replaces claimed "nothing can regress, only start - // working", which assumed every not-found path was strictly worse than the - // resource. That assumption was wrong, and three ports disagreed with it. + // CORRECTION, because the first version of this comment blamed the wrong + // thing: withdrawing this tier did NOT fix the ValidatorLightweightPicker + // screenshot difference, which persists without it. That is still an open + // question about this branch and the cause is elsewhere. + // + // The tier stays withdrawn on its own merits rather than that one. On + // master this method is `return null` on every ParparVM target, so no + // application has ever received anything from it and every caller has + // always taken its not-found path. Handing those callers a resource for the + // first time is a behaviour change for every shipping application, and it + // is a separate feature from self-hosting, which needs only the filesystem + // tier below. The javadoc this replaces claimed "nothing can regress, only + // start working" -- an assumption that every not-found path is strictly + // worse than the resource, which is not something this change established. // // The filesystem tier below stays, because it is OPT-IN: it answers only // when CN1_RESOURCE_PATH names a search root, which no application sets and From 3550d2e9c7a431e030ccc2b9d126d593062543c5 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 13 Sep 2026 14:39:59 +0300 Subject: [PATCH 56/66] Fix the iOS warning-census regression, and PROBE the inline class-init guard Two separate things, one push, because CI is the only instrument that reaches either. THE FIX, which stays: build-ios failed the native warning census with exactly one kind out of baseline -- runtime|java_io_File_runtime.m|-Wshorten-64-to-32 -- [files count] is NSUInteger, 64-bit, handed to allocArray's JAVA_INT length and then re-sent as the loop bound on every iteration. It is narrowed once and explicitly now, and the loop runs on the narrowed value so bound and index share a type. Verified with clang -Wshorten-64-to-32 -Wall: zero warnings. Worth saying why this appeared on THIS branch rather than master: the file was never compiled on the Apple targets before, because the translated java_io_File.c overwrote it -- the collision this branch fixes. Making a dead file live exposes its warnings, which is a real consequence of a real fix and not noise. THE PROBE, which comes back out either way: the inline class-init guard goes back to master's `class__X.initialized` at both emission sites, leaving the rest of the branch alone. Four native screenshot legs report ValidatorLightweightPicker as a ~4px layout shift with identical content, and the JAVASCRIPT screenshots PASS. That split is the reason for this particular probe: JavaScript shares this branch's JavaAPI and its whole front-end optimizer -- the iterator lowering included -- and differs only in the emitter and the runtime. So the JavaAPI suspects and the lowering are both weakened, and a C-emitter change is the place to look. Of those, this is the one with semantics rather than formatting. Master's guard lets a thread proceed once .initialized is set, which happens BEFORE the clinit body runs; this branch waits for __X_LOADED__, set after it completes. A static holding a computed metric read mid-initialization would differ by exactly the kind of small constant this shift looks like -- and note which way that cuts: if this probe goes green, the GOLDEN encodes a read of a partially-initialized class, and the question becomes which rendering is correct rather than how to restore the old one. Ruled out before spending a cycle on it, each by looking: the Windows resource table (generated byte-identical to master's bar an unused include, .rc identical), IdentityHashMap (only MigLayout and SensorSession use it), the java_io_File copy on the clean/Linux/Windows path (identical to master -- only the Apple variant is this branch's), and every other translator output difference (deterministic labels and local-variable order, which cannot move layout). --- .../translator/bytecodes/FusedConstructor.java | 4 ++-- .../translator/bytecodes/TypeInstruction.java | 4 ++-- vm/ByteCodeTranslator/src/java_io_File.m | 14 +++++++++++--- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedConstructor.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedConstructor.java index 8f9e0dae048..78a3303dc3c 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedConstructor.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedConstructor.java @@ -555,8 +555,8 @@ public void appendFusedAlloc(StringBuilder b, String cType, String[] lenExprs, int recvSlot, int survSlot) { b.append(" { /* FUSED construction of ").append(cType).append(" */\n"); // ACQUIRE; see the note in TypeInstruction. - b.append(" if(__builtin_expect(!__atomic_load_n(&__").append(cType) - .append("_LOADED__, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_").append(cType).append("(threadStateData);\n"); + b.append(" if(__builtin_expect(!class__").append(cType) + .append(".initialized, 0)) __STATIC_INITIALIZER_").append(cType).append("(threadStateData);\n"); for (int i = 0; i < children.size(); i++) { b.append(" int __fLen").append(i).append(" = ").append(lenExprs[i]).append(";\n"); } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java index 66bbbb76f0a..9bc19e7c1f1 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java @@ -270,9 +270,9 @@ public void appendInstruction(StringBuilder b, List l) { // in ByteCodeClass. A plain load here let a thread see the flag // set while the vtable / classToInterfaceMap rows it describes // were still invisible. - b.append("if(__builtin_expect(!__atomic_load_n(&__"); + b.append("if(__builtin_expect(!class__"); b.append(type); - b.append("_LOADED__, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_"); + b.append(".initialized, 0)) __STATIC_INITIALIZER_"); b.append(type); b.append("(threadStateData); memset(&__cn1stk_"); b.append(stackAllocId); diff --git a/vm/ByteCodeTranslator/src/java_io_File.m b/vm/ByteCodeTranslator/src/java_io_File.m index 2cd51c822c4..729d2e6c5ad 100644 --- a/vm/ByteCodeTranslator/src/java_io_File.m +++ b/vm/ByteCodeTranslator/src/java_io_File.m @@ -133,9 +133,17 @@ JAVA_OBJECT java_io_File_listImpl___java_lang_String_R_java_lang_String_1ARRAY(C type check, and it hands the collector String metadata for an array payload. cn1MainArgs has always used the array class; these three did not. Fixed on all of them, including the two that predate the Windows arm. */ - JAVA_OBJECT arr = allocArray(threadStateData, [files count], &class_array1__java_lang_String, sizeof(JAVA_OBJECT), 1); - - for (int i=0; i<[files count]; i++) { + /* [files count] is NSUInteger -- 64-bit -- while allocArray's length and the + element setter's index are JAVA_INT. Narrow ONCE and explicitly, and loop on + the narrowed value so the bound and the index have the same type. The + implicit conversion this replaces is what the native warning census caught, + and it only became visible when this file started being compiled at all: the + translated java_io_File.c used to overwrite it on the Apple targets, which is + the collision fixed earlier on this branch. */ + JAVA_INT fileCount = (JAVA_INT)[files count]; + JAVA_OBJECT arr = allocArray(threadStateData, fileCount, &class_array1__java_lang_String, sizeof(JAVA_OBJECT), 1); + + for (JAVA_INT i = 0; i < fileCount; i++) { NSString* f = [files objectAtIndex:i]; JAVA_OBJECT s = fromNSString(CN1_THREAD_STATE_PASS_ARG f); CN1_SET_ARRAY_ELEMENT_OBJECT(arr, i, s); From 8fc9a4c0c726132d931f66bf4fff39488683e3bd Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 13 Sep 2026 15:06:06 +0300 Subject: [PATCH 57/66] Guard class init on an ACQUIRE load of .initialized, not on __X_LOADED__ The bisect probe came back green on two legs, so the inline class-init guard is what moved ValidatorLightweightPicker by four pixels on all four native ports. This keeps the fix and drops the part that caused it, which turns out to be a part that was never needed. The emitted initializer orders its flags like this: class__X.vtable = malloc(...); __INIT_VTABLE_X(...); class__X.initialized = JAVA_TRUE; <- BEFORE the Java java_lang_X___CLINIT____(...); __X_LOADED__ = 1; <- AFTER The defect was VISIBILITY, not the flag's position: a plain load of .initialized let a thread see it set while the vtable and classToInterfaceMap rows it describes were still invisible, which is the SIGSEGV this branch chased. Moving the guard to __X_LOADED__ fixed that -- and, as a side effect nobody asked for, also moved the gate past the whole Java , so other threads are released at a strictly later point than master releases them. That second effect is what the screenshots saw. So the guard goes back to .initialized and gains only the ordering it was missing: master .initialized, plain load vtable race broken, timing baseline this branch __X_LOADED__, acquire race fixed, timing MOVED now .initialized, ACQUIRE race fixed, timing baseline It pairs with the release store already emitted beside the vtable setup, so the publication is correct in both directions. Verified in generated C: 34 inline sites acquire-load .initialized, against the matching __atomic_store_n(..., JAVA_TRUE, __ATOMIC_RELEASE). __X_LOADED__ keeps its acquire load at the TOP of the initializer, where it is a fast path and not a gate: reaching it means the clinit has completed, so returning early is correct and costs no monitor. This is why the choice I was about to put up -- reseed twelve per-port goldens, or drop a fix that took the Windows cross leg from 173/11 to 184/4 -- was a false one. Both horns came from my own fix being broader than the bug. Gates D and A pass, 798 files byte-identical with the negative control. --- .../tools/translator/bytecodes/FusedConstructor.java | 5 +++-- .../tools/translator/bytecodes/TypeInstruction.java | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedConstructor.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedConstructor.java index 78a3303dc3c..ab71a78e33f 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedConstructor.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedConstructor.java @@ -555,8 +555,9 @@ public void appendFusedAlloc(StringBuilder b, String cType, String[] lenExprs, int recvSlot, int survSlot) { b.append(" { /* FUSED construction of ").append(cType).append(" */\n"); // ACQUIRE; see the note in TypeInstruction. - b.append(" if(__builtin_expect(!class__").append(cType) - .append(".initialized, 0)) __STATIC_INITIALIZER_").append(cType).append("(threadStateData);\n"); + b.append(" if(__builtin_expect(!__atomic_load_n(&class__").append(cType) + .append(".initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_").append(cType) + .append("(threadStateData);\n"); for (int i = 0; i < children.size(); i++) { b.append(" int __fLen").append(i).append(" = ").append(lenExprs[i]).append(";\n"); } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java index 9bc19e7c1f1..feb8f2095aa 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java @@ -270,9 +270,9 @@ public void appendInstruction(StringBuilder b, List l) { // in ByteCodeClass. A plain load here let a thread see the flag // set while the vtable / classToInterfaceMap rows it describes // were still invisible. - b.append("if(__builtin_expect(!class__"); + b.append("if(__builtin_expect(!__atomic_load_n(&class__"); b.append(type); - b.append(".initialized, 0)) __STATIC_INITIALIZER_"); + b.append(".initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_"); b.append(type); b.append("(threadStateData); memset(&__cn1stk_"); b.append(stackAllocId); From f20c895eb84a0984bf4383e78f45ff6e319cd6b3 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 13 Sep 2026 15:27:10 +0300 Subject: [PATCH 58/66] Give the neutrality gate a vs-master mode, and drop the now-vestigial LOADED extern verify-output-neutral.sh could not have caught the codegen change that cost this branch a bisect, and the reason is structural: its two modes run the SAME translator twice, so a change present on the branch is present on both sides. It reported neutral because it WAS neutral -- against itself. Four native screenshot legs found what it could not. The new `vs-master` mode runs this branch's translator and master's over one corpus compiled against MASTER's JavaAPI, so JavaAPI is held constant and the translator is the only variable. Deterministic label names and local-variable declaration order are normalised out; everything else is a real codegen change and has to be a deliberate one. Running it immediately earned its keep. It reported 230 differing files, and one delta in them was no longer needed: __X_LOADED__ was made non-static with an extern in every class header purely so the inline guards could reach it across translation units. Those guards test .initialized again, and nothing outside a class's own .c reads the symbol -- verified by scanning the emitted tree for any file referencing another class's LOADED flag, which finds only a comment. So it goes back to internal linkage and the header extern goes away: one exported symbol per class removed from every generated project. That halves the delta to 115 files, and what remains is exactly three annotations: - if(__X_LOADED__) return; + if(__atomic_load_n(&__X_LOADED__, __ATOMIC_ACQUIRE)) return; - class__X.initialized = JAVA_TRUE; + __atomic_store_n(&class__X.initialized, JAVA_TRUE, __ATOMIC_RELEASE); - __X_LOADED__=1; + __atomic_store_n(&__X_LOADED__, 1, __ATOMIC_RELEASE); Memory ordering and nothing else -- no semantic or timing difference from master. The comment in emitClassInitializer still described the withdrawn design, saying the inline guards test __X_LOADED__. It now says what they actually do and why guarding on the completion flag was more than the visibility defect required. Verified: gates D and A pass, 798 files byte-identical with the negative control, and the vm suite is 567 tests with no failures. CI has already confirmed screenshot-comment (x64) green on the narrowed guard. --- .../tools/translator/ByteCodeClass.java | 19 +++---- vm/selfhost/verify-output-neutral.sh | 57 ++++++++++++++++++- 2 files changed, 65 insertions(+), 11 deletions(-) diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java index 03b54b1d160..efbdf68c612 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java @@ -1557,7 +1557,7 @@ public String generateCCode(List allClasses) { // while another thread was still inside the class initialiser, and then // read statics that had not been written yet. Releasing on "started" // cannot publish writes that happen after it. - b.append("int __").append(clsName).append("_LOADED__=0;\n"); + b.append("static int __").append(clsName).append("_LOADED__=0;\n"); b.append("void __STATIC_INITIALIZER_"); b.append(clsName); // ACQUIRE, not a plain load. This is the fast path of a double-checked @@ -1670,9 +1670,14 @@ public String generateCCode(List allClasses) { // initialiser re-enters itself to proceed rather than deadlock, so it has // to be set before __CLINIT__ runs, and the check above the monitor is // that recursion guard. Nothing outside this function may treat it as - // "safe to use the class" -- the inline guards test __X_LOADED__, which is - // stored after __CLINIT__ returns. The release here is still wanted for - // the vtable and classToInterfaceMap rows written just above. + // "safe to use the class" in the JLS sense -- a class under initialization + // is not finished. The release is what the INLINE GUARDS acquire against: + // they test this flag, and it is what publishes the vtable and the + // classToInterfaceMap rows written just above. Guarding them on + // __X_LOADED__ instead would also be correct about the vtable and would + // additionally hold other threads until __CLINIT__ returned -- a strictly + // later gate than master opens, which moved layout on four native ports + // and is not what the visibility defect required. b.append(".initialized, JAVA_TRUE, __ATOMIC_RELEASE);\n"); // init static fields and invoke the static initializer code block if(clInitMethod != null) { @@ -2058,12 +2063,6 @@ public String generateCHeader() { b.append("extern void __STATIC_INITIALIZER_"); b.append(clsName); b.append("(CODENAME_ONE_THREAD_STATE);\n"); - // The COMPLETION flag, for the inline guards. Set with a release store - // after __CLINIT__ returns; see the note where it is defined. - b.append("extern int __"); - b.append(clsName); - b.append("_LOADED__;\n"); - b.append("extern void __FINALIZER_"); b.append(clsName); b.append("(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT objToDelete);\n"); diff --git a/vm/selfhost/verify-output-neutral.sh b/vm/selfhost/verify-output-neutral.sh index eec550dce53..8f691c21f74 100755 --- a/vm/selfhost/verify-output-neutral.sh +++ b/vm/selfhost/verify-output-neutral.sh @@ -19,7 +19,62 @@ J8="${JDK_8_HOME:?set JDK_8_HOME to a working JDK 8}" W="$REPO/vm/selfhost/target/neutral" OUT="$W/out" -case "${1:?usage: capture | compare }" in +case "${1:?usage: capture | compare | vs-master}" in +vs-master) + # Compare THIS branch's translator against MASTER's over one corpus. + # + # This is the comparison the other two modes cannot make. They run the same + # translator twice, so a change that lands on the branch is present on both + # sides and they stay green while every emitted signature moves. That blind + # spot cost a full bisect: four native screenshot legs reported a four-pixel + # layout shift, and the cause was a codegen change this script reported as + # neutral because it was neutral -- against itself. + # + # The corpus is a small app compiled against MASTER's JavaAPI, so JavaAPI is + # held constant and the translator is the only variable. Differences in + # deterministic label names and local-variable declaration order are expected + # and are normalised out; anything else is a real codegen change and should be + # a deliberate one. + MW="${CN1_MASTER_WORKTREE:-/tmp/cn3-master}" + [ -d "$MW/vm" ] || { echo "no master worktree at $MW"; echo " git worktree add $MW origin/master"; exit 1; } + MTR="$MW/vm/ByteCodeTranslator/target/classes" + MAPI="$MW/vm/JavaAPI/target/classes" + for d in "$MTR" "$MAPI"; do + [ -d "$d" ] || { echo "missing $d -- build master's translator and JavaAPI first:"; \ + echo " (cd $MW/vm && mvn -q -B -pl ByteCodeTranslator,JavaAPI package -DskipTests)"; exit 1; } + done + APP="${CN1_NEUTRAL_APP:-/tmp/cmpcls}" + [ -d "$APP" ] || { echo "no corpus app at $APP (set CN1_NEUTRAL_APP)"; exit 1; } + ASM="$(cat "$REPO/vm/ByteCodeTranslator/target/selfhost-asm-classpath.txt")" + rm -rf "$W/m-tree" "$W/b-tree" "$OUT" + for side in m b; do + [ "$side" = m ] && TR="$MTR" || TR="$REPO/vm/ByteCodeTranslator/target/classes" + mkdir -p "$OUT" + ( cd "$W" && env -i PATH=/usr/bin:/bin HOME="$HOME" TMPDIR=/tmp LC_ALL=C \ + "$J8/bin/java" -cp "$TR:$ASM" com.codename1.tools.translator.ByteCodeTranslator \ + clean "$MAPI;$APP" "$OUT" CmpApp com.cmp CmpApp 1.0 clean none ) > "$W/$side.log" 2>&1 \ + || { echo "$side side FAILED"; tail -5 "$W/$side.log"; exit 1; } + mv "$OUT" "$W/$side-tree" + done + # The C runtime is copied verbatim and this branch edits it on purpose, so it is + # not part of the codegen question. + RUNTIME='^(cn1_globals\.[ch]|nativeMethods\.c|cn1_intrinsics\.h|java_io_File_runtime\.c|cn1-source-manifest\.txt)$' + n=0 + for f in "$W/m-tree"/dist/CmpApp-src/*.c "$W/m-tree"/dist/CmpApp-src/*.h; do + [ -f "$f" ] || continue + base="$(basename "$f")" + echo "$base" | grep -qE "$RUNTIME" && continue + other="$W/b-tree/dist/CmpApp-src/$base" + [ -f "$other" ] || { echo "ONLY IN MASTER: $base"; n=$((n+1)); continue; } + if ! diff -q <(sed -E 's/label_L[0-9]+/label_LX/g' "$f") \ + <(sed -E 's/label_L[0-9]+/label_LX/g' "$other") >/dev/null; then + [ $n -lt 12 ] && echo " differs: $base" + n=$((n+1)) + fi + done + echo "VS-MASTER: $n generated file(s) differ beyond label naming" + [ "$n" = 0 ] || echo "Each one is a codegen change against master. Confirm every one is intended." + ;; capture) TAG="${2:?}" TR="$REPO/vm/ByteCodeTranslator/target/classes" From 8c67b53ef8fac398203529069aaef29e747e3a02 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 13 Sep 2026 19:32:44 +0300 Subject: [PATCH 59/66] Review round: the gate I built to catch codegen drift could not fail Seven findings, all against code written today, and three of them against verify-output-neutral.sh itself -- the script added hours earlier precisely to catch this class of problem. Each was real. vs-master could not fail, in three independent ways: * it walked MASTER's file listing, so a generated file the branch emits and master does not was never visited and n stayed 0. It now walks the union of both trees and reports ONLY IN BRANCH / ONLY IN MASTER. * it replaced every label_L with one token, which makes a jump RETARGETED from one existing label to another compare EQUAL -- exactly the regression it exists for. Demonstrated on a fixture: the old normalisation calls the retargeted case identical; the new bijective one, renumbering by first appearance, calls it different and still absorbs the identity-hash-to- sequential rename. * it printed its findings and exited 0, so every caller read a real mismatch as a pass. It exits nonzero now. That last one is this project's own rule -- a check satisfiable by nothing happening is no check -- broken inside a checker written to enforce it. Fixing the labels exposed a fourth problem no reviewer raised: the label number also leaks into catch_L, restoreToL and tryBlockOffsetL, so every try/catch-bearing file would have reported as changed forever and buried any real difference in permanent noise. All four spellings now share one numbering, and the remaining delta is exactly the three intended memory-ordering annotations per class. The other four: Class.cn1FileResource ABANDONED the search when an earlier root held an unusable candidate. exists() is true for a directory, and the IOException path returned null instead of trying the remaining roots, so a valid resource behind a bad one was unreachable -- which defeats the point of a search path. Now isFile() and continue. build-selfhost.sh watched only '*.java' for staleness while the translator carries its C runtime as CLASSPATH RESOURCES. Verified: a cn1_globals.m edit was missed entirely by the old guard, so the self-hosted binary embedded an obsolete runtime while the JVM side used the new one -- a Gate A divergence that points at the VM and is not one. The guard now covers every file, and the staging copies every non-Java resource instead of a hand-listed four that had already drifted. bench-selfhost.sh sampled memory ONCE regardless of ROUNDS while its header promised the maximum of N samples, so one noisy run could decide the reported ratio. It now samples every round and reduces with max. bench-selfhost.sh also hard-coded /Users/shai/.../azul-25/bin/java, so anyone else running the documented command died under set -e while BUILDING the arm list, before a single measurement. JDK 25 is resolved from JDK_25_HOME or PATH. Verified: gates D and A pass, 798 files byte-identical with the negative control; vs-master exits 1 with 119 files, all of them the intended annotations; the staleness guard flips on a .m edit; the label canonicaliser distinguishes a retarget from a rename; vm suite 567 tests, 0 failures. --- vm/JavaAPI/src/java/lang/Class.java | 10 ++++- vm/selfhost/bench-selfhost.sh | 34 ++++++++++++++-- vm/selfhost/build-selfhost.sh | 24 ++++++++--- vm/selfhost/verify-output-neutral.sh | 61 +++++++++++++++++++++++----- 4 files changed, 109 insertions(+), 20 deletions(-) diff --git a/vm/JavaAPI/src/java/lang/Class.java b/vm/JavaAPI/src/java/lang/Class.java index 29ff61e1807..e3cbafc668b 100644 --- a/vm/JavaAPI/src/java/lang/Class.java +++ b/vm/JavaAPI/src/java/lang/Class.java @@ -254,11 +254,17 @@ private static java.io.InputStream cn1FileResource(String absolute) { String root = end < 0 ? path.substring(from) : path.substring(from, end); if (root.length() > 0) { java.io.File candidate = new java.io.File(root, relative); - if (candidate.exists()) { + // isFile(), not exists(): a DIRECTORY with the requested name exists + // and cannot be opened, and returning on that would abandon the + // search. Later roots still get their turn, which is the point of + // having a search path at all -- an earlier root holding an + // unusable candidate must not mask a usable one behind it. + if (candidate.isFile()) { try { return new java.io.FileInputStream(candidate); } catch (java.io.IOException err) { - return null; + // Unreadable here does not mean absent everywhere: keep going. + err = null; } } } diff --git a/vm/selfhost/bench-selfhost.sh b/vm/selfhost/bench-selfhost.sh index ff66b9dcb88..2bc6a7eb991 100755 --- a/vm/selfhost/bench-selfhost.sh +++ b/vm/selfhost/bench-selfhost.sh @@ -35,7 +35,19 @@ PARPAR="${CN1_SELFHOST_BIN:-$T/parpar-O3}" JAPI="$T/javaapi-classes" TR="$REPO/vm/ByteCodeTranslator/target/classes" ASM="$(cat "$REPO/vm/ByteCodeTranslator/target/selfhost-asm-classpath.txt")" -DEFAULT_JAVAS="/Users/shai/Library/Java/JavaVirtualMachines/azul-25/Contents/Home/bin/java,${JDK_8_HOME:-}/bin/java" +# Reference JVMs, resolved rather than hard-coded. A developer-specific absolute +# path here meant anyone else running the documented command died under `set -e` +# while BUILDING the arm list, before a single measurement -- the benchmark was +# runnable by one machine. +# SELFHOST_REF_JAVAS explicit comma-separated list, wins outright +# JDK_25_HOME a modern JDK to compare against +# java on PATH whatever this shell would run +cn1_first_java() { + for c in "${JDK_25_HOME:-}/bin/java" "$(command -v java 2>/dev/null || true)"; do + [ -n "$c" ] && [ -x "$c" ] && { echo "$c"; return; } + done +} +DEFAULT_JAVAS="$(cn1_first_java),${JDK_8_HOME:-}/bin/java" IFS=',' read -r -a REF_JAVAS <<< "${SELFHOST_REF_JAVAS:-$DEFAULT_JAVAS}" W="$T/bench"; rm -rf "$W"; mkdir -p "$W" @@ -110,7 +122,15 @@ for i in "${!ARMS[@]}"; do done # --- memory, measured separately so the probe cannot perturb the clock ---------- -declare -a PEAKS +# +# Sampled in EVERY round and reduced with max, because the header promises the +# maximum of N samples and a peak is a max. Measuring each arm once let a single +# noisy run decide the reported ratio, which is the same mistake as quoting a +# memory figure from one process: the number looked like a measurement and was a +# sample. +declare -a PEAKS PEAK_MAX +for i in "${!ARMS[@]}"; do PEAK_MAX[$i]=0; done +for round in $(seq 1 "$ROUNDS"); do for i in "${!ARMS[@]}"; do rm -rf "$W/run"; mkdir -p "$W/run" if [ "${ARMS[$i]}" = parpar ]; then @@ -121,7 +141,15 @@ for i in "${!ARMS[@]}"; do clean "$JAPI;$CLASSES" "$W/run" "$APP" "$PKG" "$APP" 1.0 clean none 2>"$W/mem.txt" >/dev/null fi PEAKS[$i]=$(awk '/peak memory footprint/{print $1}' "$W/mem.txt") - printf "mem %-8s peak %8.0f MB\n" "${NAMES[$i]}" "$(python3 -c "print(${PEAKS[$i]}/1048576.0)")" + printf "mem %-8s round %d peak %8.0f MB\n" "${NAMES[$i]}" "$round" \ + "$(python3 -c "print(${PEAKS[$i]}/1048576.0)")" + [ "${PEAKS[$i]}" -gt "${PEAK_MAX[$i]}" ] && PEAK_MAX[$i]="${PEAKS[$i]}" +done +done +for i in "${!ARMS[@]}"; do + PEAKS[$i]="${PEAK_MAX[$i]}" + printf "mem %-8s MAX over %d round(s) %8.0f MB\n" "${NAMES[$i]}" "$ROUNDS" \ + "$(python3 -c "print(${PEAKS[$i]}/1048576.0)")" done echo diff --git a/vm/selfhost/build-selfhost.sh b/vm/selfhost/build-selfhost.sh index 301ac24b42d..8108830b6ac 100755 --- a/vm/selfhost/build-selfhost.sh +++ b/vm/selfhost/build-selfhost.sh @@ -38,8 +38,15 @@ TRANSLATOR="$REPO/vm/ByteCodeTranslator/target/classes" needs_build=0 if [ ! -f "$TRANSLATOR/com/codename1/tools/translator/ByteCodeTranslator.class" ]; then needs_build=1 -elif [ -n "$(find "$REPO/vm/ByteCodeTranslator/src" -name '*.java' -newer "$TRANSLATOR" -print -quit 2>/dev/null)" ]; then - echo "translator sources are newer than $TRANSLATOR -- rebuilding" +elif [ -n "$(find "$REPO/vm/ByteCodeTranslator/src" -type f -newer "$TRANSLATOR" -print -quit 2>/dev/null)" ]; then + # -type f, not -name '*.java'. The translator carries its C runtime as CLASSPATH + # RESOURCES -- cn1_globals.m, nativeMethods.m, java_io_File.m, cn1_win_compat.c, + # xmlvm.h and the rest -- and maven copies them into target/classes. Watching + # only Java sources meant editing any of those left the old copy in place, so + # the self-hosted binary embedded an obsolete runtime while the JVM side used + # the new one. That surfaces as a Gate A divergence pointing at the VM, which is + # exactly the misdiagnosis this guard exists to prevent. + echo "translator sources or resources are newer than $TRANSLATOR -- rebuilding" needs_build=1 fi if [ "$needs_build" = 1 ]; then @@ -55,9 +62,16 @@ fi ASM_CP="$(cat "$ASM_CP_FILE")" # 2. the C runtime the translator emits from its own classpath resources. -for f in cn1_globals.h cn1_globals.m nativeMethods.m cn1_intrinsics.h; do - cp "$REPO/vm/ByteCodeTranslator/src/$f" "$TRANSLATOR/$f" -done +# +# Copy EVERY non-Java file maven would have staged, not a hand-listed four. The +# list drifts: java_io_File.m, cn1_win_compat.c and xmlvm.h are all read through +# the same classpath lookup, and a hand-written subset silently ships whichever +# ones nobody remembered. +( cd "$REPO/vm/ByteCodeTranslator/src" && find . -type f ! -name '*.java' -print ) \ + | while read -r rel; do + mkdir -p "$TRANSLATOR/$(dirname "$rel")" + cp "$REPO/vm/ByteCodeTranslator/src/$rel" "$TRANSLATOR/$rel" + done # 3. JavaAPI, rebuilt from source whenever the source set changed. # diff --git a/vm/selfhost/verify-output-neutral.sh b/vm/selfhost/verify-output-neutral.sh index 8f691c21f74..826e2fbbffa 100755 --- a/vm/selfhost/verify-output-neutral.sh +++ b/vm/selfhost/verify-output-neutral.sh @@ -19,6 +19,35 @@ J8="${JDK_8_HOME:?set JDK_8_HOME to a working JDK 8}" W="$REPO/vm/selfhost/target/neutral" OUT="$W/out" +# Renumber label_L by ORDER OF FIRST APPEARANCE within the file, rather than +# erasing every label to one token. Erasing them makes a jump that was retargeted +# from one existing label to another compare EQUAL -- which is precisely the +# code-generation regression this comparison is for. Renumbering keeps the identity +# relationships and still absorbs the switch from identity-hash names to sequential +# ones. +cn1_canon_labels() { + awk '{ + line = $0 + out = "" + # The label number leaks into DERIVED identifiers too -- catch_L, + # restoreToL, tryBlockOffsetL -- so rewriting only label_L left + # every try/catch-bearing file reporting as changed forever, which buries a + # real difference in permanent noise. All four spellings share one numbering. + while (match(line, /(label_L|catch_L|restoreToL|tryBlockOffsetL)[0-9]+/)) { + pre = substr(line, 1, RSTART - 1) + tok = substr(line, RSTART, RLENGTH) + line = substr(line, RSTART + RLENGTH) + # split the prefix from the number so both sides canonicalise together + nstart = match(tok, /[0-9]+$/) + kind = substr(tok, 1, nstart - 1) + num = substr(tok, nstart) + if (!(num in seen)) { seen[num] = ++k } + out = out pre kind seen[num] + } + print out line + }' "$1" +} + case "${1:?usage: capture | compare | vs-master}" in vs-master) # Compare THIS branch's translator against MASTER's over one corpus. @@ -58,22 +87,34 @@ vs-master) done # The C runtime is copied verbatim and this branch edits it on purpose, so it is # not part of the codegen question. - RUNTIME='^(cn1_globals\.[ch]|nativeMethods\.c|cn1_intrinsics\.h|java_io_File_runtime\.c|cn1-source-manifest\.txt)$' + RUNTIME='^(cn1_globals\\.[ch]|nativeMethods\\.c|cn1_intrinsics\\.h|java_io_File_runtime\\.c|cn1-source-manifest\\.txt)$' + # Walk the UNION of both trees, not master's listing. A file the branch emits and + # master does not would never be visited by a master-only loop, so a whole new + # generated class could appear and the gate would report neutral. + ( cd "$W/m-tree/dist/CmpApp-src" 2>/dev/null && ls ) > "$W/m.list" 2>/dev/null || : > "$W/m.list" + ( cd "$W/b-tree/dist/CmpApp-src" 2>/dev/null && ls ) > "$W/b.list" 2>/dev/null || : > "$W/b.list" + sort -u "$W/m.list" "$W/b.list" > "$W/all.list" n=0 - for f in "$W/m-tree"/dist/CmpApp-src/*.c "$W/m-tree"/dist/CmpApp-src/*.h; do - [ -f "$f" ] || continue - base="$(basename "$f")" + while read -r base; do + [ -n "$base" ] || continue echo "$base" | grep -qE "$RUNTIME" && continue - other="$W/b-tree/dist/CmpApp-src/$base" - [ -f "$other" ] || { echo "ONLY IN MASTER: $base"; n=$((n+1)); continue; } - if ! diff -q <(sed -E 's/label_L[0-9]+/label_LX/g' "$f") \ - <(sed -E 's/label_L[0-9]+/label_LX/g' "$other") >/dev/null; then + case "$base" in *.c|*.h) ;; *) continue ;; esac + mf="$W/m-tree/dist/CmpApp-src/$base"; bf="$W/b-tree/dist/CmpApp-src/$base" + if [ ! -f "$mf" ]; then echo " ONLY IN BRANCH: $base"; n=$((n+1)); continue; fi + if [ ! -f "$bf" ]; then echo " ONLY IN MASTER: $base"; n=$((n+1)); continue; fi + if ! diff -q <(cn1_canon_labels "$mf") <(cn1_canon_labels "$bf") >/dev/null; then [ $n -lt 12 ] && echo " differs: $base" n=$((n+1)) fi - done + done < "$W/all.list" echo "VS-MASTER: $n generated file(s) differ beyond label naming" - [ "$n" = 0 ] || echo "Each one is a codegen change against master. Confirm every one is intended." + if [ "$n" != 0 ]; then + echo "Each one is a codegen change against master. Confirm every one is intended." + # Exit NONZERO. Printing a finding and returning 0 makes every caller read a + # real mismatch as a passing check, which is the failure mode this whole + # script exists to prevent. + exit 1 + fi ;; capture) TAG="${2:?}" From 26d6b046575c33146a2792cd6c53c085cad4aaf5 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 13 Sep 2026 20:50:53 +0300 Subject: [PATCH 60/66] Record the three pushbacks in the code, so every review thread has an answer Three findings are correct about the facts and still should not be built. Leaving them as open threads makes that look like an oversight; the reasoning belongs where the next reader hits the code, so it is written there and the threads are closed. Class.forName's initialize flag. Honouring it needs a way to reach a class's static initializer from its Class object, and struct clazz carries no such pointer -- only newInstanceFp and enumValueOfFp. Adding one is a field on EVERY class in EVERY application, for a flag whose only in-tree caller is ASM, which passes false. Cost paid by every app, benefit claimed by none. Emit the pointer if a real caller ever appears. NativeSignatureVerifier's missing main. Nothing names it as an entry point -- check-native-signatures.sh invokes the Cli and no document spells the old command -- and adding a delegating main recreates the edge the split exists to remove: the verifier would reference the Cli, and the Cli reaches java.util.zip, which JavaAPI cannot gain while Ports/CLDC11 mirrors it. A second main also brings back ByteCodeClass's "Multiple main classes" refusal. Relative resources on a NESTED class. Fixing it at the call site means guessing where the package ends, and every guess is wrong for some real input: a package may be named like a class, and a class name may contain '_'. A guess turns today's harmless miss -- a directory named after a class does not exist, so the lookup returns null exactly as it did before this method was implemented -- into a confident wrong answer. The defect is that getName() is lossy, because ParparVM builds the runtime name from the MANGLED form; it is fixed there or not at all. --- .../translator/NativeSignatureVerifierCli.java | 8 ++++++++ vm/JavaAPI/src/java/lang/Class.java | 16 ++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifierCli.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifierCli.java index 83f082c0d93..d5ef2921d96 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifierCli.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifierCli.java @@ -41,6 +41,14 @@ * * A translation never comes through here: the verifier's in-process entry points * are what Parser calls. + * + * NativeSignatureVerifier deliberately keeps NO delegating {@code main}. Nothing + * names it as one -- scripts/check-native-signatures.sh invokes this class, and no + * document spells the old command -- and adding one would recreate the very edge + * the split removes: the verifier would reference the CLI, and the CLI reaches + * java.util.zip. A second {@code main} also brings back the "Multiple main + * classes" refusal above. Backward compatibility for an invocation nobody has is + * not worth either. */ public final class NativeSignatureVerifierCli { private NativeSignatureVerifierCli() { diff --git a/vm/JavaAPI/src/java/lang/Class.java b/vm/JavaAPI/src/java/lang/Class.java index e3cbafc668b..79da8c77e3b 100644 --- a/vm/JavaAPI/src/java/lang/Class.java +++ b/vm/JavaAPI/src/java/lang/Class.java @@ -74,6 +74,15 @@ public ClassLoader getClassLoader() { * JDBC-driver idiom. That pattern cannot work on this platform for a second * reason anyway: obfuscation rewrites class names, so a name looked up as a * string does not survive a release build. + * + * <p>WHY IT IS NOT IMPLEMENTED, rather than left as an oversight: forcing the + * initializer needs a way to reach it from a Class object, and {@code struct + * clazz} carries no static-initializer function pointer -- only newInstanceFp + * and enumValueOfFp. Adding one is a field on EVERY class in EVERY application, + * to serve a flag whose only in-tree caller is ASM, which passes + * {@code initialize = false}. The cost is paid by every app and the benefit is + * claimed by none, so this stays documented rather than built. If a real caller + * ever needs it, emit the pointer then. */ public static java.lang.Class forName(java.lang.String className, boolean initialize, ClassLoader loader) throws java.lang.ClassNotFoundException { @@ -191,6 +200,13 @@ public java.io.InputStream getResourceAsStream(java.lang.String name){ // would turn today's miss into a confidently wrong hit. The fix // belongs in the name the VM reports, which is a change to getName() // for every translated application and wants its own testing. + // + // PUSHBACK, so the next reader does not re-open this: fixing it HERE + // means guessing where the package ends, and every guess is wrong for + // some real input -- a package may legitimately be named like a class, + // and a class name may legitimately contain '_'. A guess would convert + // today's harmless miss into a confident wrong answer. The defect is + // that getName() is lossy; it is fixed there or not at all. String className = getName(); int lastDot = className.lastIndexOf('.'); absolute = lastDot < 0 ? "/" + name From 9a59f2fa1213c3c51b7b15896d2c8c8eb1b360af Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 13 Sep 2026 21:03:48 +0300 Subject: [PATCH 61/66] Two findings I resolved without reading: unique arm names, quiesced exit census The final resolve pass closed FIVE threads, not the three I had triaged -- two arrived between the query and the mutation and I closed them unread. That is the bulk-resolve mistake, and both turned out to be real, so the fixes land here and the threads are now resolved against code that actually exists. bench-selfhost.sh gave two arms the same name, and this one is MY regression from an hour earlier. Discovering JDK 25 from JDK_25_HOME or PATH instead of hard-coding it means the fallback and JDK_8_HOME can both be Java 8, so both arms label as jdk8; every tree, log and diff is filed under the name, so the second arm's `mv` lands INSIDE the first arm's directory and the correctness check then compares a tree with its own nested copy and reports a divergence that is an artefact of naming. Names are deduplicated (jdk8, jdk8#2) and each arm now prints the executable it resolved to, so a suffix is never a mystery. Checked against three same-version arms. cn1BibopExitReport walked the heap under a live collector. atexit runs with the collector still going, and cn1HeapAccounting/cn1LiveCensus/cn1AllocCensus read allObjectsInHeap, object headers and non-atomic page fields -- exactly what a sweep clears, reuses and frees. So the diagnostic could report corrupted totals or dereference a reclaimed object in the batch-program exit case it was added to measure, which is the one case where its numbers would be believed. It now waits for gcCurrentlyRunning to clear, following the shape already used for the pending-table stall. Two properties on purpose: the wait is BOUNDED, because a diagnostic must not turn a hung collector into a hung exit; and on expiry the census is SKIPPED with a message rather than run anyway, because a report read off a heap being swept is worse than no report -- it looks like data. My first attempt called cn1GcRequestStopAndJoin(), which does not exist. Caught by grepping for it rather than trusting that a plausible name was a real one. The replacement is compiled WITH -DCN1_ALLOC_CENSUS against a cleared baseline (zero errors both ways), because the normal build never compiles that block and a green build would have said nothing about it. Gates D and A pass, 798 files byte-identical with the negative control; vm suite 567 tests, 0 failures. --- vm/ByteCodeTranslator/src/cn1_globals.m | 26 +++++++++++++++++++++++++ vm/selfhost/bench-selfhost.sh | 17 +++++++++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index 4bd83e9ec75..6220cd12be2 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -5572,6 +5572,32 @@ static void cn1BibopDoInit() { // ends between collections, so the post-sweep reports alone never show the state // the process actually died holding. static void cn1BibopExitReport(void) { + // QUIESCE FIRST. The three walks below read allObjectsInHeap, object headers and + // non-atomic page fields; a concurrent sweep clears, reuses and frees exactly + // those while they are being read. atexit runs with the collector still live, so + // without this the diagnostic can report corrupted totals or dereference a + // reclaimed legacy object -- in the batch-program exit case it exists to + // measure, which is the one case where it would be believed. + // + // Waiting the cycle OUT is the established shape here (see the ablation arm in + // the pending-table stall) and is what this needs: gcCurrentlyRunning covers + // mark and sweep, so once it clears, allObjectsInHeap and the page fields are + // nobody else's. The wait is BOUNDED -- a diagnostic must not turn a hung + // collector into a hung exit -- and on expiry the census is SKIPPED rather than + // run anyway, because a report read off a heap being swept is worse than no + // report: it looks like data. + { + int waitMs = 0; + while(gcCurrentlyRunning && waitMs < 2000) { + usleep(1000); + waitMs++; + } + if(gcCurrentlyRunning) { + fprintf(stderr, "[HEAP] exit census SKIPPED: collector still running after %dms\n", + waitMs); + return; + } + } cn1HeapAccounting("exit"); cn1LiveCensus("exit"); cn1AllocCensus("exit"); diff --git a/vm/selfhost/bench-selfhost.sh b/vm/selfhost/bench-selfhost.sh index 2bc6a7eb991..4fd8ace5192 100755 --- a/vm/selfhost/bench-selfhost.sh +++ b/vm/selfhost/bench-selfhost.sh @@ -77,7 +77,22 @@ label() { ARMS=(parpar "${REF_JAVAS[@]}") declare -a NAMES -for a in "${ARMS[@]}"; do NAMES+=("$(label "$a")"); done +# Names have to be UNIQUE, not merely descriptive: every tree, log and diff is filed +# under one, so two arms sharing a label make the second `mv` land inside the first +# arm's directory and the correctness check then compares a tree against itself +# nested one level down -- a divergence that is an artefact of naming. Two arms +# collide easily now that JDK 25 is discovered rather than hard-coded: with no +# JDK_25_HOME the PATH fallback and JDK_8_HOME can both be Java 8. +for a in "${ARMS[@]}"; do + base="$(label "$a")" + name="$base"; k=2 + for prev in "${NAMES[@]}"; do + if [ "$prev" = "$name" ]; then name="${base}#${k}"; k=$((k+1)); fi + done + NAMES+=("$name") +done +# Say which executable each arm actually is, so a "#2" suffix is never a mystery. +for i in "${!ARMS[@]}"; do echo "arm : ${NAMES[$i]} -> ${ARMS[$i]}"; done echo "corpus : $CLASSES" echo "arms : ${NAMES[*]} rounds: $ROUNDS" echo "memory : peak phys_footprint (/usr/bin/time -l)" From 64e5fa500f122bb722415e77e9ad53f01b86fb24 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 13 Sep 2026 21:15:16 +0300 Subject: [PATCH 62/66] The self-hosting gates ran with the native verifier OFF, and invented label diffs Two findings, both mine, and the first is the more serious. CN1_NATIVE_VERIFY never reached the translator. parparvm-selfhost.yml sets it to strict at workflow level, and every Gate D/A process is launched through `env -i`, which starts from an EMPTY environment -- so NativeSignatureVerifier.mode() saw nothing and defaulted to OFF. The gate has been reporting a configuration it was not running in. Same shape as the gdb install that never ran and the -D handed to a CMakeLists that did not declare the variable: armed-looking and inert. Forwarded explicitly at all three env -i sites, and FORWARDED rather than hard-coded, so a local run with nothing set behaves exactly as before. Checked directly: the variable now arrives as [strict] where it previously arrived as []. The label canonicaliser generated FALSE POSITIVES. The branch numbers labels with a method-local counter, so L0 and L1 recur in every method, while master derives them from ASM identities that are distinct across a whole file. A file-wide map folds the branch's second L0 onto the first method's token and master's onto a fresh one, so identical output reports as a code-generation difference. It resets per generated function now, which is the scope the names are actually minted in. Demonstrated in both directions on a fixture: file scope calls identical code DIFFERENT, per-function calls it equal, and per-function still catches a jump retargeted from one existing label to another. That is the third distinct defect in this one script -- the first version could not fail at all, the second hid retargets, the third invented differences -- and all three were found by review rather than by the script's own use. Worth remembering before trusting the next checker written here. Gates D and A pass, 798 files byte-identical with the negative control; vs-master exits 1 on its 119 intended annotation files; vm suite 567 tests, 0 failures. --- vm/selfhost/verify-output-neutral.sh | 51 +++++++++++++++++----------- vm/selfhost/verify-selfhost.sh | 7 ++++ 2 files changed, 38 insertions(+), 20 deletions(-) diff --git a/vm/selfhost/verify-output-neutral.sh b/vm/selfhost/verify-output-neutral.sh index 826e2fbbffa..adf3a0aa4aa 100755 --- a/vm/selfhost/verify-output-neutral.sh +++ b/vm/selfhost/verify-output-neutral.sh @@ -26,26 +26,35 @@ OUT="$W/out" # relationships and still absorbs the switch from identity-hash names to sequential # ones. cn1_canon_labels() { - awk '{ - line = $0 - out = "" - # The label number leaks into DERIVED identifiers too -- catch_L, - # restoreToL, tryBlockOffsetL -- so rewriting only label_L left - # every try/catch-bearing file reporting as changed forever, which buries a - # real difference in permanent noise. All four spellings share one numbering. - while (match(line, /(label_L|catch_L|restoreToL|tryBlockOffsetL)[0-9]+/)) { - pre = substr(line, 1, RSTART - 1) - tok = substr(line, RSTART, RLENGTH) - line = substr(line, RSTART + RLENGTH) - # split the prefix from the number so both sides canonicalise together - nstart = match(tok, /[0-9]+$/) - kind = substr(tok, 1, nstart - 1) - num = substr(tok, nstart) - if (!(num in seen)) { seen[num] = ++k } - out = out pre kind seen[num] - } - print out line - }' "$1" + awk ' + # Reset the mapping at every generated FUNCTION boundary. + # + # The branch numbers labels with a METHOD-LOCAL counter, so L0 and L1 recur + # in every method, while master derives them from ASM identities that are + # distinct across the whole file. A file-wide map therefore folds the second + # method'"'"'s L0 onto the first method'"'"'s token on one side and not the other, + # and reports identical output as a codegen difference -- the mirror of the + # erase-everything bug, generating false positives instead of hiding real + # ones. Per-function scope matches how the names are actually minted. + /^[A-Za-z_][A-Za-z0-9_ \*]*\(/ { delete seen; k = 0 } + { + line = $0 + out = "" + # The number leaks into derived identifiers too -- catch_L, + # restoreToL, tryBlockOffsetL -- which must share the numbering + # WITHIN a function or every try/catch file reports as changed forever. + while (match(line, /(label_L|catch_L|restoreToL|tryBlockOffsetL)[0-9]+/)) { + pre = substr(line, 1, RSTART - 1) + tok = substr(line, RSTART, RLENGTH) + line = substr(line, RSTART + RLENGTH) + nstart = match(tok, /[0-9]+$/) + kind = substr(tok, 1, nstart - 1) + num = substr(tok, nstart) + if (!(num in seen)) { seen[num] = ++k } + out = out pre kind seen[num] + } + print out line + }' "$1" } case "${1:?usage: capture | compare | vs-master}" in @@ -80,6 +89,7 @@ vs-master) [ "$side" = m ] && TR="$MTR" || TR="$REPO/vm/ByteCodeTranslator/target/classes" mkdir -p "$OUT" ( cd "$W" && env -i PATH=/usr/bin:/bin HOME="$HOME" TMPDIR=/tmp LC_ALL=C \ + CN1_NATIVE_VERIFY="${CN1_NATIVE_VERIFY:-}" \ "$J8/bin/java" -cp "$TR:$ASM" com.codename1.tools.translator.ByteCodeTranslator \ clean "$MAPI;$APP" "$OUT" CmpApp com.cmp CmpApp 1.0 clean none ) > "$W/$side.log" 2>&1 \ || { echo "$side side FAILED"; tail -5 "$W/$side.log"; exit 1; } @@ -128,6 +138,7 @@ capture) [ -z "$newest" ] || { echo "STALE: $TR older than $newest -- run mvn package first" >&2; exit 1; } rm -rf "$W/$TAG-tree" "$OUT"; mkdir -p "$OUT" ( cd "$W" && env -i PATH=/usr/bin:/bin HOME="$HOME" TMPDIR=/tmp LC_ALL=C \ + CN1_NATIVE_VERIFY="${CN1_NATIVE_VERIFY:-}" \ CN1_RESOURCE_PATH="$REPO/vm/ByteCodeTranslator/src" \ "$J8/bin/java" -cp "$TR:$ASM" com.codename1.tools.translator.ByteCodeTranslator \ clean "$JAPI;$REPO/vm/selfhost/target/asm-classes;$REPO/vm/selfhost/target/classes" \ diff --git a/vm/selfhost/verify-selfhost.sh b/vm/selfhost/verify-selfhost.sh index a1ff647a745..28fb280724d 100755 --- a/vm/selfhost/verify-selfhost.sh +++ b/vm/selfhost/verify-selfhost.sh @@ -56,10 +56,17 @@ W="$REPO/vm/selfhost/target/verify" rm -rf "$W"; mkdir -p "$W" OUT="$W/out" +# CN1_NATIVE_VERIFY is forwarded explicitly. `env -i` starts from an EMPTY +# environment, so a workflow-level `CN1_NATIVE_VERIFY: strict` never reached the +# translator here and NativeSignatureVerifier.mode() defaulted to OFF -- the gate +# reported a mode it was not running in, which is the failure this whole script +# exists to prevent. Forwarded rather than hard-coded so a local run without it set +# behaves as it always did. run() { local tag=$1; shift mkdir -p "$OUT" ( cd "$W" && env -i PATH=/usr/bin:/bin HOME="$HOME" TMPDIR=/tmp LC_ALL=C \ + CN1_NATIVE_VERIFY="${CN1_NATIVE_VERIFY:-}" \ CN1_RESOURCE_PATH="$REPO/vm/ByteCodeTranslator/src" "$@" ) > "$W/$tag.log" 2>&1 \ || { echo "$tag FAILED"; tail -20 "$W/$tag.log"; exit 1; } mv "$OUT" "$W/$tag-tree" From d669466929d8486e7751645fb9fa96d4837786c9 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 13 Sep 2026 21:16:20 +0300 Subject: [PATCH 63/66] Fix the runtime exclusion regex: two backslashes matched nothing The RUNTIME pattern reached grep -E as \\. before each dot, which in an extended regular expression means "a literal backslash followed by any character" -- so none of cn1_globals.[ch], nativeMethods.c, cn1_intrinsics.h, java_io_File_runtime.c or cn1-source-manifest.txt was ever excluded, and the four copied runtime files this branch edits ON PURPOSE were counted as code-generation differences. A single backslash is what the shell value needs; the doubling came from the python heredoc that wrote the line. This is the gap I had already seen and not chased. The hand classification counted 115 differing generated files and vs-master reported 119, and I recorded the difference as uninteresting instead of asking what the extra four were. They were exactly these. Confirmed by the fix: the count drops 119 -> 115 and no runtime file is listed. Fourth defect in this script. The pattern across all four is the same -- each one made the gate quieter or noisier than the truth, and none was found by running it. --- vm/selfhost/verify-output-neutral.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vm/selfhost/verify-output-neutral.sh b/vm/selfhost/verify-output-neutral.sh index adf3a0aa4aa..54073a66a59 100755 --- a/vm/selfhost/verify-output-neutral.sh +++ b/vm/selfhost/verify-output-neutral.sh @@ -97,7 +97,11 @@ vs-master) done # The C runtime is copied verbatim and this branch edits it on purpose, so it is # not part of the codegen question. - RUNTIME='^(cn1_globals\\.[ch]|nativeMethods\\.c|cn1_intrinsics\\.h|java_io_File_runtime\\.c|cn1-source-manifest\\.txt)$' + # Single backslash: this value goes to grep -E, where \. is a literal dot. Two + # backslashes made it "a literal backslash followed by any character", so NOTHING + # matched and the four copied runtime files were counted as codegen differences -- + # which is the 115-versus-119 gap that should have been chased when it appeared. + RUNTIME='^(cn1_globals\.[ch]|nativeMethods\.c|cn1_intrinsics\.h|java_io_File_runtime\.c|cn1-source-manifest\.txt)$' # Walk the UNION of both trees, not master's listing. A file the branch emits and # master does not would never be visited by a master-only loop, so a whole new # generated class could appear and the gate would report neutral. From 8dbdec32d61d94c3429c5ed8f1a3b6b4c8631050 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 14 Sep 2026 04:45:13 +0300 Subject: [PATCH 64/66] Restore the absolute-path gdb install, and stop the GC loop before the census Two fixes, one of them undoing my own mistake. The gdb install. This run hit a SIGSEGV (exitValue=139, 82 of 100 screenshots, stopped in SheetScreenshotTest) and produced NO backtrace, because the step runs under `working-directory: vm` and invokes `bash scripts/ci/apt-get-install.sh gdb` -- which resolves to vm/scripts/ci/apt-get-install.sh, does not exist, and is swallowed by the trailing `|| echo`. gdb has never been installed on that leg, so every post-mortem it has ever run printed nothing while uploading a core nobody could read. 43331747d5 on this branch fixed that. I then reverted the whole workflow to master to drop my crash-hunt scaffolding and took the fix with it -- over-reverting, the same shape as reverting against a stale base earlier. Only the absolute path comes back; none of the scaffolding does. The exit census. The previous "quiesce" was check-then-act and did not close the race the review named: System's GC thread runs `while(gcShouldLoop) { gcMarkSweep(); wait(idle); }`, so waiting for gcCurrentlyRunning to fall leaves it free to raise the flag again in the gap before the walks start, or while they run. gcShouldLoop is cleared FIRST, so no new cycle can begin, and only then is waiting out the in-flight one sufficient. Set directly rather than through System.stopGC(): this runs from atexit, where calling back into Java is a larger promise than a diagnostic should make. The symbol is the generated set_static_java_lang_System_gcShouldLoop, checked against the emitted header rather than guessed -- I invented a plausible-looking cn1GcRequestStopAndJoin() earlier today and it did not exist. Compiles clean with and without -DCN1_ALLOC_CENSUS against a cleared baseline, since the normal build never compiles that block. --- .github/workflows/linux-build-run.yml | 8 +++++++- vm/ByteCodeTranslator/src/cn1_globals.m | 23 ++++++++++++++++------- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/.github/workflows/linux-build-run.yml b/.github/workflows/linux-build-run.yml index 37c133253fb..92fef870db8 100644 --- a/.github/workflows/linux-build-run.yml +++ b/.github/workflows/linux-build-run.yml @@ -283,7 +283,13 @@ jobs: # hang-stacks.txt with nothing but sample headers -- which is how four # occurrences of the suite stall ended up with no evidence at all. Still # best-effort: a runner without gdb must not fail the suite, it must say so. - bash scripts/ci/apt-get-install.sh gdb || echo "WARNING: gdb install failed" + # ABSOLUTE path. This step runs under `working-directory: vm`, so the + # relative form resolved to vm/scripts/ci/apt-get-install.sh, bash answered + # "No such file or directory", and the `|| echo` swallowed it -- gdb has + # NEVER been installed here. Every post-mortem in this job therefore + # produced nothing, including the SIGSEGV this run just hit, which uploads + # a core nobody can read. Same shape as the retry.sh path fixed elsewhere. + bash "$GITHUB_WORKSPACE/scripts/ci/apt-get-install.sh" gdb || echo "WARNING: gdb install failed" if command -v gdb >/dev/null 2>&1; then echo "gdb available: $(gdb --version | head -1)" else diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index 6220cd12be2..ca1f25f3adb 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -5579,14 +5579,23 @@ static void cn1BibopExitReport(void) { // reclaimed legacy object -- in the batch-program exit case it exists to // measure, which is the one case where it would be believed. // - // Waiting the cycle OUT is the established shape here (see the ablation arm in - // the pending-table stall) and is what this needs: gcCurrentlyRunning covers - // mark and sweep, so once it clears, allObjectsInHeap and the page fields are - // nobody else's. The wait is BOUNDED -- a diagnostic must not turn a hung - // collector into a hung exit -- and on expiry the census is SKIPPED rather than - // run anyway, because a report read off a heap being swept is worse than no - // report: it looks like data. + // STOP the loop before waiting on it. Waiting for gcCurrentlyRunning to fall was + // check-then-act and did not close the race: System's GC thread runs + // `while(gcShouldLoop) { gcMarkSweep(); wait(idle); }`, so it can raise the flag + // again the instant the wait expires -- during the gap before these walks start, + // or while they run. Clearing gcShouldLoop first means no NEW cycle can begin, + // and only then is waiting out the in-flight one sufficient. + // + // The flag is set directly rather than through System.stopGC(): this runs from + // atexit, where calling back into Java is a larger promise than a diagnostic + // should make. The GC thread observes it on its next loop test -- immediately if + // it is idling, after the current cycle if it is collecting -- and exits, which + // is exactly the ordering needed here. + set_static_java_lang_System_gcShouldLoop(JAVA_FALSE); { + // BOUNDED: a diagnostic must not turn a hung collector into a hung exit. On + // expiry the census is SKIPPED rather than run anyway, because a report read + // off a heap being swept is worse than no report -- it looks like data. int waitMs = 0; while(gcCurrentlyRunning && waitMs < 2000) { usleep(1000); From 1e5e63b860385d1f896094b43ca4566629902497 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 14 Sep 2026 09:24:45 +0300 Subject: [PATCH 65/66] Detect the reclaim-and-recycle failure the dangling check cannot see The Linux suite SIGSEGV was ArrayList.add running on an object whose class word said com_codename1_charts_compat_Canvas: the list's backing-array slot held two of Canvas's int fields, so the length load faulted. A live object had been reclaimed and its slot recycled under a reference that still pointed at it. CN1_GC_VERIFY did not catch that, and could not: it proves every traced reference RESOLVES, and a recycled slot resolves perfectly well. It holds a valid, live object -- just not the one the field was pointing at. The check that was missing is not about pointer validity at all. So mark functions now pass the field's DECLARED type, which they already knew and threw away, and cn1GcVerifyFieldType asks whether what the field holds is assignable to it: #ifdef CN1_GC_VERIFY cn1GcVerifyFieldType(threadStateData, objToMark, objInstance->..._out, cn1_class_id_java_io_OutputStream, "...out"); #endif That is exactly the question the core answered wrongly, and it names the field at the next collection instead of leaving a SIGSEGV in an unrelated method a thread and a cycle away. Arrays are skipped for now -- their id mapping is dimensional and the observed failure was a plain object field. Behind the ifdef, so a shipping build is unchanged. It COUNTS its own invocations and prints "FIELDTYPE checks=N findings=N" beside the verifier summary, and GcHeapIntegrityIntegrationTest now asserts that line exists with checks > 0. That is not decoration: I inverted the condition to prove the path executed, saw no output, and reported it as dead code -- wrongly, because the app's stderr never reaches the maven log. A detector that silently never runs is indistinguishable from a clean heap, and neither the reading nor my first conclusion could tell them apart. The counter can. GcMarkCompletenessTest is the other half: every non-static object field a class DECLARES must appear in that class's __GC_MARK_, and the base chain must exist -- without the chain, "declared by me" and "marked by me" could both be empty for a whole hierarchy and the check would pass on nothing. It has a negative control proving it notices an untraced field. It PASSES, which rules out a missing traced field as the cause of the reclaim. It first failed on my own assumption that struct obj__X lists only a class's own fields; it flattens the inherited ones, while the mark function delegates them. What is NOT fixed: the reclaim itself. The mark functions are complete, so the reference is lost somewhere else -- the SATB barriers, the root scan, or the sweep's grace rule. The instrument to name it now exists and is proven to run. Gates D and A pass, 798 files byte-identical with the negative control; vm suite 574 tests, 0 failures; cn1_globals.m compiles clean with and without CN1_GC_VERIFY. --- vm/ByteCodeTranslator/src/cn1_globals.h | 5 + vm/ByteCodeTranslator/src/cn1_globals.m | 66 ++++ .../tools/translator/ByteCodeClass.java | 30 ++ vm/ByteCodeTranslator/src/nativeMethods.m | 10 + .../GcHeapIntegrityIntegrationTest.java | 20 ++ .../translator/GcMarkCompletenessTest.java | 304 ++++++++++++++++++ 6 files changed, 435 insertions(+) create mode 100644 vm/tests/src/test/java/com/codename1/tools/translator/GcMarkCompletenessTest.java diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index 1baad97e679..c4618c211c1 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -2860,6 +2860,11 @@ extern struct clazz class_array1__JAVA_DOUBLE; extern struct clazz class_array2__JAVA_DOUBLE; extern struct clazz class_array3__JAVA_DOUBLE; +#ifdef CN1_GC_VERIFY +extern void cn1GcVerifyFieldType(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT owner, JAVA_OBJECT value, + int declaredClassId, const char* fieldName); +#endif +extern _Atomic int cn1GcFrozenForCensus; extern JAVA_OBJECT newString(CODENAME_ONE_THREAD_STATE, int length, JAVA_CHAR data[]); /** * Like newStringFromCString but DECODES, in the PLATFORM's encoding, instead of diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index ca1f25f3adb..7f32e75e6c8 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -5567,6 +5567,16 @@ static void cn1BibopDoInit() { #endif } +// Set by the exit census to stop the collector for good. Read by +// java_lang_System_gcMarkSweep__, which refuses to begin a cycle while it is set -- +// see the note there for why clearing System.gcShouldLoop alone leaves a window. +// +// Defined UNCONDITIONALLY although only the census raises it: nativeMethods.m tests +// it on every cycle, so a build without CN1_ALLOC_CENSUS must still link. The cost +// is one relaxed-path atomic load per collection, against a flag that is always 0 +// in a shipping build. +_Atomic int cn1GcFrozenForCensus = 0; + #ifdef CN1_ALLOC_CENSUS // Registered from cn1BibopDoInit under CN1_HEAP_REPORT. A batch program usually // ends between collections, so the post-sweep reports alone never show the state @@ -5591,6 +5601,11 @@ static void cn1BibopExitReport(void) { // should make. The GC thread observes it on its next loop test -- immediately if // it is idling, after the current cycle if it is collecting -- and exits, which // is exactly the ordering needed here. + // Order matters: raise the freeze BEFORE clearing the loop flag. The freeze is + // what a cycle already in flight -- or one whose thread is between the loop test + // and gcMarkSweep -- will actually honour; gcShouldLoop only stops the thread + // looping round again, and System re-raises it on its start-up path. + atomic_store_explicit(&cn1GcFrozenForCensus, 1, memory_order_release); set_static_java_lang_System_gcShouldLoop(JAVA_FALSE); { // BOUNDED: a diagnostic must not turn a hung collector into a hung exit. On @@ -10012,7 +10027,13 @@ void cn1GcVerifyChild(JAVA_OBJECT child, void* markSite) { // after the sweep, before the collector hands the world back, so the freed // memory it is looking for has had the least possible chance of being // recycled into something plausible again. +static _Atomic long cn1GcFieldTypeChecks = 0; +static _Atomic long cn1GcFieldTypeFindings = 0; + static void cn1GcVerifySummary(void) { + fprintf(stderr, "[GC-VERIFY] FIELDTYPE checks=%ld findings=%ld\n", + atomic_load_explicit(&cn1GcFieldTypeChecks, memory_order_relaxed), + atomic_load_explicit(&cn1GcFieldTypeFindings, memory_order_relaxed)); fprintf(stderr, "[GC-VERIFY] SUMMARY passes=%ld refs=%ld violations=%ld earlyFreed=%ld resurrected=%ld resurrectedDangling=%ld\n", cn1GcVerifyPasses, cn1GcVerifyTotalRefs, cn1GcVerifyTotalViolations, cn1GcVerifyEarlyFreed, cn1GcResTotal, cn1GcResDangling); @@ -11647,6 +11668,51 @@ static inline void cn1BibopStampMarked(JAVA_OBJECT obj, int markVal, int graceOn #define CN1_BIBOP_STAMP_MARKED_GRACE(o, m, snap) do {} while(0) #endif +#ifdef CN1_GC_VERIFY +/** + * Verifier builds only: is what this reference field HOLDS assignable to what it was + * DECLARED as? + * + * The existing verifier proves every traced reference resolves, and that is precisely + * why a reclaimed-then-recycled slot walks past it -- the slot holds a valid, live + * object, just not the one the field pointed at. The observed consequence was + * ArrayList.add running on a charts.compat.Canvas and faulting on the backing-array + * length load, a whole cycle and a thread away from the reclaim that caused it. + * + * Reported, not fatal: this runs inside the mark, where aborting would lose the rest + * of the census, and one line naming the field is what the failure has always + * lacked. Tagged values and null carry no header and are skipped. + */ +void cn1GcVerifyFieldType(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT owner, JAVA_OBJECT value, + int declaredClassId, const char* fieldName) { + // COUNTED, and the count is printed with the summary. A detector that silently + // never runs is indistinguishable from a clean heap -- an inverted-condition + // probe of the first version of this function produced no output at all, which + // is how that was discovered rather than shipped. + atomic_fetch_add_explicit(&cn1GcFieldTypeChecks, 1, memory_order_relaxed); + if(value == JAVA_NULL || CN1_IS_TAGGED(value)) { + return; + } + // Only ask about a pointer the collector already believes in; an unresolvable one + // is the OTHER verifier's finding and reporting it twice helps nobody. + if(cn1ConservativeResolve((void*)value) != value && !cn1GcImmortalObjContains(value)) { + return; + } + struct clazz* actual = CN1_CLASS_OF(value); + if(actual == 0) { + return; + } + if(!instanceofFunction(declaredClassId, actual->classId)) { + fprintf(stderr, + "[GC-VERIFY] TYPE CONFUSION: %s holds a %s, which is not assignable to its " + "declared type (owner %p, value %p). A live object was reclaimed and its slot " + "recycled.\n", + fieldName, actual->clsName ? actual->clsName : "?", (void*)owner, (void*)value); + atomic_fetch_add_explicit(&cn1GcFieldTypeFindings, 1, memory_order_relaxed); + } +} +#endif + void gcMarkObject(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT obj, JAVA_BOOLEAN force) { if(obj == JAVA_NULL || CN1_IS_TAGGED(obj)) { return; diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java index efbdf68c612..638b0de938a 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java @@ -1227,6 +1227,36 @@ public String generateCCode(List allClasses) { b.append(", objInstance->").append(REFERENCE_CLASS).append("_cn1Strength);\n"); continue; } + // TYPE-IDENTITY CHECK, verifier builds only. + // + // CN1_GC_VERIFY already proves every traced reference RESOLVES, which + // is why a reclaimed-and-recycled slot slips past it: the slot holds a + // perfectly valid object, just not the one the field was pointing at. + // A Linux suite core caught the consequence -- ArrayList.add running on + // an object whose class word said charts.compat.Canvas, reading the + // list's backing-array slot out of two of Canvas's int fields. + // + // The field's DECLARED type is known here and thrown away, so the + // collector has no way to notice. Passing it lets the verifier ask + // whether what the field holds is assignable to what it was declared + // as, which is exactly the question a recycled slot answers wrongly -- + // and it names the field, instead of leaving a SIGSEGV in an unrelated + // method a whole cycle later. + // + // Arrays are skipped for now: their id mapping is dimensional and the + // failure this was written for was a plain object field. + // getRuntimeDescriptor() is the mangled type for a plain object field + // and carries "[]" for an array, which is how arrays are excluded. + String fldType = fld.getRuntimeDescriptor(); + if (fldType != null && fldType.indexOf('[') < 0 + && Parser.getClassObject(fldType) != null) { + b.append("#ifdef CN1_GC_VERIFY\n"); + b.append(" cn1GcVerifyFieldType(threadStateData, objToMark, objInstance->"); + b.append(fld.getClsName()).append("_").append(fld.getFieldName()); + b.append(", cn1_class_id_").append(fldType); + b.append(", \"").append(clsName).append(".").append(fld.getFieldName()).append("\");\n"); + b.append("#endif\n"); + } b.append(" gcMarkObject(threadStateData, "); if (fld.isVolatile()) { b.append("atomic_load_explicit(&objInstance->"); diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index a7c9c2965d1..c41c81456b8 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -2537,6 +2537,16 @@ JAVA_VOID java_lang_System_gcLight__(CODENAME_ONE_THREAD_STATE) { int cn1GcProbeThrew = 0; #endif JAVA_VOID java_lang_System_gcMarkSweep__(CODENAME_ONE_THREAD_STATE) { + // FREEZE: refuse to start a cycle at all once the exit census has claimed the + // heap. Clearing System.gcShouldLoop is not sufficient on its own -- the GC + // thread may already have evaluated `while(gcShouldLoop)` and be on its way + // here, and System's start-up path re-raises that flag after its initial wait. + // Either way the census would see gcCurrentlyRunning false, start walking, and + // have the pending cycle resume and sweep underneath it. Checked here because + // this is the one door every cycle comes through. + if(atomic_load_explicit(&cn1GcFrozenForCensus, memory_order_acquire)) { + return; + } gcCurrentlyRunning = JAVA_TRUE; if(firstTimeGcThread) { firstTimeGcThread = JAVA_FALSE; diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/GcHeapIntegrityIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/GcHeapIntegrityIntegrationTest.java index 614023a7b0c..4f7826b7f68 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/GcHeapIntegrityIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/GcHeapIntegrityIntegrationTest.java @@ -169,6 +169,26 @@ private void runGate(List tempDirs) throws Exception { assertTrue(!clean.output.contains("DANGLING REFERENCE"), "The sweep left a surviving object pointing at reclaimed memory.\n" + violationExcerpt(clean.output)); + // A recycled slot is NOT dangling: it holds a live, valid object, just not + // the one the field pointed at. That is why the dangling check above passed + // through the failure a Linux core caught -- ArrayList.add running on a + // charts.compat.Canvas. cn1GcVerifyFieldType asks the other question, whether + // what a field HOLDS is assignable to what it was DECLARED as. + assertTrue(!clean.output.contains("TYPE CONFUSION"), + "A reference field holds an object of an unrelated type -- a live " + + "object was reclaimed and its slot recycled.\n" + + violationExcerpt(clean.output)); + // And prove that detector RAN. Inverting its condition on the first version + // produced no output whatever, which is how it was found to be checking + // nothing; silence from a detector that never executes is indistinguishable + // from silence from a clean heap. + java.util.regex.Matcher ft = java.util.regex.Pattern + .compile("FIELDTYPE checks=(\\d+) findings=(\\d+)").matcher(clean.output); + assertTrue(ft.find(), + "the field-type verifier never reported, so it did not run: " + clean.output); + assertTrue(Long.parseLong(ft.group(1)) > 0, + "the field-type verifier ran but checked no field, which is not a pass: " + + ft.group(0)); assertTrue(clean.output.contains("GC_VERIFY_APP_DONE"), "The workload should run to completion. Output: " + clean.output); // A workload that never finishes a collection cycle never runs the diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/GcMarkCompletenessTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/GcMarkCompletenessTest.java new file mode 100644 index 00000000000..62a89c00af8 --- /dev/null +++ b/vm/tests/src/test/java/com/codename1/tools/translator/GcMarkCompletenessTest.java @@ -0,0 +1,304 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; + +/** + * Every non-static object field a class declares must be traced by that class's + * {@code __GC_MARK_} function. + * + * A field the collector cannot see is a live object it will reclaim, and the + * failure is neither an exception nor a null dereference: the slot is recycled, + * some later object moves in, and the next method called through the stale + * reference reads ITS OWN field layout out of an unrelated object. That is + * indistinguishable from the unchecked-CHECKCAST hazard and just as invisible -- + * a SIGSEGV a long way from the cause, with no Java frame that could catch it. + * + * The shape is not hypothetical. A Linux suite core showed + * java_util_ArrayList_add running on an object whose class word said + * com_codename1_charts_compat_Canvas: the list's backing-array slot held two of + * Canvas's int fields, so the length load faulted. The list was + * Display.pendingIdleSerialCalls, reachable from a static root through a + * private final instance field, and the Canvas in its place was itself live + * (mark epoch 18, not the -1 that means fresh) -- a recycled slot, not garbage. + * + * What this checks is the ONE structural property that makes such a reclaim + * possible from the translator's side: ByteCodeClass emits a mark body from + * `fullFieldList`, filtered to non-static object fields DECLARED by the class + * (inherited ones are the base class's mark function's job). Anything that + * makes a field fall out of that filter -- a descriptor the parser types + * wrongly, a new field kind, a refactor of isObjectType -- silently stops the + * field being traced. Reading the emitted C is the only place that assumption + * is observable. + */ +class GcMarkCompletenessTest { + + /** struct obj__X { ... } -- the layout the mark function has to cover. */ + private static final Pattern STRUCT = + Pattern.compile("struct obj__(\\w+)\\s*\\{(.*?)\\n\\};", Pattern.DOTALL); + /** void __GC_MARK_X(...) { ... } */ + private static final Pattern MARKFN = + Pattern.compile("void __GC_MARK_(\\w+)\\(CODENAME_ONE_THREAD_STATE[^)]*\\)\\s*\\{(.*?)\\n\\}", + Pattern.DOTALL); + /** A JAVA_OBJECT member, i.e. exactly what the collector must follow. */ + private static final Pattern OBJ_FIELD = + Pattern.compile("^\\s*JAVA_OBJECT\\s+(\\w+)\\s*;", Pattern.MULTILINE); + + @Test + void everyDeclaredObjectFieldIsTracedByItsMarkFunction() throws Exception { + Path classes = Files.createTempDirectory("gcmark-classes"); + Path out = Files.createTempDirectory("gcmark-out"); + Path src = Files.createTempDirectory("gcmark-src"); + + // Deliberately covers the shapes that have gone wrong or could: a field + // declared on a BASE class and inherited, a collection field like the one + // the core implicated, an array field, an interface-typed field, and a + // class whose object fields sit among primitives so an offset mistake is + // visible. + Path app = src.resolve("GcMarkApp.java"); + Files.write(app, ("import java.util.*;\n" + + "class MarkBase { Object baseRef; int basePrim; }\n" + + "class MarkMid extends MarkBase { String midRef; }\n" + + "class MarkLeaf extends MarkMid {\n" + + " final ArrayList pending = new ArrayList();\n" + + " int a; Object mixedOne; long b; String[] arrayRef; int c;\n" + + " Runnable iface; Map mapRef;\n" + + "}\n" + + "public class GcMarkApp {\n" + + " static MarkLeaf keep;\n" + + " public static void main(String[] args) {\n" + + " keep = new MarkLeaf();\n" + + " keep.pending.add(new Runnable(){ public void run(){} });\n" + + " keep.mixedOne = new Object();\n" + + " keep.arrayRef = new String[2];\n" + + " keep.iface = new Runnable(){ public void run(){} };\n" + + " keep.mapRef = new HashMap();\n" + + " keep.baseRef = new Object();\n" + + " keep.midRef = \"x\";\n" + + " System.out.println(keep.pending.size());\n" + + " }\n" + + "}\n").getBytes(StandardCharsets.UTF_8)); + + CompilerHelper.CompilerConfig config = selectCompiler(); + org.junit.jupiter.api.Assumptions.assumeTrue(config != null, + "no compiler available that targets a JavaAPI-compatible bytecode level"); + + Path javaApi = Files.createTempDirectory("gcmark-java-api"); + CompilerHelper.compileJavaAPI(javaApi, config); + + List args = new ArrayList(); + args.add("-source"); args.add(config.targetVersion); + args.add("-target"); args.add(config.targetVersion); + if (CompilerHelper.useClasspath(config)) { + args.add("-classpath"); args.add(javaApi.toString()); + } else { + args.add("-bootclasspath"); args.add(javaApi.toString()); + args.add("-Xlint:-options"); + } + args.add("-nowarn"); + args.add("-d"); args.add(classes.toString()); + args.add(app.toString()); + assertTrue(CompilerHelper.compile(config.jdkHome, args) == 0, + "the fixture must compile against JavaAPI"); + + // The translator needs the class library beside the app, as every other + // integration test here stages it. + CompilerHelper.copyDirectory(javaApi, classes); + CleanTargetIntegrationTest.runTranslator(classes, out, "GcMarkApp"); + Path srcRoot = findSrcRoot(out); + + List missing = new ArrayList(); + int classesChecked = 0; + int fieldsChecked = 0; + + try (Stream files = Files.walk(srcRoot)) { + for (Path c : (Iterable) files.filter(p -> p.toString().endsWith(".c"))::iterator) { + String body = new String(Files.readAllBytes(c), StandardCharsets.ISO_8859_1); + Path header = c.resolveSibling(c.getFileName().toString().replaceAll("\\.c$", ".h")); + if (!Files.exists(header)) { + continue; + } + String head = new String(Files.readAllBytes(header), StandardCharsets.ISO_8859_1); + + Matcher mf = MARKFN.matcher(body); + while (mf.find()) { + String cls = mf.group(1); + String markBody = mf.group(2); + // struct obj__X FLATTENS the inherited fields, but the mark + // function deliberately marks only what the class DECLARES and + // chains to its base for the rest -- so requiring every struct + // member here would demand that Error re-mark Throwable's fields. + // The mangled name carries its declaring class, which is the same + // filter ByteCodeClass applies (fld.getClsName().equals(clsName)). + Set declared = new LinkedHashSet(); + for (String f : declaredObjectFields(head, cls)) { + if (f.startsWith(cls + "_")) { + declared.add(f); + } + } + if (declared.isEmpty()) { + continue; + } + classesChecked++; + for (String f : declared) { + fieldsChecked++; + // The emitted body names the field directly, whether it goes + // through gcMarkObject, gcMarkArrayObject or + // cn1GcDiscoverReference (the WeakReference referent, which is + // deliberately not traced but IS handed to the collector). + if (!markBody.contains(f)) { + missing.add(cls + "." + f); + } + } + } + } + } + + // The filter above is only sound because a class DELEGATES to its base, so + // check the delegation actually exists wherever the base declares object + // fields. Without this, "declared by me" and "marked by me" could both be + // empty for a whole hierarchy and the test would still pass. + List brokenChain = new ArrayList(); + try (Stream files2 = Files.walk(srcRoot)) { + for (Path c : (Iterable) files2.filter(p -> p.toString().endsWith(".c"))::iterator) { + String body = new String(Files.readAllBytes(c), StandardCharsets.ISO_8859_1); + Matcher mf = MARKFN.matcher(body); + while (mf.find()) { + String cls = mf.group(1); + String markBody = mf.group(2); + String base = baseOf(body, cls); + if (base != null && !base.equals("java_lang_Object") + && !markBody.contains("__GC_MARK_" + base)) { + brokenChain.add(cls + " -> " + base); + } + } + } + } + assertTrue(brokenChain.isEmpty(), + "__GC_MARK_ must chain to the base class, or the base's declared fields " + + "are traced by nobody: " + brokenChain); + + // A pass that inspected nothing is not a pass. The fixture alone declares + // eight object fields across three classes in one hierarchy. + assertTrue(classesChecked >= 3, + "expected to inspect several classes, saw " + classesChecked); + assertTrue(fieldsChecked >= 8, + "expected to inspect the fixture's object fields, saw " + fieldsChecked); + assertTrue(missing.isEmpty(), + "object field(s) declared but never traced by the class's __GC_MARK_ function -- " + + "the collector cannot see them, so it will reclaim live objects and " + + "recycle their slots: " + missing); + } + + /** + * Proves the check can fail, by deleting one field's mark from a body and + * confirming the comparison notices. A gate nobody has watched fail is not a + * gate, and this one is a string search over generated code -- exactly the kind + * that silently matches everything or nothing. + */ + @Test + void theCheckDetectsAnUntracedField() { + String head = "struct obj__Foo {\n JAVA_OBJECT Foo_kept;\n JAVA_OBJECT Foo_dropped;\n};"; + Set declared = declaredObjectFields(head, "Foo"); + assertTrue(declared.contains("Foo_kept") && declared.contains("Foo_dropped"), + "fixture parse: " + declared); + String markBody = " gcMarkObject(threadStateData, objInstance->Foo_kept, force);"; + List missing = new ArrayList(); + for (String f : declared) { + if (!markBody.contains(f)) { + missing.add(f); + } + } + assertFalse(missing.isEmpty(), "the check must notice a field that is not marked"); + assertTrue(missing.contains("Foo_dropped") && missing.size() == 1, + "it must name exactly the untraced field, got " + missing); + } + + /** The base class name from the emitted `struct clazz` initialiser, or null. */ + private static String baseOf(String body, String cls) { + Matcher m = Pattern.compile("struct clazz class__" + Pattern.quote(cls) + + "\\s*=\\s*\\{(.*?)\\};", Pattern.DOTALL).matcher(body); + if (!m.find()) { + return null; + } + Matcher b = Pattern.compile("&class__(\\w+)\\s*,\\s*(?:base_interfaces|EMPTY_INTERFACES)").matcher(m.group(1)); + return b.find() ? b.group(1) : null; + } + + private CompilerHelper.CompilerConfig selectCompiler() { + String[] preferredTargets = {"11", "17", "21", "25", "1.8"}; + for (String target : preferredTargets) { + for (CompilerHelper.CompilerConfig c : CompilerHelper.getAvailableCompilers(target)) { + if (CompilerHelper.isJavaApiCompatible(c)) { + return c; + } + } + } + return null; + } + + private static Set declaredObjectFields(String header, String cls) { + Set out = new LinkedHashSet(); + Matcher s = STRUCT.matcher(header); + while (s.find()) { + if (!s.group(1).equals(cls)) { + continue; + } + Matcher f = OBJ_FIELD.matcher(s.group(2)); + while (f.find()) { + String name = f.group(1); + // The object header's own slots are not Java fields. + if (name.startsWith("__codenameOne") || name.equals("__heapPosition")) { + continue; + } + out.add(name); + } + } + return out; + } + + private static Path findSrcRoot(Path out) throws IOException { + try (Stream w = Files.walk(out)) { + return w.filter(Files::isDirectory) + .filter(p -> p.getFileName().toString().endsWith("-src")) + .findFirst() + .orElseThrow(() -> new IOException("no generated -src directory under " + out)); + } + } +} From 1c784ef8811429650329a143823a9a2dc245f51a Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 14 Sep 2026 12:43:56 +0300 Subject: [PATCH 66/66] Make the census handshake a single atomic claim; a flag could not close it Third review on this handshake, third real window, and the previous two fixes were each narrower than the race: 1. wait for gcCurrentlyRunning to fall check-then-act; the loop starts another cycle immediately 2. clear System.gcShouldLoop first the GC thread can already be past the loop test, and start-up re-raises it 3. freeze flag + gcCurrentlyRunning the collector can load the freeze as 0 and be preempted BEFORE setting the active bit; the census then sees an idle collector and walks in behind it The error each time was treating this as a flag problem -- wait for the flag, clear it sooner, add a second one -- when the defect is that a flag and an active bit are two separate observations and the window lives BETWEEN them. No ordering of two stores removes it. So there is no flag now. One state, and every transition is a compare-exchange: IDLE --CAS--> RUNNING a collector starts a cycle IDLE --CAS--> FROZEN the census takes the heap RUNNING --CAS--> IDLE a cycle finishes, and cannot clobber FROZEN The census may walk only once it has itself won IDLE -> FROZEN, after which no cycle can begin, because beginning one means winning IDLE -> RUNNING. There is no half-state for either side to observe, so the property holds by construction rather than by my reasoning about timing -- which is the only form worth trusting after three wrong attempts at exactly that reasoning. Checked that this is not a behaviour change: System's GC thread is the ONLY caller of gcMarkSweep (its `while(gcShouldLoop)` loop), so no second entrant can observe RUNNING and the refusal path is only ever the freeze. Refusing on RUNNING as well is defence, not policy, and the code says so. Compiles clean with no defines, with -DCN1_ALLOC_CENSUS and with -DCN1_GC_VERIFY. Gates D and A pass, 798 files byte-identical with the negative control; the CN1_GC_VERIFY build passes, still reporting FIELDTYPE checks > 0; vm suite 574 tests, 0 failures. --- vm/ByteCodeTranslator/src/cn1_globals.h | 5 +++- vm/ByteCodeTranslator/src/cn1_globals.m | 36 +++++++++++++++-------- vm/ByteCodeTranslator/src/nativeMethods.m | 27 +++++++++++++++-- 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index c4618c211c1..bf870e0c375 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -2864,7 +2864,10 @@ extern struct clazz class_array3__JAVA_DOUBLE; extern void cn1GcVerifyFieldType(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT owner, JAVA_OBJECT value, int declaredClassId, const char* fieldName); #endif -extern _Atomic int cn1GcFrozenForCensus; +#define CN1_GC_CYCLE_IDLE 0 +#define CN1_GC_CYCLE_RUNNING 1 +#define CN1_GC_CYCLE_FROZEN 2 +extern _Atomic int cn1GcCycleState; extern JAVA_OBJECT newString(CODENAME_ONE_THREAD_STATE, int length, JAVA_CHAR data[]); /** * Like newStringFromCString but DECODES, in the PLATFORM's encoding, instead of diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index 7f32e75e6c8..a1af1f735fc 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -5567,15 +5567,16 @@ static void cn1BibopDoInit() { #endif } -// Set by the exit census to stop the collector for good. Read by -// java_lang_System_gcMarkSweep__, which refuses to begin a cycle while it is set -- -// see the note there for why clearing System.gcShouldLoop alone leaves a window. +// The collector's cycle claim: IDLE -> RUNNING by the collector, IDLE -> FROZEN by +// the exit census, and RUNNING -> IDLE when a cycle finishes. Every transition is a +// compare-exchange, so the two participants can never both believe they hold the +// heap -- which a freeze flag read separately from gcCurrentlyRunning could not +// guarantee, because the collector can be preempted between the two. // -// Defined UNCONDITIONALLY although only the census raises it: nativeMethods.m tests -// it on every cycle, so a build without CN1_ALLOC_CENSUS must still link. The cost -// is one relaxed-path atomic load per collection, against a flag that is always 0 -// in a shipping build. -_Atomic int cn1GcFrozenForCensus = 0; +// Defined UNCONDITIONALLY although only the census freezes: nativeMethods.m claims +// on every cycle, so a build without CN1_ALLOC_CENSUS must still link. The cost is +// one uncontended CAS per collection. +_Atomic int cn1GcCycleState = CN1_GC_CYCLE_IDLE; #ifdef CN1_ALLOC_CENSUS // Registered from cn1BibopDoInit under CN1_HEAP_REPORT. A batch program usually @@ -5605,19 +5606,30 @@ static void cn1BibopExitReport(void) { // what a cycle already in flight -- or one whose thread is between the loop test // and gcMarkSweep -- will actually honour; gcShouldLoop only stops the thread // looping round again, and System re-raises it on its start-up path. - atomic_store_explicit(&cn1GcFrozenForCensus, 1, memory_order_release); + // Stop the loop re-arming, then WIN the heap rather than wait for a flag. The + // census may only walk once it has moved the state IDLE -> FROZEN itself: after + // that no cycle can start, because starting one means winning IDLE -> RUNNING. + // Polling gcCurrentlyRunning instead left the window this replaces -- a collector + // preempted between its check and setting that flag. set_static_java_lang_System_gcShouldLoop(JAVA_FALSE); { // BOUNDED: a diagnostic must not turn a hung collector into a hung exit. On // expiry the census is SKIPPED rather than run anyway, because a report read // off a heap being swept is worse than no report -- it looks like data. int waitMs = 0; - while(gcCurrentlyRunning && waitMs < 2000) { + int frozen = 0; + while(waitMs < 2000) { + int expected = CN1_GC_CYCLE_IDLE; + if(atomic_compare_exchange_strong_explicit(&cn1GcCycleState, &expected, + CN1_GC_CYCLE_FROZEN, memory_order_acq_rel, memory_order_acquire)) { + frozen = 1; + break; + } usleep(1000); waitMs++; } - if(gcCurrentlyRunning) { - fprintf(stderr, "[HEAP] exit census SKIPPED: collector still running after %dms\n", + if(!frozen) { + fprintf(stderr, "[HEAP] exit census SKIPPED: could not freeze the collector in %dms\n", waitMs); return; } diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index c41c81456b8..af9107e6965 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -2544,8 +2544,23 @@ JAVA_VOID java_lang_System_gcMarkSweep__(CODENAME_ONE_THREAD_STATE) { // Either way the census would see gcCurrentlyRunning false, start walking, and // have the pending cycle resume and sweep underneath it. Checked here because // this is the one door every cycle comes through. - if(atomic_load_explicit(&cn1GcFrozenForCensus, memory_order_acquire)) { - return; + // CLAIM the cycle, do not merely check a flag. Loading a freeze flag and then + // setting gcCurrentlyRunning is two steps, and the collector can be preempted + // between them: the census would raise the freeze, see gcCurrentlyRunning still + // false, and start walking a heap this thread is about to sweep. The claim below + // is a single compare-exchange, so a cycle is either started or refused with + // nothing observable in between. + { + int cn1Expected = CN1_GC_CYCLE_IDLE; + if(!atomic_compare_exchange_strong_explicit(&cn1GcCycleState, &cn1Expected, + CN1_GC_CYCLE_RUNNING, memory_order_acq_rel, memory_order_acquire)) { + // In practice only the FROZEN case can be taken: System's GC thread is + // the sole caller (System.java's `while(gcShouldLoop)` loop), so no second + // entrant can observe RUNNING. Refusing on RUNNING too is defence rather + // than policy -- two concurrent cycles would be worse than a skipped one -- + // and it means this is not a behaviour change for any existing caller. + return; + } } gcCurrentlyRunning = JAVA_TRUE; if(firstTimeGcThread) { @@ -2672,6 +2687,14 @@ JAVA_VOID java_lang_System_gcMarkSweep__(CODENAME_ONE_THREAD_STATE) { // of malloc entirely for exactly this reason. lowMemoryMode = JAVA_FALSE; gcCurrentlyRunning = JAVA_FALSE; + // Release the claim. Only ever RUNNING -> IDLE: a census that froze while this + // cycle ran holds the state at FROZEN and this must not clobber it, which is why + // the transition is a compare-exchange rather than a store. + { + int cn1Running = CN1_GC_CYCLE_RUNNING; + atomic_compare_exchange_strong_explicit(&cn1GcCycleState, &cn1Running, + CN1_GC_CYCLE_IDLE, memory_order_acq_rel, memory_order_relaxed); + } } JAVA_VOID java_lang_System_exit___int(CODENAME_ONE_THREAD_STATE, JAVA_INT i) {