ParparVM: make the translator self-hosting, and fix what that exposed

.github/workflows/linux-build-run.yml

@@ -283,7 +283,13 @@ jobs:
           # hang-stacks.txt with nothing but sample headers -- which is how four
           # occurrences of the suite stall ended up with no evidence at all. Still
           # best-effort: a runner without gdb must not fail the suite, it must say so.
-          bash scripts/ci/apt-get-install.sh gdb || echo "WARNING: gdb install failed"
+          # ABSOLUTE path. This step runs under `working-directory: vm`, so the
+          # relative form resolved to vm/scripts/ci/apt-get-install.sh, bash answered
+          # "No such file or directory", and the `|| echo` swallowed it -- gdb has
+          # NEVER been installed here. Every post-mortem in this job therefore
+          # produced nothing, including the SIGSEGV this run just hit, which uploads
+          # a core nobody can read. Same shape as the retry.sh path fixed elsewhere.
+          bash "$GITHUB_WORKSPACE/scripts/ci/apt-get-install.sh" gdb || echo "WARNING: gdb install failed"
           if command -v gdb >/dev/null 2>&1; then
             echo "gdb available: $(gdb --version | head -1)"
           else

.github/workflows/parparvm-selfhost.yml

@@ -0,0 +1,121 @@
+name: ParparVM Self-Hosting
+
+# Translates the ByteCodeTranslator with itself and compares the result against
+# the same translation run on a JVM.
+#
+# What this buys that the existing suites do not: ByteCodeTranslator is a 37.6k
+# line real program that hammers collections, strings, exceptions, file I/O and
+# the GC at a scale no unit test reaches, and the emitted C is a byte-exact
+# expected value that costs nothing to maintain -- it is whatever the JVM
+# produced from the same inputs. A VM defect that changes behaviour rather than
+# crashing (a wrong hash order, a dropped write barrier, a mis-mangled symbol)
+# shows up as a diff instead of passing silently.
+#
+# Gates, cheapest first:
+#   D  native vs native, two fresh processes, same input. If the native side is
+#      not self-consistent nothing else means anything, so it runs first.
+#   A  JVM vs native over the same corpus. The headline.
+#   Negative control: after a green comparison one emitted byte is flipped and
+#      the comparator MUST report exactly that file. A comparator nobody has
+#      watched fail is not a comparator.
+#
+# Not on the PR leg by default: a full run builds the translator twice and
+# translates a large corpus several times. It runs nightly, on demand, and on a
+# PR that opts in with the `selfhost` label.
+
+on:
+  schedule:
+    # 04:20 UTC daily, off the hour to avoid the runner rush.
+    - cron: '20 4 * * *'
+  workflow_dispatch:
+  pull_request:
+    types: [ opened, synchronize, reopened, labeled ]
+    paths:
+      - 'vm/**'
+      - '.github/workflows/parparvm-selfhost.yml'
+      - '!vm/**/README.md'
+      - '!vm/**/docs/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  CN1_NATIVE_VERIFY: strict
+
+jobs:
+  selfhost:
+    # On a pull_request only when the author asked for it; the schedule and
+    # workflow_dispatch legs always run.
+    if: >-
+      github.event_name != 'pull_request' ||
+      contains(github.event.pull_request.labels.*.name, 'selfhost')
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6
+
+      - name: Install native build tools
+        run: |
+          bash scripts/ci/apt-get-update.sh
+          sudo apt-get install -y clang
+
+      - name: Set up JDK 8
+        uses: actions/setup-java@v5
+        with:
+          distribution: 'temurin'
+          java-version: '8'
+          cache: 'maven'
+      - name: Save JDK 8 path
+        run: echo "JDK_8_HOME=$JAVA_HOME" >> $GITHUB_ENV
+
+      # The translator has to exist as classes before it can translate itself.
+      - name: Build the translator
+        run: >-
+          "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -q -B
+          -pl ByteCodeTranslator -am package -DskipTests
+        working-directory: vm
+
+      - name: Resolve the ASM classpath
+        run: >-
+          "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -q -B -pl ByteCodeTranslator
+          dependency:build-classpath
+          -Dmdep.outputFile=target/selfhost-asm-classpath.txt
+        working-directory: vm
+
+      # -O1: the diff gates care about the EMITTED C, not about how well clang
+      # optimised the binary that emitted it, and -O1 links several times faster.
+      # Mark threads are set explicitly rather than left to the source default,
+      # which resolves to a single marker and makes a large corpus take hours.
+      - name: Build the self-hosted translator
+        run: vm/selfhost/build-selfhost.sh
+        env:
+          CN1_SELFHOST_CFLAGS: -DCN1_GC_MARK_THREADS=4
+
+      # The corpus is the translator's OWN classes plus ASM. verify-selfhost.sh
+      # prepends vm/selfhost/target/javaapi-classes itself, so it is not repeated
+      # here. Absolute paths: the script runs both sides under `env -i` into one
+      # fixed output directory, so a relative path would not survive.
+      - name: Gate D and Gate A, with the negative control
+        run: |
+          vm/selfhost/verify-selfhost.sh \
+            "$PWD/vm/selfhost/target/asm-classes;$PWD/vm/selfhost/target/classes" \
+            com_codename1_tools_translator_ByteCodeTranslator \
+            com.codename1.tools.translator
+
+      # Both trees, so a divergence can be inspected rather than guessed at from
+      # a one-line summary.
+      - name: Upload the compared trees on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: selfhost-trees
+          path: |
+            vm/selfhost/target/verify/jvm-tree
+            vm/selfhost/target/verify/parpar1-tree
+            vm/selfhost/target/verify/parpar2-tree
+            vm/selfhost/target/verify/*.txt
+            vm/selfhost/target/verify/*.log
+          retention-days: 7
+          if-no-files-found: ignore

scripts/check-native-signatures.sh

@@ -36,7 +36,7 @@ for arg in "$@"; do
   esac
 done
 
-if [[ ! -f "$TRANSLATOR/com/codename1/tools/translator/NativeSignatureVerifier.class" ]]; then
+if [[ ! -f "$TRANSLATOR/com/codename1/tools/translator/NativeSignatureVerifierCli.class" ]]; then
   echo "check-native-signatures: building the translator" >&2
   (cd "$REPO_ROOT/vm" && mvn -q -B -pl ByteCodeTranslator -am package -DskipTests)
 fi
@@ -95,7 +95,7 @@ for entry in "${PORTS[@]}"; do
 
   echo "== $name"
   if ! java -cp "$TRANSLATOR:$(cat "$ASM_CP_FILE")" \
-       com.codename1.tools.translator.NativeSignatureVerifier "${args[@]}"; then
+       com.codename1.tools.translator.NativeSignatureVerifierCli "${args[@]}"; then
     status=1
   fi
   checked=$((checked + 1))

scripts/copyright-header-exclusions.txt

@@ -33,3 +33,4 @@ vm/JavaAPI/src/java/util/Collections.java | Apache Harmony source retaining its
 vm/JavaAPI/src/java/util/HashMap.java | Apache Harmony source retaining its original Apache-2.0 notice
 vm/JavaAPI/src/java/util/Hashtable.java | Apache Harmony source retaining its original Apache-2.0 notice
 vm/JavaAPI/src/java/util/IdentityHashMap.java | Apache Harmony source retaining its original Apache-2.0 notice
+vm/JavaAPI/src/java/util/ArrayList.java | Apache Harmony source retaining its original Apache-2.0 notice

vm/ByteCodeTranslator/src/cn1_globals.h

@@ -2218,15 +2218,15 @@ static inline JAVA_OBJECT cn1BibopFastAllocNoZero(CODENAME_ONE_THREAD_STATE, int
 // because bibopCurrent[] is shared across all classes of the same size class).
 #if !defined(CN1_DISABLE_INLINE_ALLOC) && !defined(CN1_DISABLE_BIBOP)
 #define CN1_FAST_NEW(X) ({ \
-    if(__builtin_expect(!class__##X.initialized, 0)) __STATIC_INITIALIZER_##X(threadStateData); \
+    if(__builtin_expect(!__atomic_load_n(&class__##X.initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_##X(threadStateData); \
     JAVA_OBJECT __cn1fo = cn1BibopFastAlloc(threadStateData, sizeof(struct obj__##X), &class__##X, CN1_BIBOP_CIDX(sizeof(struct obj__##X))); \
     if(__builtin_expect(__cn1fo == (JAVA_OBJECT)0, 0)) __cn1fo = __NEW_##X(threadStateData); \
     __cn1fo; })
 // No-body-zero variant (init-before-publish). The slow-path fallback __NEW_X
 // still fully zeroes (calloc) -- correct, just un-elided on the rare page-full
 // path.
 #define CN1_FAST_NEW_NOZERO(X) ({ \
-    if(__builtin_expect(!class__##X.initialized, 0)) __STATIC_INITIALIZER_##X(threadStateData); \
+    if(__builtin_expect(!__atomic_load_n(&class__##X.initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_##X(threadStateData); \
     JAVA_OBJECT __cn1fo = cn1BibopFastAllocNoZero(threadStateData, sizeof(struct obj__##X), &class__##X, CN1_BIBOP_CIDX(sizeof(struct obj__##X))); \
     if(__builtin_expect(__cn1fo == (JAVA_OBJECT)0, 0)) __cn1fo = __NEW_##X(threadStateData); \
     __cn1fo; })