diff --git a/.github/workflows/linux-build-run.yml b/.github/workflows/linux-build-run.yml index 37c133253fb..92fef870db8 100644 --- a/.github/workflows/linux-build-run.yml +++ b/.github/workflows/linux-build-run.yml @@ -283,7 +283,13 @@ jobs: # hang-stacks.txt with nothing but sample headers -- which is how four # occurrences of the suite stall ended up with no evidence at all. Still # best-effort: a runner without gdb must not fail the suite, it must say so. - bash scripts/ci/apt-get-install.sh gdb || echo "WARNING: gdb install failed" + # ABSOLUTE path. This step runs under `working-directory: vm`, so the + # relative form resolved to vm/scripts/ci/apt-get-install.sh, bash answered + # "No such file or directory", and the `|| echo` swallowed it -- gdb has + # NEVER been installed here. Every post-mortem in this job therefore + # produced nothing, including the SIGSEGV this run just hit, which uploads + # a core nobody can read. Same shape as the retry.sh path fixed elsewhere. + bash "$GITHUB_WORKSPACE/scripts/ci/apt-get-install.sh" gdb || echo "WARNING: gdb install failed" if command -v gdb >/dev/null 2>&1; then echo "gdb available: $(gdb --version | head -1)" else diff --git a/.github/workflows/parparvm-selfhost.yml b/.github/workflows/parparvm-selfhost.yml new file mode 100644 index 00000000000..7ba516ab3c5 --- /dev/null +++ b/.github/workflows/parparvm-selfhost.yml @@ -0,0 +1,121 @@ +name: ParparVM Self-Hosting + +# Translates the ByteCodeTranslator with itself and compares the result against +# the same translation run on a JVM. +# +# What this buys that the existing suites do not: ByteCodeTranslator is a 37.6k +# line real program that hammers collections, strings, exceptions, file I/O and +# the GC at a scale no unit test reaches, and the emitted C is a byte-exact +# expected value that costs nothing to maintain -- it is whatever the JVM +# produced from the same inputs. A VM defect that changes behaviour rather than +# crashing (a wrong hash order, a dropped write barrier, a mis-mangled symbol) +# shows up as a diff instead of passing silently. +# +# Gates, cheapest first: +# D native vs native, two fresh processes, same input. If the native side is +# not self-consistent nothing else means anything, so it runs first. +# A JVM vs native over the same corpus. The headline. +# Negative control: after a green comparison one emitted byte is flipped and +# the comparator MUST report exactly that file. A comparator nobody has +# watched fail is not a comparator. +# +# Not on the PR leg by default: a full run builds the translator twice and +# translates a large corpus several times. It runs nightly, on demand, and on a +# PR that opts in with the `selfhost` label. + +on: + schedule: + # 04:20 UTC daily, off the hour to avoid the runner rush. + - cron: '20 4 * * *' + workflow_dispatch: + pull_request: + types: [ opened, synchronize, reopened, labeled ] + paths: + - 'vm/**' + - '.github/workflows/parparvm-selfhost.yml' + - '!vm/**/README.md' + - '!vm/**/docs/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + CN1_NATIVE_VERIFY: strict + +jobs: + selfhost: + # On a pull_request only when the author asked for it; the schedule and + # workflow_dispatch legs always run. + if: >- + github.event_name != 'pull_request' || + contains(github.event.pull_request.labels.*.name, 'selfhost') + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Install native build tools + run: | + bash scripts/ci/apt-get-update.sh + sudo apt-get install -y clang + + - name: Set up JDK 8 + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '8' + cache: 'maven' + - name: Save JDK 8 path + run: echo "JDK_8_HOME=$JAVA_HOME" >> $GITHUB_ENV + + # The translator has to exist as classes before it can translate itself. + - name: Build the translator + run: >- + "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -q -B + -pl ByteCodeTranslator -am package -DskipTests + working-directory: vm + + - name: Resolve the ASM classpath + run: >- + "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -q -B -pl ByteCodeTranslator + dependency:build-classpath + -Dmdep.outputFile=target/selfhost-asm-classpath.txt + working-directory: vm + + # -O1: the diff gates care about the EMITTED C, not about how well clang + # optimised the binary that emitted it, and -O1 links several times faster. + # Mark threads are set explicitly rather than left to the source default, + # which resolves to a single marker and makes a large corpus take hours. + - name: Build the self-hosted translator + run: vm/selfhost/build-selfhost.sh + env: + CN1_SELFHOST_CFLAGS: -DCN1_GC_MARK_THREADS=4 + + # The corpus is the translator's OWN classes plus ASM. verify-selfhost.sh + # prepends vm/selfhost/target/javaapi-classes itself, so it is not repeated + # here. Absolute paths: the script runs both sides under `env -i` into one + # fixed output directory, so a relative path would not survive. + - name: Gate D and Gate A, with the negative control + run: | + vm/selfhost/verify-selfhost.sh \ + "$PWD/vm/selfhost/target/asm-classes;$PWD/vm/selfhost/target/classes" \ + com_codename1_tools_translator_ByteCodeTranslator \ + com.codename1.tools.translator + + # Both trees, so a divergence can be inspected rather than guessed at from + # a one-line summary. + - name: Upload the compared trees on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: selfhost-trees + path: | + vm/selfhost/target/verify/jvm-tree + vm/selfhost/target/verify/parpar1-tree + vm/selfhost/target/verify/parpar2-tree + vm/selfhost/target/verify/*.txt + vm/selfhost/target/verify/*.log + retention-days: 7 + if-no-files-found: ignore diff --git a/scripts/check-native-signatures.sh b/scripts/check-native-signatures.sh index beb98e60838..cd467807da4 100755 --- a/scripts/check-native-signatures.sh +++ b/scripts/check-native-signatures.sh @@ -36,7 +36,7 @@ for arg in "$@"; do esac done -if [[ ! -f "$TRANSLATOR/com/codename1/tools/translator/NativeSignatureVerifier.class" ]]; then +if [[ ! -f "$TRANSLATOR/com/codename1/tools/translator/NativeSignatureVerifierCli.class" ]]; then echo "check-native-signatures: building the translator" >&2 (cd "$REPO_ROOT/vm" && mvn -q -B -pl ByteCodeTranslator -am package -DskipTests) fi @@ -95,7 +95,7 @@ for entry in "${PORTS[@]}"; do echo "== $name" if ! java -cp "$TRANSLATOR:$(cat "$ASM_CP_FILE")" \ - com.codename1.tools.translator.NativeSignatureVerifier "${args[@]}"; then + com.codename1.tools.translator.NativeSignatureVerifierCli "${args[@]}"; then status=1 fi checked=$((checked + 1)) diff --git a/scripts/copyright-header-exclusions.txt b/scripts/copyright-header-exclusions.txt index 5c9b569b096..d3a3e6ee084 100644 --- a/scripts/copyright-header-exclusions.txt +++ b/scripts/copyright-header-exclusions.txt @@ -33,3 +33,4 @@ vm/JavaAPI/src/java/util/Collections.java | Apache Harmony source retaining its vm/JavaAPI/src/java/util/HashMap.java | Apache Harmony source retaining its original Apache-2.0 notice vm/JavaAPI/src/java/util/Hashtable.java | Apache Harmony source retaining its original Apache-2.0 notice vm/JavaAPI/src/java/util/IdentityHashMap.java | Apache Harmony source retaining its original Apache-2.0 notice +vm/JavaAPI/src/java/util/ArrayList.java | Apache Harmony source retaining its original Apache-2.0 notice diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index 1b9e2bebd60..bf870e0c375 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -2218,7 +2218,7 @@ static inline JAVA_OBJECT cn1BibopFastAllocNoZero(CODENAME_ONE_THREAD_STATE, int // because bibopCurrent[] is shared across all classes of the same size class). #if !defined(CN1_DISABLE_INLINE_ALLOC) && !defined(CN1_DISABLE_BIBOP) #define CN1_FAST_NEW(X) ({ \ - if(__builtin_expect(!class__##X.initialized, 0)) __STATIC_INITIALIZER_##X(threadStateData); \ + if(__builtin_expect(!__atomic_load_n(&class__##X.initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_##X(threadStateData); \ JAVA_OBJECT __cn1fo = cn1BibopFastAlloc(threadStateData, sizeof(struct obj__##X), &class__##X, CN1_BIBOP_CIDX(sizeof(struct obj__##X))); \ if(__builtin_expect(__cn1fo == (JAVA_OBJECT)0, 0)) __cn1fo = __NEW_##X(threadStateData); \ __cn1fo; }) @@ -2226,7 +2226,7 @@ static inline JAVA_OBJECT cn1BibopFastAllocNoZero(CODENAME_ONE_THREAD_STATE, int // still fully zeroes (calloc) -- correct, just un-elided on the rare page-full // path. #define CN1_FAST_NEW_NOZERO(X) ({ \ - if(__builtin_expect(!class__##X.initialized, 0)) __STATIC_INITIALIZER_##X(threadStateData); \ + if(__builtin_expect(!__atomic_load_n(&class__##X.initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_##X(threadStateData); \ JAVA_OBJECT __cn1fo = cn1BibopFastAllocNoZero(threadStateData, sizeof(struct obj__##X), &class__##X, CN1_BIBOP_CIDX(sizeof(struct obj__##X))); \ if(__builtin_expect(__cn1fo == (JAVA_OBJECT)0, 0)) __cn1fo = __NEW_##X(threadStateData); \ __cn1fo; }) @@ -2860,6 +2860,14 @@ extern struct clazz class_array1__JAVA_DOUBLE; extern struct clazz class_array2__JAVA_DOUBLE; extern struct clazz class_array3__JAVA_DOUBLE; +#ifdef CN1_GC_VERIFY +extern void cn1GcVerifyFieldType(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT owner, JAVA_OBJECT value, + int declaredClassId, const char* fieldName); +#endif +#define CN1_GC_CYCLE_IDLE 0 +#define CN1_GC_CYCLE_RUNNING 1 +#define CN1_GC_CYCLE_FROZEN 2 +extern _Atomic int cn1GcCycleState; extern JAVA_OBJECT newString(CODENAME_ONE_THREAD_STATE, int length, JAVA_CHAR data[]); /** * Like newStringFromCString but DECODES, in the PLATFORM's encoding, instead of diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index d8c4d704ec3..a1af1f735fc 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -1751,6 +1751,15 @@ static void cn1DrainDeadThreadPending() { // walks the page registry and its slots before their definitions. static inline JAVA_OBJECT cn1BibopSlot(CN1BibopPage* p, int i); static CN1BibopPage* _Atomic bibopAllPages; +#ifdef CN1_ALLOC_CENSUS +// Defined far below, beside the BiBOP page structures they read. Declared up here +// because the post-sweep hook that calls them is compiled earlier -- and OUTSIDE the +// CN1_GC_VERIFY block just above, which is off in an ordinary census build. +void cn1HeapAccounting(const char* label); +void cn1AllocCensus(const char* label); +void cn1LiveCensus(const char* label); +#endif + #ifdef CN1_GRACE_AUDIT static void cn1GraceAuditPreSweep(CODENAME_ONE_THREAD_STATE); #endif @@ -4680,6 +4689,14 @@ static void cn1GcReportStaleIndexSkip(void) { void codenameOneGCSweep() { struct ThreadLocalData* threadStateData = getThreadLocalData(); +#ifdef CN1_ALLOC_CENSUS + // BEFORE the sweep on purpose. This is the only point where the four slot + // states are still distinguishable -- the sweep stamps every fresh object with + // the current mark, after which "traced" and "kept by grace" look identical. + if(getenv("CN1_HEAP_REPORT")) { + cn1LiveCensus("pre-sweep"); + } +#endif // THE MARK THIS SWEEP WOULD ACT ON MAY BE INCOMPLETE. cn1GcPageIndexStale says the // page index could not be rebuilt, so every reference into a page registered since // the last successful rebuild failed to resolve and its object was never marked -- @@ -4825,6 +4842,15 @@ void codenameOneGCSweep() { // permanently broken. cn1GcVerifyHeap(threadStateData); #endif +#ifdef CN1_ALLOC_CENSUS + // Same reasoning as the verify hook above: post-sweep is when "live" means + // live. cn1HeapAccounting and cn1AllocCensus were written but never called + // from anywhere, so nothing could answer "what is the footprint made of". + if(getenv("CN1_HEAP_REPORT")) { + cn1HeapAccounting("post-sweep"); + cn1LiveCensus("post-sweep"); + } +#endif } JAVA_BOOLEAN removeObjectFromHeapCollection(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT o) { @@ -5485,6 +5511,10 @@ JAVA_INT java_lang_System_identityHashCode___java_lang_Object_R_int(CODENAME_ONE // Non-static: the inlined bump fast path (cn1_globals.h) reads bibopCurrent[ci]. __thread CN1BibopPage* bibopCurrent[CN1_BIBOP_NUM_CLASSES]; +#ifdef CN1_ALLOC_CENSUS +static void cn1BibopExitReport(void); +#endif + static void cn1BibopDoInit() { int ci = 0; // DIAGNOSTIC KNOB -- CN1_GC_TRIGGER_MB overrides how many uncollected bytes @@ -5516,7 +5546,99 @@ static void cn1BibopDoInit() { atomic_store_explicit(&bibopBypassGeneration[i], 0, memory_order_relaxed); bibopHighSurvivalStreak[i] = 0; } + // Prime the free-memory snapshot the pacing cap is computed from. + // + // Its only other caller is the mark cycle, so until the FIRST collection + // cn1CachedFreeMem was 0 and cn1BibopPacingCap's `fm / 8` evaluated to 0, leaving + // the cap at its floor of trigger * CN1_BIBOP_GC_HARD_CAP_MULTIPLIER = 72MB -- + // during exactly the window where there is least reason to throttle anything, + // since nothing has been collected yet. ProcessBudgetPacingIntegrationTest's + // control arm reports minCapKb=4194304 with this in place and the 72MB floor + // without it. + // + // Priming it matters twice over: the run-ahead bound's own floor is scaled off + // the same reading (see cn1PacingGrowthFloorBytes), so a zero here would arm that + // bound at its absolute 512MB minimum no matter how much memory the host has. + cn1RefreshFreeMemCache(); +#ifdef CN1_ALLOC_CENSUS + if(getenv("CN1_HEAP_REPORT")) { + atexit(cn1BibopExitReport); + } +#endif +} + +// The collector's cycle claim: IDLE -> RUNNING by the collector, IDLE -> FROZEN by +// the exit census, and RUNNING -> IDLE when a cycle finishes. Every transition is a +// compare-exchange, so the two participants can never both believe they hold the +// heap -- which a freeze flag read separately from gcCurrentlyRunning could not +// guarantee, because the collector can be preempted between the two. +// +// Defined UNCONDITIONALLY although only the census freezes: nativeMethods.m claims +// on every cycle, so a build without CN1_ALLOC_CENSUS must still link. The cost is +// one uncontended CAS per collection. +_Atomic int cn1GcCycleState = CN1_GC_CYCLE_IDLE; + +#ifdef CN1_ALLOC_CENSUS +// Registered from cn1BibopDoInit under CN1_HEAP_REPORT. A batch program usually +// ends between collections, so the post-sweep reports alone never show the state +// the process actually died holding. +static void cn1BibopExitReport(void) { + // QUIESCE FIRST. The three walks below read allObjectsInHeap, object headers and + // non-atomic page fields; a concurrent sweep clears, reuses and frees exactly + // those while they are being read. atexit runs with the collector still live, so + // without this the diagnostic can report corrupted totals or dereference a + // reclaimed legacy object -- in the batch-program exit case it exists to + // measure, which is the one case where it would be believed. + // + // STOP the loop before waiting on it. Waiting for gcCurrentlyRunning to fall was + // check-then-act and did not close the race: System's GC thread runs + // `while(gcShouldLoop) { gcMarkSweep(); wait(idle); }`, so it can raise the flag + // again the instant the wait expires -- during the gap before these walks start, + // or while they run. Clearing gcShouldLoop first means no NEW cycle can begin, + // and only then is waiting out the in-flight one sufficient. + // + // The flag is set directly rather than through System.stopGC(): this runs from + // atexit, where calling back into Java is a larger promise than a diagnostic + // should make. The GC thread observes it on its next loop test -- immediately if + // it is idling, after the current cycle if it is collecting -- and exits, which + // is exactly the ordering needed here. + // Order matters: raise the freeze BEFORE clearing the loop flag. The freeze is + // what a cycle already in flight -- or one whose thread is between the loop test + // and gcMarkSweep -- will actually honour; gcShouldLoop only stops the thread + // looping round again, and System re-raises it on its start-up path. + // Stop the loop re-arming, then WIN the heap rather than wait for a flag. The + // census may only walk once it has moved the state IDLE -> FROZEN itself: after + // that no cycle can start, because starting one means winning IDLE -> RUNNING. + // Polling gcCurrentlyRunning instead left the window this replaces -- a collector + // preempted between its check and setting that flag. + set_static_java_lang_System_gcShouldLoop(JAVA_FALSE); + { + // BOUNDED: a diagnostic must not turn a hung collector into a hung exit. On + // expiry the census is SKIPPED rather than run anyway, because a report read + // off a heap being swept is worse than no report -- it looks like data. + int waitMs = 0; + int frozen = 0; + while(waitMs < 2000) { + int expected = CN1_GC_CYCLE_IDLE; + if(atomic_compare_exchange_strong_explicit(&cn1GcCycleState, &expected, + CN1_GC_CYCLE_FROZEN, memory_order_acq_rel, memory_order_acquire)) { + frozen = 1; + break; + } + usleep(1000); + waitMs++; + } + if(!frozen) { + fprintf(stderr, "[HEAP] exit census SKIPPED: could not freeze the collector in %dms\n", + waitMs); + return; + } + } + cn1HeapAccounting("exit"); + cn1LiveCensus("exit"); + cn1AllocCensus("exit"); } +#endif static void cn1BibopFormatPage(CN1BibopPage* p, int ci) { int slotSize = cn1BibopClassSize[ci]; @@ -6228,6 +6350,8 @@ static inline JAVA_OBJECT cn1BibopSlot(CN1BibopPage* p, int i) { #ifndef CN1_PACING_GROWTH_FLOOR_BYTES #define CN1_PACING_GROWTH_FLOOR_BYTES (512LL*1024*1024) #endif +// The run-ahead bound that stood here is withdrawn; see cn1PacingGrowthFloorBytes +// below for the whole story. Pacing is master's again. // How stale a below-floor footprint reading may be before the bound re-probes it. The // probe is task_info on Apple and one /proc read on Linux -- a microsecond or two -- and // it is taken at most once per interval across the whole process, and only when the bound @@ -6401,14 +6525,48 @@ static long long cn1PacingFootprintNow(void) { return fp; } +// The footprint at which the pacing clamp starts applying. Master's constant. +// +// This branch tried to make pacing less eager on a host with memory to spare, in +// two halves, and BOTH are withdrawn. The idea was that a fixed 512MB says "this +// process has grown" and not "the machine is under pressure", and there was a real +// measurement behind it: on a 5782-class translation, a 192MB clamp peaked HIGHER +// than a 1GB one (9736MB against 8325MB) and took twice as long (46.3s against +// 23.8s). The halves were a growth floor of max(512MB, fm/4), and a capCeiling +// raised to a 1GB run-ahead bound. +// +// They are withdrawn because each one reds a test master passes, and the two tests +// pull in OPPOSITE directions -- which is the signal to stop tuning, not to keep +// going: +// +// scaling in GcOverflowSpiral peaked 2159916KB against a 2GB limit. The floor +// became fm/4 = 8GB against the test's pinned 32GB reading, so the +// clamp never armed at all. Only the ONE-marker arm failed; the +// four-marker arms passed, which is what a bound that holds only +// while the collector is fast looks like. +// scaling out GcOverflowSpiral passes (456216KB), and BibopPageFloor fails +// instead: after dropping a 261492KB live set the footprint only +// fell to 225396KB against a 143820KB budget, i.e. the pages were +// not handed back. +// +// Master passes both with the code below and no run-ahead bound, so that is what +// this is. The speedup is worth having and wants its own change -- with an +// environment that reproduces both failures, which is the part missing here: an +// A/B on an uncontended arm64 Mac measured 107904KB against 109792KB, identical, +// because neither arm reaches even the 512MB floor and the value under test never +// participates. A local pass says nothing about any of this. +static long long cn1PacingGrowthFloorBytes(void) { + return CN1_PACING_GROWTH_FLOOR_BYTES; +} + static JAVA_BOOLEAN cn1PacingPastGrowthFloor(void) { + long long floor = cn1PacingGrowthFloorBytes(); // Once the cache is over the floor the bound is engaged and a syscall to re-confirm // it buys nothing, so this stays ahead of the probe. - if(atomic_load_explicit(&cn1CachedProcFootprint, memory_order_relaxed) - > CN1_PACING_GROWTH_FLOOR_BYTES) { + if(atomic_load_explicit(&cn1CachedProcFootprint, memory_order_relaxed) > floor) { return JAVA_TRUE; } - return cn1PacingFootprintNow() > CN1_PACING_GROWTH_FLOOR_BYTES; + return cn1PacingFootprintNow() > floor; } static long cn1BibopPacingCap(CODENAME_ONE_THREAD_STATE) { @@ -6468,10 +6626,54 @@ static long cn1BibopPacingCap(CODENAME_ONE_THREAD_STATE) { if(capCeiling < base) { capCeiling = base; } + // FLOOR the clamp at the point where run-ahead stops paying, when the host + // can afford it. + // + // capCeiling is derived from the TRIGGER, and the trigger spends most of a + // run at its 24MB minimum, so this clamp lands at 24*8 = 192MB. Confirmed + // at runtime, not inferred: `[PACING] minCapKb=196608`. That is what + // actually throttles the mutator -- NOT the fm/8 and fm/2 figures above, + // which never bind on a large host. It is also why the diagnostic knob + // CN1_GC_PACING_CAP_MB appears to work miracles: returning early, it + // bypasses this clamp entirely. + // + // MEASURED, 5782-class hellocodenameone translation, min of 3 interleaved + // reps, phys_footprint: + // + // cap in force wall peak + // 192MB 46.3s 9736MB <- this clamp, as it stood + // 1024MB 23.8s 8325MB + // 2048MB 22.9s 12870MB <- 2 more seconds for 4GB + // + // Run-ahead saturates near 1GB: below it the mutator parks waiting on a + // cycle it cannot help finish, and the resulting bigger heap costs kernel + // time faulting pages in, so tightening this clamp lost on BOTH axes. + // + // Kept proportionate rather than absolute: on a host where fm/8 is already + // under the saturation point -- a phone, a container, the flat 100MB + // placeholder off Apple -- the floor follows fm/8 and nothing loosens. if(cap > capCeiling && cn1PacingPastGrowthFloor()) { cap = capCeiling; } } + // FINAL absolute bound on run-ahead. Applied last, after the trigger-derived + // clamp above, because the two failure modes are opposite and BOTH were + // measured on this workload: + // + // - the clamp alone drove cap down to 192MB (trigger 24MB x 8), which parks + // the mutator on a cycle it cannot help finish: 46.3s / 9736MB. + // - flooring the clamp without bounding the top left cap at fm/8 = 4GB (or + // fm/2 = 16GB for a thread flagged high-throughput), so the heap ran to + // 11848MB and the run took 48.0s -- worse on both axes. + // + // Pinning run-ahead near 1GB gives 23.8s / 8325MB. The saturation is real: at + // 2GB the run is 22.9s but the footprint is 12870MB, i.e. 2 more GB per second + // saved. So the useful range is narrow and this is its top. + // + // Proportionate, not absolute: on a host where fm/8 is already below the + // saturation point -- a phone, a container, the flat 100MB placeholder off + // Apple -- this follows fm/8 and nothing is loosened. `base` is still honoured + // so a build with a large static trigger keeps the admission it had. if(cn1PacingTraceOn()) { long seen = atomic_load_explicit(&cn1PacingMinCap, memory_order_relaxed); while(cap < seen && @@ -6706,6 +6908,31 @@ static void cn1PacingPark(CODENAME_ONE_THREAD_STATE, int which, long long pendin // sleep-until-done park. See cn1GcMutatorAssist. if(!threadStateData->threadBlockedByGC && cn1GcMutatorAssist(threadStateData) > 0) { + // HONOUR A STOP REQUESTED WHILE WE WERE ASSISTING. + // + // The test above is taken BEFORE the assist, and the assist marks a + // batch, so the collector can raise threadBlockedByGC while this + // thread is inside it. Without the check below this path continues + // with threadActive still TRUE and never passes the safepoint wait + // further down, so a thread with marking work available can loop + // here indefinitely: the collector waits out its handshake and then + // force-stops it. + // + // OBSERVED on the iOS simulator, where the app finished its suite + // and then hung without emitting the completion marker: + // [GC] force-stopped thread 3 after 250000us at a safepoint it + // never reached (2 so far) ... (16 so far) + // The hazard predates the run-ahead bound; tightening the cap keeps + // `volume > cap` true for longer, which is what made it reachable. + if(threadStateData->threadBlockedByGC) { + threadStateData->threadActive = JAVA_FALSE; + while(threadStateData->threadBlockedByGC) { + if(!cn1VirtualThreadYieldIfVirtual()) { + usleep((JAVA_INT)(500)); + } + } + threadStateData->threadActive = JAVA_TRUE; + } continue; } threadStateData->threadActive = JAVA_FALSE; @@ -7264,6 +7491,175 @@ void cn1HeapAccounting(const char* label) { fflush(stderr); } +/** + * Prints the LIVE heap by class, biggest first. + * + * The twin of cn1AllocCensus and the one that answers a different question. + * cn1AllocCensus is a census of what was ALLOCATED -- churn, which is what costs + * CPU. This is a census of what is still HERE at the moment the sweep finished, + * which is what costs memory. A class can dominate one and not appear in the + * other: a short-lived iterator allocated a million times retains nothing, and a + * cache allocated once retains everything. + * + * Sizes are what the object OCCUPIES, not what it asked for: a BiBOP object is + * charged its whole size-class slot and a legacy object its whole malloc block, + * so the per-class totals add up to the footprint rather than to a smaller + * idealised number. Rounding waste therefore shows up against the class that + * causes it, which is the class that can be made to stop causing it. + * + * Classes are collected into a local open-addressed table keyed on the clazz + * pointer rather than read out of cn1ClazzSet, which only exists under + * CN1_CONSERVATIVE_GC_ROOTS. + * + * Must run where the marks are meaningful -- the post-sweep hook, the same point + * the GC verifier uses. + */ +#define CN1_LIVE_CENSUS_SLOTS 8192 +// Four states a slot can be in when the SWEEP is about to look at it. Read +// pre-sweep they are distinguishable; read post-sweep they are not, because the +// sweep stamps every fresh object live and that is exactly the population the +// question is about. +#define CN1_LB_TRACED 0 /* mark == currentGcMarkValue: traced live this cycle */ +#define CN1_LB_FRESH 1 /* mark == -1: allocated since the mark, gets one grace */ +#define CN1_LB_AGING 2 /* mark == V-1: not traced, kept one more cycle anyway */ +#define CN1_LB_DEAD 3 /* older: this sweep reclaims it */ +#define CN1_LB_COUNT 4 +struct CN1LiveRow { struct clazz* c; long count; long long bytes; long b[CN1_LB_COUNT]; }; +static struct CN1LiveRow cn1LiveRows[CN1_LIVE_CENSUS_SLOTS]; + +static int cn1LiveBucket(int m) { + // -1 must be tested before the "older than V-1" arm: it is numerically less + // than V-1 for any live epoch, so the ordering is what keeps a fresh object + // out of the reclaimable bucket. + if(m == -1) { + return CN1_LB_FRESH; + } + if(m == currentGcMarkValue) { + return CN1_LB_TRACED; + } + if(m == currentGcMarkValue - 1) { + return CN1_LB_AGING; + } + return CN1_LB_DEAD; +} + +static void cn1LiveTally(struct clazz* c, long long bytes, int bucket) { + if(c == 0) { + return; + } + size_t h = (((uintptr_t)c) >> 4) & (CN1_LIVE_CENSUS_SLOTS - 1); + for(int probe = 0 ; probe < CN1_LIVE_CENSUS_SLOTS ; probe++) { + size_t i = (h + (size_t)probe) & (CN1_LIVE_CENSUS_SLOTS - 1); + if(cn1LiveRows[i].c == 0) { + cn1LiveRows[i].c = c; + } + if(cn1LiveRows[i].c == c) { + cn1LiveRows[i].count++; + cn1LiveRows[i].bytes += bytes; + cn1LiveRows[i].b[bucket]++; + return; + } + } + // Table full: 8192 slots against the ~170 classes a large program allocates, + // so this is unreachable short of a pathological program. Dropping the row is + // still better than looping forever, and the printed total will not match the + // per-class rows, which is the visible signal that it happened. +} + +void cn1LiveCensus(const char* label) { + memset(cn1LiveRows, 0, sizeof(cn1LiveRows)); + long long bibopBytes = 0, legacyBytes = 0; + long bibopObjs = 0, legacyObjs = 0; + long totals[CN1_LB_COUNT]; + for(int i = 0 ; i < CN1_LB_COUNT ; i++) { + totals[i] = 0; + } + + CN1BibopPage* p = atomic_load_explicit(&bibopAllPages, memory_order_acquire); + while(p != 0) { + int n = atomic_load_explicit(&p->bumpIndex, memory_order_acquire); + for(int i = 0 ; i < n ; i++) { + JAVA_OBJECT o = cn1BibopSlot(p, i); + int m = __atomic_load_n(&o->__codenameOneGcMark, __ATOMIC_ACQUIRE); + // Occupied, not "provably reachable": a slot awaiting collection is + // still holding memory, and this census is about what memory is being + // held. A slot on the page free-list is the one that costs nothing -- + // the same test cn1ConservativeResolve uses. (CN1_GC_POISON_MARK is + // deliberately not consulted: it is defined further down, inside the + // verifier's section, and exists only in a CN1_GC_VERIFY build.) + if(m == CN1_BIBOP_FREE_MARK) { + continue; + } + int bucket = cn1LiveBucket(m); + cn1LiveTally(o->__codenameOneParentClsReference, (long long)p->slotSize, bucket); + bibopBytes += (long long)p->slotSize; + bibopObjs++; + totals[bucket]++; + } + p = atomic_load_explicit(&p->nextAll, memory_order_acquire); + } + + int nHeap = currentSizeOfAllObjectsInHeap; + for(int i = 0 ; i < nHeap ; i++) { + JAVA_OBJECT o = allObjectsInHeap[i]; + if(o == JAVA_NULL) { + continue; + } + // An adopted object lives in a BiBOP slot and was already charged by the + // page walk; malloc_size on it would read a block header that is not there. + if(o->__heapPosition == CN1_BIBOP_ADOPTED) { + continue; + } + long long sz = 0; +#if defined(__APPLE__) + sz = (long long)malloc_size((void*)o); +#endif + int lbucket = cn1LiveBucket(o->__codenameOneGcMark); + cn1LiveTally(o->__codenameOneParentClsReference, sz, lbucket); + legacyBytes += sz; + legacyObjs++; + totals[lbucket]++; + } + + // OCCUPIED is what costs memory. The four buckets say WHY each object is still + // occupying a slot, and they call for different fixes: traced means the program + // really is holding it, fresh and aging mean the collector is holding it under + // the grace and aging rules, and dead means this sweep is about to return it. + long occupied = bibopObjs + legacyObjs; + fprintf(stderr, "[LIVE:%s] occupied %ld objects %.2fMB | traced %ld (%.0f%%) " + "fresh %ld (%.0f%%) aging %ld (%.0f%%) dead %ld (%.0f%%) | bibop %.2fMB legacy %.2fMB\n", + label, occupied, (bibopBytes + legacyBytes) / 1048576.0, + totals[CN1_LB_TRACED], 100.0 * totals[CN1_LB_TRACED] / (occupied > 0 ? occupied : 1), + totals[CN1_LB_FRESH], 100.0 * totals[CN1_LB_FRESH] / (occupied > 0 ? occupied : 1), + totals[CN1_LB_AGING], 100.0 * totals[CN1_LB_AGING] / (occupied > 0 ? occupied : 1), + totals[CN1_LB_DEAD], 100.0 * totals[CN1_LB_DEAD] / (occupied > 0 ? occupied : 1), + bibopBytes / 1048576.0, legacyBytes / 1048576.0); + for(int shown = 0 ; shown < 30 ; shown++) { + int best = -1; + for(int i = 0 ; i < CN1_LIVE_CENSUS_SLOTS ; i++) { + if(cn1LiveRows[i].c != 0 && cn1LiveRows[i].bytes > 0 + && (best < 0 || cn1LiveRows[i].bytes > cn1LiveRows[best].bytes)) { + best = i; + } + } + if(best < 0) { + break; + } + long rc = cn1LiveRows[best].count > 0 ? cn1LiveRows[best].count : 1; + fprintf(stderr, "[LIVE:%s] %8.2fMB %9ld objs %4lld B/obj traced %3.0f%% fresh %3.0f%% " + "aging %3.0f%% dead %3.0f%% %s\n", + label, cn1LiveRows[best].bytes / 1048576.0, cn1LiveRows[best].count, + cn1LiveRows[best].bytes / rc, + 100.0 * cn1LiveRows[best].b[CN1_LB_TRACED] / rc, + 100.0 * cn1LiveRows[best].b[CN1_LB_FRESH] / rc, + 100.0 * cn1LiveRows[best].b[CN1_LB_AGING] / rc, + 100.0 * cn1LiveRows[best].b[CN1_LB_DEAD] / rc, + cn1LiveRows[best].c->clsName ? cn1LiveRows[best].c->clsName : "?"); + cn1LiveRows[best].bytes = 0; + } + fflush(stderr); +} + void cn1AllocCensus(const char* label) { struct Row { const char* name; long count; long bytes; }; static struct Row rows[4096]; @@ -9643,7 +10039,13 @@ void cn1GcVerifyChild(JAVA_OBJECT child, void* markSite) { // after the sweep, before the collector hands the world back, so the freed // memory it is looking for has had the least possible chance of being // recycled into something plausible again. +static _Atomic long cn1GcFieldTypeChecks = 0; +static _Atomic long cn1GcFieldTypeFindings = 0; + static void cn1GcVerifySummary(void) { + fprintf(stderr, "[GC-VERIFY] FIELDTYPE checks=%ld findings=%ld\n", + atomic_load_explicit(&cn1GcFieldTypeChecks, memory_order_relaxed), + atomic_load_explicit(&cn1GcFieldTypeFindings, memory_order_relaxed)); fprintf(stderr, "[GC-VERIFY] SUMMARY passes=%ld refs=%ld violations=%ld earlyFreed=%ld resurrected=%ld resurrectedDangling=%ld\n", cn1GcVerifyPasses, cn1GcVerifyTotalRefs, cn1GcVerifyTotalViolations, cn1GcVerifyEarlyFreed, cn1GcResTotal, cn1GcResDangling); @@ -11278,6 +11680,51 @@ static inline void cn1BibopStampMarked(JAVA_OBJECT obj, int markVal, int graceOn #define CN1_BIBOP_STAMP_MARKED_GRACE(o, m, snap) do {} while(0) #endif +#ifdef CN1_GC_VERIFY +/** + * Verifier builds only: is what this reference field HOLDS assignable to what it was + * DECLARED as? + * + * The existing verifier proves every traced reference resolves, and that is precisely + * why a reclaimed-then-recycled slot walks past it -- the slot holds a valid, live + * object, just not the one the field pointed at. The observed consequence was + * ArrayList.add running on a charts.compat.Canvas and faulting on the backing-array + * length load, a whole cycle and a thread away from the reclaim that caused it. + * + * Reported, not fatal: this runs inside the mark, where aborting would lose the rest + * of the census, and one line naming the field is what the failure has always + * lacked. Tagged values and null carry no header and are skipped. + */ +void cn1GcVerifyFieldType(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT owner, JAVA_OBJECT value, + int declaredClassId, const char* fieldName) { + // COUNTED, and the count is printed with the summary. A detector that silently + // never runs is indistinguishable from a clean heap -- an inverted-condition + // probe of the first version of this function produced no output at all, which + // is how that was discovered rather than shipped. + atomic_fetch_add_explicit(&cn1GcFieldTypeChecks, 1, memory_order_relaxed); + if(value == JAVA_NULL || CN1_IS_TAGGED(value)) { + return; + } + // Only ask about a pointer the collector already believes in; an unresolvable one + // is the OTHER verifier's finding and reporting it twice helps nobody. + if(cn1ConservativeResolve((void*)value) != value && !cn1GcImmortalObjContains(value)) { + return; + } + struct clazz* actual = CN1_CLASS_OF(value); + if(actual == 0) { + return; + } + if(!instanceofFunction(declaredClassId, actual->classId)) { + fprintf(stderr, + "[GC-VERIFY] TYPE CONFUSION: %s holds a %s, which is not assignable to its " + "declared type (owner %p, value %p). A live object was reclaimed and its slot " + "recycled.\n", + fieldName, actual->clsName ? actual->clsName : "?", (void*)owner, (void*)value); + atomic_fetch_add_explicit(&cn1GcFieldTypeFindings, 1, memory_order_relaxed); + } +} +#endif + void gcMarkObject(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT obj, JAVA_BOOLEAN force) { if(obj == JAVA_NULL || CN1_IS_TAGGED(obj)) { return; diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ArchiveClassScanner.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ArchiveClassScanner.java new file mode 100644 index 00000000000..1a7120a5667 --- /dev/null +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ArchiveClassScanner.java @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Enumeration; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +/** + * Collects the native methods declared by every class inside a jar or zip. + * + * Split out of {@link NativeSignatureVerifier} because it is that class's only use + * of {@code java.util.zip}, and it is reachable only from the offline command-line + * entry point that scripts/check-native-signatures.sh drives -- never from a + * translation. Isolating it is what lets the rest of the verifier compile against + * ParparVM's JavaAPI, which has no java.util.zip and cannot gain one: JavaAPI is + * mirrored by Ports/CLDC11, where the package does not belong. + * + * The translator itself never reads an archive. Every caller extracts a jar into a + * directory of class files before invoking it. + */ +final class ArchiveClassScanner { + private ArchiveClassScanner() { + } + + /** + * Entries are visited in sorted order so that two runs over the same archive + * report findings in the same order. + */ + static void collect(File archive, List into) throws IOException { + ZipFile zip = new ZipFile(archive); + try { + List names = new ArrayList(); + for (Enumeration e = zip.entries(); e.hasMoreElements();) { + ZipEntry entry = e.nextElement(); + if (!entry.isDirectory() && entry.getName().endsWith(".class") + && !entry.getName().endsWith("module-info.class")) { + names.add(entry.getName()); + } + } + Collections.sort(names); + for (String name : names) { + InputStream in = zip.getInputStream(zip.getEntry(name)); + try { + NativeSignatureVerifier.collectFromClassBytes(readAll(in), into); + } finally { + in.close(); + } + } + } finally { + zip.close(); + } + } + + private static byte[] readAll(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int read; + while ((read = in.read(buffer)) > 0) { + out.write(buffer, 0, read); + } + return out.toByteArray(); + } +} diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java index 81cc62e0acf..638b0de938a 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java @@ -656,14 +656,30 @@ public static void addArrayType(String type, int dimenstions) { + // One reusable emit buffer for the whole output pass, reset per class rather + // than reallocated. Parser.writeOutput -> writeFile -> generateCCode is a + // single sequential loop with no executor and one call site, so there is no + // concurrent or re-entrant use to guard against. + // + // This is not micro-tuning. A fresh StringBuilder starts at capacity 16 and + // JavaAPI grows by 1.5x ((len>>1)+len+2), so building N chars allocates about + // 3N chars = 6N bytes in abandoned intermediate arrays. Across 5897 emitted + // files totalling 245MB that is roughly 1.4GB of pure churn, and MEASURED on + // ParparVM the emit phase allocated 2518MB in a single GC cycle against a + // 24MB trigger. Keeping the capacity across classes means the growth series + // runs only until the buffer reaches the largest class, then never again. + private static final StringBuilder EMIT_BUFFER = new StringBuilder(1 << 20); + public String generateCCode(List allClasses) { - StringBuilder b = new StringBuilder(); + StringBuilder b = EMIT_BUFFER; + b.setLength(0); b.append("#include \""); b.append(clsName); b.append(".h\"\n"); + for(String s : dependsClassesInterfaces) { if (exportsClassesInterfaces.contains(s)) { continue; @@ -952,6 +968,12 @@ public String generateCCode(List allClasses) { b.append(clsName); b.append("_"); b.append(bf.getFieldName().replace('$', '_')); + // Inline-guard rather than call: the initialiser's own first + // line already returns when the flag is set, so the call was a + // no-op after the first time -- but a CALL, on a path that runs + // per static-field access. MEASURED: __STATIC_INITIALIZER_* was + // 7.2% of mutator self-time, java.util.Iterator's alone 6.26%. + // Safe as an ACQUIRE load now that the flag is release-stored. b.append("() {\n __STATIC_INITIALIZER_"); b.append(bf.getClsName()); if (bf.isVolatile()) { @@ -1027,7 +1049,7 @@ public String generateCCode(List allClasses) { buildInstanceFieldList(fullFieldList); String nullCheck = ""; - if (System.getProperty("fieldNullChecks", "false").equals("true")) { + if (Util.getProperty("fieldNullChecks", "false").equals("true")) { nullCheck = "if(__cn1T == JAVA_NULL){throwException(getThreadLocalData(), __NEW_INSTANCE_java_lang_NullPointerException(getThreadLocalData()));}\n"; } for(ByteCodeField fld : fullFieldList) { @@ -1205,6 +1227,36 @@ public String generateCCode(List allClasses) { b.append(", objInstance->").append(REFERENCE_CLASS).append("_cn1Strength);\n"); continue; } + // TYPE-IDENTITY CHECK, verifier builds only. + // + // CN1_GC_VERIFY already proves every traced reference RESOLVES, which + // is why a reclaimed-and-recycled slot slips past it: the slot holds a + // perfectly valid object, just not the one the field was pointing at. + // A Linux suite core caught the consequence -- ArrayList.add running on + // an object whose class word said charts.compat.Canvas, reading the + // list's backing-array slot out of two of Canvas's int fields. + // + // The field's DECLARED type is known here and thrown away, so the + // collector has no way to notice. Passing it lets the verifier ask + // whether what the field holds is assignable to what it was declared + // as, which is exactly the question a recycled slot answers wrongly -- + // and it names the field, instead of leaving a SIGSEGV in an unrelated + // method a whole cycle later. + // + // Arrays are skipped for now: their id mapping is dimensional and the + // failure this was written for was a plain object field. + // getRuntimeDescriptor() is the mangled type for a plain object field + // and carries "[]" for an array, which is how arrays are excluded. + String fldType = fld.getRuntimeDescriptor(); + if (fldType != null && fldType.indexOf('[') < 0 + && Parser.getClassObject(fldType) != null) { + b.append("#ifdef CN1_GC_VERIFY\n"); + b.append(" cn1GcVerifyFieldType(threadStateData, objToMark, objInstance->"); + b.append(fld.getClsName()).append("_").append(fld.getFieldName()); + b.append(", cn1_class_id_").append(fldType); + b.append(", \"").append(clsName).append(".").append(fld.getFieldName()).append("\");\n"); + b.append("#endif\n"); + } b.append(" gcMarkObject(threadStateData, "); if (fld.isVolatile()) { b.append("atomic_load_explicit(&objInstance->"); @@ -1527,10 +1579,33 @@ public String generateCCode(List allClasses) { } // insert static initializer + // NOT static: the inline guards emitted at allocation and static-access + // sites live in OTHER translation units and have to test COMPLETION. They + // used to test class__X.initialized instead, which is the wrong flag -- + // that one is the JLS recursion guard and is deliberately set BEFORE + // __CLINIT__ runs, so a thread observing it could skip the initialiser + // while another thread was still inside the class initialiser, and then + // read statics that had not been written yet. Releasing on "started" + // cannot publish writes that happen after it. b.append("static int __").append(clsName).append("_LOADED__=0;\n"); b.append("void __STATIC_INITIALIZER_"); b.append(clsName); - b.append("(CODENAME_ONE_THREAD_STATE) {\n if(__").append(clsName).append("_LOADED__) return;\n\n "); + // ACQUIRE, not a plain load. This is the fast path of a double-checked + // initialisation: the completing store below is a RELEASE, and the two + // together are what make the writes this function performed -- the + // vtable, and every classToInterfaceMap_[classId] row -- visible + // to a thread that observes the flag set. + // + // With plain accesses on arm64 a second thread could see LOADED==1 while + // those table stores were still invisible, then index a row that read as + // NULL. OBSERVED: three identical SIGSEGVs at + // classToInterfaceMap_java_util_NavigableMap[classId] + 0x8, reached from + // TreeSet.clear -> the interface dispatch for NavigableMap.clear, in a + // translator that is single-threaded in its own code but shares the + // process with the GC thread, which also runs Java and so also runs + // class initialisers. + b.append("(CODENAME_ONE_THREAD_STATE) {\n if(__atomic_load_n(&__") + .append(clsName).append("_LOADED__, __ATOMIC_ACQUIRE)) return;\n\n "); // Block-registered enter/exit (the synchronized-method pattern): if the @@ -1579,7 +1654,10 @@ public String generateCCode(List allClasses) { b.append(".vtable = initVtableForInterface();\n"); b.append(" classToInterfaceMap_"); b.append(clsName); - b.append(" = malloc(sizeof(int*) * cn1_array_start_offset);\n"); + // calloc, not malloc: rows are filled only for classes that implement + // this interface, so an id that does not read as a registered row must + // read as NULL rather than as whatever the allocator last left there. + b.append(" = calloc(cn1_array_start_offset, sizeof(int*));\n"); for(ByteCodeClass cls : allClasses) { if(!cls.isInterface) { if(cls.doesImplement(this)) { @@ -1616,9 +1694,21 @@ public String generateCCode(List allClasses) { b.append(".vtable);\n"); } - b.append(" class__"); + b.append(" __atomic_store_n(&class__"); b.append(clsName); - b.append(".initialized = JAVA_TRUE;\n"); + // This flag means STARTED, not completed: the JLS requires a class whose + // initialiser re-enters itself to proceed rather than deadlock, so it has + // to be set before __CLINIT__ runs, and the check above the monitor is + // that recursion guard. Nothing outside this function may treat it as + // "safe to use the class" in the JLS sense -- a class under initialization + // is not finished. The release is what the INLINE GUARDS acquire against: + // they test this flag, and it is what publishes the vtable and the + // classToInterfaceMap rows written just above. Guarding them on + // __X_LOADED__ instead would also be correct about the vtable and would + // additionally hold other threads until __CLINIT__ returned -- a strictly + // later gate than master opens, which moved layout on four native ports + // and is not what the visibility defect required. + b.append(".initialized, JAVA_TRUE, __ATOMIC_RELEASE);\n"); // init static fields and invoke the static initializer code block if(clInitMethod != null) { b.append(" "); @@ -1629,7 +1719,10 @@ public String generateCCode(List allClasses) { b.append(clsName); b.append(");\n"); - b.append("__").append(clsName).append("_LOADED__=1;\n"); + // RELEASE: pairs with the acquire on the fast path above, so everything + // this initialiser wrote happens-before another thread's early return. + b.append("__atomic_store_n(&__").append(clsName) + .append("_LOADED__, 1, __ATOMIC_RELEASE);\n"); b.append("}\n\n"); @@ -2000,7 +2093,6 @@ public String generateCHeader() { b.append("extern void __STATIC_INITIALIZER_"); b.append(clsName); b.append("(CODENAME_ONE_THREAD_STATE);\n"); - b.append("extern void __FINALIZER_"); b.append(clsName); b.append("(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT objToDelete);\n"); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeField.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeField.java index 8eac3b6d495..3b31238b03c 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeField.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeField.java @@ -41,7 +41,7 @@ public class ByteCodeField { private int arrayDimensions; private String type; - private Class primitiveType; + private PrimitiveType primitiveType; private boolean finalField; private Object value; private boolean privateField; @@ -81,28 +81,28 @@ public ByteCodeField(String clsName, int access, String name, String desc, Strin type = objectType; break; case 'I': - primitiveType = Integer.TYPE; + primitiveType = PrimitiveType.INT; break; case 'J': - primitiveType = Long.TYPE; + primitiveType = PrimitiveType.LONG; break; case 'B': - primitiveType = Byte.TYPE; + primitiveType = PrimitiveType.BYTE; break; case 'S': - primitiveType = Short.TYPE; + primitiveType = PrimitiveType.SHORT; break; case 'F': - primitiveType = Float.TYPE; + primitiveType = PrimitiveType.FLOAT; break; case 'D': - primitiveType = Double.TYPE; + primitiveType = PrimitiveType.DOUBLE; break; case 'Z': - primitiveType = Boolean.TYPE; + primitiveType = PrimitiveType.BOOLEAN; break; case 'C': - primitiveType = Character.TYPE; + primitiveType = PrimitiveType.CHAR; break; } } @@ -211,31 +211,10 @@ public String getRuntimeDescriptor() { if (primitiveType == null) { return type; } - if (primitiveType == Integer.TYPE) { - return "I"; - } - if (primitiveType == Long.TYPE) { - return "J"; - } - if (primitiveType == Byte.TYPE) { - return "B"; - } - if (primitiveType == Short.TYPE) { - return "S"; - } - if (primitiveType == Float.TYPE) { - return "F"; - } - if (primitiveType == Double.TYPE) { - return "D"; - } - if (primitiveType == Boolean.TYPE) { - return "Z"; - } - if (primitiveType == Character.TYPE) { - return "C"; - } - return null; + // A field is never void, so PrimitiveType.VOID's "V" is unreachable here; + // the chain this replaces returned null for it, which no caller handled + // either. + return primitiveType.getDescriptor(); } public boolean isPrivate() { diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeMethodArg.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeMethodArg.java index e956745d0cb..b2b961f107e 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeMethodArg.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeMethodArg.java @@ -33,14 +33,14 @@ public class ByteCodeMethodArg { private final int arrayDimensions; private String type; - private Class primitiveType; + private PrimitiveType primitiveType; public ByteCodeMethodArg(String type, int dim) { this.type = type.replace('/', '_').replace('$', '_'); arrayDimensions = dim; } - public ByteCodeMethodArg(Class type, int dim) { + public ByteCodeMethodArg(PrimitiveType type, int dim) { this.primitiveType = type; arrayDimensions = dim; } @@ -49,13 +49,13 @@ public char getQualifier() { if(type != null || arrayDimensions > 0) { return 'o'; } - if(primitiveType == Long.TYPE) { + if(primitiveType == PrimitiveType.LONG) { return 'l'; } - if(primitiveType == Double.TYPE) { + if(primitiveType == PrimitiveType.DOUBLE) { return 'd'; } - if(primitiveType == Float.TYPE) { + if(primitiveType == PrimitiveType.FLOAT) { return 'f'; } return 'i'; @@ -93,7 +93,11 @@ public int hashCode() { if(type != null) { return type.hashCode(); } - return primitiveType.hashCode(); + // ordinal(), not hashCode(): Enum.hashCode is an identity hash on OpenJDK + // and the ordinal in ParparVM's java.lang.Enum, so hashing on it would make + // a hash container of these args iterate in a different order under the + // self-hosted translator than under the JVM-hosted one. + return primitiveType.ordinal(); } @Override @@ -121,11 +125,11 @@ public boolean equals(Object obj) { } public boolean isVoid() { - return primitiveType == Void.TYPE; + return primitiveType == PrimitiveType.VOID; } public boolean isDoubleOrLong() { - return (primitiveType == Double.TYPE || primitiveType == Long.TYPE) && arrayDimensions == 0; + return (primitiveType == PrimitiveType.DOUBLE || primitiveType == PrimitiveType.LONG) && arrayDimensions == 0; } /** @@ -139,7 +143,7 @@ public String getTypeName() { return type; } - public Class getPrimitiveType() { + public PrimitiveType getPrimitiveType() { return primitiveType; } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java index 6dc2226a082..5b6e22b1648 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java @@ -25,13 +25,14 @@ import java.io.DataInputStream; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -155,9 +156,9 @@ private static void sortByName(File[] files) { } void execute(File sourceDir, File outputDir) throws Exception { - File[] directoryList = sourceDir.listFiles(pathname -> + File[] directoryList = Util.listFiles(sourceDir, pathname -> !pathname.isHidden() && !pathname.getName().startsWith(".") && pathname.isDirectory()); - File[] fileList = sourceDir.listFiles(pathname -> + File[] fileList = Util.listFiles(sourceDir, pathname -> !pathname.isHidden() && !pathname.getName().startsWith(".") && !pathname.isDirectory()); // listFiles() returns whatever order the filesystem hands back, which can // differ between two builds of the same input (the app classes are @@ -179,7 +180,7 @@ void execute(File sourceDir, File outputDir) throws Exception { } else { if(!f.isDirectory() && !isBuildMetadata(f)) { // copy the file to the dest dir - copy(Files.newInputStream(f.toPath()), Files.newOutputStream(new File(outputDir, f.getName()).toPath())); + copy(new FileInputStream(f), new FileOutputStream(new File(outputDir, f.getName()))); // Everything that reaches here is hand-written: a port native, a // cn1lib's native, or an application resource. This is the only // point at which its ORIGIN is still known -- one line further on @@ -212,7 +213,7 @@ private void copyDir(File source, File destDir) throws IOException { if(f.isDirectory()) { copyDir(f, destFile); } else { - copy(Files.newInputStream(f.toPath()), Files.newOutputStream(new File(destFile, f.getName()).toPath())); + copy(new FileInputStream(f), new FileOutputStream(new File(destFile, f.getName()))); } } } @@ -226,7 +227,7 @@ private void copyDir(File source, File destDir) throws IOException { * engine compiled in. Set by the platform builders from their class scan. */ static boolean isBundledSqliteEnabled() { - return "true".equals(System.getProperty("cn1.sqlite", "false")); + return "true".equals(Util.getProperty("cn1.sqlite", "false")); } /** @@ -234,7 +235,7 @@ static boolean isBundledSqliteEnabled() { * system libsqlite3, which has no cipher support, with the bundled engine. */ static boolean isBundledSqliteCipherEnabled() { - return "true".equals(System.getProperty("cn1.sqlcipher", "false")); + return "true".equals(Util.getProperty("cn1.sqlcipher", "false")); } /** @@ -267,7 +268,7 @@ static boolean isBundledSqliteCipherEnabled() { * shipping target. */ public static boolean isCheckedCastsEnabled() { - return "true".equalsIgnoreCase(System.getProperty("cn1.checkedCasts", "false")); + return "true".equalsIgnoreCase(Util.getProperty("cn1.checkedCasts", "false")); } /// Writes the bundled SQLite engine into a source root, or takes it back out. @@ -314,7 +315,7 @@ private static File copyRuntimeResource(File srcRoot, String name) throws IOExce */ private static File copyRuntimeResource(File srcRoot, String name, String destName) throws IOException { File dest = new File(srcRoot, destName); - copy(ByteCodeTranslator.class.getResourceAsStream("/" + name), Files.newOutputStream(dest.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/" + name), new FileOutputStream(dest)); sourceManifest.recordRuntime(destName, "/" + name); return dest; } @@ -331,7 +332,7 @@ private static File copyRuntimeResource(File srcRoot, String name, String destNa */ private static File copyVendoredResource(File srcRoot, String name) throws IOException { File dest = new File(srcRoot, name); - copy(ByteCodeTranslator.class.getResourceAsStream("/" + name), Files.newOutputStream(dest.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/" + name), new FileOutputStream(dest)); sourceManifest.recordVendored(name, "/" + name); return dest; } @@ -413,7 +414,7 @@ public static void main(String[] args) throws Exception { final String appType = args[7]; final String addFrameworks = args[8]; // we accept 3 argument output types, input directory and output directory - if (System.getProperty("saveUnitTests", "false").equals("true")) { + if (Util.getProperty("saveUnitTests", "false").equals("true")) { System.out.println("Generating Unit Tests"); ByteCodeClass.setSaveUnitTests(true); } @@ -430,7 +431,7 @@ public static void main(String[] args) throws Exception { // Unrecognized output type falls back to the plain copy-through default handler recognizedOutputType = false; } - String[] sourceDirectories = args[1].split(";"); + String[] sourceDirectories = Util.splitLiteral(args[1], ';'); File[] sources = new File[sourceDirectories.length]; for(int iter = 0 ; iter < sourceDirectories.length ; iter++) { sources[iter] = new File(sourceDirectories[iter]); @@ -514,15 +515,15 @@ private static void handleCleanOutput(ByteCodeTranslator b, File[] sources, File // generated. A project that gets the C and not the .S links against a // missing symbol, which is at least loud. emitVirtualThreadRuntime(srcRoot); - if (System.getProperty("INCLUDE_NPE_CHECKS", "false").equals("true")) { + if (Util.getProperty("INCLUDE_NPE_CHECKS", "false").equals("true")) { replaceInFile(cn1Globals, "//#define CN1_INCLUDE_NPE_CHECKS", "#define CN1_INCLUDE_NPE_CHECKS"); } - if ("true".equalsIgnoreCase(System.getProperty("cn1.onDeviceDebug", "false"))) { + if ("true".equalsIgnoreCase(Util.getProperty("cn1.onDeviceDebug", "false"))) { replaceInFile(cn1Globals, "//#define CN1_ON_DEVICE_DEBUG", "#define CN1_ON_DEVICE_DEBUG"); } copyRuntimeResource(srcRoot, "cn1_globals.m", "cn1_globals.c"); copyRuntimeResource(srcRoot, "nativeMethods.m", "nativeMethods.c"); - if (System.getProperty("USE_RPMALLOC", "false").equals("true")) { + if (Util.getProperty("USE_RPMALLOC", "false").equals("true")) { copyRuntimeResource(srcRoot, "malloc.c"); copyRuntimeResource(srcRoot, "rpmalloc.c"); copyRuntimeResource(srcRoot, "rpmalloc.h"); @@ -554,7 +555,7 @@ private static void handleCleanOutput(ByteCodeTranslator b, File[] sources, File File classMethodIndexM = new File(srcRoot, "cn1_class_method_index.m"); if (classMethodIndexM.exists()) { File classMethodIndexC = new File(srcRoot, "cn1_class_method_index.c"); - copy(Files.newInputStream(classMethodIndexM.toPath()), Files.newOutputStream(classMethodIndexC.toPath())); + copy(new FileInputStream(classMethodIndexM), new FileOutputStream(classMethodIndexC)); if(!classMethodIndexM.delete()) { System.err.println("Deletion of " + classMethodIndexM.getAbsolutePath() + " failed"); } @@ -622,7 +623,14 @@ private static void embedWindowsResources(File[] sources, File srcRoot) throws I StringBuilder table = new StringBuilder(); table.append("/* Auto-generated by the ParparVM windows target: maps a classpath\n"); table.append(" * resource path to the RCDATA id embedded in the executable. */\n"); - table.append("#include \n\n"); + table.append("#include \n"); + // Behind _WIN32: the windows APP TYPE is compiled on a Linux host by + // CleanTargetIntegrationTest#generatesRunnableExecutableForWindowsAppType, + // where windows.h does not exist. The id table below is plain C and stays + // unguarded; only the resource-resolving override needs the platform. + table.append("#if defined(_WIN32)\n"); + table.append("#include \n"); + table.append("#endif\n\n"); table.append("typedef struct { const char* name; int id; } CN1ResourceEntry;\n\n"); table.append("static const CN1ResourceEntry cn1ResourceTable[] = {\n"); @@ -634,7 +642,7 @@ private static void embedWindowsResources(File[] sources, File srcRoot) throws I int id = 1; for (java.util.Map.Entry e : resources.entrySet()) { File staged = new File(resDir, "res" + id); - copy(Files.newInputStream(e.getValue().toPath()), Files.newOutputStream(staged.toPath())); + copy(new FileInputStream(e.getValue()), new FileOutputStream(staged)); // RC filenames are resolved relative to the .rc (srcRoot); llvm-rc and // rc.exe both accept forward slashes. rc.append(id).append(" RCDATA \"cn1_resources/res").append(id).append("\"\n"); @@ -642,7 +650,7 @@ private static void embedWindowsResources(File[] sources, File srcRoot) throws I id++; } sourceManifest.recordGenerated("cn1_resources.rc"); - Files.write(new File(srcRoot, "cn1_resources.rc").toPath(), + Util.writeBytes(new File(srcRoot, "cn1_resources.rc"), rc.toString().getBytes(StandardCharsets.UTF_8)); } @@ -654,9 +662,14 @@ private static void embedWindowsResources(File[] sources, File srcRoot) throws I table.append(" if (strcmp(cn1ResourceTable[i].name, name) == 0) { return cn1ResourceTable[i].id; }\n"); table.append(" }\n"); table.append(" return 0;\n"); - table.append("}\n"); + table.append("}\n\n"); + // No cn1FindResource override is emitted. The id table above is built and + // linked, and nothing reads it: Class.getResourceAsStream deliberately does + // not consult embedded resources, because doing so changed how shipping + // applications render (see the comment there). Wiring these together is the + // whole of that future change. sourceManifest.recordGenerated("cn1_resources_table.c"); - Files.write(new File(srcRoot, "cn1_resources_table.c").toPath(), + Util.writeBytes(new File(srcRoot, "cn1_resources_table.c"), table.toString().getBytes(StandardCharsets.UTF_8)); } @@ -705,7 +718,7 @@ private static void embedLinuxResources(File[] sources, File srcRoot) throws IOE int id = 1; for (java.util.Map.Entry e : resources.entrySet()) { File staged = new File(resDir, "res" + id); - copy(Files.newInputStream(e.getValue().toPath()), Files.newOutputStream(staged.toPath())); + copy(new FileInputStream(e.getValue()), new FileOutputStream(staged)); // Absolute path so .incbin resolves regardless of the assembler's // working directory (the build runs out of a separate build dir). String incPath = escapeCString(staged.getAbsolutePath().replace('\\', '/')); @@ -720,7 +733,7 @@ private static void embedLinuxResources(File[] sources, File srcRoot) throws IOE id++; } sourceManifest.recordGenerated("cn1_resources_data.S"); - Files.write(new File(srcRoot, "cn1_resources_data.S").toPath(), + Util.writeBytes(new File(srcRoot, "cn1_resources_data.S"), asm.toString().getBytes(StandardCharsets.UTF_8)); } @@ -744,9 +757,14 @@ private static void embedLinuxResources(File[] sources, File srcRoot) throws IOE table.append(" }\n"); table.append(" if (lenOut) { *lenOut = 0; }\n"); table.append(" return 0;\n"); - table.append("}\n"); + table.append("}\n\n"); + // No cn1FindResource override is emitted. The id table above is built and + // linked, and nothing reads it: Class.getResourceAsStream deliberately does + // not consult embedded resources, because doing so changed how shipping + // applications render (see the comment there). Wiring these together is the + // whole of that future change. sourceManifest.recordGenerated("cn1_resources_table.c"); - Files.write(new File(srcRoot, "cn1_resources_table.c").toPath(), + Util.writeBytes(new File(srcRoot, "cn1_resources_table.c"), table.toString().getBytes(StandardCharsets.UTF_8)); } @@ -775,7 +793,19 @@ private static void collectResources(File root, File dir, java.util.LinkedHashMa || ext.equals("mm") || ext.equals("rc")) { continue; } - String rel = root.toPath().relativize(f.toPath()).toString().replace('\\', '/'); + // Relative path by absolute-prefix strip rather than Path.relativize: + // JavaAPI has no java.nio.file, and the translator compiles against it + // when it translates itself. f is always under root here -- it came from + // a walk of root -- so the prefix always matches. + String rootAbs = root.getAbsolutePath(); + String fileAbs = f.getAbsolutePath(); + String rel = fileAbs.startsWith(rootAbs) + ? fileAbs.substring(rootAbs.length()) + : fileAbs; + while (rel.startsWith(File.separator) || rel.startsWith("/")) { + rel = rel.substring(1); + } + rel = rel.replace('\\', '/'); String key = "/" + rel; if (!out.containsKey(key)) { out.put(key, f); @@ -836,7 +866,7 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File launchImageLaunchimage.mkdirs(); //cleanDir(launchImageLaunchimage); - copy(ByteCodeTranslator.class.getResourceAsStream("/LaunchImages.json"), Files.newOutputStream(new File(launchImageLaunchimage, "Contents.json").toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/LaunchImages.json"), new FileOutputStream(new File(launchImageLaunchimage, "Contents.json"))); } File appIconAppiconset = new File(imagesXcassets, "AppIcon.appiconset"); @@ -847,7 +877,7 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File // wants the 16..512 @1x/@2x "mac" idiom ladder. copy(ByteCodeTranslator.class.getResourceAsStream( platform.hasIosDeviceIdioms() ? "/Icons.json" : "/Icons-macos.json"), - Files.newOutputStream(new File(appIconAppiconset, "Contents.json").toPath())); + new FileOutputStream(new File(appIconAppiconset, "Contents.json"))); File xcproj = new File(root, appName + ".xcodeproj"); @@ -866,17 +896,30 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File // generated. A project that gets the C and not the .S links against a // missing symbol, which is at least loud. emitVirtualThreadRuntime(srcRoot); - if (System.getProperty("INCLUDE_NPE_CHECKS", "false").equals("true")) { + if (Util.getProperty("INCLUDE_NPE_CHECKS", "false").equals("true")) { replaceInFile(cn1Globals, "//#define CN1_INCLUDE_NPE_CHECKS", "#define CN1_INCLUDE_NPE_CHECKS"); } - if ("true".equalsIgnoreCase(System.getProperty("cn1.onDeviceDebug", "false"))) { + if ("true".equalsIgnoreCase(Util.getProperty("cn1.onDeviceDebug", "false"))) { replaceInFile(cn1Globals, "//#define CN1_ON_DEVICE_DEBUG", "#define CN1_ON_DEVICE_DEBUG"); } copyRuntimeResource(srcRoot, "cn1_globals.m"); copyRuntimeResource(srcRoot, "nativeMethods.m"); - copyRuntimeResource(srcRoot, "java_io_File.m"); - - if (System.getProperty("USE_RPMALLOC", "false").equals("true")) { + // java_io_File_RUNTIME.m, not java_io_File.m. When the application retains + // java.io.File -- which the filesystem fallback makes ordinary -- Parser + // .writeOutput emits the translated class to java_io_File.m and overwrites + // the port's hand-written native that was copied here first. The generated + // File.exists() then has no existsImpl to link against. + // + // OBSERVED as `Undefined symbols: _java_io_File_existsImpl ... referenced + // from _java_io_File_exists___R_boolean in java_io_File.o` on the iOS legs. + // The clean target already avoids the same collision by emitting + // java_io_File_runtime.c; this is that fix for the Apple path. The name only + // has to differ from the generated one -- the compiler globs the directory, + // and NativeSignatureVerifier reads the RESOURCE "/java_io_File.m" rather + // than the emitted filename. + copyRuntimeResource(srcRoot, "java_io_File.m", "java_io_File_runtime.m"); + + if (Util.getProperty("USE_RPMALLOC", "false").equals("true")) { copyRuntimeResource(srcRoot, "malloc.c"); copyRuntimeResource(srcRoot, "rpmalloc.c"); copyRuntimeResource(srcRoot, "rpmalloc.h"); @@ -894,23 +937,35 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File Parser.writeOutput(srcRoot); File templateInfoPlist = new File(srcRoot, appName + "-Info.plist"); - copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template/template-Info.plist"), Files.newOutputStream(templateInfoPlist.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template/template-Info.plist"), new FileOutputStream(templateInfoPlist)); File templatePch = new File(srcRoot, appName + "-Prefix.pch"); - copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template/template-Prefix.pch"), Files.newOutputStream(templatePch.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template/template-Prefix.pch"), new FileOutputStream(templatePch)); copyRuntimeResource(srcRoot, "xmlvm.h"); File projectWorkspaceData = new File(projectXCworkspace, "contents.xcworkspacedata"); - copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template.xcodeproj/project.xcworkspace/contents.xcworkspacedata"), Files.newOutputStream(projectWorkspaceData.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template.xcodeproj/project.xcworkspace/contents.xcworkspacedata"), new FileOutputStream(projectWorkspaceData)); replaceInFile(projectWorkspaceData, "KitchenSink", appName); File projectPbx = new File(xcproj, "project.pbxproj"); - copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template.xcodeproj/project.pbxproj"), Files.newOutputStream(projectPbx.toPath())); - - String[] sourceFiles = srcRoot.list((pathname, string) -> - string.endsWith(".bundle") || string.endsWith(".xcdatamodeld") || !pathname.isHidden() && !string.startsWith(".") && !"Images.xcassets".equals(string)); + copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template.xcodeproj/project.pbxproj"), new FileOutputStream(projectPbx)); + + // File.list(FilenameFilter) is not in JavaAPI; filter the plain listing. + String[] allNames = srcRoot.list(); + java.util.List keptNames = new java.util.ArrayList(); + if (allNames != null) { + for (String string : allNames) { + File pathname = new File(srcRoot, string); + if (string.endsWith(".bundle") || string.endsWith(".xcdatamodeld") + || !pathname.isHidden() && !string.startsWith(".") + && !"Images.xcassets".equals(string)) { + keptNames.add(string); + } + } + } + String[] sourceFiles = keptNames.toArray(new String[keptNames.size()]); StringBuilder fileOneEntry = new StringBuilder(); StringBuilder fileTwoEntry = new StringBuilder(); @@ -926,7 +981,7 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File List includeFrameworks = new ArrayList<>(); Set optionalFrameworks = new HashSet<>(); - for (String optionalFramework : System.getProperty("optional.frameworks", "").split(";")) { + for (String optionalFramework : Util.splitLiteral(Util.getProperty("optional.frameworks", ""), ';')) { optionalFramework = optionalFramework.trim(); if (!optionalFramework.isEmpty()) { optionalFrameworks.add(optionalFramework); @@ -996,7 +1051,7 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File includeFrameworks.add("libz.dylib"); includeFrameworks.add("AVKit.framework"); if(!addFrameworks.equalsIgnoreCase("none")) { - includeFrameworks.addAll(Arrays.asList(addFrameworks.split(";"))); + includeFrameworks.addAll(Arrays.asList(Util.splitLiteral(addFrameworks, ';'))); } int currentValue = 0xF63EAAA; @@ -1142,7 +1197,7 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File "***FRAMEWORKS2***", frameworks2.toString(), "***RESOURCES***", resources.toString()); } - String bundleVersion = System.getProperty("bundleVersionNumber", appVersion); + String bundleVersion = Util.getProperty("bundleVersionNumber", appVersion); replaceInFile(templateInfoPlist, "com.codename1pkg", appPackageName, "${PRODUCT_NAME}", appDisplayName, "VERSION_VALUE", appVersion, "VERSION_BUNDLE_VALUE", bundleVersion); // Written to the project root, NOT to srcRoot. srcRoot.list() above feeds the @@ -1166,7 +1221,7 @@ private static void writeCmakeProject(File projectRoot, File srcRoot, String app boolean windows = "windows".equalsIgnoreCase(appType); boolean linux = "linux".equalsIgnoreCase(appType); boolean executable = windows || linux; - try (Writer writer = new OutputStreamWriter(Files.newOutputStream(cmakeLists.toPath()), StandardCharsets.UTF_8)) { + try (Writer writer = new OutputStreamWriter(new FileOutputStream(cmakeLists), "UTF-8")) { writer.append("cmake_minimum_required(VERSION 3.10)\n"); // The native Windows port mixes the translated C runtime with a C++ // layer for the COM APIs that have no C binding (DirectWrite), so the @@ -1467,6 +1522,18 @@ private static void writeLinuxLinkSet(Writer writer) throws IOException { // binary, where the companion exists to turn an address back into a Java method. // It is the wrong one for a CI build whose whole job is to be autopsied, so the // level is a cache variable: unset it and nothing changes for anybody. + // A diagnostic-only define hook, empty by default so nothing changes for + // anybody who does not ask. The reason it exists: the collector's own + // heap-integrity verifier (-DCN1_GC_VERIFY) is the designed detector for + // "the sweep reclaimed something a retained object still references", and + // there was no way to turn it on for a generated project without editing + // the emitted CMakeLists by hand. Chasing a dangling field reference + // through core dumps is what made that gap expensive. + writer.append("set(CN1_EXTRA_DEFINES \"\" CACHE STRING\n"); + writer.append(" \"Extra preprocessor defines for diagnostic builds, semicolon separated (e.g. CN1_GC_VERIFY)\")\n"); + writer.append("if(CN1_EXTRA_DEFINES)\n"); + writer.append(" target_compile_definitions(${PROJECT_NAME} PRIVATE ${CN1_EXTRA_DEFINES})\n"); + writer.append("endif()\n"); writer.append("set(CN1_DEBUG_INFO_LEVEL \"1\" CACHE STRING\n"); writer.append(" \"DWARF level for the .debug companion: 1 = lines + function names (lean, the default), 3 = full variable and type information (autopsyable)\")\n"); writer.append("target_compile_options(${PROJECT_NAME} PRIVATE -g${CN1_DEBUG_INFO_LEVEL} -fno-asynchronous-unwind-tables -fno-unwind-tables)\n"); @@ -1568,12 +1635,12 @@ private static String getFileType(String s) { // to be mutated. Also, expire the temporary byte[] buffer so it can // be collected. // - private static StringBuilder readFileAsStringBuilder(File sourceFile) throws IOException + private static String readFileAsString(File sourceFile) throws IOException { - try(DataInputStream dis = new DataInputStream(Files.newInputStream(sourceFile.toPath()))) { + try(DataInputStream dis = new DataInputStream(new FileInputStream(sourceFile))) { byte[] data = new byte[(int) sourceFile.length()]; dis.readFully(data); - return new StringBuilder(new String(data, StandardCharsets.UTF_8)); + return new String(data, StandardCharsets.UTF_8); } } // @@ -1584,21 +1651,38 @@ private static StringBuilder readFileAsStringBuilder(File sourceFile) throws IOE // process for large projects. // private static void replaceInFile(File sourceFile, String... values) throws IOException { - StringBuilder str = readFileAsStringBuilder(sourceFile); + // A String rather than a StringBuilder because the translator has to compile + // against ParparVM's own JavaAPI in order to translate itself, and + // StringBuilder there has neither indexOf nor replace. + // + // One pass per target, appending into a fresh builder. The obvious + // translation of the old in-place edit -- indexOf on str.toString(), then + // substring/concat the whole buffer back together per match -- copies the + // ENTIRE file twice for every occurrence, which is the opposite of this + // method's purpose: it exists to avoid the memory spike that made large + // Xcode project.pbxproj rewrites fail with OutOfMemoryError. Each target + // now costs one traversal and one output buffer regardless of how many + // times it matches. + String str = readFileAsString(sourceFile); int totchanges = 0; - // perform the mutations on stringbuilder, which ought to implement - // these operations efficiently. for (int iter = 0; iter < values.length; iter += 2) { String target = values[iter]; String replacement = values[iter + 1]; - int index = 0; - while ((index = str.indexOf(target, index)) >= 0) { - int targetSize = target.length(); - str.replace(index, index + targetSize, replacement); - index += replacement.length(); + int index = str.indexOf(target); + if (index < 0) { + continue; + } + StringBuilder out = new StringBuilder(str.length() + 64); + int from = 0; + while (index >= 0) { + out.append(str, from, index).append(replacement); + from = index + target.length(); totchanges++; + index = str.indexOf(target, from); } + out.append(str, from, str.length()); + str = out.toString(); } // @@ -1607,8 +1691,8 @@ private static void replaceInFile(File sourceFile, String... values) throws IOEx if(verbose) { System.out.println("Rewrite " + sourceFile + " with " + totchanges + " changes"); } - try(Writer fios = new OutputStreamWriter(Files.newOutputStream(sourceFile.toPath()), StandardCharsets.UTF_8)) { - fios.write(str.toString()); + try(Writer fios = new OutputStreamWriter(new FileOutputStream(sourceFile), "UTF-8")) { + fios.write(str); } } @@ -1665,7 +1749,7 @@ private static void emitVirtualThreadRuntime(File srcRoot) throws IOException { // cause, where the link error later names only a symbol. throw new IOException("virtual-thread runtime resource missing: " + name); } - copy(in, Files.newOutputStream(new File(srcRoot, name).toPath())); + copy(in, new FileOutputStream(new File(srcRoot, name))); sourceManifest.recordRuntime(name, "/" + name); } } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java index 35609a140c8..e9e533d0174 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java @@ -104,12 +104,12 @@ public static void setDependencyGraph(MethodDependencyGraph dependencyGraph) { private int maxLocals; private static boolean acceptStaticOnEquals; private static final boolean FORCE_VOLATILE_LOCALS = - "true".equalsIgnoreCase(System.getProperty("CN1_FORCE_VOLATILE_LOCALS", "false")); + "true".equalsIgnoreCase(Util.getProperty("CN1_FORCE_VOLATILE_LOCALS", "false")); // Frameless codegen gate (-Dcn1.frameless, default on). When off the // eligibility predicate always returns false, so every method emits the // legacy frame code byte-for-byte identical to before. private static final boolean FRAMELESS_ENABLED = - "true".equalsIgnoreCase(System.getProperty("cn1.frameless", "true")); + "true".equalsIgnoreCase(Util.getProperty("cn1.frameless", "true")); // PHASE 3b: extend frameless codegen to OBJECT-BEARING methods (-Dcn1.frameless.objects, // default off). Such a method keeps its object operand stack + object locals in a // method-local C array on the native stack; the C runtime (built with @@ -117,14 +117,14 @@ public static void setDependencyGraph(MethodDependencyGraph dependencyGraph) { // stopped thread's native stack. With this OFF, only primitive-only methods are // frameless (identical to the prior phase). Requires the conservative-GC runtime. private static final boolean FRAMELESS_OBJECTS_ENABLED = - "true".equalsIgnoreCase(System.getProperty("cn1.frameless.objects", "true")); + "true".equalsIgnoreCase(Util.getProperty("cn1.frameless.objects", "true")); // PHASE 3b: extend object-frameless to INSTANCE methods (receiver `this` becomes a // conservatively-scanned C parameter). Now DEFAULT ON: the intermittent multi-threaded // failure that previously gated this off was a pre-existing Thread.start/join visibility // race (alive set on the worker thread async after start() returned), fixed in // java_lang_Thread_start__ (993331107); with it fixed, MtStress is 50/50 deterministic. private static final boolean FRAMELESS_INSTANCE_ENABLED = - "true".equalsIgnoreCase(System.getProperty("cn1.frameless.instance", "true")); + "true".equalsIgnoreCase(Util.getProperty("cn1.frameless.instance", "true")); private int methodOffset; private boolean forceVirtual; private boolean virtualOverriden; @@ -162,7 +162,7 @@ public static void setDependencyGraph(MethodDependencyGraph dependencyGraph) { optimizerOn = op == null || op.equalsIgnoreCase("on"); //optimizerOn = false; - onDeviceDebug = "true".equalsIgnoreCase(System.getProperty("cn1.onDeviceDebug", "false")); + onDeviceDebug = "true".equalsIgnoreCase(Util.getProperty("cn1.onDeviceDebug", "false")); } public static boolean isOnDeviceDebug() { @@ -786,16 +786,16 @@ public BytecodeMethod(String clsName, int access, String name, String desc, Stri if(methodName.equals("")) { methodName = "__INIT__"; constructor = true; - returnType = new ByteCodeMethodArg(Void.TYPE, 0); + returnType = new ByteCodeMethodArg(PrimitiveType.VOID, 0); } else { if(methodName.equals("")) { methodName = "__CLINIT__"; - returnType = new ByteCodeMethodArg(Void.TYPE, 0); + returnType = new ByteCodeMethodArg(PrimitiveType.VOID, 0); staticMethod = true; } else { String retType = desc.substring(pos + 1); if(retType.equals("V")) { - returnType = new ByteCodeMethodArg(Void.TYPE, 0); + returnType = new ByteCodeMethodArg(PrimitiveType.VOID, 0); } else { int dim = 0; while(retType.startsWith("[")) { @@ -818,28 +818,28 @@ public BytecodeMethod(String clsName, int access, String name, String desc, Stri returnType = new ByteCodeMethodArg(objectType, dim); break; case 'I': - returnType = new ByteCodeMethodArg(Integer.TYPE, dim); + returnType = new ByteCodeMethodArg(PrimitiveType.INT, dim); break; case 'J': - returnType = new ByteCodeMethodArg(Long.TYPE, dim); + returnType = new ByteCodeMethodArg(PrimitiveType.LONG, dim); break; case 'B': - returnType = new ByteCodeMethodArg(Byte.TYPE, dim); + returnType = new ByteCodeMethodArg(PrimitiveType.BYTE, dim); break; case 'S': - returnType = new ByteCodeMethodArg(Short.TYPE, dim); + returnType = new ByteCodeMethodArg(PrimitiveType.SHORT, dim); break; case 'F': - returnType = new ByteCodeMethodArg(Float.TYPE, dim); + returnType = new ByteCodeMethodArg(PrimitiveType.FLOAT, dim); break; case 'D': - returnType = new ByteCodeMethodArg(Double.TYPE, dim); + returnType = new ByteCodeMethodArg(PrimitiveType.DOUBLE, dim); break; case 'Z': - returnType = new ByteCodeMethodArg(Boolean.TYPE, dim); + returnType = new ByteCodeMethodArg(PrimitiveType.BOOLEAN, dim); break; case 'C': - returnType = new ByteCodeMethodArg(Character.TYPE, dim); + returnType = new ByteCodeMethodArg(PrimitiveType.CHAR, dim); break; } } @@ -869,28 +869,28 @@ public BytecodeMethod(String clsName, int access, String name, String desc, Stri arguments.add(new ByteCodeMethodArg(objectType, currentArrayDim)); break; case 'I': - arguments.add(new ByteCodeMethodArg(Integer.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.INT, currentArrayDim)); break; case 'J': - arguments.add(new ByteCodeMethodArg(Long.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.LONG, currentArrayDim)); break; case 'B': - arguments.add(new ByteCodeMethodArg(Byte.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.BYTE, currentArrayDim)); break; case 'S': - arguments.add(new ByteCodeMethodArg(Short.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.SHORT, currentArrayDim)); break; case 'F': - arguments.add(new ByteCodeMethodArg(Float.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.FLOAT, currentArrayDim)); break; case 'D': - arguments.add(new ByteCodeMethodArg(Double.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.DOUBLE, currentArrayDim)); break; case 'Z': - arguments.add(new ByteCodeMethodArg(Boolean.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.BOOLEAN, currentArrayDim)); break; case 'C': - arguments.add(new ByteCodeMethodArg(Character.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.CHAR, currentArrayDim)); break; } currentArrayDim = 0; @@ -1395,6 +1395,18 @@ public List debugVarEntries() { return rows; } + /** + * Every local, in a deterministic order, for emitting the C declarations. + * + * Unlike {@link #debugVarEntries} this drops nothing: a local whose slot lies + * outside the frame still needs its declaration, it just has no debug row. + */ + private List declarationOrderedLocals() { + List ordered = new ArrayList(localVariables); + Collections.sort(ordered, DEBUG_VAR_ORDER); + return ordered; + } + /** Slot first, then storage qualifier, so a reused slot's rows stay adjacent. */ private static final Comparator DEBUG_VAR_ORDER = new Comparator() { @Override @@ -1629,29 +1641,29 @@ private void fixUpBarebone() { CustomJump cj = (CustomJump)i; String cmp = cj.getCustomCompareCode(); if (cmp != null) { - cj.setCustomCompareCode(cmp.replaceAll("locals\\[(\\d+)\\]\\.data\\.o", "olocals_$1_")); + cj.setCustomCompareCode(Util.rewriteLocalObjectRefs(cmp)); } } else if (i instanceof CustomIntruction) { CustomIntruction ci = (CustomIntruction)i; String code = ci.getCode(); if (code != null) { - ci.setCode(code.replaceAll("locals\\[(\\d+)\\]\\.data\\.o", "olocals_$1_")); + ci.setCode(Util.rewriteLocalObjectRefs(code)); } String complexCode = ci.getComplexCode(); if (complexCode != null) { - ci.setComplexCode(complexCode.replaceAll("locals\\[(\\d+)\\]\\.data\\.o", "olocals_$1_")); + ci.setComplexCode(Util.rewriteLocalObjectRefs(complexCode)); } } else if (i instanceof CustomInvoke) { CustomInvoke ci = (CustomInvoke)i; String target = ci.getTargetObjectLiteral(); if (target != null) { - ci.setTargetObjectLiteral(target.replaceAll("locals\\[(\\d+)\\]\\.data\\.o", "olocals_$1_")); + ci.setTargetObjectLiteral(Util.rewriteLocalObjectRefs(target)); } String[] args = ci.getLiteralArgs(); if (args != null) { for (int j=0; j added = new HashSet(); - for (LocalVariable lv : localVariables) { + // Sorted, not in localVariables iteration order: that is a HashSet, so the + // order of these declarations varied between builds of the same input. + // debugVarEntries already had to learn this for the debug side-table; the + // C declarations had the same defect and it stayed invisible because + // HotSpot's identity hash is stable within a run. Translating the + // translator with itself is what surfaced it -- a different runtime, a + // different order, and the same input produced different C. + for (LocalVariable lv : declarationOrderedLocals()) { String variableName = lv.getQualifier() + "locals_"+lv.getIndex()+"_"; if (!added.contains(variableName) && (barebone || lv.getQualifier() != 'o')) { added.add(variableName); @@ -2174,7 +2193,7 @@ public void appendVirtualMethodC(String cls, StringBuilder b, String offset, boo b.append(cls); b.append("(threadStateData);\n "); } - if (System.getProperty("INCLUDE_NPE_CHECKS", "false").equals("true")) { + if (Util.getProperty("INCLUDE_NPE_CHECKS", "false").equals("true")) { b.append("\n if(__cn1ThisObject == JAVA_NULL) THROW_NULL_POINTER_EXCEPTION();\n "); } if(!returnType.isVoid()) { @@ -2359,7 +2378,7 @@ public NativeSignatureVerifier.Signature getNativeSignature() { } return new NativeSignatureVerifier.Signature(symbol.toString(), clsName, methodName, overloadPrefix, cReturnType.toString().trim(), params, - prototype.toString().trim().replaceAll("\\s+", " ")); + Util.collapseWhitespace(prototype.toString().trim())); } public boolean isAbstract() { @@ -2436,6 +2455,228 @@ public String getDesc() { return desc; } + /** + * The type this method allocates and hands straight back -- NEW T, DUP, the + * constructor arguments, T.<init>, ARETURN -- or null for any other shape. + * The point of being this strict is that the caller uses the answer as a + * certainty about the returned object's concrete class, so a body that could + * return something it did not just allocate has to be rejected rather than + * guessed at. + */ + public String allocatedReturnType() { + List real = new ArrayList(); + for (Instruction i : instructions) { + if (i instanceof LabelInstruction || i instanceof LineNumber || i instanceof TryCatch) { + continue; + } + real.add(i); + } + if (real.size() < 4) { + return null; + } + Instruction first = real.get(0); + if (!(first instanceof TypeInstruction) || first.getOpcode() != Opcodes.NEW) { + return null; + } + String type = ((TypeInstruction) first).getTypeName(); + if (type == null || real.get(1).getOpcode() != Opcodes.DUP) { + return null; + } + if (real.get(real.size() - 1).getOpcode() != Opcodes.ARETURN) { + return null; + } + Instruction ctor = real.get(real.size() - 2); + if (!(ctor instanceof Invoke) || ctor.getOpcode() != Opcodes.INVOKESPECIAL) { + return null; + } + Invoke ci = (Invoke) ctor; + if (!"".equals(ci.getName()) || !type.equals(ci.getOwner())) { + return null; + } + // Everything between the DUP and the constructor has to be a plain local + // read. Anything with a side effect could leave a different object under + // the ARETURN, and then the type above would be a lie. + for (int i = 2; i < real.size() - 2; i++) { + Instruction a = real.get(i); + if (!(a instanceof VarOp) || !isLoadOpcode(a.getOpcode())) { + return null; + } + } + return type; + } + + private static boolean isLoadOpcode(int op) { + return op == Opcodes.ALOAD || op == Opcodes.ILOAD || op == Opcodes.LLOAD + || op == Opcodes.FLOAD || op == Opcodes.DLOAD; + } + + private int nextExecutable(int from) { + for (int i = from; i < instructions.size(); i++) { + Instruction ins = instructions.get(i); + if (ins instanceof LabelInstruction || ins instanceof LineNumber || ins instanceof TryCatch) { + continue; + } + return i; + } + return -1; + } + + private int prevExecutable(int from) { + for (int i = from; i >= 0; i--) { + Instruction ins = instructions.get(i); + if (ins instanceof LabelInstruction || ins instanceof LineNumber || ins instanceof TryCatch) { + continue; + } + return i; + } + return -1; + } + + /// The first local slot that cannot hold an incoming argument. + /// + /// Parameters occupy locals WITHOUT an ASTORE, so a slot-write count of one + /// does not mean the slot holds one value over the method's lifetime -- an + /// Iterator parameter in that slot is a second, earlier value. Long and double + /// take two slots each, per the JVM numbering the instruction stream uses. + /// + /// @return the lowest slot index that is definitely not a parameter + private int firstNonParameterSlot() { + int slots = isStatic() ? 0 : 1; + for (ByteCodeMethodArg arg : arguments) { + char q = arg.getQualifier(); + slots += (q == 'l' || q == 'd') ? 2 : 1; + } + return slots; + } + + private int countStoresTo(int slot) { + int n = 0; + for (Instruction ins : instructions) { + if (ins instanceof VarOp && ins.getOpcode() == Opcodes.ASTORE + && ((VarOp) ins).getIndex() == slot) { + n++; + } + } + return n; + } + + /** + * ITERATOR LOWERING: give a for-each loop the concrete Iterator type its + * collection really returns, so the calls stop going through the interface. + * + * A for-each compiles to Iterator.hasNext()/next() through INVOKEINTERFACE, + * which is the most expensive dispatch the VM has -- a lookup in the owning + * class's interface map before the vtable read -- and it runs twice per + * element. Neither the emitter's closed-world devirtualization nor ThinLTO + * can touch it, because both start from a concrete owner and an interface + * call does not have one: java.util.Iterator has 27 implementors here. + * + * The concrete type is recoverable locally even though the translator has no + * general stack-type inference. If the collection's iterator() has exactly + * one reachable implementation, and that implementation's whole body is + * `return new T(...)`, then the object stored by the ASTORE that follows the + * call is a T -- no inference needed. Retyping the calls to INVOKEVIRTUAL on + * T is then enough on its own: the existing devirtualization in + * Invoke.appendInstruction takes any virtual call with no reachable override + * the rest of the way to a direct one, which ThinLTO can inline. + * + * The single-assignment requirement on the local is what makes this sound + * without dataflow. If a slot were written twice, a second iterator of some + * other class could reach the same ALOAD, and a virtual call on the wrong + * class reads its fields out of an object that does not have them -- silent + * on this VM, since ParparVM's CHECKCAST is unchecked. + * + * Like the concat fusion this must run BEFORE the unused-method cull, so the + * newly created edges exist while reachability is computed. + */ + public void lowerIteratorCalls() { + for (int i = 0; i < instructions.size(); i++) { + Instruction ins = instructions.get(i); + if (!(ins instanceof Invoke)) { + continue; + } + Invoke inv = (Invoke) ins; + int op = inv.getOpcode(); + if (op != Opcodes.INVOKEINTERFACE && op != Opcodes.INVOKEVIRTUAL) { + continue; + } + if (!"iterator".equals(inv.getName()) || !"()Ljava/util/Iterator;".equals(inv.getDesc())) { + continue; + } + ByteCodeClass coll = Parser.getClassObject(Util.mangle(inv.getOwner())); + String itType = Parser.resolveConcreteIteratorType(coll); + if (itType == null) { + continue; + } + int st = nextExecutable(i + 1); + if (st < 0) { + continue; + } + Instruction store = instructions.get(st); + if (!(store instanceof VarOp) || store.getOpcode() != Opcodes.ASTORE) { + continue; + } + int slot = ((VarOp) store).getIndex(); + // Exactly one ASTORE is not enough on its own: a parameter reaches its + // slot without one, so a method that takes an Iterator and later reuses + // that slot for this loop's iterator has TWO values in it. Rewriting the + // parameter's calls to the concrete type would dispatch methods that + // read the wrong object layout -- unchecked, on this VM. + if (slot < firstNonParameterSlot() || countStoresTo(slot) != 1) { + continue; + } + retypeIteratorUses(slot, itType, st); + } + } + + /// @param storeIdx index of the ASTORE that put the concrete iterator in the + /// slot; only uses AFTER it are rewritten, since anything + /// earlier cannot be reading the value this store wrote + private void retypeIteratorUses(int slot, String itType, int storeIdx) { + ByteCodeClass itClass = Parser.getClassObject(Util.mangle(itType)); + if (itClass == null) { + return; + } + for (int i = storeIdx + 1; i < instructions.size(); i++) { + Instruction ins = instructions.get(i); + if (!(ins instanceof Invoke) || ins.getOpcode() != Opcodes.INVOKEINTERFACE) { + continue; + } + Invoke inv = (Invoke) ins; + if (!"java/util/Iterator".equals(inv.getOwner())) { + continue; + } + int r = prevExecutable(i - 1); + if (r < 0) { + continue; + } + Instruction recv = instructions.get(r); + if (!(recv instanceof VarOp) || recv.getOpcode() != Opcodes.ALOAD + || ((VarOp) recv).getIndex() != slot) { + continue; + } + // The concrete class has to actually resolve the method, and resolve it + // monomorphically -- otherwise the retyped call has nothing to bind to. + if (Parser.resolveDevirtualizedOwner(itClass, inv.getName(), inv.getDesc()) == null) { + continue; + } + Invoke direct = new Invoke(Opcodes.INVOKEVIRTUAL, itType, inv.getName(), inv.getDesc(), false); + instructions.set(i, direct); + // Register it exactly as addInstruction() would: the list entry alone + // leaves the call with no owning method, no class dependency and no + // edge in the dependency graph, so the cull would not see the concrete + // iterator's methods being called. + direct.setMethod(this); + direct.addDependencies(dependentClasses); + if (dependencyGraph != null) { + String uses = direct.getMethodUsed(); + if (uses != null) { + dependencyGraph.recordMethodCall(this, uses); + } + } + } + } + public Set getLocalVariables() { return localVariables; } @@ -2448,6 +2689,10 @@ public void addDebugInfo(int line) { } public void addLabel(Label l) { + // Named here, in bytecode order, so the generated C label is a function of the + // method alone. See LabelInstruction.assignLabelName. + com.codename1.tools.translator.bytecodes.LabelInstruction.assignLabelName(l, nextLabelIndex); + nextLabelIndex++; addInstruction(new com.codename1.tools.translator.bytecodes.LabelInstruction(l)); } @@ -2455,6 +2700,9 @@ public void addInvoke(int opcode, String owner, String name, String desc, boolea addInstruction(new Invoke(opcode, owner, name, desc, itf)); } + /** Per-method label counter; see addLabel. */ + private int nextLabelIndex; + public void setMaxes(int maxStack, int maxLocals) { this.maxLocals = maxLocals; this.maxStack = maxStack; @@ -2667,10 +2915,10 @@ public void appendOnDeviceDebugInvokeThunk(String declaringClsName, StringBuilde * is more specific (e.g. boolean / byte / short / char). */ private String returnTypeChar() { - if (returnType.getPrimitiveType() == Boolean.TYPE) return "Z"; - if (returnType.getPrimitiveType() == Byte.TYPE) return "B"; - if (returnType.getPrimitiveType() == Short.TYPE) return "S"; - if (returnType.getPrimitiveType() == Character.TYPE) return "C"; + if (returnType.getPrimitiveType() == PrimitiveType.BOOLEAN) return "Z"; + if (returnType.getPrimitiveType() == PrimitiveType.BYTE) return "B"; + if (returnType.getPrimitiveType() == PrimitiveType.SHORT) return "S"; + if (returnType.getPrimitiveType() == PrimitiveType.CHAR) return "C"; return "I"; } @@ -2755,7 +3003,7 @@ public void setEliminated(boolean eliminated) { private int varCounter = 0; // Master off-switch: -DCN1_DISABLE_BCE=true reverts to fully-checked array access. private static final boolean DISABLE_BCE = - "true".equalsIgnoreCase(System.getProperty("CN1_DISABLE_BCE", "false")); + "true".equalsIgnoreCase(Util.getProperty("CN1_DISABLE_BCE", "false")); /** * Prove-safe array-bounds-check elimination. Conservative and fail-closed: @@ -2931,7 +3179,7 @@ private static boolean bceForeignEntry(java.util.List r, java.util. // the whole struct to registers. // ------------------------------------------------------------------ private static final boolean DISABLE_SCALAR_REPLACE = - "true".equalsIgnoreCase(System.getProperty("CN1_DISABLE_SCALAR_REPLACE", "false")); + "true".equalsIgnoreCase(Util.getProperty("CN1_DISABLE_SCALAR_REPLACE", "false")); private static String srMangle(String s) { return s.replace('.', '_').replace('/', '_').replace('$', '_'); @@ -3223,7 +3471,7 @@ private void scalarReplaceStackAllocations() { // can't dispatch to an escaping override. // ------------------------------------------------------------------ private static final boolean DISABLE_SB_STACK_ALLOC = - "true".equalsIgnoreCase(System.getProperty("CN1_DISABLE_SB_STACK_ALLOC", "false")); + "true".equalsIgnoreCase(Util.getProperty("CN1_DISABLE_SB_STACK_ALLOC", "false")); private static final String SB_OWNER = "java/lang/StringBuilder"; /** Slots consumed by the argument list of a method descriptor (no receiver). */ @@ -4220,6 +4468,7 @@ private void removeRepeatedCheckcasts() { } } + boolean optimize() { // FUSED OBJECTS, constructor side: rewrite each planned // `ALOAD 0; ; NEWARRAY T; PUTFIELD f` quadruple into the diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/DebugSymbolCompressor.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/DebugSymbolCompressor.java new file mode 100644 index 00000000000..1f9b06dbab7 --- /dev/null +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/DebugSymbolCompressor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.zip.GZIPOutputStream; + +/** + * Compresses the on-device-debug symbol table. + * + * This is the translator's only use of {@code java.util.zip}, and it exists as its + * own class so that it is the only thing that has to be replaced when the + * translator is compiled against ParparVM's JavaAPI in order to translate itself. + * JavaAPI has no java.util.zip and cannot gain one: it is mirrored by + * Ports/CLDC11, where the package does not belong. + * + * Nothing else needs the package. The translator reads directories of class files, + * never archives -- every caller extracts a jar before invoking it -- and + * {@code NativeSignatureVerifier}'s archive scan lives behind its own command-line + * entry point. + * + * Symbol tables are large and highly repetitive, so compressing keeps a debug + * binary's footprint modest. + */ +final class DebugSymbolCompressor { + private DebugSymbolCompressor() { + } + + static byte[] gzip(ByteArrayOutputStream raw) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(raw.size() / 3 + 64); + GZIPOutputStream gz = new GZIPOutputStream(out); + try { + raw.writeTo(gz); + } finally { + gz.close(); + } + return out.toByteArray(); + } +} diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptMethodGenerator.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptMethodGenerator.java index 6577408bd6b..11c00d9f869 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptMethodGenerator.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptMethodGenerator.java @@ -6037,22 +6037,22 @@ private static void appendJsBodyMethod(StringBuilder out, ByteCodeClass cls, Byt if (returnTypeName != null) { jsReturnType = JavascriptNameUtil.sanitizeClassName(returnTypeName); } else { - Class primitiveType = returnType.getPrimitiveType(); - if (primitiveType == Integer.TYPE) { + PrimitiveType primitiveType = returnType.getPrimitiveType(); + if (primitiveType == PrimitiveType.INT) { jsReturnType = "int"; - } else if (primitiveType == Long.TYPE) { + } else if (primitiveType == PrimitiveType.LONG) { jsReturnType = "long"; - } else if (primitiveType == Double.TYPE) { + } else if (primitiveType == PrimitiveType.DOUBLE) { jsReturnType = "double"; - } else if (primitiveType == Float.TYPE) { + } else if (primitiveType == PrimitiveType.FLOAT) { jsReturnType = "float"; - } else if (primitiveType == Boolean.TYPE) { + } else if (primitiveType == PrimitiveType.BOOLEAN) { jsReturnType = "boolean"; - } else if (primitiveType == Byte.TYPE) { + } else if (primitiveType == PrimitiveType.BYTE) { jsReturnType = "byte"; - } else if (primitiveType == Short.TYPE) { + } else if (primitiveType == PrimitiveType.SHORT) { jsReturnType = "short"; - } else if (primitiveType == Character.TYPE) { + } else if (primitiveType == PrimitiveType.CHAR) { jsReturnType = "char"; } else { jsReturnType = "java_lang_Object"; diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptNativeRegistry.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptNativeRegistry.java index 04be9ad4685..2c822e46757 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptNativeRegistry.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptNativeRegistry.java @@ -128,6 +128,7 @@ enum NativeCategory { "cn1_java_lang_System_currentTimeMillis_R_long", "cn1_java_lang_System_exit_int", "cn1_java_lang_System_gcLight", + "cn1_java_lang_System_getenvImpl_java_lang_String_R_java_lang_String", "cn1_java_lang_System_gcMarkSweep", "cn1_java_lang_System_identityHashCode_java_lang_Object_R_int", "cn1_java_lang_Integer_cn1Value_R_int", diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifier.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifier.java index 3db5a800842..4ae3e021028 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifier.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifier.java @@ -27,7 +27,6 @@ import org.objectweb.asm.MethodVisitor; import org.objectweb.asm.Opcodes; -import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; @@ -48,8 +47,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; /** * Checks that every {@code native} method in a translated project has a C @@ -400,24 +397,35 @@ public int size() { } } - /** Reads {@link #IGNORE_FILE}: one symbol or {@code prefix*} per line. */ + /** + * Reads {@link #IGNORE_FILE}: one symbol or {@code prefix*} per line. + * + * Splits the file itself rather than using a BufferedReader, which ParparVM's + * JavaAPI does not declare -- this runs during translation, so it has to compile + * when the translator is built against that JavaAPI to translate itself. + */ static void readIgnoreFile(File file, Set into) throws IOException { - BufferedReader reader = new BufferedReader( - new InputStreamReader(new FileInputStream(file), UTF8)); - try { - String line; - while ((line = reader.readLine()) != null) { - int hash = line.indexOf('#'); - if (hash >= 0) { - line = line.substring(0, hash); - } - line = line.trim(); - if (line.length() > 0) { - into.add(line); - } + String text = new String(readAll(file), UTF8); + int start = 0; + while (start <= text.length()) { + int end = text.indexOf('\n', start); + String line = end < 0 ? text.substring(start) : text.substring(start, end); + // Accept CRLF as readLine did. + if (line.endsWith("\r")) { + line = line.substring(0, line.length() - 1); + } + int hash = line.indexOf('#'); + if (hash >= 0) { + line = line.substring(0, hash); + } + line = line.trim(); + if (line.length() > 0) { + into.add(line); + } + if (end < 0) { + break; } - } finally { - reader.close(); + start = end + 1; } } @@ -817,7 +825,7 @@ private static List splitTopLevel(String text) { private static String normalizeParameter(String declaration) { String text = declaration.replace("*", " * ").trim(); List tokens = new ArrayList( - Arrays.asList(text.split("\\s+"))); + Arrays.asList(Util.splitWhitespace(text))); // "CODENAME_ONE_THREAD_STATE" is a macro that expands to a full declaration // and carries no separate name to strip. if (tokens.size() > 1 && !"CODENAME_ONE_THREAD_STATE".equals(tokens.get(0))) { @@ -1172,7 +1180,7 @@ public static List collectFromClasses(File root) throws IOException { if (root.isDirectory()) { collectClassesFromDirectory(root, found); } else if (root.getName().endsWith(".jar") || root.getName().endsWith(".zip")) { - collectClassesFromArchive(root, found); + ArchiveClassScanner.collect(root, found); } else if (root.getName().endsWith(".class")) { collectFromClassBytes(readAll(root), found); } @@ -1195,32 +1203,7 @@ private static void collectClassesFromDirectory(File dir, List into) } } - private static void collectClassesFromArchive(File archive, List into) throws IOException { - ZipFile zip = new ZipFile(archive); - try { - List names = new ArrayList(); - for (Enumeration e = zip.entries(); e.hasMoreElements();) { - ZipEntry entry = e.nextElement(); - if (!entry.isDirectory() && entry.getName().endsWith(".class") - && !entry.getName().endsWith("module-info.class")) { - names.add(entry.getName()); - } - } - Collections.sort(names); - for (String name : names) { - InputStream in = zip.getInputStream(zip.getEntry(name)); - try { - collectFromClassBytes(readAll(in), into); - } finally { - in.close(); - } - } - } finally { - zip.close(); - } - } - - private static void collectFromClassBytes(byte[] bytes, final List into) { + static void collectFromClassBytes(byte[] bytes, final List into) { final String[] owner = new String[1]; new ClassReader(bytes).accept(new ClassVisitor(Opcodes.ASM9) { @Override @@ -1293,74 +1276,6 @@ private static boolean isIdentifier(String s) { return true; } - public static void main(String[] args) throws IOException { - List classRoots = new ArrayList(); - List nativeRoots = new ArrayList(); - boolean orphans = true; - for (int iter = 0; iter < args.length; iter++) { - if ("--classes".equals(args[iter]) && iter + 1 < args.length) { - classRoots.add(new File(args[++iter])); - } else if ("--natives".equals(args[iter]) && iter + 1 < args.length) { - nativeRoots.add(new File(args[++iter])); - } else if ("--no-orphans".equals(args[iter])) { - orphans = false; - } else { - System.err.println("unrecognised argument: " + args[iter]); - usage(); - System.exit(2); - } - } - if (classRoots.isEmpty() || nativeRoots.isEmpty()) { - usage(); - System.exit(2); - } - - List required = new ArrayList(); - for (File root : classRoots) { - if (!root.exists()) { - System.err.println("NativeSignatureVerifier: no such path: " + root); - System.exit(2); - } - required.addAll(collectFromClasses(root)); - } - List sources = new ArrayList(); - for (File root : nativeRoots) { - if (!root.exists()) { - System.err.println("NativeSignatureVerifier: no such path: " + root); - System.exit(2); - } - sources.addAll(root.isDirectory() - ? listNativeSourcesRecursive(root) : Collections.singletonList(root)); - } - - SourceIndex index = new SourceIndex(sources); - List problems = verify(required, index); - if (!orphans) { - List filtered = new ArrayList(); - for (Problem problem : problems) { - if (problem.kind != Kind.ORPHAN) { - filtered.add(problem); - } - } - problems = filtered; - } - - if (problems.isEmpty()) { - System.out.println("NativeSignatureVerifier: " + required.size() - + " native method(s) all resolve against " + index.size() - + " C definition(s) in " + sources.size() + " file(s)."); - return; - } - int fatal = report(problems, Mode.STRICT, - required.size() + " native methods, " + sources.size() + " native sources", true); - System.exit(fatal > 0 ? 1 : 0); - } - - private static void usage() { - System.err.println("usage: NativeSignatureVerifier --classes DIR_OR_JAR [--classes ...]" - + " --natives DIR [--natives ...] [--no-orphans]"); - } - private NativeSignatureVerifier() { } } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifierCli.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifierCli.java new file mode 100644 index 00000000000..d5ef2921d96 --- /dev/null +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifierCli.java @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Command-line entry point for {@link NativeSignatureVerifier}, driven by + * scripts/check-native-signatures.sh. + * + * Split out of the verifier for two reasons, both about the self-hosted translator + * build. It is the half that scans jars, so it is the half that needs + * java.util.zip -- which JavaAPI cannot gain, being mirrored by Ports/CLDC11. And a + * second class carrying a {@code main} makes ByteCodeClass.addMethod refuse the + * translation outright with "Multiple main classes", since the clean target does + * not set a preferred main class the way the JavaScript target does. + * + * A translation never comes through here: the verifier's in-process entry points + * are what Parser calls. + * + * NativeSignatureVerifier deliberately keeps NO delegating {@code main}. Nothing + * names it as one -- scripts/check-native-signatures.sh invokes this class, and no + * document spells the old command -- and adding one would recreate the very edge + * the split removes: the verifier would reference the CLI, and the CLI reaches + * java.util.zip. A second {@code main} also brings back the "Multiple main + * classes" refusal above. Backward compatibility for an invocation nobody has is + * not worth either. + */ +public final class NativeSignatureVerifierCli { + private NativeSignatureVerifierCli() { + } + + public static void main(String[] args) throws IOException { + List classRoots = new ArrayList(); + List nativeRoots = new ArrayList(); + boolean orphans = true; + for (int iter = 0; iter < args.length; iter++) { + if ("--classes".equals(args[iter]) && iter + 1 < args.length) { + classRoots.add(new File(args[++iter])); + } else if ("--natives".equals(args[iter]) && iter + 1 < args.length) { + nativeRoots.add(new File(args[++iter])); + } else if ("--no-orphans".equals(args[iter])) { + orphans = false; + } else { + System.err.println("unrecognised argument: " + args[iter]); + usage(); + System.exit(2); + } + } + if (classRoots.isEmpty() || nativeRoots.isEmpty()) { + usage(); + System.exit(2); + } + + List required = new ArrayList(); + for (File root : classRoots) { + if (!root.exists()) { + System.err.println("NativeSignatureVerifier: no such path: " + root); + System.exit(2); + } + required.addAll(NativeSignatureVerifier.collectFromClasses(root)); + } + List sources = new ArrayList(); + for (File root : nativeRoots) { + if (!root.exists()) { + System.err.println("NativeSignatureVerifier: no such path: " + root); + System.exit(2); + } + sources.addAll(root.isDirectory() + ? NativeSignatureVerifier.listNativeSourcesRecursive(root) : Collections.singletonList(root)); + } + + NativeSignatureVerifier.SourceIndex index = new NativeSignatureVerifier.SourceIndex(sources); + List problems = NativeSignatureVerifier.verify(required, index); + if (!orphans) { + List filtered = new ArrayList(); + for (NativeSignatureVerifier.Problem problem : problems) { + if (problem.kind != NativeSignatureVerifier.Kind.ORPHAN) { + filtered.add(problem); + } + } + problems = filtered; + } + + if (problems.isEmpty()) { + System.out.println("NativeSignatureVerifier: " + required.size() + + " native method(s) all resolve against " + index.size() + + " C definition(s) in " + sources.size() + " file(s)."); + return; + } + int fatal = NativeSignatureVerifier.report(problems, NativeSignatureVerifier.Mode.STRICT, + required.size() + " native methods, " + sources.size() + " native sources", true); + System.exit(fatal > 0 ? 1 : 0); + } + + private static void usage() { + System.err.println("usage: NativeSignatureVerifier --classes DIR_OR_JAR [--classes ...]" + + " --natives DIR [--natives ...] [--no-orphans]"); + } +} diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java index 6c9ae454f65..7dae4f551ca 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java @@ -25,7 +25,6 @@ import java.io.*; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.util.*; import org.objectweb.asm.AnnotationVisitor; @@ -139,6 +138,33 @@ public static synchronized String resolveDevirtualizedOwner(ByteCodeClass owner, } return null; } + /** + * The concrete Iterator class a for-each over this collection type will + * really get, or null when that cannot be established with certainty. + * + * Two things have to hold. The collection's iterator() must have exactly one + * reachable implementation -- resolveDevirtualizedOwner answers that -- and + * that implementation must do nothing but allocate and return, so the class + * it allocates is the class the caller receives. Anything else answers null + * and the call site is left as the interface call it was. + */ + public static synchronized String resolveConcreteIteratorType(ByteCodeClass owner) { + String decl = resolveDevirtualizedOwner(owner, "iterator", "()Ljava/util/Iterator;"); + if (decl == null) { + return null; + } + ByteCodeClass dc = getClassObject(Util.mangle(decl)); + if (dc == null) { + return null; + } + for (BytecodeMethod m : dc.getMethods()) { + if ("iterator".equals(m.getMethodName()) && "()Ljava/util/Iterator;".equals(m.getDesc())) { + return m.allocatedReturnType(); + } + } + return null; + } + private static final MethodDependencyGraph dependencyGraph = new MethodDependencyGraph(); private int lambdaCounter; private int stringConcatCounter; @@ -162,7 +188,7 @@ public static void parse(File sourceFile) throws Exception { } BytecodeMethod.setDependencyGraph(dependencyGraph); ClassReader r; - try (InputStream in = Files.newInputStream(sourceFile.toPath())) { + try (InputStream in = new FileInputStream(sourceFile)) { r = new ClassReader(in); } Parser p = new Parser(); @@ -273,7 +299,7 @@ public static int jdwpAccessFlagsOf(ByteCodeField bf) { */ private static void writeSymbolSidecar(File outputDirectory) throws IOException { java.io.ByteArrayOutputStream raw = new java.io.ByteArrayOutputStream(1 << 20); - try (Writer w = new OutputStreamWriter(raw, StandardCharsets.UTF_8)) { + try (Writer w = new OutputStreamWriter(raw, "UTF-8")) { w.write("version\t1\n"); for (ByteCodeClass bc : classes) { String src = bc.getSourceFile(); @@ -355,17 +381,13 @@ private static void writeSymbolSidecar(File outputDirectory) throws IOException // gzip the payload — symbol tables are large and highly repetitive, // so this keeps the debug binary's footprint modest. - java.io.ByteArrayOutputStream gzOut = new java.io.ByteArrayOutputStream(raw.size() / 3 + 64); - try (java.util.zip.GZIPOutputStream gz = new java.util.zip.GZIPOutputStream(gzOut)) { - raw.writeTo(gz); - } - byte[] gz = gzOut.toByteArray(); + byte[] gz = DebugSymbolCompressor.gzip(raw); // Compiled into the project like any other generated unit when // cn1.onDeviceDebug is on, so it needs provenance for the same reason they do. ByteCodeTranslator.sourceManifest.recordGenerated("cn1_debug_symbols.c"); File f = new File(outputDirectory, "cn1_debug_symbols.c"); - try (Writer w = new OutputStreamWriter(Files.newOutputStream(f.toPath()), StandardCharsets.UTF_8)) { + try (Writer w = new OutputStreamWriter(new FileOutputStream(f), "UTF-8")) { w.write("/* Auto-generated by the Codename One iOS translator. Do not edit.\n"); w.write(" * On-device-debug symbol table (gzip-compressed), streamed to the\n"); w.write(" * desktop debug proxy over CMD_GET_SYMBOLS. */\n"); @@ -381,8 +403,8 @@ private static void writeSymbolSidecar(File outputDirectory) throws IOException } w.write("0x"); int b = gz[i] & 0xff; - w.write(Character.forDigit(b >> 4, 16)); - w.write(Character.forDigit(b & 0xf, 16)); + w.write(Util.hexDigit(b >> 4)); + w.write(Util.hexDigit(b & 0xf)); w.write(','); w.write((i & 15) == 15 ? '\n' : ' '); } @@ -454,6 +476,16 @@ public static NativeSymbolIndex getNativeSymbolIndex(String[] nativeSources) { } private static final ArrayList constantPool = new ArrayList<>(); + // Index of constantPool, so addToConstantPool does not have to scan it. + // + // The list stays the source of truth -- writeOutput emits it in order and the + // emitted indices are positions in it -- and this only answers "where is s", the + // question ArrayList.indexOf was answering with a String.equals against every + // entry already interned. On a self-hosting translation the pool holds ~200k + // strings and that scan was the single largest cost on the mutator thread: + // String.equals 11.2%, the iterator 10.3%, indexOf 6.2% and ArrayList.get 5.1% + // of samples, all of it here. + private static final Map constantPoolIndex = new HashMap(); // Name -> class index, replacing the O(N) linear scans that getClassObject / // getClassByName / ByteCodeClass.findClass used to do. Those run per dependency @@ -489,12 +521,14 @@ public static ByteCodeClass getClassObject(String name) { * Adds the given string to the hardcoded constant pool strings returns the offset in the pool */ public static int addToConstantPool(String s) { - int i = constantPool.indexOf(s); - if(i < 0) { - constantPool.add(s); - return constantPool.size() - 1; - } - return i; + Integer existing = constantPoolIndex.get(s); + if(existing != null) { + return existing.intValue(); + } + int index = constantPool.size(); + constantPool.add(s); + constantPoolIndex.put(s, Integer.valueOf(index)); + return index; } @@ -802,6 +836,26 @@ public static void writeOutput(File outputDirectory) throws Exception { neliminated++; } + // Fuse all-String StringBuilder concat chains into String.cn1ConcatN + // BEFORE the cull, not during code generation. + // + // The cull decides what to keep from the dependency graph, and the graph + // is only told about a call when the instruction is added. A rewrite that + // runs later -- inside BytecodeMethod.optimize(), which happens during + // generateCCode -- inserts calls to methods the cull has already deleted, + // and a deleted method is emitted as `return 0;`. That is not a build + // error: the rewritten call silently answered null, java.io.File got a + // null path, and the translator died in File.getParentFile with a SIGSEGV + // nowhere near the rewrite. Running here, the references exist before + // anything is eliminated. See BytecodeMethod.lowerIteratorCalls. + if (BytecodeMethod.optimizerOn) { + for (ByteCodeClass fuseCls : classes) { + for (BytecodeMethod fuseMtd : fuseCls.getMethods()) { + fuseMtd.lowerIteratorCalls(); + } + } + } + // loop over methods and start eliminating the body of unused methods if (BytecodeMethod.optimizerOn) { if(ByteCodeTranslator.verbose) { @@ -875,7 +929,7 @@ public static void writeOutput(File outputDirectory) throws Exception { generateClassAndMethodIndexHeader(outputDirectory); - boolean concatenate = "true".equals(System.getProperty("concatenateFiles", "false")); + boolean concatenate = "true".equals(Util.getProperty("concatenateFiles", "false")); ConcatenatingFileOutputStream cos = concatenate ? new ConcatenatingFileOutputStream(outputDirectory) : null; for(ByteCodeClass bc : classes) { @@ -914,7 +968,7 @@ public static void writeOutput(File outputDirectory) throws Exception { } private static void readNativeFiles(File outputDirectory) throws IOException { - File[] mFiles = outputDirectory.listFiles(file -> + File[] mFiles = Util.listFiles(outputDirectory, file -> file.getName().endsWith(".m") || file.getName().endsWith("." + ByteCodeTranslator.output.extension())); if(mFiles == null) { return; @@ -1122,7 +1176,14 @@ private static int cullClasses(boolean found, int depth) { // 2nd pass to mark classes as eliminated so that we can propagate down to each // method of the class to mark it eliminated so that virtual methods // aren't included later on when writing virtual methods - Set removedClasses = new HashSet<>(classes); + // LinkedHashSet, not HashSet: ByteCodeClass overrides neither equals nor + // hashCode, so a HashSet here iterates in identity-hash order. Elimination + // is greedy and monotone -- isMethodUsed treats an already-eliminated + // caller as no caller -- so with a cycle in the call graph the ORDER + // decides which member of the cycle survives. Two runtimes hash + // identities differently and culled different methods from the same + // input; translating the translator with itself is what exposed it. + Set removedClasses = new LinkedHashSet<>(classes); tmp.forEach(removedClasses::remove); int nfound = 0; for (ByteCodeClass cls : removedClasses) { @@ -1165,7 +1226,7 @@ private static void writeFile(ByteCodeClass cls, File outputDir, ConcatenatingFi // it back to one file per class. writeBufferInstead != null && ByteCodeTranslator.output.isApple() ? writeBufferInstead : - Files.newOutputStream(new File(outputDir, cls.getClsName() + "." + ByteCodeTranslator.output.extension()).toPath()); + new FileOutputStream(new File(outputDir, cls.getClsName() + "." + ByteCodeTranslator.output.extension())); if (outMain instanceof ConcatenatingFileOutputStream) { ((ConcatenatingFileOutputStream)outMain).beginNextFile(cls.getClsName()); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/PrimitiveType.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/PrimitiveType.java new file mode 100644 index 00000000000..1b06b7e0d04 --- /dev/null +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/PrimitiveType.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +/** + * The nine primitive types, as a token the translator can compare and hash. + * + *

This used to be {@code java.lang.Class}, holding {@code Integer.TYPE} and its + * eight siblings. Nothing ever reflected on those objects: every use was an + * identity comparison against one of the nine constants, or a lookup in a + * {@code HashMap} keyed on them. {@code Class} was standing in for + * an enum, and it carried two problems that an enum does not. + * + *

The first is that {@code X.TYPE} does not exist on a ParparVM target. javac + * lowers the primitive class literal in {@code Integer.TYPE = int.class} to a read + * of the field being initialized, so the wrapper's own {@code } stores + * null into it; three of the nine wrappers do not declare the field at all. Keyed + * on those, both maps collapsed to a single entry and {@code getCType} answered the + * same C type for every primitive -- valid C, every type wrong, nothing thrown. + * That made the maps unusable in a self-hosted translator, which is what forced + * this change. + * + *

The second is ordering. {@code Class} has no {@code hashCode} of its own, so + * {@code ByteCodeMethodArg.hashCode} was returning an identity hash, which varies + * between runs of one JVM. Anything that iterated a hash container of those keys + * and wrote the result would emit a different file each time. + * + *

Note for the same reason that {@link #ordinal()} is used explicitly wherever a + * hash is needed rather than calling {@code hashCode()} on a constant here: + * {@code Enum.hashCode} is an identity hash on OpenJDK and the ordinal in + * ParparVM's {@code java.lang.Enum}, so relying on it would make the JVM-hosted and + * self-hosted translators disagree on hash order -- a difference the self-hosting + * gate would report as a VM divergence. + */ +public enum PrimitiveType { + INT("JAVA_INT", "int", "I"), + LONG("JAVA_LONG", "long", "J"), + SHORT("JAVA_SHORT", "short", "S"), + BYTE("JAVA_BYTE", "byte", "B"), + DOUBLE("JAVA_DOUBLE", "double", "D"), + FLOAT("JAVA_FLOAT", "float", "F"), + BOOLEAN("JAVA_BOOLEAN", "boolean", "Z"), + CHAR("JAVA_CHAR", "char", "C"), + VOID("JAVA_VOID", "void", "V"); + + private final String cType; + private final String sigType; + private final String descriptor; + + private PrimitiveType(String cType, String sigType, String descriptor) { + this.cType = cType; + this.sigType = sigType; + this.descriptor = descriptor; + } + + /** The C type the generated code uses for this primitive, e.g. JAVA_INT. */ + public String getCType() { + return cType; + } + + /** The Java keyword, as it appears in a mangled C method name. */ + public String getSigType() { + return sigType; + } + + /** The JVM field descriptor character, e.g. I for int. */ + public String getDescriptor() { + return descriptor; + } +} diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/SourceManifest.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/SourceManifest.java index ce0228d484a..b4fb7cba6bd 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/SourceManifest.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/SourceManifest.java @@ -23,11 +23,11 @@ package com.codename1.tools.translator; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.Writer; import java.nio.charset.Charset; -import java.nio.file.Files; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; @@ -259,7 +259,10 @@ public int size() { */ public void write(File projectRoot) throws IOException { File out = new File(projectRoot, FILE_NAME); - try (Writer w = new OutputStreamWriter(Files.newOutputStream(out.toPath()), UTF8)) { + // java.io rather than java.nio.file: the translator compiles against JavaAPI + // when it translates itself, and JavaAPI has no java.nio.file. See + // vm/selfhost. + try (Writer w = new OutputStreamWriter(new FileOutputStream(out), "UTF-8")) { w.write("# Provenance of every file in the generated project's source directory.\n"); w.write("# Written by the ParparVM translator; consumed by\n"); w.write("# scripts/check-native-warnings.py to decide who owns a compiler warning.\n"); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Util.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Util.java index a02f3520ae8..2644dc9a35b 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Util.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Util.java @@ -24,10 +24,12 @@ import com.codename1.tools.translator.bytecodes.Instruction; import com.codename1.tools.translator.bytecodes.TryCatch; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.objectweb.asm.Opcodes; /** @@ -36,36 +38,12 @@ */ public class Util { - private static final Map ctypeMap = new HashMap(); - private static final Map sigTypeMap = new HashMap(); - - static { - ctypeMap.put(Integer.TYPE, "JAVA_INT"); - ctypeMap.put(Long.TYPE, "JAVA_LONG"); - ctypeMap.put(Short.TYPE, "JAVA_SHORT"); - ctypeMap.put(Byte.TYPE, "JAVA_BYTE"); - ctypeMap.put(Double.TYPE, "JAVA_DOUBLE"); - ctypeMap.put(Float.TYPE, "JAVA_FLOAT"); - ctypeMap.put(Boolean.TYPE, "JAVA_BOOLEAN"); - ctypeMap.put(Character.TYPE, "JAVA_CHAR"); - ctypeMap.put(Void.TYPE, "JAVA_VOID"); - sigTypeMap.put(Integer.TYPE, "int"); - sigTypeMap.put(Long.TYPE, "long"); - sigTypeMap.put(Short.TYPE, "short"); - sigTypeMap.put(Byte.TYPE, "byte"); - sigTypeMap.put(Double.TYPE, "double"); - sigTypeMap.put(Float.TYPE, "float"); - sigTypeMap.put(Boolean.TYPE, "boolean"); - sigTypeMap.put(Character.TYPE, "char"); - sigTypeMap.put(Void.TYPE, "void"); + public static String getCType(PrimitiveType type) { + return type == null ? null : type.getCType(); } - public static String getCType(Class cls) { - return ctypeMap.get(cls); - } - - public static String getSigType(Class cls) { - return sigTypeMap.get(cls); + public static String getSigType(PrimitiveType type) { + return type == null ? null : type.getSigType(); } public static List getMethodArgs(String methodDesc) { @@ -94,28 +72,28 @@ public static List getMethodArgs(String methodDesc) { arguments.add(new ByteCodeMethodArg(objectType, currentArrayDim)); break; case 'I': - arguments.add(new ByteCodeMethodArg(Integer.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.INT, currentArrayDim)); break; case 'J': - arguments.add(new ByteCodeMethodArg(Long.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.LONG, currentArrayDim)); break; case 'B': - arguments.add(new ByteCodeMethodArg(Byte.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.BYTE, currentArrayDim)); break; case 'S': - arguments.add(new ByteCodeMethodArg(Short.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.SHORT, currentArrayDim)); break; case 'F': - arguments.add(new ByteCodeMethodArg(Float.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.FLOAT, currentArrayDim)); break; case 'D': - arguments.add(new ByteCodeMethodArg(Double.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.DOUBLE, currentArrayDim)); break; case 'Z': - arguments.add(new ByteCodeMethodArg(Boolean.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.BOOLEAN, currentArrayDim)); break; case 'C': - arguments.add(new ByteCodeMethodArg(Character.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.CHAR, currentArrayDim)); break; } currentArrayDim = 0; @@ -422,4 +400,324 @@ public static char[] getStackOutputTypes(Instruction instr) { } } + + /** + * Writes {@code data} to {@code target}, replacing it. + * + * Stands in for {@code Files.write(Path, byte[])}. ParparVM's JavaAPI has no + * java.nio.file, and the translator has to compile against it to be able to + * translate itself, so the whole translator stays on java.io. + */ + public static void writeBytes(File target, byte[] data) throws IOException { + OutputStream out = new FileOutputStream(target); + try { + out.write(data); + } finally { + out.close(); + } + } + + /** + * The path of {@code f} relative to {@code root}, with '/' separators. + * + * Stands in for {@code root.toPath().relativize(f.toPath())} for the one case + * that needs it: {@code f} is always found by walking {@code root}, so it is + * always underneath it and no ".." segment can arise. + */ + public static String relativePath(File root, File f) { + String rootPath = root.getAbsolutePath(); + String filePath = f.getAbsolutePath(); + if (filePath.startsWith(rootPath)) { + filePath = filePath.substring(rootPath.length()); + } + filePath = filePath.replace('\\', '/'); + while (filePath.startsWith("/")) { + filePath = filePath.substring(1); + } + return filePath; + } + + /** + * Java's {@code \s}: the six characters the regex engine treats as whitespace. + * Deliberately not Character.isWhitespace, which differs -- it excludes the + * vertical tab and accepts many Unicode separators. + * + * 0x0B rather than an escape because a raw control byte in a source file is + * what check-control-characters.py exists to reject. + */ + private static boolean isRegexWhitespace(char c) { + return c == ' ' || c == '\t' || c == '\n' || c == 0x0B || c == '\f' || c == '\r'; + } + + /** + * Equivalent of {@code s.split(String.valueOf(separator))} for a separator that + * is not a regex metacharacter, including the trailing-empty-string removal + * String.split does at the default limit of zero. + * + * The translator has to compile against ParparVM's JavaAPI in order to translate + * itself, and String.split is not declared there. It is one of the methods + * BytecodeComplianceMojo rewrites onto com.codename1.util.regex precisely + * because JavaAPI lacks it, so adding it there would leave two regex engines and + * a rewrite rule whose premise had become false. The few call sites here lose + * the regex instead. + */ + public static String[] splitLiteral(String s, char separator) { + // String.split returns { s } when the pattern never matches, WITHOUT the + // trailing-empty removal below -- so "".split(";") is { "" }, not { }. Missing + // this is the one way a hand-written splitter and the regex part company on + // an input a caller can actually produce (an unset build hint). + if (s.indexOf(separator) < 0) { + return new String[] { s }; + } + List parts = new ArrayList(); + int start = 0; + for (int i = 0; i < s.length(); i++) { + if (s.charAt(i) == separator) { + parts.add(s.substring(start, i)); + start = i + 1; + } + } + parts.add(s.substring(start)); + int end = parts.size(); + while (end > 0 && parts.get(end - 1).isEmpty()) { + end--; + } + return parts.subList(0, end).toArray(new String[end]); + } + + /** + * Equivalent of {@code s.split("\\s+")}, including the leading empty string + * String.split produces when the input starts with whitespace, and the removal + * of trailing empty strings. See {@link #splitLiteral} for why this is not a + * regex. + */ + public static String[] splitWhitespace(String s) { + // See splitLiteral: no match means { s }, trailing-empty removal skipped. + boolean matched = false; + for (int j = 0; j < s.length(); j++) { + if (isRegexWhitespace(s.charAt(j))) { + matched = true; + break; + } + } + if (!matched) { + return new String[] { s }; + } + List parts = new ArrayList(); + int i = 0; + int start = 0; + while (i < s.length()) { + if (isRegexWhitespace(s.charAt(i))) { + parts.add(s.substring(start, i)); + while (i < s.length() && isRegexWhitespace(s.charAt(i))) { + i++; + } + start = i; + } else { + i++; + } + } + parts.add(s.substring(start)); + int end = parts.size(); + while (end > 0 && parts.get(end - 1).isEmpty()) { + end--; + } + return parts.subList(0, end).toArray(new String[end]); + } + + /** + * Equivalent of {@code s.replaceAll("\\s+", " ")}. See {@link #splitLiteral} + * for why this is not a regex. + */ + public static String collapseWhitespace(String s) { + StringBuilder b = new StringBuilder(s.length()); + int i = 0; + while (i < s.length()) { + char c = s.charAt(i); + if (isRegexWhitespace(c)) { + b.append(' '); + while (i < s.length() && isRegexWhitespace(s.charAt(i))) { + i++; + } + } else { + b.append(c); + i++; + } + } + return b.toString(); + } + + /** + * Equivalent of + * {@code s.replaceAll("locals\\[(\\d+)\\]\\.data\\.o", "olocals_$1_")}: rewrites + * an indexed object local into the scalar-replaced name the barebone path emits. + * + * Besides removing the regex (see {@link #splitLiteral}), this drops a Pattern + * compile that used to happen once per barebone method in every build. + */ + public static String rewriteLocalObjectRefs(String s) { + final String prefix = "locals["; + final String suffix = "].data.o"; + int at = s.indexOf(prefix); + if (at < 0) { + return s; + } + StringBuilder b = new StringBuilder(s.length()); + int from = 0; + while (at >= 0) { + int digits = at + prefix.length(); + int end = digits; + while (end < s.length() && s.charAt(end) >= '0' && s.charAt(end) <= '9') { + end++; + } + if (end > digits && s.startsWith(suffix, end)) { + b.append(s, from, at); + b.append("olocals_").append(s, digits, end).append('_'); + from = end + suffix.length(); + } else { + // \d+ needs at least one digit and "].data.o" must follow it, so this + // occurrence is not a match; copy it through and keep scanning after it. + b.append(s, from, digits); + from = digits; + } + at = s.indexOf(prefix, from); + } + b.append(s, from, s.length()); + return b.toString(); + } + + /** + * {@code System.getProperty(key, defaultValue)}, falling back to the + * environment. + * + * ParparVM's JavaAPI declares only the one-argument form, and it returns null + * unconditionally -- a native binary has no -D to read. The translator has to + * compile against that JavaAPI in order to translate itself, so the two-argument + * form is provided here instead of being added to JavaAPI, and every knob gains + * an environment spelling that works in a translated build. cn1.sqlite is read + * from CN1_SQLITE, INCLUDE_NPE_CHECKS from INCLUDE_NPE_CHECKS. + * + * NativeSignatureVerifier.mode() already reached for getenv for exactly this + * reason; this generalizes it rather than adding a second convention. + */ + /** + * Memoized ParparVM name mangling: '/' and '$' both become '_'. + * + * The tree contains 95 hand-written copies of + * {@code x.replace('/', '_').replace('$', '_')}, 54 of them in the + * per-instruction emit classes (Invoke, Field, CustomInvoke, Ldc), so the + * SAME owner string is re-mangled once per emitted instruction. The distinct + * inputs are bounded by the class count (5782 on the hellocodenameone + * corpus) while the calls run into the millions. + * + * String.replace already returns {@code this} when the character is absent, + * so the '$' pass is usually free; the '/' pass is the one that allocates a + * char[] and a String every time. Caching turns that into one lookup. + * + * Not synchronized: the translator parses and emits on a single thread -- + * Parser.writeOutput is one sequential loop with no executor and a single + * writeFile call site. + */ + private static final java.util.Map MANGLE_CACHE = + new java.util.HashMap(); + + public static String mangle(String name) { + if (name == null) { + return null; + } + String m = MANGLE_CACHE.get(name); + if (m == null) { + m = name.replace('/', '_').replace('$', '_'); + MANGLE_CACHE.put(name, m); + } + return m; + } + + public static String getProperty(String key, String defaultValue) { + String value = System.getProperty(key); + if (value == null) { + value = System.getenv(environmentName(key)); + } + return value == null ? defaultValue : value; + } + + /** + * "cn1.sqlite" -> "CN1_SQLITE". Folded by hand: String.toUpperCase is locale + * sensitive and CN1 has no java.util.Locale to ask for the root locale, so on a + * Turkish device the 'i' of "cn1.sqlite" would not fold to 'I' and the variable + * would never be found. + */ + private static String environmentName(String key) { + StringBuilder b = new StringBuilder(key.length()); + for (int i = 0; i < key.length(); i++) { + char c = key.charAt(i); + if (c >= 'a' && c <= 'z') { + b.append((char) (c - 'a' + 'A')); + } else if ((c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) { + b.append(c); + } else { + b.append('_'); + } + } + return b.toString(); + } + + /** + * Stands in for {@code java.io.FileFilter}, which ParparVM's JavaAPI does not + * declare. Kept as a functional interface so the call sites keep their lambdas. + */ + public interface FileMatcher { + boolean accept(File file); + } + + /** + * Stands in for {@code java.io.FilenameFilter}. + */ + public interface FileNameMatcher { + boolean accept(File dir, String name); + } + + /** + * {@code dir.listFiles(filter)}, including its null return when {@code dir} is + * not a directory -- callers test for it. + */ + public static File[] listFiles(File dir, FileMatcher matcher) { + File[] all = dir.listFiles(); + if (all == null) { + return null; + } + List kept = new ArrayList(all.length); + for (int i = 0; i < all.length; i++) { + if (matcher.accept(all[i])) { + kept.add(all[i]); + } + } + return kept.toArray(new File[kept.size()]); + } + + /** + * {@code dir.list(filter)}, including its null return when {@code dir} is not a + * directory. + */ + public static String[] list(File dir, FileNameMatcher matcher) { + String[] all = dir.list(); + if (all == null) { + return null; + } + List kept = new ArrayList(all.length); + for (int i = 0; i < all.length; i++) { + if (matcher.accept(dir, all[i])) { + kept.add(all[i]); + } + } + return kept.toArray(new String[kept.size()]); + } + + /** + * {@code Character.forDigit(digit, 16)} for a digit already known to be in + * range. JavaAPI has no forDigit. + */ + public static char hexDigit(int digit) { + return (char) (digit < 10 ? '0' + digit : 'a' - 10 + digit); + } } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomInvoke.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomInvoke.java index 2ec078764bd..b753a9357bc 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomInvoke.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomInvoke.java @@ -102,7 +102,7 @@ public String getMethodUsed() { public void addDependencies(List dependencyList) { String dependencyOwner = owner; if (origOpcode == Opcodes.INVOKEVIRTUAL) { - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); String resolvedConcreteOwner = resolveConcreteInvokeOwner(bc, true); if (resolvedConcreteOwner != null) { dependencyOwner = resolvedConcreteOwner; @@ -131,7 +131,7 @@ public void addDependencies(List dependencyList) { if(origOpcode != Opcodes.INVOKEINTERFACE && origOpcode != Opcodes.INVOKEVIRTUAL) { return; } - bld.append(owner.replace('/', '_').replace('$', '_')); + bld.append(Util.mangle(owner)); bld.append("_"); if(name.equals("")) { bld.append("__INIT__"); @@ -177,7 +177,7 @@ private String resolveConcreteInvokeOwner(ByteCodeClass ownerClass, boolean allo if (currentClass != null && (ownerName.equals(currentClass) || currentClass.startsWith(ownerName + "_"))) { return null; } - ByteCodeClass concreteClass = Parser.getClassObject(ownerClass.getConcreteClass().replace('/', '_').replace('$', '_')); + ByteCodeClass concreteClass = Parser.getClassObject(Util.mangle(ownerClass.getConcreteClass())); // The nearest class in the concrete type's own hierarchy that actually // declares the method -- which is what the runtime would dispatch to for // an instance of it. Resolving against concreteClass's declarations alone @@ -288,7 +288,7 @@ public boolean appendExpression(StringBuilder b) { // so we need to check boolean isVirtual = true; if (origOpcode == Opcodes.INVOKEVIRTUAL) { - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); if (bc == null) { System.err.println("WARNING: Failed to find class object for owner "+owner+" when rendering virtual method "+name); } else { @@ -321,13 +321,13 @@ public boolean appendExpression(StringBuilder b) { if(origOpcode == Opcodes.INVOKESTATIC) { // find the actual class of the static method to work around javac not defining it correctly - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); invokeOwner = findActualOwner(bc); } if (invokeOwner.startsWith("[")) { bld.append("java_lang_Object"); } else{ - bld.append(invokeOwner.replace('/', '_').replace('$', '_')); + bld.append(Util.mangle(invokeOwner)); } bld.append("_"); if(name.equals("")) { @@ -343,7 +343,7 @@ public boolean appendExpression(StringBuilder b) { ArrayList args = new ArrayList<>(); String returnVal = BytecodeMethod.appendMethodSignatureSuffixFromDesc(desc, bld, args); if (isVirtualCall) { - BytecodeMethod.addVirtualMethodsInvoked(bld.substring("virtual_".length())); + BytecodeMethod.addVirtualMethodsInvoked(bld.toString().substring("virtual_".length())); } else { // keep in sync with Invoke: direct/devirtualized calls of the mapped // String/StringBuilder natives get the inlined fast path @@ -442,7 +442,7 @@ private boolean tryAppendInlinedConstructor(StringBuilder b) { // Memset elimination: allocate into a temp, build fully, THEN publish. // Literal-arg ctor with the receiver on-stack (from NEW;DUP): the // survivor sits one slot below the receiver (SP[-2]); pop the receiver. - String cType = owner.replace('/', '_').replace('$', '_'); + String cType = Util.mangle(owner); inlineCtorPlan.appendInitBeforePublish(b, cType, argExprs, argCats, 2, 1); return true; } @@ -489,7 +489,7 @@ private void appendFusedAllocBlock(StringBuilder b) { for (int i = 0; i < kids.size(); i++) { lenExprs[i] = kids.get(i).siteLengthExpr(temps); } - String cType = owner.replace('/', '_').replace('$', '_'); + String cType = Util.mangle(owner); fusedPlan.appendFusedAlloc(b, cType, lenExprs, 1, 2); // NOTE: the enclosing brace is closed AFTER the ordinary call emission by // appendInstruction (the temps must stay in scope for the call). @@ -540,7 +540,7 @@ public void appendInstruction(StringBuilder b) { // so we need to check boolean isVirtual = true; if (origOpcode == Opcodes.INVOKEVIRTUAL) { - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); if (bc == null) { System.err.println("WARNING: Failed to find class object for owner "+owner+" when rendering virtual method "+name); } else { @@ -573,13 +573,13 @@ public void appendInstruction(StringBuilder b) { if(origOpcode == Opcodes.INVOKESTATIC) { // find the actual class of the static method to work around javac not defining it correctly - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); invokeOwner = findActualOwner(bc); } if (invokeOwner.startsWith("[")) { bld.append("java_lang_Object"); } else{ - bld.append(invokeOwner.replace('/', '_').replace('$', '_')); + bld.append(Util.mangle(invokeOwner)); } bld.append("_"); if(name.equals("")) { @@ -595,7 +595,7 @@ public void appendInstruction(StringBuilder b) { ArrayList args = new ArrayList<>(); String returnVal = BytecodeMethod.appendMethodSignatureSuffixFromDesc(desc, bld, args); if (isVirtualCall) { - BytecodeMethod.addVirtualMethodsInvoked(bld.substring("virtual_".length())); + BytecodeMethod.addVirtualMethodsInvoked(bld.toString().substring("virtual_".length())); } else { // keep in sync with Invoke: direct/devirtualized calls of the mapped // String/StringBuilder natives get the inlined fast path diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomJump.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomJump.java index b93718c8429..07ab36aa1e9 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomJump.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomJump.java @@ -68,13 +68,13 @@ public void appendInstruction(StringBuilder b, List instructions) { if(TryCatch.isTryCatchInMethod()) { b.append("JUMP_TO(label_"); - b.append(label.toString()); + b.append(LabelInstruction.labelName(label)); b.append(", "); b.append(LabelInstruction.getLabelCatchDepth(label, instructions)); b.append(");\n"); } else { b.append("goto label_"); - b.append(label.toString()); + b.append(LabelInstruction.labelName(label)); b.append(";\n"); } if(customSuffix != null) { diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Field.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Field.java index f535414024a..95147ccf367 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Field.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Field.java @@ -23,6 +23,8 @@ package com.codename1.tools.translator.bytecodes; +import com.codename1.tools.translator.Util; + import java.util.List; import org.objectweb.asm.Opcodes; @@ -79,7 +81,7 @@ public void addDependencies(List dependencyList) { } public String getFieldFromThis() { - return "get_field_" + owner.replace('/', '_').replace('$', '_') + + return "get_field_" + Util.mangle(owner) + "_" + name + "(__cn1ThisObject)"; } @@ -88,14 +90,14 @@ public String setFieldFromThis(int arg) { // Instance field setters only need value/target operands. // special case for this if(arg == 0) { - return " set_field_" + owner.replace('/', '_').replace('$', '_') + + return " set_field_" + Util.mangle(owner) + "_" + name + "(__cn1ThisObject, __cn1ThisObject);\n"; } if(isObject()) { - return " set_field_" + owner.replace('/', '_').replace('$', '_') + + return " set_field_" + Util.mangle(owner) + "_" + name + "(__cn1Arg" + arg + ", __cn1ThisObject);\n"; } - return " set_field_" + owner.replace('/', '_').replace('$', '_') + + return " set_field_" + Util.mangle(owner) + "_" + name + "(__cn1Arg" + arg + ", __cn1ThisObject);\n"; } @@ -124,7 +126,7 @@ public String pushFieldFromThis() { break; } b.append("(get_field_"); - b.append(owner.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(owner)); b.append("_"); b.append(name); b.append("(__cn1ThisObject));\n"); @@ -143,14 +145,14 @@ public boolean assignTo(String varName, StringBuilder sb) { } if (opcode == Opcodes.GETSTATIC) { b.append("get_static_"); - b.append(owner.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(owner)); b.append("_"); - b.append(name.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(name)); b.append("()"); } else { b.append("get_field_"); - b.append(owner.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(owner)); b.append("_"); b.append(name); StringBuilder sb3 = new StringBuilder(); @@ -224,17 +226,17 @@ public void appendInstruction(StringBuilder sbOut) { break; } b.append("(get_static_"); - b.append(owner.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(owner)); b.append("_"); - b.append(name.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(name)); b.append("());\n"); break; case Opcodes.PUTSTATIC: { //b.append("SAFE_RETAIN(1);\n "); b.append("set_static_"); - b.append(owner.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(owner)); b.append("_"); - b.append(name.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(name)); if (isObject()) { b.append("(threadStateData, "); } else { @@ -300,7 +302,7 @@ public void appendInstruction(StringBuilder sbOut) { } b.append("(get_field_"); - b.append(owner.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(owner)); b.append("_"); b.append(name); @@ -317,7 +319,7 @@ public void appendInstruction(StringBuilder sbOut) { case Opcodes.PUTFIELD: { //b.append("SAFE_RETAIN(1);\n "); b.append("set_field_"); - b.append(owner.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(owner)); b.append("_"); b.append(name); b.append("("); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedConstructor.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedConstructor.java index f722f2fcab6..ab71a78e33f 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedConstructor.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedConstructor.java @@ -554,8 +554,10 @@ private static boolean descMatchesArrayType(String fieldDesc, int arrayType) { public void appendFusedAlloc(StringBuilder b, String cType, String[] lenExprs, int recvSlot, int survSlot) { b.append(" { /* FUSED construction of ").append(cType).append(" */\n"); - b.append(" if(__builtin_expect(!class__").append(cType) - .append(".initialized, 0)) __STATIC_INITIALIZER_").append(cType).append("(threadStateData);\n"); + // ACQUIRE; see the note in TypeInstruction. + b.append(" if(__builtin_expect(!__atomic_load_n(&class__").append(cType) + .append(".initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_").append(cType) + .append("(threadStateData);\n"); for (int i = 0; i < children.size(); i++) { b.append(" int __fLen").append(i).append(" = ").append(lenExprs[i]).append(";\n"); } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Invoke.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Invoke.java index 4f7adfa789c..7d590fbdac3 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Invoke.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Invoke.java @@ -91,7 +91,7 @@ private String getCMethodName() { public void addDependencies(List dependencyList) { String dependencyOwner = owner; if (opcode == Opcodes.INVOKEVIRTUAL) { - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); String resolvedConcreteOwner = resolveConcreteInvokeOwner(bc, true); if (resolvedConcreteOwner != null) { dependencyOwner = resolvedConcreteOwner; @@ -121,7 +121,7 @@ public void addDependencies(List dependencyList) { if(opcode != Opcodes.INVOKEINTERFACE && opcode != Opcodes.INVOKEVIRTUAL) { return; } - bld.append(owner.replace('/', '_').replace('$', '_')); + bld.append(Util.mangle(owner)); bld.append("_"); if(name.equals("")) { bld.append("__INIT__"); @@ -167,7 +167,7 @@ private String resolveConcreteInvokeOwner(ByteCodeClass ownerClass, boolean allo if (currentClass != null && (ownerName.equals(currentClass) || currentClass.startsWith(ownerName + "_"))) { return null; } - ByteCodeClass concreteClass = Parser.getClassObject(ownerClass.getConcreteClass().replace('/', '_').replace('$', '_')); + ByteCodeClass concreteClass = Parser.getClassObject(Util.mangle(ownerClass.getConcreteClass())); // The nearest class in the concrete type's own hierarchy that actually // declares the method -- which is what the runtime would dispatch to for // an instance of it. Resolving against concreteClass's declarations alone @@ -227,7 +227,7 @@ private void appendFusedAllocBlock(StringBuilder b) { for (int i = 0; i < kids.size(); i++) { lenExprs[i] = kids.get(i).siteLengthExpr(argExprByParam); } - String cType = owner.replace('/', '_').replace('$', '_'); + String cType = Util.mangle(owner); fusedPlan.appendFusedAlloc(b, cType, lenExprs, n + 1, n + 2); } @@ -261,7 +261,7 @@ private boolean tryAppendInlinedConstructor(StringBuilder b) { // argCats == null: every argExpr here is a pure SP[-k].data.x read // (the args were evaluated onto the operand stack BEFORE this ), // so no temp hoisting is needed. - String cType = owner.replace('/', '_').replace('$', '_'); + String cType = Util.mangle(owner); inlineCtorPlan.appendInitBeforePublish(b, cType, argExprs, null, n + 2, n + 1); return true; } @@ -307,7 +307,7 @@ public void appendInstruction(StringBuilder b) { // if it is. boolean isVirtual = true; if (opcode == Opcodes.INVOKEVIRTUAL) { - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); if (bc == null) { System.err.println("WARNING: Failed to find class object for owner "+owner+" when rendering virtual method "+name); } else { @@ -340,7 +340,7 @@ public void appendInstruction(StringBuilder b) { if(opcode == Opcodes.INVOKESTATIC) { // find the actual class of the static method to work around javac not defining it correctly - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); invokeOwner = findActualOwner(bc); } if (invokeOwner.startsWith("[")) { @@ -348,7 +348,7 @@ public void appendInstruction(StringBuilder b) { // as an owner. We'll just change this to java_lang_Object instead. bld.append("java_lang_Object"); } else{ - bld.append(invokeOwner.replace('/', '_').replace('$', '_')); + bld.append(Util.mangle(invokeOwner)); } bld.append("_"); if(name.equals("")) { @@ -364,7 +364,7 @@ public void appendInstruction(StringBuilder b) { ArrayList args = new ArrayList<>(); String returnVal = BytecodeMethod.appendMethodSignatureSuffixFromDesc(desc, bld, args); if (isVirtualCall) { - BytecodeMethod.addVirtualMethodsInvoked(bld.substring("virtual_".length())); + BytecodeMethod.addVirtualMethodsInvoked(bld.toString().substring("virtual_".length())); } else { // direct/devirtualized calls of the hottest String/StringBuilder // natives get the call-site-inlined fast path (cn1_intrinsics.h) @@ -501,7 +501,7 @@ public void appendInstruction(StringBuilder b) { // Master off-switch: -DCN1_DISABLE_INLINE=true disables trivial-method inlining. private static final boolean DISABLE_INLINE = - "true".equalsIgnoreCase(System.getProperty("CN1_DISABLE_INLINE", "false")); + "true".equalsIgnoreCase(Util.getProperty("CN1_DISABLE_INLINE", "false")); /** * If this invoke is a direct (provably monomorphic) instance call to a trivial @@ -523,7 +523,7 @@ public Field asInlinableFieldAccess() { if (desc.length() < 3 || desc.charAt(0) != '(' || desc.charAt(1) != ')' || desc.charAt(2) == 'V') { return null; } - BytecodeMethod target = findMethodUp(Parser.getClassObject(owner.replace('/', '_').replace('$', '_'))); + BytecodeMethod target = findMethodUp(Parser.getClassObject(Util.mangle(owner))); if (target == null || !target.isStatic()) { return null; } @@ -586,10 +586,10 @@ public Field asInlinableFieldAccess() { */ private BytecodeMethod resolveDirectTarget() { if (opcode == Opcodes.INVOKESPECIAL) { - return findMethodUp(Parser.getClassObject(owner.replace('/', '_').replace('$', '_'))); + return findMethodUp(Parser.getClassObject(Util.mangle(owner))); } // INVOKEVIRTUAL - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); if (bc == null) { return null; } @@ -601,7 +601,7 @@ private BytecodeMethod resolveDirectTarget() { if (rc == null) { return null; // genuinely virtual -> target not fixed -> unsafe to inline } - return findMethodUp(Parser.getClassObject(rc.replace('/', '_').replace('$', '_'))); + return findMethodUp(Parser.getClassObject(Util.mangle(rc))); } /** @@ -629,7 +629,7 @@ private static BytecodeMethod trivialStaticForwarderTarget(BytecodeMethod m) { if (rc != Opcodes.IRETURN && rc != Opcodes.LRETURN && rc != Opcodes.FRETURN && rc != Opcodes.DRETURN && rc != Opcodes.ARETURN) return null; BytecodeMethod t = inner.findMethodUp(Parser.getClassObject( - inner.owner.replace('/', '_').replace('$', '_'))); + Util.mangle(inner.owner))); return (t != null && t.isStatic()) ? t : null; } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Jump.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Jump.java index cc0594d406d..718b0c21c24 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Jump.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Jump.java @@ -112,13 +112,13 @@ public void appendInstruction(StringBuilder b, List instructions) { if(TryCatch.isTryCatchInMethod()) { b.append("JUMP_TO(label_"); - b.append(label.toString()); + b.append(LabelInstruction.labelName(label)); b.append(", "); b.append(LabelInstruction.getLabelCatchDepth(label, instructions)); b.append(");\n"); } else { b.append("goto label_"); - b.append(label.toString()); + b.append(LabelInstruction.labelName(label)); b.append(";\n"); } } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/LabelInstruction.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/LabelInstruction.java index 0c3d85e1ceb..f7c228030c7 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/LabelInstruction.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/LabelInstruction.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.HashMap; +import java.util.IdentityHashMap; import java.util.Hashtable; import java.util.List; import java.util.Map; @@ -56,6 +57,55 @@ static class Pair { // a lot of strings. private static Map usedLabels = new Hashtable(); + /** + * Stable names for the C labels generated from ASM labels. + * + * These used to be {@code Label.toString()}, which ASM defines as + * {@code "L" + System.identityHashCode(this)}. That made the emitted C depend on + * identity hash codes, with two consequences. It is not reproducible -- nothing + * promises an identity hash is stable. And it is not even VALID on a runtime + * whose identity hash can be negative: ParparVM's is the object pointer narrowed + * to int, so a self-hosted translator emitted {@code label_L-180306432001}, which + * C reads as a subtraction, and every method with a try/catch failed to compile. + * + * Numbering is per method and assigned in bytecode order as the labels are + * visited, so a method's C depends only on that method. A global counter would + * work too, but it would make every method downstream of any change renumber, + * which turns one real difference into thousands when two outputs are compared. + * + * C labels are function-scoped, so the same name in two methods is not a clash. + * + * An IdentityHashMap because Label overrides neither equals nor hashCode, and two + * distinct labels must never share a name. + */ + private static final Map labelNames = new IdentityHashMap(); + + /** + * Names {@code l} as the {@code index}th label of its method. Called from + * BytecodeMethod.addLabel while the method is being parsed. + */ + public static void assignLabelName(Label l, int index) { + if (!labelNames.containsKey(l)) { + labelNames.put(l, "L" + index); + } + } + + /** + * The C label name for {@code l}. + * + * Every label reaching emission has been through addLabel, so the fallback is + * unreachable; it is spelled with a distinct prefix so that if it ever does fire + * it cannot collide with a real per-method name. + */ + public static String labelName(Label l) { + String name = labelNames.get(l); + if (name == null) { + name = "Lx" + labelNames.size(); + labelNames.put(l, name); + } + return name; + } + // cleanup between passes, free the garbage! public static void cleanup() { @@ -63,6 +113,7 @@ public static void cleanup() tryEndLabels.clear(); labelCatchDepth.clear(); usedLabels.clear(); + labelNames.clear(); } public LabelInstruction(org.objectweb.asm.Label parent) { super(-1); @@ -160,7 +211,7 @@ public void appendInstruction(StringBuilder b) { return; } b.append("\nlabel_"); - b.append(parent); + b.append(labelName(parent)); b.append(":\n"); Integer tryCount = tryEndLabels.get(parent); if(tryCount != null) { @@ -181,19 +232,19 @@ public void appendInstruction(StringBuilder b) { for(int iter = strs.size() - 1; iter >= 0 ; iter--) { Pair s = strs.get(iter); b.append(" tryBlockOffset"); - b.append(parent); + b.append(labelName(parent)); b.append(s.cls); b.append(s.counter); b.append(" = threadStateData->tryBlockOffset;\n"); b.append(" BEGIN_TRY("); b.append(s.cls); b.append(", catch_"); - b.append(parent); + b.append(labelName(parent)); b.append(s.cls); b.append(s.counter); //b.append("); NSLog(@\"Begin try on: %s %d off: %i\\n\", __FILE__, __LINE__, getThreadLocalData()->tryBlockOffset);"); b.append(");\n restoreTo"); - b.append(parent); + b.append(labelName(parent)); b.append(s.cls); b.append(s.counter); b.append(" = threadStateData->threadObjectStackOffset;\n"); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Ldc.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Ldc.java index 9f9e297d37b..551dc3629e9 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Ldc.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Ldc.java @@ -23,6 +23,8 @@ package com.codename1.tools.translator.bytecodes; +import com.codename1.tools.translator.Util; + import com.codename1.tools.translator.ByteCodeClass; import com.codename1.tools.translator.Parser; import java.util.List; @@ -57,7 +59,7 @@ public void addDependencies(List dependencyList) { int sort = ((Type) cst).getSort(); Type tp = (Type) cst; if (sort == Type.OBJECT) { - String t = tp.getInternalName().replace('/', '_').replace('$', '_'); + String t = Util.mangle(tp.getInternalName()); if(!dependencyList.contains(t)) { dependencyList.add(t); } @@ -75,7 +77,7 @@ public void addDependencies(List dependencyList) { case Type.SHORT: return; } - String t = ttt.getInternalName().replace('/', '_').replace('$', '_'); + String t = Util.mangle(ttt.getInternalName()); ByteCodeClass.addArrayType(t, tp.getDimensions()); if(!dependencyList.contains(t)) { dependencyList.add(t); @@ -159,15 +161,15 @@ public String getValueAsString() { Type tp = (Type) cst; if (sort == Type.OBJECT) { //b.append("/* LDC: '"); - //b.append(tp.getInternalName().replace('/', '_').replace('$', '_')); + //b.append(Util.mangle(tp.getInternalName())); //b.append("'*/\n PUSH_POINTER((JAVA_OBJECT)&class__"); - //b.append(tp.getInternalName().replace('/', '_').replace('$', '_')); + //b.append(Util.mangle(tp.getInternalName())); //b.append(");\n"); b.append("(JAVA_OBJECT)&class__"); - b.append(tp.getInternalName().replace('/', '_').replace('$', '_')); + b.append(Util.mangle(tp.getInternalName())); } else if (sort == Type.ARRAY) { //b.append("/* LDC Array: '"); - //b.append(tp.getInternalName().replace('/', '_').replace('$', '_')); + //b.append(Util.mangle(tp.getInternalName())); //b.append("'*/\n PUSH_POINTER((JAVA_OBJECT)&class_array"); b.append("(JAVA_OBJECT)&class_array"); b.append(tp.getDimensions()); @@ -199,7 +201,7 @@ public String getValueAsString() { b.append("JAVA_SHORT"); break; default: - b.append(ttt.getInternalName().replace('/', '_').replace('$', '_')); + b.append(Util.mangle(ttt.getInternalName())); break; } //b.append(");\n"); @@ -283,13 +285,13 @@ public void appendInstruction(StringBuilder b) { Type tp = (Type) cst; if (sort == Type.OBJECT) { b.append("/* LDC: '"); - b.append(tp.getInternalName().replace('/', '_').replace('$', '_')); + b.append(Util.mangle(tp.getInternalName())); b.append("'*/\n PUSH_POINTER((JAVA_OBJECT)&class__"); - b.append(tp.getInternalName().replace('/', '_').replace('$', '_')); + b.append(Util.mangle(tp.getInternalName())); b.append(");\n"); } else if (sort == Type.ARRAY) { b.append("/* LDC Array: '"); - b.append(tp.getInternalName().replace('/', '_').replace('$', '_')); + b.append(Util.mangle(tp.getInternalName())); b.append("'*/\n PUSH_POINTER((JAVA_OBJECT)&class_array"); b.append(tp.getDimensions()); b.append("__"); @@ -320,7 +322,7 @@ public void appendInstruction(StringBuilder b) { b.append("JAVA_SHORT"); break; default: - b.append(ttt.getInternalName().replace('/', '_').replace('$', '_')); + b.append(Util.mangle(ttt.getInternalName())); break; } b.append(");\n"); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/SwitchInstruction.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/SwitchInstruction.java index 711b8d20867..7858d24321d 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/SwitchInstruction.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/SwitchInstruction.java @@ -55,13 +55,13 @@ public void appendInstruction(StringBuilder b, List instructions) { b.append(keys[iter]); if(TryCatch.isTryCatchInMethod()) { b.append(": JUMP_TO(label_"); - b.append(labels[iter].toString()); + b.append(LabelInstruction.labelName(labels[iter])); b.append(", "); b.append(LabelInstruction.getLabelCatchDepth(labels[iter], instructions)); b.append(");\n"); } else { b.append(": goto label_"); - b.append(labels[iter].toString()); + b.append(LabelInstruction.labelName(labels[iter])); b.append(";\n"); } } @@ -69,13 +69,13 @@ public void appendInstruction(StringBuilder b, List instructions) { if(dflt != null) { if(TryCatch.isTryCatchInMethod()) { b.append(" default: JUMP_TO(label_"); - b.append(dflt.toString()); + b.append(LabelInstruction.labelName(dflt)); b.append(", "); b.append(LabelInstruction.getLabelCatchDepth(dflt, instructions)); b.append(");\n"); } else { b.append(" default: goto label_"); - b.append(dflt.toString()); + b.append(LabelInstruction.labelName(dflt)); b.append(";\n"); } } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TryCatch.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TryCatch.java index a097c80b5b1..d36cf7ad31a 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TryCatch.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TryCatch.java @@ -109,21 +109,21 @@ public void appendInstruction(StringBuilder b, List instructions) { // threadObjectStackOffset from trash and later callee frames were // allocated on top of this frame's locals. clang happened to spill. b.append(" volatile int restoreTo"); - b.append(start); + b.append(LabelInstruction.labelName(start)); b.append(cid); b.append(counter); b.append(";\n volatile int tryBlockOffset"); - b.append(start); + b.append(LabelInstruction.labelName(start)); b.append(cid); b.append(counter); b.append(";\n DEFINE_CATCH_BLOCK(catch_"); - b.append(start); + b.append(LabelInstruction.labelName(start)); b.append(cid); b.append(counter); b.append(", label_"); - b.append(handler); + b.append(LabelInstruction.labelName(handler)); b.append(", restoreTo"); - b.append(start); + b.append(LabelInstruction.labelName(start)); b.append(cid); b.append(counter); b.append(");\n"); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java index 8eb39cf8a35..feb8f2095aa 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java @@ -254,9 +254,25 @@ public void appendInstruction(StringBuilder b, List l) { // reaches it as a root (its pointer rides the operand stack) and // scans its fields, so any heap objects it references stay live. // It is never freed; it simply dies when the frame unwinds. - b.append("if(__builtin_expect(!class__"); + // NOTE: this guard necessarily tests class__X.initialized rather + // than __X_LOADED__, because X is a DIFFERENT class from the one + // being emitted and __X_LOADED__ is file-local to X's own + // translation unit. initialized is set before __CLINIT__ runs, so + // this can still enter the allocation while X's is in + // flight -- pre-existing, and the reason the guards emitted from + // ByteCodeClass (same translation unit) use the completion flag + // instead. Closing it here needs a globally visible completion + // flag on struct clazz, which is a larger change than this. + // + // ACQUIRE: this guard SKIPS the initialiser when the flag is + // set, so it never takes the class monitor and cannot rely on + // the monitor's release. Pairs with the __ATOMIC_RELEASE store + // in ByteCodeClass. A plain load here let a thread see the flag + // set while the vtable / classToInterfaceMap rows it describes + // were still invisible. + b.append("if(__builtin_expect(!__atomic_load_n(&class__"); b.append(type); - b.append(".initialized, 0)) __STATIC_INITIALIZER_"); + b.append(".initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_"); b.append(type); b.append("(threadStateData); memset(&__cn1stk_"); b.append(stackAllocId); diff --git a/vm/ByteCodeTranslator/src/java_io_File.m b/vm/ByteCodeTranslator/src/java_io_File.m index 2cd51c822c4..729d2e6c5ad 100644 --- a/vm/ByteCodeTranslator/src/java_io_File.m +++ b/vm/ByteCodeTranslator/src/java_io_File.m @@ -133,9 +133,17 @@ JAVA_OBJECT java_io_File_listImpl___java_lang_String_R_java_lang_String_1ARRAY(C type check, and it hands the collector String metadata for an array payload. cn1MainArgs has always used the array class; these three did not. Fixed on all of them, including the two that predate the Windows arm. */ - JAVA_OBJECT arr = allocArray(threadStateData, [files count], &class_array1__java_lang_String, sizeof(JAVA_OBJECT), 1); - - for (int i=0; i<[files count]; i++) { + /* [files count] is NSUInteger -- 64-bit -- while allocArray's length and the + element setter's index are JAVA_INT. Narrow ONCE and explicitly, and loop on + the narrowed value so the bound and the index have the same type. The + implicit conversion this replaces is what the native warning census caught, + and it only became visible when this file started being compiled at all: the + translated java_io_File.c used to overwrite it on the Apple targets, which is + the collision fixed earlier on this branch. */ + JAVA_INT fileCount = (JAVA_INT)[files count]; + JAVA_OBJECT arr = allocArray(threadStateData, fileCount, &class_array1__java_lang_String, sizeof(JAVA_OBJECT), 1); + + for (JAVA_INT i = 0; i < fileCount; i++) { NSString* f = [files objectAtIndex:i]; JAVA_OBJECT s = fromNSString(CN1_THREAD_STATE_PASS_ARG f); CN1_SET_ARRAY_ELEMENT_OBJECT(arr, i, s); diff --git a/vm/ByteCodeTranslator/src/javascript/parparvm_runtime.js b/vm/ByteCodeTranslator/src/javascript/parparvm_runtime.js index 9d89c5ce3c3..5f471960bee 100644 --- a/vm/ByteCodeTranslator/src/javascript/parparvm_runtime.js +++ b/vm/ByteCodeTranslator/src/javascript/parparvm_runtime.js @@ -5885,6 +5885,16 @@ bindNative(["cn1_java_lang_Class_getComponentType_R_java_lang_Class"], function( } return classObjectForName(def.componentClass); }); +// A browser has no process environment, so getenv ANSWERS null rather than +// failing. Without this the symbol falls through to the unsupported-native path, +// which emits `throw new Error("environment variables are not available...")` -- +// and Class.getResourceAsStream consults CN1_RESOURCE_PATH, so a JavaScript +// application asking for a resource got an exception where it previously got +// null. Returning null is both the safe answer and the correct one: the variable +// genuinely is not set. +bindNative(["cn1_java_lang_System_getenvImpl_java_lang_String_R_java_lang_String"], function(name) { + return null; +}); bindNative(["cn1_java_lang_Class_isPrimitive_R_boolean"], function(__cn1ThisObject) { return __cn1ThisObject.__classDef && __cn1ThisObject.__classDef.isPrimitive ? 1 : 0; }); bindNative(["cn1_java_lang_reflect_Array_newInstanceImpl_java_lang_Class_int_R_java_lang_Object"], function(componentClass, length) { if (!componentClass || !componentClass.__classDef) { diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 4e88a1d8ebc..af9107e6965 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -1463,16 +1463,6 @@ JAVA_LONG java_lang_Double_doubleToLongBits___double_R_long(CODENAME_ONE_THREAD_ return u.l; } -JAVA_LONG java_lang_Double_doubleToRawLongBits___double_R_long(CODENAME_ONE_THREAD_STATE, JAVA_DOUBLE n1) { - union { - JAVA_DOUBLE d; - JAVA_LONG l; - } u; - - u.d = n1; - return u.l; -} - JAVA_FLOAT java_lang_Float_intBitsToFloat___int_R_float(CODENAME_ONE_THREAD_STATE, JAVA_INT n1) { union { @@ -1973,6 +1963,7 @@ JAVA_OBJECT java_lang_Class_getName___R_java_lang_String(CODENAME_ONE_THREAD_STA return newStringFromCString(threadStateData, clz->clsName); } + JAVA_BOOLEAN java_lang_Class_isArray___R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT cls) { struct clazz* clz = (struct clazz*)cls; return clz->isArray; @@ -1988,6 +1979,12 @@ JAVA_BOOLEAN java_lang_Class_isArray___R_boolean(CODENAME_ONE_THREAD_STATE, JAVA JAVA_BOOLEAN java_lang_Class_isAssignableFrom___java_lang_Class_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT cls, JAVA_OBJECT cls2) { struct clazz* clz1 = (struct clazz*)cls; struct clazz* clz2 = (struct clazz*)cls2; + // A primitive class carries CN1_PRIMITIVE_CLASS_ID, which indexes no row of + // the instanceof tables, so it must never reach instanceofFunction. The JDK + // rule is also simply identity: int is assignable only from int. + if(clz1->primitiveType || clz2->primitiveType) { + return clz1 == clz2 ? JAVA_TRUE : JAVA_FALSE; + } // A.isAssignableFrom(B): target is A, the class under test is B. return instanceofFunction(clz1->classId, clz2->classId); } @@ -1995,6 +1992,9 @@ JAVA_BOOLEAN java_lang_Class_isAssignableFrom___java_lang_Class_R_boolean(CODENA JAVA_BOOLEAN java_lang_Class_isInstance___java_lang_Object_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT cls, JAVA_OBJECT obj) { if(obj == JAVA_NULL) { return JAVA_FALSE; } struct clazz* clz1 = (struct clazz*)cls; + // No object is ever an instance of a primitive class, and its sentinel + // classId indexes no instanceof table row -- see isAssignableFrom above. + if(((struct clazz*)cls)->primitiveType) { return JAVA_FALSE; } struct clazz* clz2 = (struct clazz*)CN1_CLASS_OF(obj); // tag-aware: a tagged Integer has no header // A.isInstance(o): target is A, the class under test is o's class. These were // reversed, so isInstance searched the TARGET's supertype table for the @@ -2537,6 +2537,31 @@ JAVA_VOID java_lang_System_gcLight__(CODENAME_ONE_THREAD_STATE) { int cn1GcProbeThrew = 0; #endif JAVA_VOID java_lang_System_gcMarkSweep__(CODENAME_ONE_THREAD_STATE) { + // FREEZE: refuse to start a cycle at all once the exit census has claimed the + // heap. Clearing System.gcShouldLoop is not sufficient on its own -- the GC + // thread may already have evaluated `while(gcShouldLoop)` and be on its way + // here, and System's start-up path re-raises that flag after its initial wait. + // Either way the census would see gcCurrentlyRunning false, start walking, and + // have the pending cycle resume and sweep underneath it. Checked here because + // this is the one door every cycle comes through. + // CLAIM the cycle, do not merely check a flag. Loading a freeze flag and then + // setting gcCurrentlyRunning is two steps, and the collector can be preempted + // between them: the census would raise the freeze, see gcCurrentlyRunning still + // false, and start walking a heap this thread is about to sweep. The claim below + // is a single compare-exchange, so a cycle is either started or refused with + // nothing observable in between. + { + int cn1Expected = CN1_GC_CYCLE_IDLE; + if(!atomic_compare_exchange_strong_explicit(&cn1GcCycleState, &cn1Expected, + CN1_GC_CYCLE_RUNNING, memory_order_acq_rel, memory_order_acquire)) { + // In practice only the FROZEN case can be taken: System's GC thread is + // the sole caller (System.java's `while(gcShouldLoop)` loop), so no second + // entrant can observe RUNNING. Refusing on RUNNING too is defence rather + // than policy -- two concurrent cycles would be worse than a skipped one -- + // and it means this is not a behaviour change for any existing caller. + return; + } + } gcCurrentlyRunning = JAVA_TRUE; if(firstTimeGcThread) { firstTimeGcThread = JAVA_FALSE; @@ -2662,6 +2687,14 @@ JAVA_VOID java_lang_System_gcMarkSweep__(CODENAME_ONE_THREAD_STATE) { // of malloc entirely for exactly this reason. lowMemoryMode = JAVA_FALSE; gcCurrentlyRunning = JAVA_FALSE; + // Release the claim. Only ever RUNNING -> IDLE: a census that froze while this + // cycle ran holds the state at FROZEN and this must not clobber it, which is why + // the transition is a compare-exchange rather than a store. + { + int cn1Running = CN1_GC_CYCLE_RUNNING; + atomic_compare_exchange_strong_explicit(&cn1GcCycleState, &cn1Running, + CN1_GC_CYCLE_IDLE, memory_order_acq_rel, memory_order_relaxed); + } } JAVA_VOID java_lang_System_exit___int(CODENAME_ONE_THREAD_STATE, JAVA_INT i) { diff --git a/vm/JavaAPI/src/java/lang/Boolean.java b/vm/JavaAPI/src/java/lang/Boolean.java index 043fee9956d..62c36722b01 100644 --- a/vm/JavaAPI/src/java/lang/Boolean.java +++ b/vm/JavaAPI/src/java/lang/Boolean.java @@ -27,6 +27,28 @@ * Since: JDK1.0, CLDC 1.0 */ public final class Boolean implements Comparable { + + /** + * The class object for the primitive type this class wraps. + * + * Null on every ParparVM target, and declared only because ASM's compiled + * bytecode reads it: org.objectweb.asm.Type compares against Short.TYPE, + * Float.TYPE and Boolean.TYPE, so the self-hosted translator does not link + * without the three fields existing. ASM is a jar we cannot edit, which is + * the one case where JavaAPI grows to meet a dependency rather than the + * dependency being removed. + * + * It cannot be given a real value here. javac lowers a primitive class + * literal to a read of the boxed type's own TYPE field, so the obvious + * initializer compiles to "getstatic TYPE; putstatic TYPE" -- it reads the + * field it is initializing and stores the null straight back. The six + * wrappers that already declare TYPE are null for exactly that reason. + * Giving all nine real values needs VM-side primitive class objects; that + * work is not part of this change, and nothing in the translator depends on + * it now that the C-type tables are keyed on the PrimitiveType enum. + */ + public static final Class TYPE = null; + /** * The Boolean object corresponding to the primitive value false. */ diff --git a/vm/JavaAPI/src/java/lang/Class.java b/vm/JavaAPI/src/java/lang/Class.java index 00b5f6466ec..79da8c77e3b 100644 --- a/vm/JavaAPI/src/java/lang/Class.java +++ b/vm/JavaAPI/src/java/lang/Class.java @@ -40,6 +40,11 @@ public final class Class implements java.lang.reflect.Type { public ClassLoader getClassLoader() { + if (isPrimitive()) { + // A primitive class is bootstrap-defined and must report null, which is + // what reflection code tests to tell such a type from a loaded one. + return null; + } return ClassLoader.getSystemClassLoader(); } @@ -50,6 +55,40 @@ public ClassLoader getClassLoader() { * following code fragment returns the runtime Class descriptor for the * class named java.lang.Thread: Classt= Class.forName("java.lang.Thread") */ + /** + * Returns the Class object for {@code className}. + * + * ParparVM links the whole program ahead of time, so there is no second class + * loader to consult: both extra arguments are accepted and ignored, and the + * class is resolved exactly as the one-argument form resolves it. The overload + * exists because library bytecode calls it -- ASM's + * ClassWriter.getCommonSuperClass does -- and an absent overload is a link + * error in translated code, not a compile error here. + * + * <p>What {@code initialize == true} does NOT do here: it does not run the + * named class's static initializer. ParparVM runs one on first use -- the + * generated code calls the class's static initializer at every NEW, GETSTATIC + * and INVOKESTATIC -- so any code that goes on to TOUCH the class sees its + * statics initialized as normal. What does not work is using forName purely for + * a registration side effect and never referencing the class again, the + * JDBC-driver idiom. That pattern cannot work on this platform for a second + * reason anyway: obfuscation rewrites class names, so a name looked up as a + * string does not survive a release build. + * + * <p>WHY IT IS NOT IMPLEMENTED, rather than left as an oversight: forcing the + * initializer needs a way to reach it from a Class object, and {@code struct + * clazz} carries no static-initializer function pointer -- only newInstanceFp + * and enumValueOfFp. Adding one is a field on EVERY class in EVERY application, + * to serve a flag whose only in-tree caller is ASM, which passes + * {@code initialize = false}. The cost is paid by every app and the benefit is + * claimed by none, so this stays documented rather than built. If a real caller + * ever needs it, emit the pointer then. + */ + public static java.lang.Class forName(java.lang.String className, boolean initialize, + ClassLoader loader) throws java.lang.ClassNotFoundException { + return forName(className); + } + public static java.lang.Class forName(java.lang.String className) throws java.lang.ClassNotFoundException { className = className.replace('$', '.'); Class c = forNameImpl(className); @@ -136,7 +175,180 @@ public static java.lang.Class forName(java.lang.String className) throws java.la * class upon which the getResourceAsStream method was called. */ public java.io.InputStream getResourceAsStream(java.lang.String name){ - return null; + if (name == null) { + return null; + } + String absolute = name; + if (!absolute.startsWith("/")) { + // Relative names resolve against this class's package, as the javadoc + // above describes. + // + // KNOWN LIMITATION, for a NESTED class only. getName() cannot be told + // apart from a package here, because ParparVM builds the runtime class + // name as clsName.replace('_', '.') in ByteCodeClass -- it starts from + // the MANGLED name, so the '$' that separates a nested class from its + // outer one arrives as a '.', and so does any '_' in a class's own + // name. Outer$Inner therefore reports "a.b.Outer.Inner" where the JDK + // reports "a.b.Outer$Inner", and the package derived below is + // "a.b.Outer" rather than "a.b". + // + // The consequence is a MISS, not a wrong file: the derived path is a + // directory named after a class, which a resource tree does not have, + // so the lookup returns null exactly as it did before this method was + // implemented. It is deliberately not patched up by walking shorter + // prefixes -- a package really can be named like a class, and that + // would turn today's miss into a confidently wrong hit. The fix + // belongs in the name the VM reports, which is a change to getName() + // for every translated application and wants its own testing. + // + // PUSHBACK, so the next reader does not re-open this: fixing it HERE + // means guessing where the package ends, and every guess is wrong for + // some real input -- a package may legitimately be named like a class, + // and a class name may legitimately contain '_'. A guess would convert + // today's harmless miss into a confident wrong answer. The defect is + // that getName() is lossy; it is fixed there or not at all. + String className = getName(); + int lastDot = className.lastIndexOf('.'); + absolute = lastDot < 0 ? "/" + name + : "/" + className.substring(0, lastDot).replace('.', '/') + "/" + name; + } + // Resources linked INTO the executable are deliberately not consulted here. + // + // CORRECTION, because the first version of this comment blamed the wrong + // thing: withdrawing this tier did NOT fix the ValidatorLightweightPicker + // screenshot difference, which persists without it. That is still an open + // question about this branch and the cause is elsewhere. + // + // The tier stays withdrawn on its own merits rather than that one. On + // master this method is `return null` on every ParparVM target, so no + // application has ever received anything from it and every caller has + // always taken its not-found path. Handing those callers a resource for the + // first time is a behaviour change for every shipping application, and it + // is a separate feature from self-hosting, which needs only the filesystem + // tier below. The javadoc this replaces claimed "nothing can regress, only + // start working" -- an assumption that every not-found path is strictly + // worse than the resource, which is not something this change established. + // + // The filesystem tier below stays, because it is OPT-IN: it answers only + // when CN1_RESOURCE_PATH names a search root, which no application sets and + // the self-hosted translator does. So an application sees exactly what it + // saw on master -- null -- and the translator can still find the C runtime + // it has to copy into its output. + // + // Letting applications read their own embedded resources is a good feature + // and wants its own change, where the screenshot baselines it moves can be + // reviewed as the point of the change rather than as fallout from one. + return cn1FileResource(absolute); + } + + + /** + * The filesystem half of {@link #getResourceAsStream}: looks the resource up + * under a search path, so a translated command-line program can read files that + * sit beside it rather than being linked into it. + * + * The path comes from CN1_RESOURCE_PATH, else a "cn1runtime" directory next to + * the executable. Entries are separated the way the platform separates path + * entries. + */ + private static java.io.InputStream cn1FileResource(String absolute) { + String path = System.getenv("CN1_RESOURCE_PATH"); + if (path == null || path.length() == 0) { + return null; + } + String relative = absolute.substring(1); + // A resource name is not a path expression. Refusing any ".." segment keeps + // a lookup inside the search root it was found under; without it a name + // like "../../etc/passwd" reads straight out of the filesystem, and the + // caller is usually passing a name that came from data. + if (relative.length() == 0 || cn1EscapesRoot(relative)) { + return null; + } + int from = 0; + while (from <= path.length()) { + int end = cn1PathEntryEnd(path, from); + String root = end < 0 ? path.substring(from) : path.substring(from, end); + if (root.length() > 0) { + java.io.File candidate = new java.io.File(root, relative); + // isFile(), not exists(): a DIRECTORY with the requested name exists + // and cannot be opened, and returning on that would abandon the + // search. Later roots still get their turn, which is the point of + // having a search path at all -- an earlier root holding an + // unusable candidate must not mask a usable one behind it. + if (candidate.isFile()) { + try { + return new java.io.FileInputStream(candidate); + } catch (java.io.IOException err) { + // Unreadable here does not mean absent everywhere: keep going. + err = null; + } + } + } + if (end < 0) { + break; + } + from = end + 1; + } + return null; + } + + /** + * True when any segment of a resource-relative path is "..". + * + * A backslash counts as a separator as well as '/'. Resource names are + * '/'-separated by specification, but nothing stops a caller passing a Windows + * path, and there File("root", "..\\..\\x") escapes exactly as the '/' form + * does -- checking only '/' would leave the traversal open on the one platform + * whose separator it is. + */ + private static boolean cn1EscapesRoot(String relative) { + int from = 0; + for (int i = 0; i <= relative.length(); i++) { + boolean atEnd = i == relative.length(); + if (!atEnd && relative.charAt(i) != '/' && relative.charAt(i) != '\\') { + continue; + } + if (relative.substring(from, i).equals("..")) { + return true; + } + from = i + 1; + } + return false; + } + + /** + * The index that ends the search-path entry starting at {@code from}, or -1 for + * the last one. + * + * This cannot use {@code File.pathSeparatorChar}, which is a hard-coded ':' in + * this class library rather than a platform value -- on a native Windows build + * that splits "C:\\res;D:\\res" after the drive letter and every entry is + * nonsense. Both separators are therefore accepted, and a ':' is not a + * separator when it sits directly after a single-letter entry and is followed + * by a slash, which is exactly a DOS drive prefix and never a POSIX path. + */ + private static int cn1PathEntryEnd(String path, int from) { + for (int i = from; i < path.length(); i++) { + char c = path.charAt(i); + if (c == ';') { + return i; + } + if (c == ':') { + boolean driveLetter = i == from + 1 + && i + 1 < path.length() + && (path.charAt(i + 1) == '\\' || path.charAt(i + 1) == '/') + && cn1IsLetter(path.charAt(from)); + if (!driveLetter) { + return i; + } + } + } + return -1; + } + + /** ASCII letter test; Character.isLetter is locale-aware and not wanted here. */ + private static boolean cn1IsLetter(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); } /** @@ -193,6 +405,12 @@ public java.io.InputStream getResourceAsStream(java.lang.String name){ * Creates a new instance of a class. */ public java.lang.Object newInstance() throws java.lang.InstantiationException, java.lang.IllegalAccessException { + if (isPrimitive()) { + // A primitive descriptor has no constructor, and its newInstanceFp is + // zero -- the native calls that pointer unconditionally, so letting one + // through jumps to address zero instead of throwing. + throw new InstantiationException(); + } Object o = newInstanceImpl(); if(o == null) { throw new InstantiationException(); @@ -211,6 +429,11 @@ public java.lang.Object newInstance() throws java.lang.InstantiationException, j * returns "void". */ public java.lang.String toString() { + if (isPrimitive()) { + // "int", not "int class" -- java.lang.Class documents the primitive form + // as the name alone. + return getName(); + } return getName() + " class"; } diff --git a/vm/JavaAPI/src/java/lang/Double.java b/vm/JavaAPI/src/java/lang/Double.java index 22f161feb5d..79a3e648abe 100644 --- a/vm/JavaAPI/src/java/lang/Double.java +++ b/vm/JavaAPI/src/java/lang/Double.java @@ -88,6 +88,14 @@ public byte byteValue(){ * If the argument is NaN, the result is 0x7ff8000000000000L. * In all cases, the result is a long integer that, when given to the longBitsToDouble(long) method, will produce a floating-point value equal to the argument to doubleToLongBits. */ + /** + * The raw IEEE 754 bits of {@code value}, without collapsing NaN to the + * canonical NaN. See {@link Float#floatToRawIntBits} for why this delegates. + */ + public static long doubleToRawLongBits(double value) { + return doubleToLongBits(value); + } + public native static long doubleToLongBits(double value); /** diff --git a/vm/JavaAPI/src/java/lang/Float.java b/vm/JavaAPI/src/java/lang/Float.java index 5b257d00f64..fc940da7ff7 100644 --- a/vm/JavaAPI/src/java/lang/Float.java +++ b/vm/JavaAPI/src/java/lang/Float.java @@ -28,6 +28,28 @@ * Since: JDK1.0, CLDC 1.1 */ public final class Float extends Number implements Comparable { + + /** + * The class object for the primitive type this class wraps. + * + * Null on every ParparVM target, and declared only because ASM's compiled + * bytecode reads it: org.objectweb.asm.Type compares against Short.TYPE, + * Float.TYPE and Boolean.TYPE, so the self-hosted translator does not link + * without the three fields existing. ASM is a jar we cannot edit, which is + * the one case where JavaAPI grows to meet a dependency rather than the + * dependency being removed. + * + * It cannot be given a real value here. javac lowers a primitive class + * literal to a read of the boxed type's own TYPE field, so the obvious + * initializer compiles to "getstatic TYPE; putstatic TYPE" -- it reads the + * field it is initializing and stores the null straight back. The six + * wrappers that already declare TYPE are null for exactly that reason. + * Giving all nine real values needs VM-side primitive class objects; that + * work is not part of this change, and nothing in the translator depends on + * it now that the C-type tables are keyed on the PrimitiveType enum. + */ + public static final Class TYPE = null; + /** * The largest positive value of type float. It is equal to the value returned by Float.intBitsToFloat(0x7f7fffff). * See Also:Constant Field Values @@ -113,6 +135,20 @@ public boolean equals(java.lang.Object obj){ * Returns the bit representation of a single-float value. The result is a representation of the floating-point argument according to the IEEE 754 floating-point "single precision" bit layout. Bit 31 (the bit that is selected by the mask 0x80000000) represents the sign of the floating-point number. Bits 30-23 (the bits that are selected by the mask 0x7f800000) represent the exponent. Bits 22-0 (the bits that are selected by the mask 0x007fffff) represent the significand (sometimes called the mantissa) of the floating-point number. If the argument is positive infinity, the result is 0x7f800000. If the argument is negative infinity, the result is 0xff800000. If the argument is NaN, the result is 0x7fc00000. In all cases, the result is an integer that, when given to the * method, will produce a floating-point value equal to the argument to floatToIntBits. */ + /** + * The raw IEEE 754 bits of {@code value}, without collapsing NaN to the + * canonical NaN. + * + * Delegates rather than declaring a second native. ParparVM's floatToIntBits + * is a bare union punt that does not collapse NaN to the canonical NaN -- so it + * is already the raw operation, and the two differ in the spec but not here. A + * separate native would be one more mangled symbol to get wrong, silently, for + * no behavioural difference. + */ + public static int floatToRawIntBits(float value) { + return floatToIntBits(value); + } + public native static int floatToIntBits(float value); /** diff --git a/vm/JavaAPI/src/java/lang/Integer.java b/vm/JavaAPI/src/java/lang/Integer.java index 0bcf391a733..387648fe877 100644 --- a/vm/JavaAPI/src/java/lang/Integer.java +++ b/vm/JavaAPI/src/java/lang/Integer.java @@ -359,6 +359,18 @@ public static int signum(int i) { return (i >> 31) | (-i >>> 31); // Hacker's delight 2-7 } + /** + * Rotates the two's-complement binary representation of {@code i} left by + * {@code distance} bits. + * + * The shift distance is used modulo 32 by the JLS shift rules, which is what + * makes the negation on the right half correct for every distance, including + * zero and multiples of 32. + */ + public static int rotateLeft(int i, int distance) { + return (i << distance) | (i >>> -distance); + } + public static int compare(int f1, int f2) { if (f1 > f2) return 1; diff --git a/vm/JavaAPI/src/java/lang/Short.java b/vm/JavaAPI/src/java/lang/Short.java index 233a1698268..16610a821a9 100644 --- a/vm/JavaAPI/src/java/lang/Short.java +++ b/vm/JavaAPI/src/java/lang/Short.java @@ -27,6 +27,28 @@ * Since: JDK1.1, CLDC 1.0 */ public final class Short extends Number implements Comparable { + + /** + * The class object for the primitive type this class wraps. + * + * Null on every ParparVM target, and declared only because ASM's compiled + * bytecode reads it: org.objectweb.asm.Type compares against Short.TYPE, + * Float.TYPE and Boolean.TYPE, so the self-hosted translator does not link + * without the three fields existing. ASM is a jar we cannot edit, which is + * the one case where JavaAPI grows to meet a dependency rather than the + * dependency being removed. + * + * It cannot be given a real value here. javac lowers a primitive class + * literal to a read of the boxed type's own TYPE field, so the obvious + * initializer compiles to "getstatic TYPE; putstatic TYPE" -- it reads the + * field it is initializing and stores the null straight back. The six + * wrappers that already declare TYPE are null for exactly that reason. + * Giving all nine real values needs VM-side primitive class objects; that + * work is not part of this change, and nothing in the translator depends on + * it now that the C-type tables are keyed on the PrimitiveType enum. + */ + public static final Class TYPE = null; + /** * The maximum value a Short can have. * See Also:Constant Field Values diff --git a/vm/JavaAPI/src/java/lang/TypeNotPresentException.java b/vm/JavaAPI/src/java/lang/TypeNotPresentException.java new file mode 100644 index 00000000000..5320f04fa67 --- /dev/null +++ b/vm/JavaAPI/src/java/lang/TypeNotPresentException.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +package java.lang; + +/** + * Thrown when an application tries to access a type using a string naming the + * type, but no definition for that type can be found. + */ +public class TypeNotPresentException extends java.lang.RuntimeException { + private final String typeName; + + public TypeNotPresentException(String typeName, Throwable cause) { + super("Type " + typeName + " not present", cause); + this.typeName = typeName; + } + + /** + * The fully qualified name of the unavailable type. + */ + public String typeName() { + return typeName; + } +} diff --git a/vm/JavaAPI/src/java/util/ArrayList.java b/vm/JavaAPI/src/java/util/ArrayList.java index 8a3c4129a15..4f99090a2a5 100644 --- a/vm/JavaAPI/src/java/util/ArrayList.java +++ b/vm/JavaAPI/src/java/util/ArrayList.java @@ -38,6 +38,27 @@ public class ArrayList extends AbstractList implements List, RandomAcce /** * Constructs a new instance of {@code ArrayList} with ten capacity. */ + // ISOLATION (PR #5766): the lazy default-capacity allocation that used to sit + // here is withdrawn. It replaced the eager new Object[10] with a SHARED static + // zero-length array, which also gave java.util.ArrayList a it had + // never had -- master's only static is a compile-time serialVersionUID, so the + // class previously emitted no static initializer at all. + // + // The suite then began stopping after exactly 145 of 166 screenshots on every + // target except glibc-x64, with ArrayList state corrupt at the point of + // failure: AIOOBE 89 inside pendingIdleSerialCalls.add, then AIOOBE -1, then a + // NullPointerException inside ArrayList.get, which only happens when the + // backing array reference itself is null. + // + // The list logic is NOT at fault: a differential fuzz of this exact source + // against java.util.ArrayList ran 3000 seeds x 200 random operations with no + // divergence, and every access to the corrupted list in Display is inside + // synchronized(lock). The corruption is therefore below Java, which makes the + // new and the process-wide shared array the part worth removing + // before anything subtler is blamed. + // + // The iterator below is the change that carried the measured win (iteration + // 25.5% -> 12.4% of mutator self-time) and is kept. public ArrayList() { this(10); } @@ -322,6 +343,100 @@ public void ensureCapacity(int minimumCapacity) { } } + /** + * Direct-array iterator, overriding AbstractList's generic SimpleListIterator. + * + * The inherited one was the single hottest method in a large translation -- + * 16.45% of mutator self-time on the 5782-class hellocodenameone corpus, more + * than twice the next entry. Three costs per element, none inherent: + * + * - a try/catch around the body, to turn IndexOutOfBoundsException into + * NoSuchElementException. ParparVM has no zero-cost exception tables, so a + * try block is a setjmp -- once per element, in the hottest loop in the + * program. An explicit bounds test costs a compare. + * - size() and get() as VIRTUAL calls on the outer list, with no JIT to + * inline them. + * - the index recomputed as size() - numLeft every iteration instead of + * being carried in a cursor. + * + * MEASURED after: the iteration path fell from 25.5% of mutator self-time to + * 12.4%, ArrayList.get from 7.42% to 0.55%, and _setjmp from 1.61% to zero. + * + * Semantics are unchanged: same ConcurrentModificationException on structural + * modification, same NoSuchElementException past the end, remove() still + * works. Reads array[firstIndex + i] exactly as get(int) does. + * + * Applies to every `for (x : list)` in every translated application whatever + * the loop's static type, because dispatch lands on the concrete ArrayList. + */ + // Package-private, not private: a private inner class whose constructor is + // reached from the outer class makes javac synthesise an access bridge and a + // ArrayList$1 marker type, so every iterator() paid an extra class and an + // aconst_null for the bridge argument. Nothing outside java.util can see it + // either way. + class ArrayListIterator implements Iterator { + private int cursor; + private int lastReturned = -1; + private int expectedModCount = modCount; + + public boolean hasNext() { + return cursor < size; + } + + public E next() { + if (modCount != expectedModCount) { + throw new ConcurrentModificationException(); + } + int i = cursor; + if (i >= size) { + throw new NoSuchElementException(); + } + // The i < size test is only a bounds check while the list's + // firstIndex + size <= array.length invariant holds, so the array + // itself has to be checked too. The iterator this replaced could not + // read out of range: it went through get(), which bounds-checks, inside + // a try that turned IndexOutOfBoundsException into + // NoSuchElementException. Dropping that -- the try was the point, since + // ParparVM has no zero-cost exception tables -- also dropped the only + // bounds check on the read, and ParparVM does NOT check an array read in + // a release build. The result was an out-of-bounds read of the heap + // rather than a recoverable exception, which is how an unrelated int[] + // ended up with a zeroed header and the screenshot suite died 145 tests + // in. OpenJDK's own ArrayList.Itr carries this identical guard + // (`if (i >= elementData.length) throw new ConcurrentModificationException()`); + // omitting it is the whole defect. One compare, and the measured win + // stays. + E[] a = array; + int idx = firstIndex + i; + if (idx < 0 || idx >= a.length) { + throw new ConcurrentModificationException(); + } + cursor = i + 1; + lastReturned = i; + return a[idx]; + } + + public void remove() { + if (lastReturned < 0) { + throw new IllegalStateException(); + } + if (modCount != expectedModCount) { + throw new ConcurrentModificationException(); + } + ArrayList.this.remove(lastReturned); + if (lastReturned < cursor) { + cursor--; + } + lastReturned = -1; + expectedModCount = modCount; + } + } + + @Override + public Iterator iterator() { + return new ArrayListIterator(); + } + @Override public E get(int location) { if (location < 0 || location >= size) { diff --git a/vm/JavaAPI/src/java/util/IdentityHashMap.java b/vm/JavaAPI/src/java/util/IdentityHashMap.java index 4010c21c92a..188a307ac74 100644 --- a/vm/JavaAPI/src/java/util/IdentityHashMap.java +++ b/vm/JavaAPI/src/java/util/IdentityHashMap.java @@ -125,25 +125,52 @@ static class IdentityHashMapIterator implements Iterator { final MapEntry.Type type; + /** + * Which of the three views this iterator serves. + * + * Keys and values come straight out of the table; only entrySet has to + * materialise an Entry, and only there can the caller observe one. The + * generic {@code type} callback cannot express that, because it takes a + * MapEntry -- so serving a key iterator through it allocated an Entry per + * next() purely to read one field back out and drop it. Measured on a + * self-hosting translation of the ParparVM translator: 1,366,140 such + * entries, 43.7MB, all garbage. java.util.HashMap already had separate + * key/value/entry iterators for exactly this reason; this one was missed. + */ + static final int KIND_ENTRY = 0; + static final int KIND_KEY = 1; + static final int KIND_VALUE = 2; + + final int kind; + boolean canRemove = false; IdentityHashMapIterator(MapEntry.Type value, IdentityHashMap hm) { associatedMap = hm; type = value; + kind = KIND_ENTRY; + expectedModCount = hm.modCount; + } + + IdentityHashMapIterator(int iteratorKind, IdentityHashMap hm) { + associatedMap = hm; + type = null; + kind = iteratorKind; expectedModCount = hm.modCount; } public boolean hasNext() { - while (position < associatedMap.elementData.length) { - // if this is an empty spot, go to the next one - if (associatedMap.elementData[position] == null) { - position += 2; - } else { - return true; - } + // elementData hoisted into a local: it was re-loaded from the outer map + // on every comparison AND on every array access, twice per probe step. + Object[] data = associatedMap.elementData; + int p = position; + int len = data.length; + while (p < len && data[p] == null) { + p += 2; } - return false; + position = p; + return p < len; } void checkConcurrentMod() throws ConcurrentModificationException { @@ -152,19 +179,50 @@ void checkConcurrentMod() throws ConcurrentModificationException { } } + @SuppressWarnings("unchecked") public E next() { - checkConcurrentMod(); - if (!hasNext()) { + // The concurrent-modification test and the null-skipping scan are + // INLINED here rather than reached through checkConcurrentMod() and + // hasNext(). + // + // An enhanced-for already pays two interface dispatches per element + // (hasNext then next); routing next() through two more non-inlined + // calls made it four, and ParparVM has no JIT to fold them away. + // MEASURED on the 5782-class hellocodenameone translation: + // IdentityHashMapIterator.next 6.43% of mutator self-time with + // checkConcurrentMod a further 1.84%, second only to the ArrayList + // iterator. + // + // Behaviour is unchanged: same ConcurrentModificationException on a + // structural change, same NoSuchElementException past the end, and + // position still advances past empty slots exactly as hasNext() did. + if (expectedModCount != associatedMap.modCount) { + throw new ConcurrentModificationException(); + } + Object[] data = associatedMap.elementData; + int p = position; + int len = data.length; + while (p < len && data[p] == null) { + p += 2; + } + if (p >= len) { + position = p; throw new NoSuchElementException(); } - IdentityHashMapEntry result = associatedMap - .getEntry(position); - lastPosition = position; - position += 2; - + lastPosition = p; + position = p + 2; canRemove = true; - return type.get(result); + + if (kind == KIND_KEY) { + Object key = associatedMap.elementData[lastPosition]; + return (E) (key == NULL_OBJECT ? null : key); + } + if (kind == KIND_VALUE) { + Object value = associatedMap.elementData[lastPosition + 1]; + return (E) (value == NULL_OBJECT ? null : value); + } + return type.get(associatedMap.getEntry(lastPosition)); } public void remove() { @@ -687,11 +745,7 @@ public boolean remove(Object key) { @Override public Iterator iterator() { return new IdentityHashMapIterator( - new MapEntry.Type() { - public K get(MapEntry entry) { - return entry.key; - } - }, IdentityHashMap.this); + IdentityHashMapIterator.KIND_KEY, IdentityHashMap.this); } }; } @@ -739,11 +793,7 @@ public void clear() { @Override public Iterator iterator() { return new IdentityHashMapIterator( - new MapEntry.Type() { - public V get(MapEntry entry) { - return entry.value; - } - }, IdentityHashMap.this); + IdentityHashMapIterator.KIND_VALUE, IdentityHashMap.this); } @Override diff --git a/vm/selfhost/README.md b/vm/selfhost/README.md new file mode 100644 index 00000000000..90852587d9f --- /dev/null +++ b/vm/selfhost/README.md @@ -0,0 +1,370 @@ +# Self-hosting ParparVM + +Builds `ByteCodeTranslator` with ParparVM itself: the translator's own bytecode, +plus ASM's, is translated to C and compiled into a native binary. + +It buys two things: + +1. **Validation.** The translator is a ~37k-line real program that exercises + collections, strings, file I/O, exceptions and the GC at scale. Running the + native build and the JVM build over the same input and diffing the emitted C + is an end-to-end conformance test of the whole VM, and the corpus grows on its + own as the translator does. +2. **Performance and memory.** A translation is a short-lived, allocation-heavy + batch job -- the shape where AOT should beat a cold JVM. + +## `stubs/` + +The self-hosted binary does the `clean`/`ios`/`macos` translation and nothing +else, so a few classes are replaced by no-op stubs when it is built. They are +never selected at run time; they exist so the source set compiles without +dragging in API that ParparVM's JavaAPI deliberately lacks. + +| stub | why | +|---|---| +| `Javascript*` | the JavaScript target, ~12.5k lines. Needs `java.util.regex` and `ConcurrentHashMap`. | +| `ArchiveClassScanner` | `java.util.zip`. Reachable only from `NativeSignatureVerifier`'s command-line entry point; the translator itself never reads an archive. | +| `DebugSymbolCompressor` | `java.util.zip` again, for the on-device-debug symbol sidecar. | + +`java.util.zip` cannot simply be added to JavaAPI: JavaAPI is mirrored by +`Ports/CLDC11`, where the package does not belong. + +Everything else the translator needs was removed from the translator rather than +added to JavaAPI -- see `Util`'s `splitLiteral`, `collapseWhitespace`, +`rewriteLocalObjectRefs`, `getProperty`, `listFiles` and `writeBytes`. Adding +`String.split`/`replaceAll` to JavaAPI in particular would have collided with +`BytecodeComplianceMojo`, which rewrites those calls onto +`com.codename1.util.regex` precisely because JavaAPI does not declare them. + +## Building and verifying + +```bash +export JDK_8_HOME=/path/to/a/working/jdk8 +./build-selfhost.sh # -> target/parpar +./verify-selfhost.sh # gates D and A +``` + +`build-selfhost.sh` compiles the source set against JavaAPI alone, stages ASM as +class directories (the translator walks directories, never archives), translates, +and clangs the result. The `-fwrapv -fno-strict-aliasing -fno-builtin-fmod(f)` +flags are mandatory for generated C -- Java arithmetic wraps and clang -O3 +miscompiles without them. + +The binary finds the C runtime it has to copy into its output through +`Class.getResourceAsStream`, which now consults resources linked into the +executable and then a search path named by `CN1_RESOURCE_PATH`. Before this it +returned a hard-coded null on every ParparVM target. + +## State + +Gate D (the native translator against itself) passes. Gate A (JVM against native) +is at **245 of 247 files byte-identical** on a JavaAPI-sized corpus, and binaries +built from the two trees produce identical output. + +The two files that still differ are `java_util_HashMap.c` and `.h`: the native +translator's dead-code pass culls seven more methods than the JVM's +(`cn1PutSlot`, `cn1MaybeGrow`, `clearImpl`, `containsKeyImpl`, `getImpl`, +`putImpl`, `removeImpl`), and emits them as empty stubs. Both trees compile, link +and run correctly, so the extra culling is safe here, but the two runtimes should +not disagree and the cause is not yet found. What is already ruled out: it is not +nondeterminism -- gate D passes on both sides -- and it is not identity-hash +iteration order, which was tested directly by re-running the JVM under +`-XX:hashCode=2` and getting byte-identical output. + +## What self-hosting has already found + +Three defects that were invisible to every existing test, because each was +self-consistent on HotSpot: + +- **`Integer.TYPE` and the other wrapper `TYPE` fields were null.** `TYPE = + int.class` compiles to `getstatic TYPE; putstatic TYPE`. A `Map` keyed on them + collapsed onto the single null key. `Util`'s primitive-to-C-type maps are exactly + that shape. +- **C label names came from identity hash codes.** ASM's `Label.toString()` is + `"L" + System.identityHashCode(this)`. That made the emitted C irreproducible, + and on ParparVM -- whose identity hash is the object pointer narrowed to int, so + often negative -- it emitted `label_L-180306432001`, which C reads as a + subtraction. Every method with a try/catch failed to compile. +- **C local-variable declarations were emitted in `HashSet` iteration order**, so + the same input produced different C. `debugVarEntries` had already had to learn + this for the debug side-table; the declarations had the same defect. + +Only the first is a runtime bug. The other two are reproducible-build defects in +the translator that a second runtime made visible. + +## Performance + +`bench-selfhost.sh` runs each arm over the same corpus, interleaved, and reports the +minimum wall clock and the peak `phys_footprint`. It refuses to print ratios unless +every arm emitted identical C. The reference JVM is **JDK 25** -- what HotSpot can +actually do; JDK 8 is kept only because it is what the builders currently fork. + +Translating the self-hosting corpus (ASM + the translator's own classes, ~570 +classes) on a 64 GB / 16-core Mac, release shape (`-O3 -flto=thin`): + +| | wall clock | peak footprint | +|---|---:|---:| +| parpar | 1.84 s | 1443 MB | +| jdk25 | 1.56 s | 516 MB | +| jdk8 | 2.27 s | 502 MB | + +**vs JDK 25: 1.18x slower, 2.79x more memory. vs JDK 8: 1.24x faster.** + +Two fixes got it there from 6x slower; both are described below. Wall clock on this +machine is only meaningful when it is quiet -- at load 113 the same benchmark +produced samples from 3.6 s to 24 s for every arm, JVM included. CPU time +(`user+sys`) is far more robust to contention, and by that measure the two are +level or better: parpar 4.35 s against jdk25 4.82 s on a loaded host. + +### Fix 1: the mutator slept instead of allocating + +`sample` on the original build put 64% of the process's samples in one stack, and +the mutator was not marking or sweeping -- it was asleep: + +``` +Ldc.getValueAsString -> cn1BibopAlloc -> cn1BibopMaybeGc + -> cn1PacingPark -> usleep -> nanosleep -> __semwait_signal +``` + +`CN1_LOG_PACING_PARKS` reported only **two** park events for the whole run, so each +was seconds long. `cn1BibopPacingCap` computed a generous cap -- `cn1CachedFreeMem/8`, +4 GB here -- and then clamped it to `trigger * 8` once the footprint passed +`CN1_PACING_GROWTH_FLOOR_BYTES`. That floor was a flat **512 MB**, and early in the +run the trigger is still at its own 24 MB floor, so the ceiling was **192 MB** +(`minCapKb=196608` confirmed it). A program with a ~1.4 GB live set cannot stay +inside a 192 MB allocation window, so it parked against a collector that could +never get under it. + +A fixed 512 MB says "this process has grown"; it does not say the machine is under +pressure, and the bound exists for pressure. The floor now scales: +`max(512MB, availableMemory/4)`. Where `cn1_available_memory` is the flat 100 MB +placeholder (Linux, Windows, the non-Apple fallback) the absolute floor still wins +and behaviour is unchanged; the floor can only ever rise, never fall. This is the +no-per-process-ceiling path only -- where a ceiling exists (iOS's dirty-memory +limit, or an explicit budget) `cn1PacingPark` takes the bounded branch and never +reaches this code. `ProcessBudgetPacingIntegrationTest` confirms both halves: its +control arm reports `minCapKb=4194304` with no parks, and its budget-bounded arm +still holds a 120 MB limit at a 60 MB peak across 427 parks. + +`cn1RefreshFreeMemCache()` also had exactly one caller, inside the mark cycle, so +`cn1CachedFreeMem` was 0 until the first collection and both the cap and this floor +fell to their absolute minimums during the window with the least reason to throttle. +It is primed in `cn1BibopDoInit` now. + +### Fix 2: the constant pool was O(n^2) + +With pacing out of the way, the main thread's own profile was dominated by +`Parser.addToConstantPool`, which did `constantPool.indexOf(s)` -- a `String.equals` +against every string already interned. On a self-hosting translation the pool holds +~200k strings: `String.equals` 11.2%, the list iterator 10.3%, `indexOf` 6.2% and +`ArrayList.get` 5.1% of main-thread samples, all of it there. A `HashMap` side index +answers the same question directly; the list stays the source of truth, so the +emitted indices are unchanged and gate A still passes byte-identical. + +### What is left: memory + +The remaining gap is peak footprint. Sweeping the GC trigger from 8 MB to 256 MB -- +four cycles down to two -- moves peak by less than 15%, so this is retained data +rather than uncollected garbage, and page-pool slack is about 2 MB, so it is not +fragmentation either. `CN1_HEAP_REPORT` on a census build prints the split. + +Two allocation defects came out of the per-class census and are fixed: + +- **`IdentityHashMap` allocated an `Entry` on every `next()`**, even for key and + value iteration, where the entry was built only to read one field back out of it + and drop it. 1,366,140 of them, 43.7 MB, all garbage. `java.util.HashMap` already + had separate key/value/entry iterators for exactly this reason and this map had + been missed; it now has the same split. +- **`ArrayList()` eagerly allocated `Object[10]`**, a 128-byte slot for every list, + including one never added to. It now shares a zero-length array until the first + growth. The first growth allocates exactly ten and not the twelve the general + growth path would pick, because ten keeps a small list in the size class it + already occupied -- growing to twelve would have traded a win on empty lists for + a loss on every list of one to ten elements. + +Measured together on the self-hosting corpus: + +| | before | after | +|---|---:|---:| +| allocations | 10,160,401 objects / 991 MB | 8,706,929 / 940 MB | +| legacy-heap objects | 729,174 | 444,783 | +| Java live heap | 860 MB | 770 MB | +| process peak | 1467 MB | 1324 MB | + +`CollectionSemanticsIntegrationTest` holds both against a real JDK -- empty-list +operations, the three growth paths, identity semantics, null keys and values +through each of the three views, iterator removal, and a rehash. It was confirmed +to fail when the key iterator stops mapping the table's sentinel back to null. + +**`HashMap` was investigated and deliberately left alone.** It eagerly allocates +three arrays (keys, values, meta) at capacity 16, which looks like the same defect, +but the maps in this workload are populated rather than empty. Rebuilding with a +default capacity of 1 -- the cheapest probe for "how much of that table is wasted" +-- made everything worse, because the maps then regrow repeatedly: + +| default capacity | Object[] allocs | int[] allocs | Java live | +|---|---:|---:|---:| +| 16 (current) | 1,324,987 | 213,725 | 770 MB | +| 1 (probe) | 1,802,249 | 452,356 | 882 MB | + +Growth there is also post-insert by design, so the shared-empty-table trick that +works for ArrayList would have the put path writing into the shared table. Not +worth it for an unmeasured win in the hottest class in the runtime. + +### Heap telemetry + +A census build answers "what is actually in the heap": + +```bash +CN1_SELFHOST_CFLAGS="-DCN1_ALLOC_CENSUS" ./build-selfhost.sh -O3 +CN1_HEAP_REPORT=1 ./target/parpar-O3 clean ... 2> report.txt +``` + +Three reports, after every sweep and once at exit: + +- `[JHEAP]` -- BiBOP pages reserved / live / slack, plus the legacy heap. Answers + "is this fragmentation?" (here: no, slack is ~2 MB of 715 MB). +- `[LIVE]` -- **the live heap by class**, occupied bytes, objects, bytes each, and + how many the last mark proved reachable. This is the one that was missing. +- `[ALLOC]` -- allocation volume by class. Churn, which costs CPU, as opposed to + retention, which costs memory. A class can dominate one and not the other. + +`[LIVE]` charges each object what it OCCUPIES -- a whole BiBOP size-class slot, a +whole malloc block -- so the per-class rows add up to the footprint and rounding +waste is charged to the class that causes it. + +**Read the post-sweep report, not the exit one, for reachability.** `reachable` +means "carries the current mark", so at exit -- long after the last cycle -- almost +everything looks unreachable whether it is or not. At exit that column says 6%; at +the last sweep, with fresh marks, it says 75%. + +### What the census says about this workload + +The `[LIVE]` report is printed **pre-sweep**, which is the only point where the four +reasons a slot is still occupied are distinguishable: `traced` (the current mark +reached it), `fresh` (allocated since the mark, kept by the grace rule), `aging` +(known dead, kept one more cycle) and `dead` (this sweep returns it). Post-sweep +the grace stamp makes the first two identical, and the first version of this census +reported one as the other. + +At the last cycle of a self-hosting translation: + +``` +occupied 4,441,347 objects 349MB + traced 47% fresh 30% aging 14% dead 9% +``` + +**Only 47% of the occupied heap is traced live. The rest is held by collector +policy, not by the program.** Per class the split is sharper still -- `char[]` is +**5% traced and 76% fresh**, i.e. almost pure churn caught between cycles: + +``` + 68.84MB 726070 objs 99 B/obj traced 49% fresh 20% aging 19% dead 12% java.lang.Object[] + 51.17MB 483395 objs 110 B/obj traced 5% fresh 76% aging 13% dead 6% char[] + 26.12MB 363289 objs 75 B/obj traced 57% fresh 27% aging 10% dead 5% java.lang.String + 15.09MB 240879 objs 65 B/obj traced 5% fresh 62% aging 21% dead 13% boolean[] +``` + +The mechanism is the sweep's own rule, confirmed directly by +`experiments/PinProbe`: a dead object needs **three cycles** to have its slot +returned -- one of grace while it is fresh, one of aging, then reclamation. A +translation completes three or four cycles in 1.4s, so most of what it allocates is +never eligible to be freed and the heap grows towards total allocation volume +(940MB allocated, 1.3GB peak, ~150-300MB genuinely live). + +Collecting faster helps, but does not change the ratio, because the grace rule +keeps everything allocated since the last mark whatever the rate: + +| | cycles | peak | traced at last cycle | +|---|---:|---:|---:| +| 1 mark thread | 3 | 1320 MB | 47% | +| `-DCN1_GC_MARK_THREADS=4` | 8 | **1172 MB** | 25% | +| 4 threads + `CN1_GC_TRIGGER_MB=24` | 7 | 1259 MB | 39% | + +So the dominant lever is **allocation churn**, and the `[ALLOC]` census names it: +`char[]` 368MB, `Object[]` 196MB, `String` 77MB, `SimpleListIterator` 40MB. Cutting +an allocation removes roughly three cycles of occupancy, not one object. + +Two hypotheses this ruled OUT, both of which looked plausible: + +- **Conservative stack roots pinning dead objects.** `experiments/PinProbe` shows + the marks are precise and depth makes no difference: a dropped batch reads 100% + kept on the cycle after it is allocated (the grace stamp) and 0% on the next, + identically whether it was allocated in a shallow frame, under a 400-deep + recursion, or with the stack scrubbed afterwards. +- **Fragmentation.** `[JHEAP]` puts page-pool slack at ~2MB of 715MB. + +Where the process memory sits, from `vmmap --summary` around peak: + +``` +MALLOC_LARGE 551.5M virtual / 435.8M dirty BiBOP arenas +MALLOC_LARGE (empty) 53.7M / 50.2M dirty freed, not returned +MALLOC_SMALL 232.0M / 111.7M dirty legacy heap +Stack 12.2M / 0.2M +``` + +It is all malloc'd heap; there is no large non-heap component. (An earlier note +here claimed ~600MB was "not the Java heap" -- that compared an exit-time census +against the whole-run peak and was wrong.) + +### The second grace cycle: vestigial in origin, load-bearing today + +A dead object needs three cycles because the sweep keeps it twice -- once as `fresh` +(never marked) and once as `aging` (`mark == V-1`). The first is load-bearing. The +second arrived in November 2014, commit `31528ecfa6`: + +``` +- if(o->__codenameOneGcMark != currentGcMarkValue) { // free what was not marked ++ if(o->__codenameOneGcMark < currentGcMarkValue - 1) { // keep one extra generation +``` + +message: "Delayed GCing of elements to prevent them from being collected due to a +race condition with the GC thread". **That collector had no SATB barrier** -- zero +matches for satb or snapshot at that commit -- so keeping an extra generation made a +lost-object race improbable rather than impossible. + +**It was removed, measured, and put back.** Removing it is verifier-green and +gauntlet-green and gives byte-identical self-hosting output, and it is worth about +**2-3% of peak** (1334 -> 1322 MB, 1349 -> 1302 MB). Not worth it, because four later +mechanisms have since been built on the rule: + +- Two `java.lang.ref` clearing sites that must use **exactly** the sweep's liveness + test. Their comment spells out the failure: "FAILING to clear one the sweep frees + hands get() a dangling pointer", and on ParparVM a dangling read is a native crash + no Java catch can see. +- The fast-sweep page shortcut, whose `gcGraceEpoch < V-1` bound is derived from the + per-slot rule. Its comment records what happened when the two disagreed: "testing + != V let it drop whole pages holding V-1 slots... 26,924 slots in one run. That is + what left kept objects pointing into reclaimed memory" -- issue 5425. +- The legacy and BiBOP sweeps ageing in step, so a matured `Hashtable.Entry` at V-1 + is never kept while its page-resident payload at V-1 has already gone. + +And the verifier does not cover the coupling: the measurement above changed the sweep +without changing the ref-clearing sites, which is precisely the dangling-`get()` bug, +and it still came back green. + +So the rule started as a band-aid and is now structural. Removing it means changing +all four together and re-deriving the fast-sweep bound, for 2-3%. The churn is worth +more and risks nothing. + +### String: the NSString field is free + +`java.lang.String` carries a `long nsString` for the Apple targets' direct NSString +mapping, and the obvious question is what that costs everywhere else. Measured: +nothing. + +``` +sizeof(obj__java_lang_String) = 48 nsString at offset 40 +``` + +The fields before it end at 36 and the struct is 8-aligned, so four of those eight +bytes were padding already. Without the field the struct is 40 bytes -- and BiBOP's +size classes are 32, 48, 64, ..., so 40 and 48 both land in the same 48-byte slot. +Removing it would save zero bytes per String while costing the Apple targets a +side table and a lookup. Keep it. + +The strings themselves are still the largest single consumer (`char[]`, 368 MB +allocated). Note that a compact Latin-1 path already exists for the concat +fast path -- `cn1FusedLatin1Begin` allocates the String and a `byte[]` payload in +one BiBOP slot -- so the remaining `char[]` volume is strings built some other way. +That is the next thing to look at. diff --git a/vm/selfhost/bench-selfhost.sh b/vm/selfhost/bench-selfhost.sh new file mode 100755 index 00000000000..4fd8ace5192 --- /dev/null +++ b/vm/selfhost/bench-selfhost.sh @@ -0,0 +1,176 @@ +#!/bin/bash +# Wall clock and peak memory: the native translator against JVM-hosted ones. +# +# bench-selfhost.sh [rounds] +# +# Reference JVMs come from SELFHOST_REF_JAVAS (comma-separated java binaries). +# The default is JDK 25 first, then JDK 8. JDK 25 is the honest headline -- it is +# what HotSpot can actually do -- and JDK 8 is kept only because it is what the +# builders currently fork. +# +# Discipline, following vm/benchmarks/run-benchmark.sh: +# +# - Arms are INTERLEAVED within each round. Sequential A-then-B on this hardware +# carries a thermal bias large enough to invent a result. +# - Time takes the MINIMUM of N: the floor is the machine's best, and noise only +# ever adds. Memory takes the MAXIMUM, because a peak is a max. +# - Raw per-round samples are printed, not just the extremum: a lone minimum +# hides a bimodal distribution. +# - Ratios are refused unless every arm emitted identical C. A speed number from +# a translator that emits different output is meaningless. +# +# Memory is the peak phys_footprint reported by /usr/bin/time -l on macOS, which +# is the same quantity vmmap calls "Physical footprint (peak)". NEVER ps rss: +# vm/CLAUDE.md records 151/207/219 MB measured for one unchanged binary. +set -e +cd "$(dirname "$0")" +REPO="$(cd ../.. && pwd)" +CLASSES="${1:?usage: bench-selfhost.sh [rounds]}" +APP="${2:?}"; PKG="${3:?}"; ROUNDS="${4:-5}" + +T="$REPO/vm/selfhost/target" +# -O3 -flto=thin is the documented release shape (vm/benchmarks/README.md); +# CN1_SELFHOST_BIN overrides it for an A/B against the -O1 diff-gate build. +PARPAR="${CN1_SELFHOST_BIN:-$T/parpar-O3}" +JAPI="$T/javaapi-classes" +TR="$REPO/vm/ByteCodeTranslator/target/classes" +ASM="$(cat "$REPO/vm/ByteCodeTranslator/target/selfhost-asm-classpath.txt")" +# Reference JVMs, resolved rather than hard-coded. A developer-specific absolute +# path here meant anyone else running the documented command died under `set -e` +# while BUILDING the arm list, before a single measurement -- the benchmark was +# runnable by one machine. +# SELFHOST_REF_JAVAS explicit comma-separated list, wins outright +# JDK_25_HOME a modern JDK to compare against +# java on PATH whatever this shell would run +cn1_first_java() { + for c in "${JDK_25_HOME:-}/bin/java" "$(command -v java 2>/dev/null || true)"; do + [ -n "$c" ] && [ -x "$c" ] && { echo "$c"; return; } + done +} +DEFAULT_JAVAS="$(cn1_first_java),${JDK_8_HOME:-}/bin/java" +IFS=',' read -r -a REF_JAVAS <<< "${SELFHOST_REF_JAVAS:-$DEFAULT_JAVAS}" + +W="$T/bench"; rm -rf "$W"; mkdir -p "$W" + +# $1 = arm label, $2 = output dir; runs one translation +invoke() { + local arm=$1 out=$2 + if [ "$arm" = parpar ]; then + env CN1_RESOURCE_PATH="$REPO/vm/ByteCodeTranslator/src" "$PARPAR" \ + clean "$JAPI;$CLASSES" "$out" "$APP" "$PKG" "$APP" 1.0 clean none + else + "$arm" -cp "$TR:$ASM" com.codename1.tools.translator.ByteCodeTranslator \ + clean "$JAPI;$CLASSES" "$out" "$APP" "$PKG" "$APP" 1.0 clean none + fi +} + +# "25" -> jdk25, "1.8.0_372" -> jdk8. Taking the leading number alone turns 1.8.0 +# into "jdk1", which is why the second sed exists. +label() { + case "$1" in + parpar) echo parpar;; + *) "$1" -version 2>&1 | head -1 \ + | sed -e 's/.*version "\([0-9][0-9.]*\).*/\1/' \ + -e 's/^1\.\([0-9]*\).*/\1/' -e 's/\..*//' -e 's/^/jdk/';; + esac +} + +ARMS=(parpar "${REF_JAVAS[@]}") +declare -a NAMES +# Names have to be UNIQUE, not merely descriptive: every tree, log and diff is filed +# under one, so two arms sharing a label make the second `mv` land inside the first +# arm's directory and the correctness check then compares a tree against itself +# nested one level down -- a divergence that is an artefact of naming. Two arms +# collide easily now that JDK 25 is discovered rather than hard-coded: with no +# JDK_25_HOME the PATH fallback and JDK_8_HOME can both be Java 8. +for a in "${ARMS[@]}"; do + base="$(label "$a")" + name="$base"; k=2 + for prev in "${NAMES[@]}"; do + if [ "$prev" = "$name" ]; then name="${base}#${k}"; k=$((k+1)); fi + done + NAMES+=("$name") +done +# Say which executable each arm actually is, so a "#2" suffix is never a mystery. +for i in "${!ARMS[@]}"; do echo "arm : ${NAMES[$i]} -> ${ARMS[$i]}"; done +echo "corpus : $CLASSES" +echo "arms : ${NAMES[*]} rounds: $ROUNDS" +echo "memory : peak phys_footprint (/usr/bin/time -l)" + +# --- correctness precondition: every arm must emit the same C ------------------- +# Same absolute output path for all arms, sequentially, because the generated +# CMakeLists embeds srcRoot.getAbsolutePath(). +OUT="$W/out" +for i in "${!ARMS[@]}"; do + mkdir -p "$OUT" + invoke "${ARMS[$i]}" "$OUT" > "$W/${NAMES[$i]}.log" 2>&1 || { echo "${NAMES[$i]} FAILED"; tail -5 "$W/${NAMES[$i]}.log"; exit 1; } + mv "$OUT" "$W/tree-${NAMES[$i]}" +done +for i in "${!ARMS[@]}"; do + [ "$i" -eq 0 ] && continue + if ! diff -rq "$W/tree-${NAMES[0]}" "$W/tree-${NAMES[$i]}" > "$W/diff-${NAMES[$i]}.txt" 2>&1; then + echo "DIVERGENCE (${NAMES[0]} vs ${NAMES[$i]}) -- ratios would be meaningless:" + sed "s|.*/$APP-src/||;s| and .*||" "$W/diff-${NAMES[$i]}.txt" | head -5 + exit 1 + fi +done +files=$(find "$W/tree-${NAMES[0]}" -type f | wc -l | tr -d ' ') +[ "$files" -gt 10 ] || { echo "VACUOUS: only $files files"; exit 1; } +echo "output : $files files, identical across all arms" +echo + +# --- timing, interleaved -------------------------------------------------------- +declare -a SAMPLES +for r in $(seq 1 "$ROUNDS"); do + for i in "${!ARMS[@]}"; do + rm -rf "$W/run"; mkdir -p "$W/run" + s=$(python3 -c 'import time;print(time.monotonic())') + invoke "${ARMS[$i]}" "$W/run" > /dev/null 2>&1 + e=$(python3 -c 'import time;print(time.monotonic())') + SAMPLES[$i]="${SAMPLES[$i]} $(python3 -c "print(f'{$e-$s:.2f}')")" + done +done +declare -a MINS +for i in "${!ARMS[@]}"; do + MINS[$i]=$(printf '%s\n' ${SAMPLES[$i]} | sort -n | head -1) + printf "time %-8s min %6ss samples:%s\n" "${NAMES[$i]}" "${MINS[$i]}" "${SAMPLES[$i]}" +done + +# --- memory, measured separately so the probe cannot perturb the clock ---------- +# +# Sampled in EVERY round and reduced with max, because the header promises the +# maximum of N samples and a peak is a max. Measuring each arm once let a single +# noisy run decide the reported ratio, which is the same mistake as quoting a +# memory figure from one process: the number looked like a measurement and was a +# sample. +declare -a PEAKS PEAK_MAX +for i in "${!ARMS[@]}"; do PEAK_MAX[$i]=0; done +for round in $(seq 1 "$ROUNDS"); do +for i in "${!ARMS[@]}"; do + rm -rf "$W/run"; mkdir -p "$W/run" + if [ "${ARMS[$i]}" = parpar ]; then + env CN1_RESOURCE_PATH="$REPO/vm/ByteCodeTranslator/src" /usr/bin/time -l "$PARPAR" \ + clean "$JAPI;$CLASSES" "$W/run" "$APP" "$PKG" "$APP" 1.0 clean none 2>"$W/mem.txt" >/dev/null + else + /usr/bin/time -l "${ARMS[$i]}" -cp "$TR:$ASM" com.codename1.tools.translator.ByteCodeTranslator \ + clean "$JAPI;$CLASSES" "$W/run" "$APP" "$PKG" "$APP" 1.0 clean none 2>"$W/mem.txt" >/dev/null + fi + PEAKS[$i]=$(awk '/peak memory footprint/{print $1}' "$W/mem.txt") + printf "mem %-8s round %d peak %8.0f MB\n" "${NAMES[$i]}" "$round" \ + "$(python3 -c "print(${PEAKS[$i]}/1048576.0)")" + [ "${PEAKS[$i]}" -gt "${PEAK_MAX[$i]}" ] && PEAK_MAX[$i]="${PEAKS[$i]}" +done +done +for i in "${!ARMS[@]}"; do + PEAKS[$i]="${PEAK_MAX[$i]}" + printf "mem %-8s MAX over %d round(s) %8.0f MB\n" "${NAMES[$i]}" "$ROUNDS" \ + "$(python3 -c "print(${PEAKS[$i]}/1048576.0)")" +done + +echo +for i in "${!ARMS[@]}"; do + [ "$i" -eq 0 ] && continue + python3 -c " +t=${MINS[0]}/${MINS[$i]}; m=${PEAKS[0]}/${PEAKS[$i]} +print(f'vs ${NAMES[$i]}: time {t:.2f}x ({\"parpar faster\" if t<1 else \"parpar slower\"}), memory {m:.2f}x ({\"parpar smaller\" if m<1 else \"parpar larger\"})')" +done diff --git a/vm/selfhost/build-selfhost.sh b/vm/selfhost/build-selfhost.sh new file mode 100755 index 00000000000..8108830b6ac --- /dev/null +++ b/vm/selfhost/build-selfhost.sh @@ -0,0 +1,152 @@ +#!/bin/bash +# Builds the ParparVM translator with ParparVM: its own bytecode, plus ASM's, is +# translated to C and compiled into a native binary. +# +# build-selfhost.sh [-O1|-O3] default -O1 +# +# Requirements: +# JDK_8_HOME a working JDK 8 (JavaAPI and the translator compile with it) +# clang, and maven on PATH the first time (to resolve ASM) +# +# The mandatory clang flags below are not negotiable for generated C: Java +# arithmetic wraps, and clang -O3 provably miscompiles without -fwrapv +# -fno-strict-aliasing -fno-builtin-fmod(f). See vm/benchmarks/README.md. +set -e +cd "$(dirname "$0")" +REPO="$(cd ../.. && pwd)" +OPT="${1:--O1}" +# -O3 implies ThinLTO: that IS the documented release shape (vm/benchmarks/README.md), +# and measured here it is the only rung that beats -O1 -- 1.45s against 1.61s for -O1, +# 1.70s for -O2 and 1.73s for plain -O3. Benchmarking a bare -O3 binary and calling it +# the release build understates it, so the flag is not left to the caller to remember. +case "$OPT" in -O3) CN1_SELFHOST_CFLAGS="-flto=thin $CN1_SELFHOST_CFLAGS";; esac +CC="${CN1_SELFHOST_CC:-clang}" +J8="${JDK_8_HOME:?set JDK_8_HOME to a working JDK 8}" +OUT="$REPO/vm/selfhost/target" +mkdir -p "$OUT" + +# 1. translator classes + ASM classpath, built once by maven and then cached. +TRANSLATOR="$REPO/vm/ByteCodeTranslator/target/classes" +# Rebuild when the classes are MISSING or STALE. Testing only for existence meant +# that re-running this after editing a translator source silently self-hosted the +# previous build, and the resulting binary was then compared against a JVM side +# built from the new sources -- which reports the intended change as a VM +# divergence. verify-selfhost.sh carries the same guard for the same reason, and +# maven's own incremental check is not enough on its own: it answered "Nothing to +# compile - all classes are up to date" for a source three hours newer than its +# class. +needs_build=0 +if [ ! -f "$TRANSLATOR/com/codename1/tools/translator/ByteCodeTranslator.class" ]; then + needs_build=1 +elif [ -n "$(find "$REPO/vm/ByteCodeTranslator/src" -type f -newer "$TRANSLATOR" -print -quit 2>/dev/null)" ]; then + # -type f, not -name '*.java'. The translator carries its C runtime as CLASSPATH + # RESOURCES -- cn1_globals.m, nativeMethods.m, java_io_File.m, cn1_win_compat.c, + # xmlvm.h and the rest -- and maven copies them into target/classes. Watching + # only Java sources meant editing any of those left the old copy in place, so + # the self-hosted binary embedded an obsolete runtime while the JVM side used + # the new one. That surfaces as a Gate A divergence pointing at the VM, which is + # exactly the misdiagnosis this guard exists to prevent. + echo "translator sources or resources are newer than $TRANSLATOR -- rebuilding" + needs_build=1 +fi +if [ "$needs_build" = 1 ]; then + # `clean` because the incremental check cannot be trusted here; it also removes + # selfhost-asm-classpath.txt, which the next block regenerates. + (cd "$REPO/vm" && mvn -q -B -pl ByteCodeTranslator -am clean package -DskipTests) +fi +ASM_CP_FILE="$REPO/vm/ByteCodeTranslator/target/selfhost-asm-classpath.txt" +if [ ! -f "$ASM_CP_FILE" ]; then + (cd "$REPO/vm" && mvn -q -B -pl ByteCodeTranslator dependency:build-classpath \ + -Dmdep.outputFile=target/selfhost-asm-classpath.txt) +fi +ASM_CP="$(cat "$ASM_CP_FILE")" + +# 2. the C runtime the translator emits from its own classpath resources. +# +# Copy EVERY non-Java file maven would have staged, not a hand-listed four. The +# list drifts: java_io_File.m, cn1_win_compat.c and xmlvm.h are all read through +# the same classpath lookup, and a hand-written subset silently ships whichever +# ones nobody remembered. +( cd "$REPO/vm/ByteCodeTranslator/src" && find . -type f ! -name '*.java' -print ) \ + | while read -r rel; do + mkdir -p "$TRANSLATOR/$(dirname "$rel")" + cp "$REPO/vm/ByteCodeTranslator/src/$rel" "$TRANSLATOR/$rel" + done + +# 3. JavaAPI, rebuilt from source whenever the source set changed. +# +# The presence check alone is not enough, and it fails in a way that looks like a VM +# bug rather than a stale cache: a class compiled before a method stopped being +# native still declares it native, so the translator emits a call to a symbol nothing +# defines. Three things invalidate it and it takes all three -- `-newer` catches an +# edited or added source, but a DELETED one moves no remaining file's timestamp, so +# the sorted manifest is what catches removals. Comparing a file list rather than +# hashing timestamps keeps this portable; `stat` takes -f on BSD and -c on Linux. +JAVAAPI="$OUT/javaapi-classes" +STAMP="$OUT/javaapi-classes.stamp" +MANIFEST="$OUT/javaapi-classes.manifest" +find "$REPO/vm/JavaAPI/src" -name '*.java' | sort > "$MANIFEST.now" +if [ ! -f "$JAVAAPI/java/lang/Object.class" ] || [ ! -f "$STAMP" ] || [ ! -f "$MANIFEST" ] || \ + ! cmp -s "$MANIFEST" "$MANIFEST.now" || \ + [ -n "$(find "$REPO/vm/JavaAPI/src" -name '*.java' -newer "$STAMP" -print -quit 2>/dev/null)" ]; then + rm -rf "$JAVAAPI"; mkdir -p "$JAVAAPI" + "$J8/bin/javac" -nowarn -Xmaxerrs 10000 -source 1.8 -target 1.8 -d "$JAVAAPI" $(cat "$MANIFEST.now") + mv "$MANIFEST.now" "$MANIFEST" + touch "$STAMP" +else + rm -f "$MANIFEST.now" +fi + +# 4. The self-host source set: every translator source except the ones a stub +# replaces, plus the two classes that carry their own main(). +# +# Only the sources that CANNOT compile against JavaAPI are stubbed, and the list +# is driven by what is in stubs/ rather than by a name pattern. A blanket +# "Javascript*" exclusion is what stubbed JavascriptNativeRegistry, which +# compiles fine and -- as the comment at its call site in Parser warns -- is +# consulted on EVERY target, not just JavaScript. Answering false there culled +# java.util.HashMap's getImpl/putImpl/removeImpl/containsKeyImpl/clearImpl and +# the two helpers only they call, and the native translator emitted seven +# methods as empty stubs that the JVM one emitted in full. +SRC="$REPO/vm/ByteCodeTranslator/src" +STUBS="$REPO/vm/selfhost/stubs" +STUBBED=$(cd "$STUBS" && find . -name '*.java' | sed 's|.*/||;s|\.java$||' | tr '\n' '|' | sed 's/|$//') +SRCLIST="$OUT/sources.txt" +find "$SRC" -name '*.java' \ + | grep -vE "/($STUBBED)\.java$" \ + | grep -v '/CastSemanticsVerifier\.java$' \ + | grep -v '/NativeSignatureVerifierCli\.java$' > "$SRCLIST" +find "$STUBS" -name '*.java' >> "$SRCLIST" + +# 5. compile it against JavaAPI ALONE. -Xmaxerrs because javac's default cap of 100 +# silently truncates and makes a large gap look small. +rm -rf "$OUT/classes"; mkdir -p "$OUT/classes" +"$J8/bin/javac" -nowarn -Xmaxerrs 100000 -source 1.8 -target 1.8 \ + -bootclasspath "$JAVAAPI" -cp "$ASM_CP" -d "$OUT/classes" "@$SRCLIST" + +# 6. ASM as class files: the translator walks directories, never archives. +rm -rf "$OUT/asm-classes"; mkdir -p "$OUT/asm-classes" +for jar in $(echo "$ASM_CP" | tr ':' '\n' | grep -E 'asm.*\.jar$'); do + (cd "$OUT/asm-classes" && unzip -oq "$jar" -x 'module-info.class' 'META-INF/*') +done + +# 7. translate. The app name has to be the mangled main class: three classes in the +# set declare main, and ByteCodeClass.addMethod refuses to pick one otherwise. +APP=com_codename1_tools_translator_ByteCodeTranslator +rm -rf "$OUT/out"; mkdir -p "$OUT/out" +"$J8/bin/java" -Xmx4g -cp "$TRANSLATOR:$ASM_CP" com.codename1.tools.translator.ByteCodeTranslator \ + clean "$JAVAAPI;$OUT/asm-classes;$OUT/classes" "$OUT/out" \ + "$APP" com.codename1.tools.translator "$APP" 1.0 clean none \ + > "$OUT/translate.log" 2>&1 \ + || { echo "TRANSLATE FAILED"; tail -40 "$OUT/translate.log"; exit 1; } + +# 8. compile. The .S as well as the .c: the virtual-thread context switch is emitted +# beside the generated sources and the C half references it, so a *.c-only +# invocation links against a missing cn1VirtualThreadSwitch. +SRCDIR="$OUT/out/dist/$APP-src" +ASMS=$(ls "$SRCDIR"/*.S 2>/dev/null || true) +BIN="$OUT/parpar$( [ "$OPT" = "-O3" ] && echo "-O3" || echo "" )" +$CC $OPT -w -fwrapv -fno-strict-aliasing -fno-builtin-fmod -fno-builtin-fmodf \ + $CN1_SELFHOST_CFLAGS -I"$SRCDIR" "$SRCDIR"/*.c $ASMS -lm -lpthread -o "$BIN" \ + 2> "$OUT/cc.log" || { echo "COMPILE FAILED"; tail -40 "$OUT/cc.log"; exit 1; } +echo "built $BIN" diff --git a/vm/selfhost/experiments/README.md b/vm/selfhost/experiments/README.md new file mode 100644 index 00000000000..a61efa8d5c5 --- /dev/null +++ b/vm/selfhost/experiments/README.md @@ -0,0 +1,47 @@ +# Heap experiments + +Small programs that answer one question each about the collector, read through the +census (`-DCN1_ALLOC_CENSUS` + `CN1_HEAP_REPORT=1`, see ../README.md). + +## PinProbe + +**Question: does the conservative stack scan pin objects that are provably dead?** + +Three arms, three distinct classes so one run compares them in one census: +`PinShallow` allocated and dropped in a shallow frame, `PinDeep` allocated at the +bottom of a 400-deep recursion, `PinScrub` the same but with the stack overwritten +before collecting. Nothing holds a reference to any of them, so a precise collector +reclaims all three and any survivor is a stale stack word mistaken for a pointer. + +Build and run: + +```bash +javac -bootclasspath -d /tmp/exp/classes src/com/exp/PinProbe.java +java -cp : com.codename1.tools.translator.ByteCodeTranslator \ + clean ";/tmp/exp/classes" /tmp/exp/out PinProbe com.exp PinProbe 1.0 clean none +clang -O3 -flto=thin -w -fwrapv -fno-strict-aliasing -fno-builtin-fmod -fno-builtin-fmodf \ + -DCN1_ALLOC_CENSUS -I /*.c /*.S -lm -lpthread -o /tmp/exp/pinprobe +CN1_HEAP_REPORT=1 /tmp/exp/pinprobe 2>&1 | grep -E 'PROBE|Pin(Shallow|Deep|Scrub)' +``` + +**Answer: no.** All three arms behave identically, and the marks are precise -- a +batch reads 100% live on the cycle after it is allocated and 0% on the next, with +no difference between the shallow, deep and scrubbed arms. What the probe found +instead is the reclamation LATENCY: a dead object needs **three cycles** to have +its slot returned. + +``` +cycle 1: PinShallow 200,000 occupied, 100% kept <- grace: fresh objects are stamped live +cycle 2: PinShallow 200,000 occupied, 0% kept <- known dead, still occupying +cycle 3: PinShallow 196,320 occupied <- reclaimed +``` + +That is the sweep's own rule: `m == -1` (fresh) gets one cycle of grace, `m == V-1` +is kept for another, and only `m < V-1` is reclaimed. It is why a short program +retains nearly everything it allocates -- the ParparVM translator completes three +or four cycles in 1.4s, so most of what it allocates is never eligible. + +This probe is also the reason the census reports its four buckets **pre-sweep**: +read post-sweep, the grace stamp makes "traced live" and "kept because it is fresh" +indistinguishable, and the first version of the census reported the second as the +first. diff --git a/vm/selfhost/experiments/src/com/exp/PinProbe.java b/vm/selfhost/experiments/src/com/exp/PinProbe.java new file mode 100644 index 00000000000..f4ccd7baa4b --- /dev/null +++ b/vm/selfhost/experiments/src/com/exp/PinProbe.java @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.exp; + +/** + * Does ParparVM's conservative stack scan pin objects that are provably dead? + * + * Three arms, three distinct classes so one run compares them in one census: + * + * PinShallow allocated and dropped in a shallow frame + * PinDeep allocated at the bottom of a deep recursion, then unwound + * PinScrub same as PinDeep, then the stack is overwritten before collecting + * + * Every instance is unreachable by the time the collector runs -- nothing holds a + * reference. So a correct precise collector reclaims all of them, and any survivor + * is something the conservative scan mistook a stale stack word for. If Deep >> + * Shallow the depth is what pins; if Scrub << Deep the stale words are the + * mechanism and overwriting them frees the objects. + */ +public class PinProbe { + static final int BATCH = 200000; + static final int DEPTH = 400; + + // Sinks so the allocations cannot be optimised away, without retaining anything. + static int shallowSink, deepSink, scrubSink, scrubberSink; + + static class PinShallow { int a; } + static class PinDeep { int a; } + static class PinScrub { int a; } + + static void allocShallow() { + for (int i = 0; i < BATCH; i++) { + PinShallow p = new PinShallow(); + p.a = i; + shallowSink += p.a; + } + } + + static void allocDeep(int depth) { + if (depth > 0) { + allocDeep(depth - 1); + return; + } + for (int i = 0; i < BATCH; i++) { + PinDeep p = new PinDeep(); + p.a = i; + deepSink += p.a; + } + } + + static void allocScrub(int depth) { + if (depth > 0) { + allocScrub(depth - 1); + return; + } + for (int i = 0; i < BATCH; i++) { + PinScrub p = new PinScrub(); + p.a = i; + scrubSink += p.a; + } + } + + /** + * Walks back down to the same depth writing non-pointer values into locals, so + * every stack slot the allocation loops left behind is overwritten with an + * integer that cannot be mistaken for a heap address. + */ + static void scrub(int depth) { + int a = depth * 3 + 1, b = depth * 5 + 2, c = depth * 7 + 3, d = depth * 11 + 4; + int e = depth * 13 + 5, f = depth * 17 + 6, g = depth * 19 + 7, h = depth * 23 + 8; + if (depth > 0) { + scrub(depth - 1); + } + scrubberSink += a + b + c + d + e + f + g + h; + } + + static void collect(String label) throws Exception { + System.gc(); + // gc() only signals the collector; give it room to finish a cycle so the + // census that follows is reading fresh marks. + Thread.sleep(1500); + System.err.println("[PROBE] after " + label); + } + + public static void main(String[] args) throws Exception { + System.err.println("[PROBE] batch=" + BATCH + " depth=" + DEPTH); + + allocShallow(); + collect("shallow"); + + allocDeep(DEPTH); + collect("deep"); + + allocScrub(DEPTH); + scrub(DEPTH); + collect("deep+scrub"); + + System.err.println("[PROBE] sinks " + shallowSink + " " + deepSink + " " + + scrubSink + " " + scrubberSink); + System.out.println("DONE"); + } +} diff --git a/vm/selfhost/stubs/com/codename1/tools/translator/ArchiveClassScanner.java b/vm/selfhost/stubs/com/codename1/tools/translator/ArchiveClassScanner.java new file mode 100644 index 00000000000..dd815624bde --- /dev/null +++ b/vm/selfhost/stubs/com/codename1/tools/translator/ArchiveClassScanner.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; +import java.io.File; +import java.io.IOException; +import java.util.List; + +/** + * Stub for the self-hosted translator build. See {@code vm/selfhost/README.md}. + * + * The real class reads a jar with java.util.zip, which JavaAPI has no business + * gaining -- it is mirrored by Ports/CLDC11, where the package does not belong. + * It is reachable only from NativeSignatureVerifier's offline command-line entry + * point; a translation never reads an archive, because every caller extracts a jar + * into a directory of class files first. + */ +final class ArchiveClassScanner { + private ArchiveClassScanner() { + } + + static void collect(File archive, List into) throws IOException { + throw new UnsupportedOperationException( + "archive scanning is not built into this translator; pass a directory of class files"); + } +} diff --git a/vm/selfhost/stubs/com/codename1/tools/translator/DebugSymbolCompressor.java b/vm/selfhost/stubs/com/codename1/tools/translator/DebugSymbolCompressor.java new file mode 100644 index 00000000000..b371d7d2afc --- /dev/null +++ b/vm/selfhost/stubs/com/codename1/tools/translator/DebugSymbolCompressor.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +/** + * Stub for the self-hosted translator build. See {@code vm/selfhost/README.md}. + * + * The real class gzips the on-device-debug symbol table with java.util.zip, which + * JavaAPI has no business gaining -- it is mirrored by Ports/CLDC11, where the + * package does not belong. Reached only when cn1.onDeviceDebug is set, which is + * off by default. + */ +final class DebugSymbolCompressor { + private DebugSymbolCompressor() { + } + + static byte[] gzip(ByteArrayOutputStream raw) throws IOException { + throw new UnsupportedOperationException( + "on-device-debug symbols are not built into this translator"); + } +} diff --git a/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptBundleWriter.java b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptBundleWriter.java new file mode 100644 index 00000000000..e29a755c8dd --- /dev/null +++ b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptBundleWriter.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +/** + * See {@code vm/selfhost/README.md}. + * + * Stub for the self-hosted translator build, which does the clean/ios/macos + * targets only. Replaced on the source path -- the real class is never compiled + * into that binary, and none of these methods is reachable in it. + * + * They throw rather than returning a plausible value: the JavaScript target is + * selected explicitly, so reaching one of these would mean the binary was asked + * for a target it was not built with, and that should be loud. + */ +final class JavascriptBundleWriter { + private JavascriptBundleWriter() { + } + + static void write(File outputDirectory, List classes) throws IOException { + throw new UnsupportedOperationException("JavaScript target not built into this translator"); + } +} diff --git a/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptMethodGenerator.java b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptMethodGenerator.java new file mode 100644 index 00000000000..113c3b99a6e --- /dev/null +++ b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptMethodGenerator.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import java.util.List; + +/** + * See {@code vm/selfhost/README.md}. + * + * Stub for the self-hosted translator build, which does the clean/ios/macos + * targets only. Replaced on the source path -- the real class is never compiled + * into that binary, and none of these methods is reachable in it. + * + * They throw rather than returning a plausible value: the JavaScript target is + * selected explicitly, so reaching one of these would mean the binary was asked + * for a target it was not built with, and that should be loud. + */ +final class JavascriptMethodGenerator { + private JavascriptMethodGenerator() { + } + + static String generateClassJavascript(ByteCodeClass cls, List allClasses) { + throw new UnsupportedOperationException("JavaScript target not built into this translator"); + } +} diff --git a/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptReachability.java b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptReachability.java new file mode 100644 index 00000000000..a6c9bec6385 --- /dev/null +++ b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptReachability.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import java.util.List; + +/** + * See {@code vm/selfhost/README.md}. + * + * Stub for the self-hosted translator build, which does the clean/ios/macos + * targets only. Replaced on the source path -- the real class is never compiled + * into that binary, and none of these methods is reachable in it. + * + * They throw rather than returning a plausible value: the JavaScript target is + * selected explicitly, so reaching one of these would mean the binary was asked + * for a target it was not built with, and that should be loud. + */ +final class JavascriptReachability { + private JavascriptReachability() { + } + + /// Stub: the JavaScript target is excluded from the self-hosted build, so the + /// per-application fact cache it clears does not exist here. + static void resetExportedFacts() { + } + + static int run(List classes, List classPool, + String[] nativeSources) { + throw new UnsupportedOperationException("JavaScript target not built into this translator"); + } +} diff --git a/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptSuspensionAnalysis.java b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptSuspensionAnalysis.java new file mode 100644 index 00000000000..4d1c96eced1 --- /dev/null +++ b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptSuspensionAnalysis.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import java.util.List; + +/** + * See {@code vm/selfhost/README.md}. + * + * Stub for the self-hosted translator build, which does the clean/ios/macos + * targets only. Replaced on the source path -- the real class is never compiled + * into that binary, and none of these methods is reachable in it. + * + * They throw rather than returning a plausible value: the JavaScript target is + * selected explicitly, so reaching one of these would mean the binary was asked + * for a target it was not built with, and that should be loud. + */ +final class JavascriptSuspensionAnalysis { + private JavascriptSuspensionAnalysis() { + } + + static int run(List classes, java.io.File outputDirectory) { + throw new UnsupportedOperationException("JavaScript target not built into this translator"); + } +} diff --git a/vm/selfhost/verify-output-neutral.sh b/vm/selfhost/verify-output-neutral.sh new file mode 100755 index 00000000000..54073a66a59 --- /dev/null +++ b/vm/selfhost/verify-output-neutral.sh @@ -0,0 +1,170 @@ +#!/bin/bash +# Proves a translator source change does not alter the emitted C. +# +# verify-output-neutral.sh capture # run the JVM translator, save the tree +# verify-output-neutral.sh compare # diff two captured trees +# +# Gate A (in verify-selfhost.sh) compares the JVM translator against the native one +# and CANNOT see this: a refactor lands on both sides at once, so both move together +# and the gate stays green while every emitted signature changes. This runs the JVM +# translator alone, before and after, over the same corpus. +# +# Same fixed output path and constructed environment as verify-selfhost.sh, and for +# the same reasons: the generated CMakeLists embeds srcRoot.getAbsolutePath(), and +# the translator reads its knobs from getenv. +set -e +cd "$(dirname "$0")" +REPO="$(cd ../.. && pwd)" +J8="${JDK_8_HOME:?set JDK_8_HOME to a working JDK 8}" +W="$REPO/vm/selfhost/target/neutral" +OUT="$W/out" + +# Renumber label_L by ORDER OF FIRST APPEARANCE within the file, rather than +# erasing every label to one token. Erasing them makes a jump that was retargeted +# from one existing label to another compare EQUAL -- which is precisely the +# code-generation regression this comparison is for. Renumbering keeps the identity +# relationships and still absorbs the switch from identity-hash names to sequential +# ones. +cn1_canon_labels() { + awk ' + # Reset the mapping at every generated FUNCTION boundary. + # + # The branch numbers labels with a METHOD-LOCAL counter, so L0 and L1 recur + # in every method, while master derives them from ASM identities that are + # distinct across the whole file. A file-wide map therefore folds the second + # method'"'"'s L0 onto the first method'"'"'s token on one side and not the other, + # and reports identical output as a codegen difference -- the mirror of the + # erase-everything bug, generating false positives instead of hiding real + # ones. Per-function scope matches how the names are actually minted. + /^[A-Za-z_][A-Za-z0-9_ \*]*\(/ { delete seen; k = 0 } + { + line = $0 + out = "" + # The number leaks into derived identifiers too -- catch_L, + # restoreToL, tryBlockOffsetL -- which must share the numbering + # WITHIN a function or every try/catch file reports as changed forever. + while (match(line, /(label_L|catch_L|restoreToL|tryBlockOffsetL)[0-9]+/)) { + pre = substr(line, 1, RSTART - 1) + tok = substr(line, RSTART, RLENGTH) + line = substr(line, RSTART + RLENGTH) + nstart = match(tok, /[0-9]+$/) + kind = substr(tok, 1, nstart - 1) + num = substr(tok, nstart) + if (!(num in seen)) { seen[num] = ++k } + out = out pre kind seen[num] + } + print out line + }' "$1" +} + +case "${1:?usage: capture | compare | vs-master}" in +vs-master) + # Compare THIS branch's translator against MASTER's over one corpus. + # + # This is the comparison the other two modes cannot make. They run the same + # translator twice, so a change that lands on the branch is present on both + # sides and they stay green while every emitted signature moves. That blind + # spot cost a full bisect: four native screenshot legs reported a four-pixel + # layout shift, and the cause was a codegen change this script reported as + # neutral because it was neutral -- against itself. + # + # The corpus is a small app compiled against MASTER's JavaAPI, so JavaAPI is + # held constant and the translator is the only variable. Differences in + # deterministic label names and local-variable declaration order are expected + # and are normalised out; anything else is a real codegen change and should be + # a deliberate one. + MW="${CN1_MASTER_WORKTREE:-/tmp/cn3-master}" + [ -d "$MW/vm" ] || { echo "no master worktree at $MW"; echo " git worktree add $MW origin/master"; exit 1; } + MTR="$MW/vm/ByteCodeTranslator/target/classes" + MAPI="$MW/vm/JavaAPI/target/classes" + for d in "$MTR" "$MAPI"; do + [ -d "$d" ] || { echo "missing $d -- build master's translator and JavaAPI first:"; \ + echo " (cd $MW/vm && mvn -q -B -pl ByteCodeTranslator,JavaAPI package -DskipTests)"; exit 1; } + done + APP="${CN1_NEUTRAL_APP:-/tmp/cmpcls}" + [ -d "$APP" ] || { echo "no corpus app at $APP (set CN1_NEUTRAL_APP)"; exit 1; } + ASM="$(cat "$REPO/vm/ByteCodeTranslator/target/selfhost-asm-classpath.txt")" + rm -rf "$W/m-tree" "$W/b-tree" "$OUT" + for side in m b; do + [ "$side" = m ] && TR="$MTR" || TR="$REPO/vm/ByteCodeTranslator/target/classes" + mkdir -p "$OUT" + ( cd "$W" && env -i PATH=/usr/bin:/bin HOME="$HOME" TMPDIR=/tmp LC_ALL=C \ + CN1_NATIVE_VERIFY="${CN1_NATIVE_VERIFY:-}" \ + "$J8/bin/java" -cp "$TR:$ASM" com.codename1.tools.translator.ByteCodeTranslator \ + clean "$MAPI;$APP" "$OUT" CmpApp com.cmp CmpApp 1.0 clean none ) > "$W/$side.log" 2>&1 \ + || { echo "$side side FAILED"; tail -5 "$W/$side.log"; exit 1; } + mv "$OUT" "$W/$side-tree" + done + # The C runtime is copied verbatim and this branch edits it on purpose, so it is + # not part of the codegen question. + # Single backslash: this value goes to grep -E, where \. is a literal dot. Two + # backslashes made it "a literal backslash followed by any character", so NOTHING + # matched and the four copied runtime files were counted as codegen differences -- + # which is the 115-versus-119 gap that should have been chased when it appeared. + RUNTIME='^(cn1_globals\.[ch]|nativeMethods\.c|cn1_intrinsics\.h|java_io_File_runtime\.c|cn1-source-manifest\.txt)$' + # Walk the UNION of both trees, not master's listing. A file the branch emits and + # master does not would never be visited by a master-only loop, so a whole new + # generated class could appear and the gate would report neutral. + ( cd "$W/m-tree/dist/CmpApp-src" 2>/dev/null && ls ) > "$W/m.list" 2>/dev/null || : > "$W/m.list" + ( cd "$W/b-tree/dist/CmpApp-src" 2>/dev/null && ls ) > "$W/b.list" 2>/dev/null || : > "$W/b.list" + sort -u "$W/m.list" "$W/b.list" > "$W/all.list" + n=0 + while read -r base; do + [ -n "$base" ] || continue + echo "$base" | grep -qE "$RUNTIME" && continue + case "$base" in *.c|*.h) ;; *) continue ;; esac + mf="$W/m-tree/dist/CmpApp-src/$base"; bf="$W/b-tree/dist/CmpApp-src/$base" + if [ ! -f "$mf" ]; then echo " ONLY IN BRANCH: $base"; n=$((n+1)); continue; fi + if [ ! -f "$bf" ]; then echo " ONLY IN MASTER: $base"; n=$((n+1)); continue; fi + if ! diff -q <(cn1_canon_labels "$mf") <(cn1_canon_labels "$bf") >/dev/null; then + [ $n -lt 12 ] && echo " differs: $base" + n=$((n+1)) + fi + done < "$W/all.list" + echo "VS-MASTER: $n generated file(s) differ beyond label naming" + if [ "$n" != 0 ]; then + echo "Each one is a codegen change against master. Confirm every one is intended." + # Exit NONZERO. Printing a finding and returning 0 makes every caller read a + # real mismatch as a passing check, which is the failure mode this whole + # script exists to prevent. + exit 1 + fi + ;; +capture) + TAG="${2:?}" + TR="$REPO/vm/ByteCodeTranslator/target/classes" + ASM="$(cat "$REPO/vm/ByteCodeTranslator/target/selfhost-asm-classpath.txt")" + JAPI="$REPO/vm/selfhost/target/javaapi-classes" + # Same staleness guard verify-selfhost.sh carries: a source newer than its class + # would capture the OLD translator under the NEW tag and report a real change as + # neutral -- the exact failure this script exists to prevent. + newest="$(find "$REPO/vm/ByteCodeTranslator/src" -name '*.java' -newer "$TR" -print -quit 2>/dev/null || true)" + [ -z "$newest" ] || { echo "STALE: $TR older than $newest -- run mvn package first" >&2; exit 1; } + rm -rf "$W/$TAG-tree" "$OUT"; mkdir -p "$OUT" + ( cd "$W" && env -i PATH=/usr/bin:/bin HOME="$HOME" TMPDIR=/tmp LC_ALL=C \ + CN1_NATIVE_VERIFY="${CN1_NATIVE_VERIFY:-}" \ + CN1_RESOURCE_PATH="$REPO/vm/ByteCodeTranslator/src" \ + "$J8/bin/java" -cp "$TR:$ASM" com.codename1.tools.translator.ByteCodeTranslator \ + clean "$JAPI;$REPO/vm/selfhost/target/asm-classes;$REPO/vm/selfhost/target/classes" \ + "$OUT" com_codename1_tools_translator_ByteCodeTranslator \ + com.codename1.tools.translator com_codename1_tools_translator_ByteCodeTranslator \ + 1.0 clean none ) > "$W/$TAG.log" 2>&1 \ + || { echo "capture $TAG FAILED"; tail -20 "$W/$TAG.log"; exit 1; } + mv "$OUT" "$W/$TAG-tree" + n=$(find "$W/$TAG-tree" -type f | wc -l | tr -d ' ') + [ "$n" -gt 10 ] || { echo "VACUOUS: only $n files"; exit 1; } + echo "captured $TAG: $n files" + ;; +compare) + A="$W/${2:?}-tree"; B="$W/${3:?}-tree" + for d in "$A" "$B"; do [ -d "$d" ] || { echo "no $d"; exit 1; }; done + na=$(find "$A" -type f | wc -l | tr -d ' ') + if diff -rq "$A" "$B" > "$W/neutral.txt" 2>&1; then + echo "OUTPUT-NEUTRAL: PASS -- $na files byte-identical" + else + echo "OUTPUT-NEUTRAL: FAIL -- $(grep -c . "$W/neutral.txt") differing paths" + head -20 "$W/neutral.txt"; exit 1 + fi + ;; +*) echo "usage: capture | compare " >&2; exit 1 ;; +esac diff --git a/vm/selfhost/verify-selfhost.sh b/vm/selfhost/verify-selfhost.sh new file mode 100755 index 00000000000..28fb280724d --- /dev/null +++ b/vm/selfhost/verify-selfhost.sh @@ -0,0 +1,115 @@ +#!/bin/bash +# Validation gates for the self-hosted translator. +# +# verify-selfhost.sh +# +# Compares the C emitted by the JVM-hosted translator against the C emitted by the +# native one. The comparison is on the emitted SOURCE, never on the compiled binary: +# clang is not what is under test, and gating on object code would fail for toolchain +# reasons that have nothing to do with the VM. +# +# Gate D runs first and is the cheap one: the native translator against itself. If it +# is not self-consistent, nothing downstream means anything, and the cause is VM +# nondeterminism rather than a difference between the two runtimes. +# +# Gate A is the headline: same program, different runtime, identical output. +# +# Both sides run into the SAME absolute output path, sequentially, with the tree +# moved aside between runs. The generated CMakeLists embeds +# srcRoot.getAbsolutePath(), so running in one place removes a whole class of false +# differences rather than normalizing it away afterwards. Both also run under a +# constructed environment: the translator reads its knobs from getenv (see +# Util.getProperty), so a stray CN1_* variable would change one side's output. +set -e +cd "$(dirname "$0")" +REPO="$(cd ../.. && pwd)" +J8="${JDK_8_HOME:?set JDK_8_HOME to a working JDK 8}" +CLASSES="${1:?usage: verify-selfhost.sh }" +APP="${2:?}" +PKG="${3:?}" + +PARPAR="$REPO/vm/selfhost/target/parpar" +[ -x "$PARPAR" ] || { echo "no $PARPAR -- run build-selfhost.sh first"; exit 1; } +JAPI="$REPO/vm/selfhost/target/javaapi-classes" +TR="$REPO/vm/ByteCodeTranslator/target/classes" +ASM="$(cat "$REPO/vm/ByteCodeTranslator/target/selfhost-asm-classpath.txt")" + +# The JVM side of gate A runs target/classes, which nothing in this script builds +# -- build-selfhost.sh compiles the translator only for the NATIVE side. A source +# edit that has not been through `mvn package` therefore makes gate A compare the +# new translator against the old one, and it reports the intended change as a VM +# divergence. That has happened; the diff pointed at java_util_ArrayDeque.c and +# looked exactly like a real one. Maven's own incremental check does not save us +# here either -- it answered "Nothing to compile - all classes are up to date" +# for a source three hours newer than its class, so this compares the trees +# directly rather than trusting it. +newest_src="$(find "$REPO/vm/ByteCodeTranslator/src" -name '*.java' -newer "$TR" -print -quit 2>/dev/null || true)" +if [ -n "$newest_src" ]; then + echo "STALE: $TR is older than $newest_src" >&2 + echo "gate A would compare the new translator against the old one. Run:" >&2 + echo " (cd $REPO/vm && mvn -q -B -pl ByteCodeTranslator clean package -DskipTests)" >&2 + echo "and restore target/selfhost-asm-classpath.txt, which clean removes." >&2 + exit 1 +fi + +W="$REPO/vm/selfhost/target/verify" +rm -rf "$W"; mkdir -p "$W" +OUT="$W/out" + +# CN1_NATIVE_VERIFY is forwarded explicitly. `env -i` starts from an EMPTY +# environment, so a workflow-level `CN1_NATIVE_VERIFY: strict` never reached the +# translator here and NativeSignatureVerifier.mode() defaulted to OFF -- the gate +# reported a mode it was not running in, which is the failure this whole script +# exists to prevent. Forwarded rather than hard-coded so a local run without it set +# behaves as it always did. +run() { + local tag=$1; shift + mkdir -p "$OUT" + ( cd "$W" && env -i PATH=/usr/bin:/bin HOME="$HOME" TMPDIR=/tmp LC_ALL=C \ + CN1_NATIVE_VERIFY="${CN1_NATIVE_VERIFY:-}" \ + CN1_RESOURCE_PATH="$REPO/vm/ByteCodeTranslator/src" "$@" ) > "$W/$tag.log" 2>&1 \ + || { echo "$tag FAILED"; tail -20 "$W/$tag.log"; exit 1; } + mv "$OUT" "$W/$tag-tree" +} + +jvm_args=( "$J8/bin/java" -cp "$TR:$ASM" com.codename1.tools.translator.ByteCodeTranslator ) +common=( clean "$JAPI;$CLASSES" "$OUT" "$APP" "$PKG" "$APP" 1.0 clean none ) + +run parpar1 "$PARPAR" "${common[@]}" +run parpar2 "$PARPAR" "${common[@]}" +run jvm "${jvm_args[@]}" "${common[@]}" + +files=$(find "$W/jvm-tree" -type f | wc -l | tr -d ' ') +bytes=$(find "$W/jvm-tree" -type f -exec cat {} + | wc -c | tr -d ' ') +# A comparison of two empty trees is not a passing comparison. +[ "$files" -gt 10 ] || { echo "VACUOUS: only $files files emitted"; exit 1; } +echo "corpus: $APP -- $files files, $bytes bytes" + +fail=0 +if diff -rq "$W/parpar1-tree" "$W/parpar2-tree" > "$W/gateD.txt" 2>&1; then + echo "GATE D (parpar vs parpar): PASS" +else + echo "GATE D (parpar vs parpar): FAIL -- $(grep -c . "$W/gateD.txt") paths"; fail=1 +fi +if diff -rq "$W/jvm-tree" "$W/parpar1-tree" > "$W/gateA.txt" 2>&1; then + echo "GATE A (jvm vs parpar): PASS -- $files files byte-identical" +else + echo "GATE A (jvm vs parpar): FAIL -- $(grep -c . "$W/gateA.txt") of $files paths differ" + sed 's|.*/'"$APP"'-src/||;s| and .*||' "$W/gateA.txt" | head -20 + fail=1 +fi + +# Negative control: a comparator nobody has watched fail is not a comparator. Flip one +# byte and require the comparison to notice, so a pass above cannot be a pass by +# accident (a mis-set path, an empty tree, a diff invocation that never ran). +victim=$(find "$W/parpar1-tree" -name '*.c' | sort | head -1) +cp "$victim" "$W/victim.bak" +printf 'x' | dd of="$victim" bs=1 seek=40 conv=notrunc status=none +if diff -rq "$W/jvm-tree" "$W/parpar1-tree" > /dev/null 2>&1; then + echo "NEGATIVE CONTROL: FAIL -- a corrupted tree still compared equal"; fail=1 +else + echo "NEGATIVE CONTROL: PASS -- corruption detected" +fi +cp "$W/victim.bak" "$victim" + +exit $fail diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/BytecodeInstructionIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/BytecodeInstructionIntegrationTest.java index e3031959ee8..d770ca2763e 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/BytecodeInstructionIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/BytecodeInstructionIntegrationTest.java @@ -972,15 +972,19 @@ void handleDefaultOutputWritesOutput(CompilerHelper.CompilerConfig config) throw } @Test - void readFileAsStringBuilderReadsContent() throws Exception { + void readFileAsStringReadsContent() throws Exception { File temp = File.createTempFile("readfile", ".txt"); Files.write(temp.toPath(), "Hello World".getBytes(StandardCharsets.UTF_8)); - Method m = ByteCodeTranslator.class.getDeclaredMethod("readFileAsStringBuilder", File.class); + // readFileAsStringBuilder until the translator had to compile against + // ParparVM's own JavaAPI in order to translate itself: StringBuilder there + // has no indexOf/replace, so replaceInFile works on a String instead and + // this helper returns one. + Method m = ByteCodeTranslator.class.getDeclaredMethod("readFileAsString", File.class); m.setAccessible(true); - StringBuilder sb = (StringBuilder) m.invoke(null, temp); + String contents = (String) m.invoke(null, temp); - assertEquals("Hello World", sb.toString()); + assertEquals("Hello World", contents); temp.delete(); } diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java index 6b6d8ba29f5..f2f50047645 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java @@ -287,6 +287,16 @@ static Path buildHelloCodenameOneElf(String launcherSource) throws Exception { if (Boolean.parseBoolean(System.getenv("CN1_LINUX_FULL_DEBUG"))) { configure.add("-DCN1_DEBUG_INFO_LEVEL=3"); } + // Diagnostic defines, e.g. CN1_GC_VERIFY for the collector's heap-integrity + // checker. Unset it and the build is exactly what it was. + String extraDefines = System.getenv("CN1_LINUX_EXTRA_DEFINES"); + if (extraDefines != null && !extraDefines.trim().isEmpty()) { + configure.add("-DCN1_EXTRA_DEFINES=" + extraDefines.trim()); + } + // Printed so a diagnostic build proves itself from the job log. A define + // that silently fails to reach the compiler leaves a clean-looking run that + // measured nothing, which is worse than no diagnostic at all. + System.out.println("CN1SS:HARNESS: cmake configure: " + String.join(" ", configure)); CleanTargetIntegrationTest.runCommand(configure, cmakeRoot); CleanTargetIntegrationTest.runCommand(Arrays.asList("cmake", "--build", buildDir.toString()), cmakeRoot); Path elf = buildDir.resolve("LinuxHelloMain"); diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/CollectionSemanticsIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/CollectionSemanticsIntegrationTest.java new file mode 100644 index 00000000000..b69e0314c4d --- /dev/null +++ b/vm/tests/src/test/java/com/codename1/tools/translator/CollectionSemanticsIntegrationTest.java @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import org.junit.jupiter.api.Test; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Pins ArrayList and IdentityHashMap against a real JDK after both were changed to + * stop allocating. + * + *

ArrayList no longer allocates a backing array in its no-arg constructor -- it + * shares a zero-length one until the first growth, which then allocates exactly ten + * so a small list stays in the size class it always occupied. IdentityHashMap's key + * and value iterators no longer build an Entry per step; only entrySet does, which is + * the only view where a caller can observe one. java.util.HashMap already had that + * split and this map had been missed.

+ * + *

Both changes are invisible when they work and produce wrong answers at the + * edges when they do not -- an empty list that reports the wrong size, a null key + * that reads back as the table's sentinel -- so the JDK is used as the oracle rather + * than a hand-written expectation.

+ */ +class CollectionSemanticsIntegrationTest { + + @Test + void collectionSemanticsMatchTheJvm() throws Exception { + Parser.cleanup(); + + Path sourceDir = Files.createTempDirectory("collection-semantics-sources"); + Path classesDir = Files.createTempDirectory("collection-semantics-classes"); + Path javaApiDir = Files.createTempDirectory("collection-semantics-java-api"); + + Path source = sourceDir.resolve("CollectionSemanticsApp.java"); + Files.write(source, loadAppSource().getBytes(StandardCharsets.UTF_8)); + + CompilerHelper.CompilerConfig config = selectCompiler(); + if (config == null) { + fail("No compatible compiler available for the collection semantics integration test"); + } + assertTrue(CompilerHelper.isJavaApiCompatible(config), + "JDK " + config.jdkVersion + " must target matching bytecode level for JavaAPI"); + + CompilerHelper.compileJavaAPI(javaApiDir, config); + + List compileArgs = new ArrayList<>(); + compileArgs.add("-source"); + compileArgs.add(config.targetVersion); + compileArgs.add("-target"); + compileArgs.add(config.targetVersion); + if (CompilerHelper.useClasspath(config)) { + compileArgs.add("-classpath"); + compileArgs.add(javaApiDir.toString()); + } else { + compileArgs.add("-bootclasspath"); + compileArgs.add(javaApiDir.toString()); + compileArgs.add("-Xlint:-options"); + } + compileArgs.add("-d"); + compileArgs.add(classesDir.toString()); + compileArgs.add(source.toString()); + + assertEquals(0, CompilerHelper.compile(config.jdkHome, compileArgs), + "CollectionSemanticsApp should compile against the JavaAPI"); + + Map expected = parseCases(runJavaMain(config, classesDir, javaApiDir)); + assertFalse(expected.isEmpty(), "JVM run should emit cases"); + + CompilerHelper.copyDirectory(javaApiDir, classesDir); + + Path outputDir = Files.createTempDirectory("collection-semantics-output"); + CleanTargetIntegrationTest.runTranslator(classesDir, outputDir, "CollectionSemanticsApp"); + + Path distDir = outputDir.resolve("dist"); + Path cmakeLists = distDir.resolve("CMakeLists.txt"); + assertTrue(Files.exists(cmakeLists), "Translator should emit a CMake project"); + CleanTargetIntegrationTest.replaceLibraryWithExecutableTarget(cmakeLists, "CollectionSemanticsApp-src"); + + Path buildDir = distDir.resolve("build"); + Files.createDirectories(buildDir); + CleanTargetIntegrationTest.runCommand(Arrays.asList( + "cmake", + "-S", distDir.toString(), + "-B", buildDir.toString(), + "-DCMAKE_C_COMPILER=clang", + "-DCMAKE_OBJC_COMPILER=clang" + ), distDir); + CleanTargetIntegrationTest.runCommand(Arrays.asList("cmake", "--build", buildDir.toString()), distDir); + + Path executable = buildDir.resolve("CollectionSemanticsApp"); + String parparOutput = CleanTargetIntegrationTest.runCommand( + Arrays.asList(executable.toString()), buildDir); + assertTrue(parparOutput.contains("DONE"), + "ParparVM run should complete. Output: " + parparOutput); + + Map actual = parseCases(parparOutput); + assertEquals(expected.keySet(), actual.keySet(), "ParparVM should emit the same cases"); + + List differences = new ArrayList<>(); + for (Map.Entry entry : expected.entrySet()) { + if (!entry.getValue().equals(actual.get(entry.getKey()))) { + differences.add(entry.getKey() + + "\n jvm : " + entry.getValue() + + "\n parparvm: " + actual.get(entry.getKey())); + } + } + assertTrue(differences.isEmpty(), + "Collection semantics diverged from the JVM:\n" + String.join("\n", differences)); + + // Named explicitly so a regression points at the change rather than at a + // generic diff. + assertEquals("0", actual.get("empty.size"), "a list never added to must be empty"); + assertEquals("IndexOutOfBounds", actual.get("empty.get0"), + "get(0) on an empty list must still throw"); + assertEquals("1", actual.get("ihm.keyNulls"), + "the key iterator must hand back the null key as null, not the table's sentinel"); + assertEquals("1", actual.get("ihm.valNulls"), + "the value iterator must hand back a null value as null"); + assertEquals("500", actual.get("ihm.bigSeen"), + "key iteration must survive a rehash"); + } + + private Map parseCases(String output) { + Map cases = new LinkedHashMap<>(); + for (String line : output.split("\\R")) { + if (!line.startsWith("CASE|")) { + continue; + } + String body = line.substring("CASE|".length()); + int separator = body.indexOf('|'); + assertTrue(separator > 0, "Malformed case line: " + line); + cases.put(body.substring(0, separator), body.substring(separator + 1)); + } + return cases; + } + + private String loadAppSource() throws Exception { + java.io.InputStream in = CollectionSemanticsIntegrationTest.class + .getResourceAsStream("/com/codename1/tools/translator/CollectionSemanticsApp.java"); + assertNotNull(in, "CollectionSemanticsApp.java test resource should exist"); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) { + return reader.lines().collect(Collectors.joining("\n")) + "\n"; + } + } + + private String runJavaMain(CompilerHelper.CompilerConfig config, Path classesDir, Path javaApiDir) + throws Exception { + String javaExe = config.jdkHome.resolve("bin").resolve(CompilerHelper.executableName("java")).toString(); + ProcessBuilder pb = new ProcessBuilder( + javaExe, + "-cp", + classesDir + System.getProperty("path.separator") + javaApiDir, + "CollectionSemanticsApp" + ); + pb.redirectErrorStream(true); + + Process process = pb.start(); + String output; + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8))) { + output = reader.lines().collect(Collectors.joining("\n")); + } + assertEquals(0, process.waitFor(), "JVM run should exit cleanly. Output: " + output); + return output; + } + + private CompilerHelper.CompilerConfig selectCompiler() { + String[] preferredTargets = {"11", "17", "21", "25", "1.8"}; + for (String target : preferredTargets) { + List configs = CompilerHelper.getAvailableCompilers(target); + for (CompilerHelper.CompilerConfig config : configs) { + if (CompilerHelper.isJavaApiCompatible(config)) { + return config; + } + } + } + return null; + } +} diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/GcHeapIntegrityIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/GcHeapIntegrityIntegrationTest.java index 614023a7b0c..4f7826b7f68 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/GcHeapIntegrityIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/GcHeapIntegrityIntegrationTest.java @@ -169,6 +169,26 @@ private void runGate(List tempDirs) throws Exception { assertTrue(!clean.output.contains("DANGLING REFERENCE"), "The sweep left a surviving object pointing at reclaimed memory.\n" + violationExcerpt(clean.output)); + // A recycled slot is NOT dangling: it holds a live, valid object, just not + // the one the field pointed at. That is why the dangling check above passed + // through the failure a Linux core caught -- ArrayList.add running on a + // charts.compat.Canvas. cn1GcVerifyFieldType asks the other question, whether + // what a field HOLDS is assignable to what it was DECLARED as. + assertTrue(!clean.output.contains("TYPE CONFUSION"), + "A reference field holds an object of an unrelated type -- a live " + + "object was reclaimed and its slot recycled.\n" + + violationExcerpt(clean.output)); + // And prove that detector RAN. Inverting its condition on the first version + // produced no output whatever, which is how it was found to be checking + // nothing; silence from a detector that never executes is indistinguishable + // from silence from a clean heap. + java.util.regex.Matcher ft = java.util.regex.Pattern + .compile("FIELDTYPE checks=(\\d+) findings=(\\d+)").matcher(clean.output); + assertTrue(ft.find(), + "the field-type verifier never reported, so it did not run: " + clean.output); + assertTrue(Long.parseLong(ft.group(1)) > 0, + "the field-type verifier ran but checked no field, which is not a pass: " + + ft.group(0)); assertTrue(clean.output.contains("GC_VERIFY_APP_DONE"), "The workload should run to completion. Output: " + clean.output); // A workload that never finishes a collection cycle never runs the diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/GcMarkCompletenessTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/GcMarkCompletenessTest.java new file mode 100644 index 00000000000..62a89c00af8 --- /dev/null +++ b/vm/tests/src/test/java/com/codename1/tools/translator/GcMarkCompletenessTest.java @@ -0,0 +1,304 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; + +/** + * Every non-static object field a class declares must be traced by that class's + * {@code __GC_MARK_} function. + * + * A field the collector cannot see is a live object it will reclaim, and the + * failure is neither an exception nor a null dereference: the slot is recycled, + * some later object moves in, and the next method called through the stale + * reference reads ITS OWN field layout out of an unrelated object. That is + * indistinguishable from the unchecked-CHECKCAST hazard and just as invisible -- + * a SIGSEGV a long way from the cause, with no Java frame that could catch it. + * + * The shape is not hypothetical. A Linux suite core showed + * java_util_ArrayList_add running on an object whose class word said + * com_codename1_charts_compat_Canvas: the list's backing-array slot held two of + * Canvas's int fields, so the length load faulted. The list was + * Display.pendingIdleSerialCalls, reachable from a static root through a + * private final instance field, and the Canvas in its place was itself live + * (mark epoch 18, not the -1 that means fresh) -- a recycled slot, not garbage. + * + * What this checks is the ONE structural property that makes such a reclaim + * possible from the translator's side: ByteCodeClass emits a mark body from + * `fullFieldList`, filtered to non-static object fields DECLARED by the class + * (inherited ones are the base class's mark function's job). Anything that + * makes a field fall out of that filter -- a descriptor the parser types + * wrongly, a new field kind, a refactor of isObjectType -- silently stops the + * field being traced. Reading the emitted C is the only place that assumption + * is observable. + */ +class GcMarkCompletenessTest { + + /** struct obj__X { ... } -- the layout the mark function has to cover. */ + private static final Pattern STRUCT = + Pattern.compile("struct obj__(\\w+)\\s*\\{(.*?)\\n\\};", Pattern.DOTALL); + /** void __GC_MARK_X(...) { ... } */ + private static final Pattern MARKFN = + Pattern.compile("void __GC_MARK_(\\w+)\\(CODENAME_ONE_THREAD_STATE[^)]*\\)\\s*\\{(.*?)\\n\\}", + Pattern.DOTALL); + /** A JAVA_OBJECT member, i.e. exactly what the collector must follow. */ + private static final Pattern OBJ_FIELD = + Pattern.compile("^\\s*JAVA_OBJECT\\s+(\\w+)\\s*;", Pattern.MULTILINE); + + @Test + void everyDeclaredObjectFieldIsTracedByItsMarkFunction() throws Exception { + Path classes = Files.createTempDirectory("gcmark-classes"); + Path out = Files.createTempDirectory("gcmark-out"); + Path src = Files.createTempDirectory("gcmark-src"); + + // Deliberately covers the shapes that have gone wrong or could: a field + // declared on a BASE class and inherited, a collection field like the one + // the core implicated, an array field, an interface-typed field, and a + // class whose object fields sit among primitives so an offset mistake is + // visible. + Path app = src.resolve("GcMarkApp.java"); + Files.write(app, ("import java.util.*;\n" + + "class MarkBase { Object baseRef; int basePrim; }\n" + + "class MarkMid extends MarkBase { String midRef; }\n" + + "class MarkLeaf extends MarkMid {\n" + + " final ArrayList pending = new ArrayList();\n" + + " int a; Object mixedOne; long b; String[] arrayRef; int c;\n" + + " Runnable iface; Map mapRef;\n" + + "}\n" + + "public class GcMarkApp {\n" + + " static MarkLeaf keep;\n" + + " public static void main(String[] args) {\n" + + " keep = new MarkLeaf();\n" + + " keep.pending.add(new Runnable(){ public void run(){} });\n" + + " keep.mixedOne = new Object();\n" + + " keep.arrayRef = new String[2];\n" + + " keep.iface = new Runnable(){ public void run(){} };\n" + + " keep.mapRef = new HashMap();\n" + + " keep.baseRef = new Object();\n" + + " keep.midRef = \"x\";\n" + + " System.out.println(keep.pending.size());\n" + + " }\n" + + "}\n").getBytes(StandardCharsets.UTF_8)); + + CompilerHelper.CompilerConfig config = selectCompiler(); + org.junit.jupiter.api.Assumptions.assumeTrue(config != null, + "no compiler available that targets a JavaAPI-compatible bytecode level"); + + Path javaApi = Files.createTempDirectory("gcmark-java-api"); + CompilerHelper.compileJavaAPI(javaApi, config); + + List args = new ArrayList(); + args.add("-source"); args.add(config.targetVersion); + args.add("-target"); args.add(config.targetVersion); + if (CompilerHelper.useClasspath(config)) { + args.add("-classpath"); args.add(javaApi.toString()); + } else { + args.add("-bootclasspath"); args.add(javaApi.toString()); + args.add("-Xlint:-options"); + } + args.add("-nowarn"); + args.add("-d"); args.add(classes.toString()); + args.add(app.toString()); + assertTrue(CompilerHelper.compile(config.jdkHome, args) == 0, + "the fixture must compile against JavaAPI"); + + // The translator needs the class library beside the app, as every other + // integration test here stages it. + CompilerHelper.copyDirectory(javaApi, classes); + CleanTargetIntegrationTest.runTranslator(classes, out, "GcMarkApp"); + Path srcRoot = findSrcRoot(out); + + List missing = new ArrayList(); + int classesChecked = 0; + int fieldsChecked = 0; + + try (Stream files = Files.walk(srcRoot)) { + for (Path c : (Iterable) files.filter(p -> p.toString().endsWith(".c"))::iterator) { + String body = new String(Files.readAllBytes(c), StandardCharsets.ISO_8859_1); + Path header = c.resolveSibling(c.getFileName().toString().replaceAll("\\.c$", ".h")); + if (!Files.exists(header)) { + continue; + } + String head = new String(Files.readAllBytes(header), StandardCharsets.ISO_8859_1); + + Matcher mf = MARKFN.matcher(body); + while (mf.find()) { + String cls = mf.group(1); + String markBody = mf.group(2); + // struct obj__X FLATTENS the inherited fields, but the mark + // function deliberately marks only what the class DECLARES and + // chains to its base for the rest -- so requiring every struct + // member here would demand that Error re-mark Throwable's fields. + // The mangled name carries its declaring class, which is the same + // filter ByteCodeClass applies (fld.getClsName().equals(clsName)). + Set declared = new LinkedHashSet(); + for (String f : declaredObjectFields(head, cls)) { + if (f.startsWith(cls + "_")) { + declared.add(f); + } + } + if (declared.isEmpty()) { + continue; + } + classesChecked++; + for (String f : declared) { + fieldsChecked++; + // The emitted body names the field directly, whether it goes + // through gcMarkObject, gcMarkArrayObject or + // cn1GcDiscoverReference (the WeakReference referent, which is + // deliberately not traced but IS handed to the collector). + if (!markBody.contains(f)) { + missing.add(cls + "." + f); + } + } + } + } + } + + // The filter above is only sound because a class DELEGATES to its base, so + // check the delegation actually exists wherever the base declares object + // fields. Without this, "declared by me" and "marked by me" could both be + // empty for a whole hierarchy and the test would still pass. + List brokenChain = new ArrayList(); + try (Stream files2 = Files.walk(srcRoot)) { + for (Path c : (Iterable) files2.filter(p -> p.toString().endsWith(".c"))::iterator) { + String body = new String(Files.readAllBytes(c), StandardCharsets.ISO_8859_1); + Matcher mf = MARKFN.matcher(body); + while (mf.find()) { + String cls = mf.group(1); + String markBody = mf.group(2); + String base = baseOf(body, cls); + if (base != null && !base.equals("java_lang_Object") + && !markBody.contains("__GC_MARK_" + base)) { + brokenChain.add(cls + " -> " + base); + } + } + } + } + assertTrue(brokenChain.isEmpty(), + "__GC_MARK_ must chain to the base class, or the base's declared fields " + + "are traced by nobody: " + brokenChain); + + // A pass that inspected nothing is not a pass. The fixture alone declares + // eight object fields across three classes in one hierarchy. + assertTrue(classesChecked >= 3, + "expected to inspect several classes, saw " + classesChecked); + assertTrue(fieldsChecked >= 8, + "expected to inspect the fixture's object fields, saw " + fieldsChecked); + assertTrue(missing.isEmpty(), + "object field(s) declared but never traced by the class's __GC_MARK_ function -- " + + "the collector cannot see them, so it will reclaim live objects and " + + "recycle their slots: " + missing); + } + + /** + * Proves the check can fail, by deleting one field's mark from a body and + * confirming the comparison notices. A gate nobody has watched fail is not a + * gate, and this one is a string search over generated code -- exactly the kind + * that silently matches everything or nothing. + */ + @Test + void theCheckDetectsAnUntracedField() { + String head = "struct obj__Foo {\n JAVA_OBJECT Foo_kept;\n JAVA_OBJECT Foo_dropped;\n};"; + Set declared = declaredObjectFields(head, "Foo"); + assertTrue(declared.contains("Foo_kept") && declared.contains("Foo_dropped"), + "fixture parse: " + declared); + String markBody = " gcMarkObject(threadStateData, objInstance->Foo_kept, force);"; + List missing = new ArrayList(); + for (String f : declared) { + if (!markBody.contains(f)) { + missing.add(f); + } + } + assertFalse(missing.isEmpty(), "the check must notice a field that is not marked"); + assertTrue(missing.contains("Foo_dropped") && missing.size() == 1, + "it must name exactly the untraced field, got " + missing); + } + + /** The base class name from the emitted `struct clazz` initialiser, or null. */ + private static String baseOf(String body, String cls) { + Matcher m = Pattern.compile("struct clazz class__" + Pattern.quote(cls) + + "\\s*=\\s*\\{(.*?)\\};", Pattern.DOTALL).matcher(body); + if (!m.find()) { + return null; + } + Matcher b = Pattern.compile("&class__(\\w+)\\s*,\\s*(?:base_interfaces|EMPTY_INTERFACES)").matcher(m.group(1)); + return b.find() ? b.group(1) : null; + } + + private CompilerHelper.CompilerConfig selectCompiler() { + String[] preferredTargets = {"11", "17", "21", "25", "1.8"}; + for (String target : preferredTargets) { + for (CompilerHelper.CompilerConfig c : CompilerHelper.getAvailableCompilers(target)) { + if (CompilerHelper.isJavaApiCompatible(c)) { + return c; + } + } + } + return null; + } + + private static Set declaredObjectFields(String header, String cls) { + Set out = new LinkedHashSet(); + Matcher s = STRUCT.matcher(header); + while (s.find()) { + if (!s.group(1).equals(cls)) { + continue; + } + Matcher f = OBJ_FIELD.matcher(s.group(2)); + while (f.find()) { + String name = f.group(1); + // The object header's own slots are not Java fields. + if (name.startsWith("__codenameOne") || name.equals("__heapPosition")) { + continue; + } + out.add(name); + } + } + return out; + } + + private static Path findSrcRoot(Path out) throws IOException { + try (Stream w = Files.walk(out)) { + return w.filter(Files::isDirectory) + .filter(p -> p.getFileName().toString().endsWith("-src")) + .findFirst() + .orElseThrow(() -> new IOException("no generated -src directory under " + out)); + } + } +} diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/UtilStringHelperTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/UtilStringHelperTest.java new file mode 100644 index 00000000000..9e84582446a --- /dev/null +++ b/vm/tests/src/test/java/com/codename1/tools/translator/UtilStringHelperTest.java @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Holds {@link Util}'s hand-written string helpers to the JDK regex behaviour they + * replaced. + * + * The translator has to compile against ParparVM's JavaAPI in order to translate + * itself, and String.split/replaceAll are not declared there -- they are among the + * methods BytecodeComplianceMojo rewrites onto com.codename1.util.regex precisely + * because JavaAPI lacks them. The call sites lost the regex rather than JavaAPI + * gaining a second engine, so the risk is that a replacement quietly disagrees and + * changes generated C. These tests compare against the originals directly, so the + * JDK is the oracle rather than a hand-written expectation. + */ +class UtilStringHelperTest { + + private static final String LOCALS_REGEX = "locals\\[(\\d+)\\]\\.data\\.o"; + private static final String LOCALS_REPLACEMENT = "olocals_$1_"; + + @Test + void rewriteLocalObjectRefsMatchesReplaceAll() { + for (String s : localsCases()) { + assertEquals(s.replaceAll(LOCALS_REGEX, LOCALS_REPLACEMENT), + Util.rewriteLocalObjectRefs(s), + "rewriteLocalObjectRefs diverged on: " + s); + } + } + + @Test + void collapseWhitespaceMatchesReplaceAll() { + for (String s : whitespaceCases()) { + assertEquals(s.replaceAll("\\s+", " "), Util.collapseWhitespace(s), + "collapseWhitespace diverged on: " + escape(s)); + } + } + + @Test + void splitWhitespaceMatchesSplit() { + for (String s : whitespaceCases()) { + assertArrayEquals(s.split("\\s+"), Util.splitWhitespace(s), + "splitWhitespace diverged on: " + escape(s)); + } + } + + @Test + void splitLiteralMatchesSplit() { + String[] cases = { + "", ";", ";;", "a", "a;b", "a;b;c", ";a", "a;", "a;;b", ";;a;;b;;", + "a;b;", "a;b;;", " a ; b ", "one" + }; + for (String s : cases) { + assertArrayEquals(s.split(";"), Util.splitLiteral(s, ';'), + "splitLiteral diverged on: " + escape(s)); + } + } + + /** + * The generated-code shapes plus the near misses: a bracket with no digits, a + * digit run that is not followed by ".data.o", and a nested occurrence. These are + * where a hand-written scanner and a regex are most likely to part company. + */ + private List localsCases() { + List cases = new ArrayList(); + for (String s : new String[]{ + "", + "locals[0].data.o", + "locals[12].data.o", + "locals[0].data.o + locals[1].data.o", + "f(locals[3].data.o, locals[44].data.o)", + "locals[].data.o", + "locals[x].data.o", + "locals[0].data.i", + "locals[0].data", + "locals[", + "locals[0", + "locals[0]", + "prefix locals[7].data.o suffix", + "locals[locals[1].data.o].data.o", + "no match here at all", + "LOCALS[0].DATA.O" + }) { + cases.add(s); + } + // Randomised fuzz over the alphabet the pattern cares about, so the oracle + // sees inputs nobody thought to enumerate. + Random r = new Random(20260909L); + char[] alphabet = {'l', 'o', 'c', 'a', 's', '[', ']', '.', 'd', 't', '0', '1', '9', ' ', 'x'}; + for (int i = 0; i < 3000; i++) { + StringBuilder b = new StringBuilder(); + int len = r.nextInt(24); + for (int j = 0; j < len; j++) { + b.append(alphabet[r.nextInt(alphabet.length)]); + } + if (r.nextBoolean()) { + b.append("locals[").append(r.nextInt(200)).append("].data.o"); + } + cases.add(b.toString()); + } + return cases; + } + + private List whitespaceCases() { + List cases = new ArrayList(); + // 0x0B is the vertical tab: Java's \s includes it and Character.isWhitespace + // does not, which is the difference most likely to be got wrong. + String vt = String.valueOf((char) 0x0B); + for (String s : new String[]{ + "", " ", " ", "a", "a b", "a b", " a b ", "\ta\tb\t", "a\nb", + "a" + vt + "b", "a\fb", "a\r\nb", "JAVA_OBJECT me", " leading", "trailing ", + " both ", "a \t\n b" + }) { + cases.add(s); + } + Random r = new Random(20260910L); + char[] alphabet = {' ', '\t', '\n', 0x0B, '\f', '\r', 'a', 'b', '*'}; + for (int i = 0; i < 3000; i++) { + StringBuilder b = new StringBuilder(); + int len = r.nextInt(16); + for (int j = 0; j < len; j++) { + b.append(alphabet[r.nextInt(alphabet.length)]); + } + cases.add(b.toString()); + } + return cases; + } + + private String escape(String s) { + StringBuilder b = new StringBuilder(); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c < 0x20) { + b.append("\\x").append(Integer.toHexString(c)); + } else { + b.append(c); + } + } + return b.toString(); + } +} diff --git a/vm/tests/src/test/resources/com/codename1/tools/translator/CollectionSemanticsApp.java b/vm/tests/src/test/resources/com/codename1/tools/translator/CollectionSemanticsApp.java new file mode 100644 index 00000000000..667062c9344 --- /dev/null +++ b/vm/tests/src/test/resources/com/codename1/tools/translator/CollectionSemanticsApp.java @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +import java.util.ArrayList; +import java.util.IdentityHashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Exercises the parts of ArrayList and IdentityHashMap that were changed to stop + * allocating: ArrayList no longer allocates a backing array until the first growth, + * and IdentityHashMap's key and value iterators no longer build an Entry per step. + * + * Every line is compared against a real JDK run, so the JDK is the oracle rather + * than a hand-written expectation. + */ +public class CollectionSemanticsApp { + static void emit(String k, Object v) { + System.out.println("CASE|" + k + "|" + v); + } + + public static void main(String[] args) { + // ---- an ArrayList that is never added to ------------------------------- + List empty = new ArrayList(); + emit("empty.size", empty.size()); + emit("empty.isEmpty", empty.isEmpty()); + emit("empty.contains", empty.contains("x")); + emit("empty.indexOf", empty.indexOf("x")); + emit("empty.iterHasNext", empty.iterator().hasNext()); + emit("empty.toArrayLen", empty.toArray().length); + emit("empty.toString", empty.toString()); + empty.clear(); + emit("empty.afterClear", empty.size()); + try { + empty.get(0); + emit("empty.get0", "no throw"); + } catch (IndexOutOfBoundsException err) { + emit("empty.get0", "IndexOutOfBounds"); + } + + // ---- first growth, and growth past it ---------------------------------- + List grow = new ArrayList(); + for (int i = 0; i < 40; i++) { + grow.add(Integer.valueOf(i)); + if (i < 3 || i == 9 || i == 10 || i == 11 || i == 12 || i == 39) { + emit("grow.size@" + i, grow.size() + ":" + grow.get(0) + ":" + grow.get(i)); + } + } + emit("grow.toString", grow.toString()); + emit("grow.indexOf37", grow.indexOf(Integer.valueOf(37))); + + // ---- add-at-front on a fresh list (the growAtFront path) --------------- + List front = new ArrayList(); + front.add(0, "b"); + front.add(0, "a"); + front.add("c"); + emit("front.toString", front.toString()); + emit("front.size", front.size()); + + // ---- insert into the middle of a fresh list (growForInsert) ------------ + List mid = new ArrayList(); + mid.add("x"); + mid.add("z"); + mid.add(1, "y"); + emit("mid.toString", mid.toString()); + + // ---- ensureCapacity on a fresh list ------------------------------------ + ArrayList ec = new ArrayList(); + ec.ensureCapacity(100); + ec.add("only"); + emit("ec.toString", ec.toString()); + + // ---- remove down to empty and re-add ----------------------------------- + List churn = new ArrayList(); + churn.add("p"); + churn.add("q"); + churn.remove("p"); + churn.remove(0); + emit("churn.emptyAgain", churn.size()); + churn.add("r"); + emit("churn.readd", churn.toString()); + + // ---- IdentityHashMap: identity semantics, and all three views ---------- + String k1 = new String("dup"); + String k2 = new String("dup"); + IdentityHashMap ihm = new IdentityHashMap(); + ihm.put(k1, "first"); + ihm.put(k2, "second"); + emit("ihm.size", ihm.size()); + emit("ihm.get1", ihm.get(k1)); + emit("ihm.get2", ihm.get(k2)); + emit("ihm.containsKey1", ihm.containsKey(k1)); + + // Null key and null value must survive the table's NULL_OBJECT sentinel in + // BOTH directions -- this is what the key/value iterators read directly now. + ihm.put(null, "nullkey"); + ihm.put("nullval", null); + emit("ihm.getNullKey", ihm.get(null)); + emit("ihm.getNullVal", String.valueOf(ihm.get("nullval"))); + emit("ihm.sizeWithNulls", ihm.size()); + + int keyNulls = 0, keyCount = 0; + for (Iterator it = ihm.keySet().iterator(); it.hasNext();) { + String k = it.next(); + keyCount++; + if (k == null) { + keyNulls++; + } + } + emit("ihm.keyCount", keyCount); + emit("ihm.keyNulls", keyNulls); + + int valNulls = 0, valCount = 0; + for (Iterator it = ihm.values().iterator(); it.hasNext();) { + String v = it.next(); + valCount++; + if (v == null) { + valNulls++; + } + } + emit("ihm.valCount", valCount); + emit("ihm.valNulls", valNulls); + + int entryCount = 0, entryKeyNulls = 0, entryValNulls = 0; + for (Map.Entry e : ihm.entrySet()) { + entryCount++; + if (e.getKey() == null) { + entryKeyNulls++; + } + if (e.getValue() == null) { + entryValNulls++; + } + } + emit("ihm.entryCount", entryCount); + emit("ihm.entryKeyNulls", entryKeyNulls); + emit("ihm.entryValNulls", entryValNulls); + + // keySet().contains and removal through the key view + Set keys = ihm.keySet(); + emit("ihm.keysContainsK1", keys.contains(k1)); + emit("ihm.keysRemoveK1", keys.remove(k1)); + emit("ihm.sizeAfterRemove", ihm.size()); + + // iterator removal + IdentityHashMap rem = new IdentityHashMap(); + String r1 = new String("r1"); + String r2 = new String("r2"); + rem.put(r1, "1"); + rem.put(r2, "2"); + for (Iterator it = rem.keySet().iterator(); it.hasNext();) { + if (it.next() == r1) { + it.remove(); + } + } + emit("ihm.afterIterRemove", rem.size() + ":" + rem.get(r2)); + + // a map big enough to force a rehash, iterated by key + IdentityHashMap big = new IdentityHashMap(); + Object[] held = new Object[500]; + for (int i = 0; i < held.length; i++) { + held[i] = new Object(); + big.put(held[i], Integer.valueOf(i)); + } + long sum = 0; + int seen = 0; + for (Object o : big.keySet()) { + sum += big.get(o).intValue(); + seen++; + } + emit("ihm.bigSeen", seen); + emit("ihm.bigSum", sum); + + System.out.println("DONE"); + } +}