Merge pull request #85 from thisisthepy/work/jar-libpython

Fix: Real libpython in the desktop jar, staged prefix and packaged home; extract into a cache

Author: b-re-w

docs/SPEC.md

@@ -89,6 +89,19 @@ fail, then implement.
   from the prefix a property names, or from a Compose resources directory). The
   `createDistributable` run itself is a manual check —
   `docs/platforms/desktop-packaged-app.md`.
+- **L-9a** (issue #74) Desktop packaging and loading of `libpython`: every `lib/**` entry of the desktop jar is a real
+  file (never 0 bytes -- Gradle extracts the python-build-standalone symlink `libpython3.14.so` as an empty file, so
+  symlinks are resolved and the real library is packed under the `System.mapLibraryName` name, no `.so.1.0`); linux
+  libraries are stripped of debug info when an ELF-capable `strip` exists on the build host (best-effort). The
+  classpath library is extracted to `<user cache>/python-multiplatform/<python version>/<platform>/` (macOS
+  `~/Library/Caches`, Windows `%LOCALAPPDATA%`, else `$XDG_CACHE_HOME` or `~/.cache`; fallback
+  `java.io.tmpdir/python-multiplatform-<user>`; override `-Dpython.multiplatform.cache`), never the working
+  directory, and reused when its size matches. The same holds for the plugin's staged prefix and the packaged
+  `python-multiplatform-home`: `stagePythonHome` extracts the archive itself and materialises every symlink/hard
+  link as a copy of its in-archive target (Gradle's `tarTree` yields 0-byte files), the staging stamp changed so
+  old prefixes are re-extracted, and the packaged copy fails the build on any 0-byte `libpython*`.
+  `Status: implemented` -- `PM/desktopTest/.../ffi/DesktopJarLibrariesTest.kt`, `ExtractLibraryCacheTest.kt`,
+  `GP/StagedPrefixLinksTest.kt`.
 - **L-10** Desktop: the library uses no JDK module beyond `java.base`, so it runs on a `jlink`ed
   runtime such as the one Compose Desktop's `createDistributable` bundles (issue #77). In particular
   native memory is reached through `java.lang.foreign`, never `sun.misc.Unsafe` (module

python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt

@@ -678,6 +678,9 @@ class PythonBindingsPlugin : Plugin<Project> {
                     exclude(PACKAGED_PREFIX_EXCLUDES)
                     into(PACKAGED_HOME_DIRECTORY)
                 }
+                doLast {
+                    requireNoEmptyLibraries(destinationDir.resolve(PACKAGED_HOME_DIRECTORY))
+                }
             }
     }
 

python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonHomeStaging.kt

@@ -202,7 +202,149 @@ internal fun stagingStamp(
     platform: String,
     freeThreaded: Boolean,
 ): String = "python-multiplatform $pythonVersion+$pbsRelease $platform " +
-    (if (freeThreaded) "freethreaded" else "default")
+    (if (freeThreaded) "freethreaded" else "default") +
+    // Bumped when the extraction itself changes. "links-materialised" (issue #74): Gradle's
+    // `tarTree` turned the archive's `libpython3.14.so -> libpython3.14.so.1.0` into a 0-byte file,
+    // so every prefix staged before this is wrong and must be extracted again.
+    " links-materialised"
+
+/**
+ * Extracts a python-build-standalone `.tar.gz` into [root], turning every symbolic and hard link
+ * into a copy of its in-archive target (issue #74).
+ *
+ * Gradle's `tarTree` extracts a symlink as an empty regular file, so `lib/libpython3.14.so` -- a
+ * link to `libpython3.14.so.1.0` -- came out 0 bytes and `System.load` failed with "file too
+ * short". A copy rather than a link also survives being copied into a jar or an app image, which
+ * flatten links. Dependency-free: `GZIPInputStream` plus a minimal ustar/GNU/pax header reader.
+ * A link whose target is not inside the archive, or that escapes [root], is skipped.
+ */
+internal fun extractTarGzMaterialisingLinks(archive: File, root: File) {
+    val rootPath = root.canonicalFile.toPath()
+    fun inside(rel: String): java.nio.file.Path? {
+        val p = rootPath.resolve(rel).normalize()
+        return if (p.startsWith(rootPath) && p != rootPath) p else null
+    }
+    class Link(val path: java.nio.file.Path, val target: String, val hard: Boolean)
+    val links = mutableListOf<Link>()
+
+    java.util.zip.GZIPInputStream(archive.inputStream().buffered(1 shl 16)).use { input ->
+        val header = ByteArray(512)
+        var longName: String? = null
+        var longLink: String? = null
+        var paxPath: String? = null
+        var paxLink: String? = null
+
+        fun readFully(buf: ByteArray, len: Int): Boolean {
+            var n = 0
+            while (n < len) { val r = input.read(buf, n, len - n); if (r < 0) return false; n += r }
+            return true
+        }
+        fun str(off: Int, len: Int): String {
+            var end = off
+            while (end < off + len && header[end] != 0.toByte()) end++
+            return String(header, off, end - off, Charsets.UTF_8)
+        }
+        fun octal(off: Int, len: Int): Long = str(off, len).trim().ifEmpty { "0" }.toLong(8)
+        fun readData(size: Long): ByteArray {
+            val padded = (size + 511) / 512 * 512
+            val buf = ByteArray(padded.toInt())
+            check(readFully(buf, buf.size)) { "truncated tar entry in $archive" }
+            return buf.copyOf(size.toInt())
+        }
+
+        while (true) {
+            if (!readFully(header, 512)) break
+            if (header.all { it == 0.toByte() }) break
+            var name = str(0, 100)
+            val prefix = str(345, 155)
+            if (prefix.isNotEmpty() && str(257, 5) == "ustar") name = "$prefix/$name"
+            val mode = octal(100, 8)
+            val size = octal(124, 12)
+            val type = header[156].toInt().toChar()
+            var linkName = str(157, 100)
+            when (type) {
+                'L' -> { longName = String(readData(size), Charsets.UTF_8).trimEnd('\u0000'); continue }
+                'K' -> { longLink = String(readData(size), Charsets.UTF_8).trimEnd('\u0000'); continue }
+                'x' -> {
+                    String(readData(size), Charsets.UTF_8).lineSequence().forEach { rec ->
+                        val kv = rec.substringAfter(' ', "")
+                        when {
+                            kv.startsWith("path=") -> paxPath = kv.removePrefix("path=")
+                            kv.startsWith("linkpath=") -> paxLink = kv.removePrefix("linkpath=")
+                        }
+                    }
+                    continue
+                }
+                'g' -> { readData(size); continue }
+            }
+            (paxPath ?: longName)?.let { name = it }
+            (paxLink ?: longLink)?.let { linkName = it }
+            longName = null; longLink = null; paxPath = null; paxLink = null
+
+            val dest = inside(name.trimEnd('/')) ?: run {
+                if (size > 0) readData(size)
+                null
+            } ?: continue
+            when (type) {
+                '5' -> java.nio.file.Files.createDirectories(dest)
+                '2' -> links += Link(dest, linkName, hard = false)
+                '1' -> links += Link(dest, linkName, hard = true)
+                '0', '\u0000', '7' -> {
+                    java.nio.file.Files.createDirectories(dest.parent)
+                    dest.toFile().outputStream().use { out ->
+                        var remaining = size
+                        val buf = ByteArray(1 shl 16)
+                        while (remaining > 0) {
+                            val r = input.read(buf, 0, minOf(buf.size.toLong(), remaining).toInt())
+                            check(r >= 0) { "truncated tar entry $name in $archive" }
+                            out.write(buf, 0, r); remaining -= r
+                        }
+                    }
+                    var pad = (512 - size % 512) % 512
+                    while (pad > 0) { val r = input.skip(pad); if (r <= 0) { input.read(); pad-- } else pad -= r }
+                    if (mode and 0b001_001_001L != 0L) dest.toFile().setExecutable(true, false)
+                }
+                else -> if (size > 0) readData(size)
+            }
+        }
+    }
+
+    // Links last, repeated until stable, so a link to a link (or to a later entry) resolves.
+    var pending: List<Link> = links
+    while (pending.isNotEmpty()) {
+        val unresolved = mutableListOf<Link>()
+        for (link in pending) {
+            val source = if (link.hard) inside(link.target)
+            else link.path.parent.resolve(link.target).normalize().takeIf { it.startsWith(rootPath) }
+            val file = source?.toFile()
+            if (file == null || !file.exists() || (file.isFile && pending.any { it.path == source })) {
+                unresolved += link; continue
+            }
+            if (file.isDirectory) file.copyRecursively(link.path.toFile(), overwrite = true)
+            else {
+                java.nio.file.Files.createDirectories(link.path.parent)
+                file.copyTo(link.path.toFile(), overwrite = true)
+                link.path.toFile().setExecutable(file.canExecute(), false)
+            }
+        }
+        if (unresolved.size == pending.size) break // remaining targets are outside the archive
+        pending = unresolved
+    }
+}
+
+/**
+ * Fails when a packaged prefix carries a zero-length shared library -- a symlink stub is never a
+ * library, and shipping one fails at the user's first launch with "file too short" (issue #74).
+ */
+internal fun requireNoEmptyLibraries(dir: File) {
+    if (!dir.isDirectory) return
+    val empty = dir.walkTopDown()
+        .filter { it.isFile && it.length() == 0L && it.name.startsWith("libpython") }
+        .map { it.relativeTo(dir).path }.toList()
+    if (empty.isNotEmpty()) {
+        throw GradleException("packaged CPython prefix contains zero-length libraries: $empty (symlink stubs?)")
+    }
+}
 
 /**
  * The directory a packaged desktop application carries its CPython prefix in, inside Compose
@@ -334,10 +476,8 @@ abstract class StagePythonHomeTask : DefaultTask() {
         }
 
         logger.lifecycle("Staging CPython $version ($target) into $prefix")
-        fs.copy {
-            from(archives.tarTree(archives.gzip(archive)))
-            into(root)
-        }
+        // Not `tarTree`: it extracts the archive's symlinks as empty files (issue #74).
+        extractTarGzMaterialisingLinks(archive, root)
 
         if (!marker.isFile) {
             throw GradleException(

python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/StagedPrefixLinksTest.kt

@@ -0,0 +1,89 @@
+package python.multiplatform.gradle
+
+import org.gradle.api.GradleException
+import java.io.ByteArrayOutputStream
+import java.io.File
+import java.nio.file.Files
+import java.util.zip.GZIPOutputStream
+import kotlin.test.Test
+import kotlin.test.assertContentEquals
+import kotlin.test.assertFailsWith
+import kotlin.test.assertTrue
+
+/**
+ * Issue #74. Gradle's `tarTree` extracts the archive's `lib/libpython3.14.so -> libpython3.14.so.1.0`
+ * as a 0-byte file, which the staged prefix, the packaged `python-multiplatform-home` and the
+ * desktop jar then all carried. Expected red before the fix: the extractor did not exist; with
+ * `tarTree` the staged link is empty.
+ */
+class StagedPrefixLinksTest {
+
+    private fun entry(out: ByteArrayOutputStream, name: String, type: Char, data: ByteArray = ByteArray(0), link: String = "") {
+        val h = ByteArray(512)
+        fun put(off: Int, s: String) = s.toByteArray().copyInto(h, off)
+        put(0, name)
+        put(100, "0000755\u0000")
+        put(124, String.format("%011o\u0000", data.size))
+        h[156] = type.code.toByte()
+        put(157, link)
+        put(257, "ustar\u000000")
+        for (i in 148 until 156) h[i] = ' '.code.toByte()
+        put(148, String.format("%06o\u0000 ", h.sumOf { it.toInt() and 0xff }))
+        out.write(h)
+        out.write(data)
+        out.write(ByteArray((512 - data.size % 512) % 512))
+    }
+
+    private fun fakeArchive(dir: File, payload: ByteArray): File {
+        val tar = ByteArrayOutputStream()
+        entry(tar, "python/", '5')
+        entry(tar, "python/lib/", '5')
+        entry(tar, "python/lib/libpython3.14.so.1.0", '0', payload)
+        entry(tar, "python/lib/libpython3.14.so", '2', link = "libpython3.14.so.1.0")
+        entry(tar, "python/lib/libpython3.so", '2', link = "libpython3.14.so") // link to a link
+        tar.write(ByteArray(1024))
+        val f = File(dir, "fake.tar.gz")
+        GZIPOutputStream(f.outputStream()).use { it.write(tar.toByteArray()) }
+        return f
+    }
+
+    @Test
+    fun symlinksBecomeCopiesOfTheirTargets() {
+        val work = Files.createTempDirectory(File("../.tmp").apply { mkdirs() }.toPath(), "links").toFile()
+        try {
+            val payload = ByteArray(5000) { (it % 251).toByte() }
+            val root = File(work, "out")
+            extractTarGzMaterialisingLinks(fakeArchive(work, payload), root)
+
+            val real = File(root, "python/lib/libpython3.14.so.1.0")
+            val link = File(root, "python/lib/libpython3.14.so")
+            val chained = File(root, "python/lib/libpython3.so")
+            assertContentEquals(payload, real.readBytes())
+            assertTrue(link.length() > 0, "staged symlink is empty")
+            assertContentEquals(real.readBytes(), link.readBytes())
+            assertContentEquals(real.readBytes(), chained.readBytes())
+            assertTrue(!Files.isSymbolicLink(link.toPath()))
+        } finally {
+            work.deleteRecursively()
+        }
+    }
+
+    @Test
+    fun theStampChangedSoOldPrefixesAreReExtracted() {
+        assertTrue("links-materialised" in stagingStamp("3.14.7", "20260807", "linux-x86_64", false))
+    }
+
+    @Test
+    fun aPackagedHomeWithAnEmptyLibraryFailsTheBuild() {
+        val work = Files.createTempDirectory(File("../.tmp").apply { mkdirs() }.toPath(), "packaged").toFile()
+        try {
+            val lib = File(work, "lib").apply { mkdirs() }
+            File(lib, "libpython3.14.so.1.0").writeBytes(ByteArray(10))
+            requireNoEmptyLibraries(work) // clean tree passes
+            File(lib, "libpython3.14.so").writeBytes(ByteArray(0))
+            assertFailsWith<GradleException> { requireNoEmptyLibraries(work) }
+        } finally {
+            work.deleteRecursively()
+        }
+    }
+}