Merge pull request #239 from thisisthepy/feat/klib-member-functions

Klib walker: bind member functions of classes and interfaces (#237)

Author: b-re-w

docs/SPEC.md

@@ -410,7 +410,19 @@ All in `PM/commonTest`, so they run wherever the interpreter loads.
   declined (#73). `Status: implemented`, `GP/artifact/TextStateBindingTest.kt`,
   `python-multiplatform-compose/fixtures/compose/.../TextFieldStateRenderTest.kt`.
 - **B-2** The walker on **klibs** (Kotlin/Native libraries). `Status: partial`. It reads
-  top-level functions only: no class constructor, property or member function yet (B-1, #215).
+  top-level functions and, since #237, the **member functions** of the klib's classes; no class
+  constructor or property yet. A member binds as #215's `METHOD` `Owner.name` (receiver in `args[0]`,
+  `(args[0] as Owner).name(...)`, served on the proxy, never a module attribute) when its class can
+  cross as a handle and is an interface, a final class or an enum class: the pinned compiler's klib
+  ABI records modality but no visibility, so a member of an open, abstract or sealed class, which
+  may be `protected`, is declined with that reason, and an `object`'s functions (statics on the jar
+  side, #53) are declined too. Also declined, with reasons: members of a generic class, `suspend`,
+  `@Deprecated`, `@PublishedApi`, `@Composable`, member extensions and context receivers, a member
+  named like a property, data-class `copy`/`componentN`, and anything without a klib boundary type;
+  `Any`'s members and an enum's static `valueOf`/`values`/`entries` are not bound. From
+  `kotlinx-coroutines-core` this binds `CompletableJob.complete`, `Job.start`,
+  `Semaphore.tryAcquire`/`release` and others; `GP/artifact/KlibMemberFunctionTest.kt`, and on the iOS
+  simulator `WalkedKlibArtifactMemberFunctionTest`.
   Since #234 a **Kotlin object crosses as a handle**: a type whose class is a public, non-generic,
   non-value `class`, `interface`, `enum class` or `object` of the walked klib is an `OBJECT` slot
   or result, non-null or nullable, cast in generated source as the jar producer casts one, and a

python-multiplatform-gradle-plugin/fixtures/klib-artifact/src/nativeTest/kotlin/fixture/artifact/WalkedKlibArtifactMemberFunctionTest.kt

@@ -0,0 +1,73 @@
+package fixture.artifact
+
+import python.multiplatform.ffi.Python3
+import python.multiplatform.ffi.pythonx.PythonxAdapter
+import python.multiplatform.ffi.upcall.PythonProxySource
+import python.multiplatform.generated.FunctionTable
+import python.multiplatform.generated.artifacts.ArtifactTable
+import python.multiplatform.reflection.UpcallTable
+import python.native.ffi.UpcallEntry
+import kotlin.test.AfterTest
+import kotlin.test.BeforeTest
+import kotlin.test.Test
+
+/**
+ * Issue #237 on Kotlin/Native: member functions of the walked klib's interfaces, called from Python on
+ * the handles its factories return, through the generated table and the binding layer -- #215's path,
+ * from a klib.
+ *
+ * - `Job(None)` is a `CompletableJob`: `complete()` is `True` the first time and `False` after, and
+ *   `start()`, declared on its supertype `Job`, is reached through the ancestry the table carries.
+ * - A `Semaphore` of two permits gives two and refuses a third until one is released.
+ *
+ * Red before #237: `complete` and `try_acquire` are `AttributeError`s, because no member was bound.
+ */
+class WalkedKlibArtifactMemberFunctionTest {
+
+    @BeforeTest
+    fun installBothLayers() {
+        Python3.initialize(silent = true)
+        UpcallTable.clear()
+        UpcallTable.install(FunctionTable.fragments + ArtifactTable.fragments)
+        val mainGlobals = Python3.import("__main__").getAttr("__dict__")
+        check(UpcallEntry.publish(mainGlobals.pointer)) { "the upcall bootstrap could not be published" }
+        PythonProxySource.install()
+        PythonxAdapter.install(emptySet())
+    }
+
+    @AfterTest
+    fun cleanup() {
+        UpcallTable.clear()
+    }
+
+    @Test
+    fun aJobsMembersRunOnTheKotlinJob() {
+        Python3.exec(
+            """
+            from kotlinx.coroutines import Job
+            _km_job = Job(None)
+            assert _km_job.complete() is True, 'complete() did not complete the job'
+            assert _km_job.complete() is False, 'a completed job completed twice'
+            _km_lazy = Job(None)
+            assert _km_lazy.start() is False, 'Job() is already active, so start() is False'
+            del _km_job, _km_lazy
+            """.trimIndent(),
+        )
+    }
+
+    @Test
+    fun aSemaphoresPermitsAreCountedByKotlin() {
+        Python3.exec(
+            """
+            from kotlinx.coroutines.sync import Semaphore
+            _km_s = Semaphore(2, 0)
+            assert _km_s.try_acquire() is True
+            assert _km_s.try_acquire() is True
+            assert _km_s.try_acquire() is False, 'a third permit out of two'
+            _km_s.release()
+            assert _km_s.try_acquire() is True, 'a released permit was not given back'
+            del _km_s
+            """.trimIndent(),
+        )
+    }
+}

python-multiplatform-gradle-plugin/fixtures/klib-artifact/src/nativeTest/kotlin/fixture/artifact/WalkedKlibArtifactTableTest.kt

@@ -57,23 +57,33 @@ class WalkedKlibArtifactTableTest {
     /**
      * The fixed text: everything `KlibScanner` bound out of the whole `kotlinx-coroutines-core` klib
      * under the `kotlinx.coroutines` namespace. Zero before issue #234, when no Kotlin object could
-     * cross the klib boundary; since then the factories whose types are the klib's own classes.
-     * `KlibScannerTest.aRealThirdPartyKlibIsScannedWithoutCrashingAndBindsItsHandleFactories` pins the
+     * cross the klib boundary; since then the factories whose types are the klib's own classes, and
+     * since #237 the members of its interfaces and final classes.
+     * `KlibScannerTest.aRealThirdPartyKlibIsScannedWithoutCrashingAndBindsItsHandleFactoriesAndMembers` pins the
      * same list walking the klib directly; if the two disagree, the wiring between `KlibScanner` and
      * this generated table is what broke.
      */
     @Test
-    fun theWalkerBindsExactlyTheKlibsHandleFactories() {
+    fun theWalkerBindsExactlyTheKlibsHandleFactoriesAndMembers() {
         ArtifactTable.registerInto()
         assertEquals(
             listOf(
+                "kotlinx.coroutines.ChildJob.parentCancelled",
+                "kotlinx.coroutines.CompletableJob.complete",
+                "kotlinx.coroutines.DisposableHandle.dispose",
                 "kotlinx.coroutines.Job",
+                "kotlinx.coroutines.Job.attachChild",
+                "kotlinx.coroutines.Job.start",
                 "kotlinx.coroutines.MainScope",
+                "kotlinx.coroutines.Runnable.run",
                 "kotlinx.coroutines.SupervisorJob",
+                "kotlinx.coroutines.TimeoutCancellationException.createCopy",
                 "kotlinx.coroutines.newFixedThreadPoolContext",
                 "kotlinx.coroutines.newSingleThreadContext",
                 "kotlinx.coroutines.sync.Mutex",
                 "kotlinx.coroutines.sync.Semaphore",
+                "kotlinx.coroutines.sync.Semaphore.release",
+                "kotlinx.coroutines.sync.Semaphore.tryAcquire",
             ),
             UpcallTable.entries().map { it.name }.filter { it.startsWith("kotlinx.") }.sorted(),
         )

python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/KlibScanner.kt

@@ -6,6 +6,8 @@ import org.jetbrains.kotlin.library.abi.AbiClassifierReference
 import org.jetbrains.kotlin.library.abi.AbiDeclarationContainer
 import org.jetbrains.kotlin.library.abi.AbiCompoundName
 import org.jetbrains.kotlin.library.abi.AbiFunction
+import org.jetbrains.kotlin.library.abi.AbiModality
+import org.jetbrains.kotlin.library.abi.AbiProperty
 import org.jetbrains.kotlin.library.abi.AbiQualifiedName
 import org.jetbrains.kotlin.library.abi.AbiType
 import org.jetbrains.kotlin.library.abi.AbiTypeArgument
@@ -187,6 +189,7 @@ internal object KlibScanner {
             return emptyList()
         }
 
+        val classes = classesOf(abi.topLevelDeclarations)
         val candidates = abi.topLevelDeclarations.declarations
             .filterIsInstance<AbiFunction>()
             .filter { matchesInclude(it.qualifiedName, includePrefixes) }
@@ -196,10 +199,132 @@ internal object KlibScanner {
             // receiver handle to be worth anything. Filtered rather than declined for the same reason
             // `ArtifactScanner` never builds a model for `<init>`.
             .filter { !it.isConstructor }
-            .mapNotNull { candidateOrNull(it, classesOf(abi.topLevelDeclarations)) }
+            .mapNotNull { candidateOrNull(it, classes) }
 
-        // Same rule, same implementation as the JVM producer -- see this object's KDoc.
-        return ArtifactScanner.disambiguateOverloads(candidates)
+        // Issue #237: the member functions of the klib's classes, as #215 binds them from a jar.
+        val members = classes.values
+            .filter { matchesInclude(it.qualifiedName, includePrefixes) }
+            .filter { !ArtifactScanner.isExcludedPackage(it.qualifiedName.packageName.toString(), excludePackages) }
+            .flatMap { memberCandidatesOf(it, classes) }
+
+        // Same rule, same implementation as the JVM producer -- see this object's KDoc. A member's key
+        // is `Owner.name`, so its overload set is its class's alone.
+        return ArtifactScanner.disambiguateOverloads(candidates + members)
+    }
+
+    /**
+     * The member functions of [cls] (issue #237), as `METHOD` entries `Owner.name` -- the shape #215
+     * gives a jar's members and KSP gives its own: `args[0]` is the receiver, not counted in `arity`,
+     * and the generated call is `(args[0] as Owner).name(...)`. The binding layer serves them as
+     * methods of the receiver's proxy, and never as a module attribute.
+     *
+     * The class must be able to cross as a handle ([handleClassOrNull]); a generic one's members are
+     * declined, as on the jar side. A member is declined, with its reason, when it is `suspend`,
+     * `@Deprecated`, `@PublishedApi`, `@Composable`, a member extension or one with context receivers,
+     * named like a property of its class (the property's name is the attribute), or typed by something
+     * without a klib boundary type. Constructors are not read here (stage 3).
+     */
+    private fun memberCandidatesOf(cls: AbiClass, classes: Map<String, AbiClass>): List<Candidate> {
+        val owner = cls.qualifiedName.dotted()
+        val ownerType = KotlinTypeModel(owner)
+        val generic = cls.typeParameters.isNotEmpty()
+        if (!generic && cls.kind != AbiClassKind.OBJECT && handleClassOrNull(ownerType, classes) == null) return emptyList()
+        val propertyNames = cls.declarations.filterIsInstance<AbiProperty>()
+            .mapTo(HashSet()) { it.qualifiedName.relativeName.toString().substringAfterLast('.') }
+        // The pinned compiler's klib ABI records modality but no visibility, and lists the `protected`
+        // members of an open, abstract or sealed class beside its public ones; source outside a subclass
+        // cannot call a `protected` one. An interface has none, and a final class's are effectively
+        // private, so only those members bind. An `object`'s functions are statics on the jar side (#53),
+        // called without a receiver, which this `METHOD` shape is not.
+        val unbindable = when {
+            cls.kind == AbiClassKind.OBJECT -> "a function of an object: bound as a static on the jar side (#53), not yet from a klib"
+            cls.kind == AbiClassKind.CLASS && cls.modality != AbiModality.FINAL ->
+                "a member of a non-final class: this klib ABI does not say whether it is protected"
+            else -> null
+        }
+        val enum = cls.kind == AbiClassKind.ENUM_CLASS
+        return cls.declarations.filterIsInstance<AbiFunction>()
+            .filter { !it.isConstructor }
+            // `Any`'s members are not the class's API, as on the jar side; an enum class's
+            // `valueOf`/`values`/`entries` are statics the ABI lists among its members, and no
+            // instance call reaches them.
+            .filter { function -> !isAnyMember(function) && !(enum && function.leafName() in ENUM_STATICS) }
+            .mapNotNull { function -> memberCandidateOrNull(function, owner, ownerType, generic, unbindable, propertyNames, classes) }
+    }
+
+    private fun memberCandidateOrNull(
+        function: AbiFunction,
+        owner: String,
+        ownerType: KotlinTypeModel,
+        generic: Boolean,
+        unbindable: String?,
+        propertyNames: Set<String>,
+        classes: Map<String, AbiClass>,
+    ): Candidate? {
+        val name = function.leafName()
+        val model = declarationModelOf(function)?.copy(
+            simpleName = name,
+            owner = owner,
+            ownerIsClass = true,
+            receiver = ownerType,
+            receiverBoundaryTag = OBJECT_TAG,
+            kind = METHOD_KIND,
+        ) ?: return null
+        val declined = { reason: String -> Candidate(null, model.copy(declineReason = reason)) }
+        when {
+            function.isSuspend -> return declined("suspend")
+            generic -> return declined("a member of a generic class: not bound yet")
+            unbindable != null -> return declined(unbindable)
+            function.hasAnnotation(DEPRECATED) ->
+                return declined("@Deprecated: its level is not in this klib ABI, and a HIDDEN one cannot be called from source")
+            function.hasAnnotation(PUBLISHED_API) ->
+                return declined("@PublishedApi internal -- binary-visible for inlining, not part of the module's public Kotlin API")
+            function.hasAnnotation(COMPOSABLE) -> return declined("a @Composable member function: not bound on a proxy")
+            function.hasExtensionReceiverParameter -> return declined("a member extension: it needs two receivers")
+            function.contextReceiverParametersCount > 0 ->
+                return declined("${function.contextReceiverParametersCount} context receiver(s), whose types this klib ABI does not expose")
+            name in propertyNames -> return declined("shares its name with a property of the class, which keeps the attribute")
+            // A data class's `copy`/`componentN` are synthetics the jar side never binds (U-4), and this
+            // ABI cannot tell them from a hand-written member of the same name.
+            name == "copy" || DATA_COMPONENT.matches(name) ->
+                return declined("a data-class synthetic (copy/componentN), or a member this klib ABI cannot tell from one")
+        }
+        val returnType = klibBoundaryTypeOf(model.returnType, classes)
+            ?: return declined("no boundary type for return ${model.returnType.render()}")
+        val boundaryTypes = model.parameters.map { parameter ->
+            klibBoundaryTypeOf(parameter.type, classes)
+                ?: return declined("no boundary type for parameter ${parameter.type.render()}")
+        }
+        if (boundaryTypes.any { it.isReturnOnly }) return declined("a parameter of a return-only boundary type")
+
+        val key = "$owner.$name"
+        val argumentExpressions = boundaryTypes.mapIndexed { index, type -> type.read("args[${index + 1}]") }
+        val body = returnType.wrapReturn("(args[0] as $owner).$name(${argumentExpressions.joinToString(", ")})")
+        val callable = ArtifactCallable(
+            name = key,
+            arity = boundaryTypes.size,
+            paramTags = boundaryTypes.map { it.tag },
+            returnTag = returnType.tag,
+            // The receiver is read even with no declared parameter, so the body always takes `args`.
+            lambdaBody = "{ args -> $body }",
+            receiverTypeName = owner,
+            paramTypeNames = model.parameters.map { it.type.qualifiedName },
+            returnTypeName = model.returnType.qualifiedName,
+            returnSupertypes = if (returnType.tag == OBJECT_TAG) supertypesOf(model.returnType.qualifiedName, classes) else emptyList(),
+            // Every argument is passed, as for a top-level klib entry (see [candidateOrNull]).
+            paramHasDefault = List(boundaryTypes.size) { false },
+            kind = METHOD_KIND,
+        )
+        return Candidate(
+            callable = callable,
+            declaration = model.copy(
+                bindingName = key,
+                returnBoundaryTag = returnType.tag,
+                parameters = model.parameters.mapIndexed { index, parameter ->
+                    parameter.copy(boundaryTag = boundaryTypes[index].tag, declaresDefault = false)
+                },
+            ),
+        )
     }
 
     /**
@@ -471,6 +596,21 @@ internal object KlibScanner {
 
     private const val OBJECT_TAG = "OBJECT"
 
+    private const val METHOD_KIND = "METHOD"
+
+    private val ENUM_STATICS = setOf("valueOf", "values", "entries")
+
+    private val DATA_COMPONENT = Regex("component[1-9][0-9]*")
+
+    private fun AbiFunction.leafName(): String = qualifiedName.relativeName.toString().substringAfterLast('.')
+
+    /** `toString()`, `hashCode()` and `equals(other)`: what every class has from `Any`. */
+    private fun isAnyMember(function: AbiFunction): Boolean = when (function.leafName()) {
+        "toString", "hashCode" -> function.valueParameters.isEmpty()
+        "equals" -> function.valueParameters.size == 1
+        else -> false
+    }
+
     /**
      * The walked klib's class [type] names, when a handle of it can cross (issue #234): a public
      * (the ABI lists nothing else) `class`, `interface`, `enum class` or `object` that is not a value