Skip to content

Commit 3d4e277

Browse files
b-re-wb-re-w
authored
Merge pull request #55 from thisisthepy/work/include-provider
Feat: Expose each target's CPython include directory as a Gradle Provider
2 parents d6d3978 + 4706f11 commit 3d4e277

5 files changed

Lines changed: 269 additions & 0 deletions

File tree

‎docs/SPEC.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,10 @@ All in `PM/commonTest`, so they run wherever the interpreter loads.
253253
3.14t runs the whole desktop suite with `-PpythonFreeThreaded=true` (236 tests, 0 failures; ROADMAP §9).
254254
Only desktop has free-threaded prebuilts; Android and iOS do not. `Py_LIMITED_API` is not defined, so
255255
`abi3t` is not a blocker.
256+
- **T-3** `python-multiplatform` publishes per-target, per-flavour CPython include directories as
257+
`Provider<Directory>` carrying the extraction task (`cpythonIncludeDirectories` extension; see
258+
`docs/platforms/python-version-acquisition.md` §7). `Status: partial` (only the configured flavour is
259+
extracted) — `GP/CPythonIncludeDirectoriesTest.kt`.
256260

257261
## 8. Measurement
258262

‎docs/platforms/python-version-acquisition.md‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,3 +59,34 @@ CPython 3.14.2 for `wasm32-emscripten` (Emscripten 5.0.3, matched to the `pyemsc
5959
PEP 783) under the git-ignored `.caches/`, and `python-multiplatform-wasm-runtime` is the published zip of
6060
the result. This is deliberately a different patch release from the native `pythonVersion`; see
6161
[`wasm-design.md`](wasm-design.md).
62+
63+
## 7. Include directories as a build output
64+
65+
Compiling a C extension against the embedded CPython needs that CPython's headers, not the host's.
66+
`python-multiplatform` exposes them as a project extension named `cpythonIncludeDirectories`
67+
(`python.multiplatform.gradle.CPythonIncludeDirectories`, in the Gradle plugin module).
68+
69+
* **Key:** `(target, flavour)`. Targets: `macos-aarch64`, `macos-x86_64`, `linux-x86_64`,
70+
`windows-x86_64`, `android-aarch64`, `android-x86_64`, `ios-arm64`, `ios-arm64_x86_64-simulator`
71+
(the iOS names are XCFramework slices). Flavour: `CPythonFlavour.GIL` (default) or `FREE_THREADED`
72+
(desktop only).
73+
* **Value:** `Provider<Directory>` of the directory that directly contains `Python.h`, i.e. the `-I`
74+
directory: `<ver>/<target>[-freethreaded]/python/include/python3.14[t]` on desktop (Windows has no
75+
`python3.14` level), `<ver>/android-*/prefix/include/python3.14`,
76+
`<ver>/ios/Python.xcframework/<slice>/include/python3.14`.
77+
* **Task dependency:** the provider carries the matching `downloadPython_*` task; using it as a task
78+
input makes Gradle download and extract first.
79+
* **Only the configured flavour is extracted.** `-PpythonFreeThreaded=true` selects the free-threaded
80+
desktop tree; asking for the other flavour throws a `GradleException` naming that property
81+
(`isAvailable` tests it without throwing).
82+
83+
```kotlin
84+
// consumer build.gradle.kts
85+
evaluationDependsOn(":python-multiplatform")
86+
val includes = project(":python-multiplatform").extensions.getByType<CPythonIncludeDirectories>()
87+
tasks.register<Exec>("compileExt") {
88+
val inc = includes.includeDir("macos-aarch64") // or ("linux-x86_64", CPythonFlavour.FREE_THREADED)
89+
inputs.dir(inc)
90+
commandLine("cc", "-I${inc.get().asFile}", "-c", "ext.c")
91+
}
92+
```

‎python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/CPythonIncludeDirectories.kt‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
package python.multiplatform.gradle
2+
3+
import org.gradle.api.GradleException
4+
import org.gradle.api.file.Directory
5+
import org.gradle.api.provider.Provider
6+
import org.gradle.api.tasks.TaskProvider
7+
8+
/** Which CPython build a header directory belongs to. The two are different trees with different `Py_GIL_DISABLED` layouts. */
9+
enum class CPythonFlavour { GIL, FREE_THREADED }
10+
11+
/**
12+
* Where, inside the build-time acquisition tree (`python-multiplatform/build/python-standalone/
13+
* extracted/<pythonVersion>/`), the headers of each target's CPython live. Pure functions, so the
14+
* layout is pinned by tests without downloading anything.
15+
*
16+
* The value is the directory that **directly contains `Python.h`** -- the one a compiler wants as
17+
* `-I`. That is one level below the `include/` directory on every target except Windows
18+
* (`include/python3.14/`, `include/python3.14t/` for free-threaded), which is why the issue's
19+
* table (`.../include`) and this differ by that last segment.
20+
*/
21+
object CPythonIncludeLayout {
22+
val desktopTargets = listOf("macos-aarch64", "macos-x86_64", "linux-x86_64", "windows-x86_64")
23+
val androidTargets = listOf("android-aarch64", "android-x86_64")
24+
25+
/** XCFramework slice directory names; the simulator slice is a fat arm64+x86_64 one. */
26+
val iosTargets = listOf("ios-arm64", "ios-arm64_x86_64-simulator")
27+
28+
val allTargets: List<String> = desktopTargets + androidTargets + iosTargets
29+
30+
fun supports(target: String, flavour: CPythonFlavour): Boolean =
31+
target in allTargets && (flavour == CPythonFlavour.GIL || target in desktopTargets)
32+
33+
/** The extraction task that produces [target] -- the name `python-multiplatform/build.gradle.kts` registers. */
34+
fun downloadTaskName(target: String): String = when (target) {
35+
in iosTargets -> "downloadPython_ios"
36+
in desktopTargets, in androidTargets -> "downloadPython_${target.replace("-", "_")}"
37+
else -> throw IllegalArgumentException("unknown CPython target '$target'")
38+
}
39+
40+
/** Path of the include directory relative to the version-keyed extraction root. */
41+
fun relativePath(target: String, flavour: CPythonFlavour, pythonVersion: String): String {
42+
require(supports(target, flavour)) { "no $flavour CPython for target '$target'" }
43+
val parts = pythonVersion.split('.')
44+
require(parts.size >= 2) { "python version '$pythonVersion' has no major.minor" }
45+
val tag = "${parts[0]}.${parts[1]}" + if (flavour == CPythonFlavour.FREE_THREADED) "t" else ""
46+
return when (target) {
47+
"windows-x86_64" ->
48+
(if (flavour == CPythonFlavour.FREE_THREADED) "windows-x86_64-freethreaded" else "windows-x86_64") +
49+
"/python/include"
50+
in desktopTargets -> {
51+
val dir = if (flavour == CPythonFlavour.FREE_THREADED) "$target-freethreaded" else target
52+
"$dir/python/include/python$tag"
53+
}
54+
in androidTargets -> "$target/prefix/include/python$tag"
55+
else -> "ios/Python.xcframework/$target/include/python$tag"
56+
}
57+
}
58+
}
59+
60+
/**
61+
* The public output: one `Provider<Directory>` per (target, flavour), each carrying the extraction
62+
* task as its build dependency, so anything that wires it as an input (`-I`, an `@InputDirectory`)
63+
* triggers the download and unpack.
64+
*
65+
* Only the flavour the build was configured for (`-PpythonFreeThreaded`) has an extraction task;
66+
* asking for the other one fails with a message naming the property, rather than returning a path
67+
* nothing will ever create.
68+
*
69+
* @param extractedRoot the version-keyed extraction directory
70+
* @param extractionTask the task that fills [extractedRoot] for a target+flavour, or null when this build does not extract it
71+
*/
72+
class CPythonIncludeDirectories(
73+
private val extractedRoot: Directory,
74+
private val pythonVersion: String,
75+
private val extractionTask: (target: String, flavour: CPythonFlavour) -> TaskProvider<*>?,
76+
) {
77+
val targets: List<String> get() = CPythonIncludeLayout.allTargets
78+
79+
/** Whether [includeDir] would succeed for this pair in this build. */
80+
fun isAvailable(target: String, flavour: CPythonFlavour = CPythonFlavour.GIL): Boolean =
81+
CPythonIncludeLayout.supports(target, flavour) && extractionTask(target, flavour) != null
82+
83+
/** The directory holding `Python.h` for [target]; resolving it makes Gradle run the extraction task first. */
84+
fun includeDir(target: String, flavour: CPythonFlavour = CPythonFlavour.GIL): Provider<Directory> {
85+
if (!CPythonIncludeLayout.supports(target, flavour)) {
86+
throw GradleException(
87+
"No $flavour CPython include directory for '$target'. Targets: ${CPythonIncludeLayout.allTargets}; " +
88+
"free-threaded exists for desktop targets only.",
89+
)
90+
}
91+
val task = extractionTask(target, flavour) ?: throw GradleException(
92+
"The $flavour CPython for '$target' is not extracted by this build. " +
93+
"Only the flavour selected with -PpythonFreeThreaded is downloaded.",
94+
)
95+
val relative = CPythonIncludeLayout.relativePath(target, flavour, pythonVersion)
96+
// `TaskProvider.map` keeps the task as the producer, so the dependency travels with the value.
97+
return task.map { extractedRoot.dir(relative) }
98+
}
99+
}

‎python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/CPythonIncludeDirectoriesTest.kt‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
package python.multiplatform.gradle
2+
3+
import org.gradle.api.GradleException
4+
import org.gradle.testfixtures.ProjectBuilder
5+
import java.io.File
6+
import kotlin.test.Test
7+
import kotlin.test.assertEquals
8+
import kotlin.test.assertFailsWith
9+
import kotlin.test.assertFalse
10+
import kotlin.test.assertNotEquals
11+
import kotlin.test.assertTrue
12+
13+
/** Issue #46: per-target CPython include directories as public providers. */
14+
class CPythonIncludeDirectoriesTest {
15+
16+
private val version = "3.14.7"
17+
18+
@Test
19+
fun layoutMatchesTheObservedAcquisitionTree() {
20+
val l = CPythonIncludeLayout
21+
assertEquals("macos-aarch64/python/include/python3.14", l.relativePath("macos-aarch64", CPythonFlavour.GIL, version))
22+
assertEquals("linux-x86_64-freethreaded/python/include/python3.14t", l.relativePath("linux-x86_64", CPythonFlavour.FREE_THREADED, version))
23+
assertEquals("windows-x86_64/python/include", l.relativePath("windows-x86_64", CPythonFlavour.GIL, version))
24+
assertEquals("windows-x86_64-freethreaded/python/include", l.relativePath("windows-x86_64", CPythonFlavour.FREE_THREADED, version))
25+
assertEquals("android-aarch64/prefix/include/python3.14", l.relativePath("android-aarch64", CPythonFlavour.GIL, version))
26+
assertEquals("ios/Python.xcframework/ios-arm64/include/python3.14", l.relativePath("ios-arm64", CPythonFlavour.GIL, version))
27+
}
28+
29+
@Test
30+
fun freeThreadedIsDistinctFromDefaultOnEveryDesktopTarget() {
31+
for (t in CPythonIncludeLayout.desktopTargets) {
32+
assertNotEquals(
33+
CPythonIncludeLayout.relativePath(t, CPythonFlavour.GIL, version),
34+
CPythonIncludeLayout.relativePath(t, CPythonFlavour.FREE_THREADED, version),
35+
)
36+
}
37+
}
38+
39+
@Test
40+
fun freeThreadedDoesNotExistOffDesktop() {
41+
assertFalse(CPythonIncludeLayout.supports("android-aarch64", CPythonFlavour.FREE_THREADED))
42+
assertFalse(CPythonIncludeLayout.supports("ios-arm64", CPythonFlavour.FREE_THREADED))
43+
assertFailsWith<IllegalArgumentException> {
44+
CPythonIncludeLayout.relativePath("ios-arm64", CPythonFlavour.FREE_THREADED, version)
45+
}
46+
}
47+
48+
@Test
49+
fun downloadTaskNamesMatchTheRootBuild() {
50+
assertEquals("downloadPython_macos_aarch64", CPythonIncludeLayout.downloadTaskName("macos-aarch64"))
51+
assertEquals("downloadPython_android_x86_64", CPythonIncludeLayout.downloadTaskName("android-x86_64"))
52+
assertEquals("downloadPython_ios", CPythonIncludeLayout.downloadTaskName("ios-arm64_x86_64-simulator"))
53+
}
54+
55+
/** Registry over fake extraction tasks that really create `Python.h`; only [configured] desktop flavour is extracted. */
56+
private fun fixture(configured: CPythonFlavour): Pair<org.gradle.api.Project, CPythonIncludeDirectories> {
57+
val project = ProjectBuilder.builder().build()
58+
val root = project.layout.projectDirectory.dir("extracted/$version")
59+
val tasks = CPythonIncludeLayout.allTargets.associateWith { target ->
60+
// Android and iOS are always GIL; desktop follows the configured flavour.
61+
val flavour = if (target in CPythonIncludeLayout.desktopTargets) configured else CPythonFlavour.GIL
62+
project.tasks.register("fake_${target.replace('-', '_')}") {
63+
doLast {
64+
root.dir(CPythonIncludeLayout.relativePath(target, flavour, version)).asFile
65+
.also { it.mkdirs() }.resolve("Python.h").writeText("/* $target */")
66+
}
67+
}
68+
}
69+
val dirs = CPythonIncludeDirectories(root, version) { target, flavour ->
70+
val extracted = if (target in CPythonIncludeLayout.desktopTargets) configured else CPythonFlavour.GIL
71+
if (flavour == extracted) tasks[target] else null
72+
}
73+
return project to dirs
74+
}
75+
76+
@Test
77+
fun everyTargetResolvesToADistinctDirectoryHoldingPythonH() {
78+
val (project, dirs) = fixture(CPythonFlavour.GIL)
79+
val seen = mutableSetOf<File>()
80+
for (t in dirs.targets) {
81+
val producer = project.tasks.getByName("fake_${t.replace('-', '_')}")
82+
val provider = dirs.includeDir(t)
83+
// A consumer that takes the provider as an input must depend on the extraction task.
84+
val consumer = project.tasks.register("use_${t.replace('-', '_')}") { inputs.dir(provider) }.get()
85+
assertTrue(
86+
producer in consumer.taskDependencies.getDependencies(consumer),
87+
"$t: provider does not carry its extraction task",
88+
)
89+
producer.actions.forEach { it.execute(producer) }
90+
val dir = provider.get().asFile
91+
assertTrue(File(dir, "Python.h").isFile, "$t: Python.h missing in $dir")
92+
assertTrue(seen.add(dir), "$t: duplicate directory $dir")
93+
}
94+
}
95+
96+
@Test
97+
fun theOtherFlavourFailsLoudlyNamingTheProperty() {
98+
val (_, dirs) = fixture(CPythonFlavour.GIL)
99+
assertFalse(dirs.isAvailable("macos-aarch64", CPythonFlavour.FREE_THREADED))
100+
val e = assertFailsWith<GradleException> { dirs.includeDir("macos-aarch64", CPythonFlavour.FREE_THREADED) }
101+
assertTrue("pythonFreeThreaded" in e.message.orEmpty())
102+
assertFailsWith<GradleException> { dirs.includeDir("ios-arm64", CPythonFlavour.FREE_THREADED) }
103+
}
104+
105+
@Test
106+
fun configuredFreeThreadedExposesFreeThreadedKey() {
107+
val (_, dirs) = fixture(CPythonFlavour.FREE_THREADED)
108+
assertTrue(dirs.isAvailable("linux-x86_64", CPythonFlavour.FREE_THREADED))
109+
assertTrue(dirs.includeDir("linux-x86_64", CPythonFlavour.FREE_THREADED).get().asFile.path.endsWith("python3.14t"))
110+
}
111+
}

‎python-multiplatform/build.gradle.kts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinNativeLink
1818
import org.jetbrains.kotlin.konan.target.Family
1919
import org.jetbrains.kotlin.konan.target.KonanTarget.*
2020
import org.jetbrains.kotlin.konan.target.linker
21+
import python.multiplatform.gradle.CPythonFlavour
22+
import python.multiplatform.gradle.CPythonIncludeDirectories
23+
import python.multiplatform.gradle.CPythonIncludeLayout
2124
import python.multiplatform.gradle.GenerateWasmProxyExportsTask
2225

2326

@@ -614,6 +617,27 @@ val downloadAllPythonBuilds by tasks.registering {
614617
dependsOn(downloadPython_ios)
615618
}
616619

620+
/**
621+
* Public build output (issue #46): the include directory of the CPython this build acquires, per
622+
* target and flavour, each carrying its extraction task. Consumers (pypackpack's compile slot,
623+
* TypedPython's C build) read it with
624+
* `project(":python-multiplatform").extensions.getByType<CPythonIncludeDirectories>().includeDir("macos-aarch64")`
625+
* after `evaluationDependsOn(":python-multiplatform")`. Only the desktop flavour selected with
626+
* `-PpythonFreeThreaded` has a task; Android and iOS are always GIL. See
627+
* `docs/platforms/python-version-acquisition.md` §7.
628+
*/
629+
extensions.add(
630+
"cpythonIncludeDirectories",
631+
CPythonIncludeDirectories(
632+
layout.projectDirectory.dir(extractedDir.absolutePath),
633+
configuredPythonVersion,
634+
) { target, flavour ->
635+
val desktop = target in CPythonIncludeLayout.desktopTargets
636+
val extractedFlavour = if (desktop && pythonFreeThreaded) CPythonFlavour.FREE_THREADED else CPythonFlavour.GIL
637+
if (flavour == extractedFlavour) tasks.named(CPythonIncludeLayout.downloadTaskName(target)) else null
638+
},
639+
)
640+
617641
val androidBuildDir = "$projectDir/build/android"
618642

619643
// =================================================================================================

0 commit comments

Comments
 (0)