Skip to content

Commit d8eafe0

Browse files
authored
Merge pull request #235 from thisisthepy/feat/dev-agent
Dev agent: debug-only channel into the running interpreter (#233)
2 parents e25804c + 0335742 commit d8eafe0

8 files changed

Lines changed: 1006 additions & 1 deletion

File tree

‎.github/workflows/desktop.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,11 @@ jobs:
4242
restore-keys: |
4343
${{ runner.os }}-konan-
4444
45+
# The dev agent's protocol (SPEC L-12, #233): stdlib-only, so the runner's own python3 runs it.
46+
- name: Dev agent protocol tests
47+
working-directory: python-multiplatform/src/test/python
48+
run: python3 -m unittest -v test_devagent
49+
4550
- name: Run Desktop Tests
4651
# Using --console=plain so logs are readable in GitHub Actions
4752
run: ./gradlew :python-multiplatform:desktopTest --console=plain

‎docs/SPEC.md‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,28 @@ fail, then implement.
135135
source directories, file selection, a staged fake tree); `PM/iosSimulatorArm64Test/.../env/IosPythonHomeTest.kt`
136136
(resolution order; the simulator test task's `PYTHONHOME` still wins). The installed-app run on the
137137
simulator is a manual check, `docs/platforms/ios-app-bundle.md`.
138+
- **L-12** The dev agent (#233; toolchain `docs/design/live-connection.md`, option B): when the launcher sets
139+
`PYTHON_MULTIPLATFORM_DEV_AGENT` to `port=<n>;token=<t>[;overlay=<dir>]` **and** an installed payload root
140+
holds the marker `python_multiplatform_devagent.enable`, `Python3.initialize()` loads the embedded agent
141+
(`src/commonMain/python/python_multiplatform_devagent.py`, generated into Kotlin by `generateDevAgentSource`)
142+
as the module `python_multiplatform_devagent` and it listens on 127.0.0.1 only. The marker is the build-time
143+
switch: this library is one artifact for debug and release, so the app's build decides, and toolchain stages
144+
the marker into debug variants only. Without it the variable is ignored with a warning and no port opens. The
145+
token must be at least 43 characters (`secrets.token_urlsafe(32)`), or the agent refuses to start. The listening
146+
socket cannot be bound by another socket while it listens: `SO_EXCLUSIVEADDRUSE` on Windows; on POSIX,
147+
`SO_REUSEADDR` only so that a restarted app can rebind the launcher's port past TIME_WAIT. A connection's first
148+
message must be `hello` with the token. Then, one JSON object per line: `execute` runs code in `__main__`'s
149+
persistent namespace, streams stdout and stderr, returns the last expression's value and reports errors with
150+
their traceback; `complete` completes from that namespace; `push` writes `.py` files into a writable overlay
151+
first on `sys.path` and re-imports the loaded ones, children first, through `typedpython.importgate` when the
152+
app has it (N-7b), keeping the old module when a re-import is refused or fails. A malformed request is printed
153+
and does not stop the app. `Status: partial`: desktop
154+
(`PM/desktopTest/.../env/PythonDevAgentTest.kt`: the embedded agent answers over loopback; a child JVM with the
155+
marker and the variable has an agent on that port, one without the variable has none, one without the marker
156+
(a release build) opens no port with the variable set, and a short token opens no port; the protocol, token
157+
length, port takeover and restart: `python-multiplatform/src/test/python/test_devagent.py`, CI `Dev agent
158+
protocol tests`, macOS only so far); nothing sets the variable or stages the marker on Android or iOS yet
159+
(toolchain's M2, M3); no interrupt; `execute` runs on the agent's thread (no main-thread dispatch yet).
138160

139161
## 2. Low-level C API (downcall surface)
140162

‎python-multiplatform/build.gradle.kts‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -719,6 +719,41 @@ val generatePythonProperties by tasks.registering {
719719
}
720720
}
721721

722+
/**
723+
* The dev agent (SPEC L-12, #233) is Python, kept as a real file -- `src/commonMain/python/
724+
* python_multiplatform_devagent.py`, which `src/test/python/test_devagent.py` tests with plain CPython -- and
725+
* embedded here as a Kotlin string constant, the way the runtime's other helper sources are, so every platform
726+
* carries it with no resource or packaging step.
727+
*/
728+
val generateDevAgentSource by tasks.registering {
729+
val source = layout.projectDirectory.file("src/commonMain/python/python_multiplatform_devagent.py")
730+
val outputDir = layout.buildDirectory.dir("generated/devagent/kotlin")
731+
inputs.file(source)
732+
outputs.dir(outputDir)
733+
doLast {
734+
val escaped = buildString {
735+
source.asFile.readText().forEach { c ->
736+
when (c) {
737+
'\\' -> append("\\\\")
738+
'"' -> append("\\\"")
739+
'$' -> append("\\$")
740+
'\n' -> append("\\n")
741+
'\r' -> {}
742+
else -> append(c)
743+
}
744+
}
745+
}
746+
val file = outputDir.get().asFile.resolve("python/multiplatform/env/DevAgentSource.kt")
747+
file.parentFile.mkdirs()
748+
file.writeText(
749+
"// Generated by :python-multiplatform:generateDevAgentSource from\n" +
750+
"// src/commonMain/python/python_multiplatform_devagent.py. Do not edit.\n" +
751+
"package python.multiplatform.env\n\n" +
752+
"internal const val DEV_AGENT_SOURCE: String = \"" + escaped + "\"\n",
753+
)
754+
}
755+
}
756+
722757
val androidBuildDir = "$projectDir/build/android"
723758

724759
// =================================================================================================
@@ -1303,7 +1338,9 @@ kotlin {
13031338

13041339
sourceSets {
13051340
val jvmMain by creating
1306-
val commonMain by getting
1341+
val commonMain by getting {
1342+
kotlin.srcDir(generateDevAgentSource)
1343+
}
13071344
val commonTest by getting {
13081345
dependencies {
13091346
implementation(libs.kotlin.test)
Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
package python.multiplatform.env
2+
3+
import python.multiplatform.ffi.Python3
4+
import python.native.ffi.PyImport_ExecCodeModule
5+
import python.native.ffi.Py_CompileString
6+
import python.native.ffi.Py_DecRef
7+
import python.multiplatform.ffi.pyErrorOrGeneric
8+
9+
/**
10+
* The dev agent (SPEC L-12, #233): a debug-only channel from a notebook or a hot reload into this
11+
* interpreter, designed in toolchain's `docs/design/live-connection.md` (option B).
12+
*
13+
* [Python3.initialize] calls [startIfRequested]. It does nothing unless the launcher set
14+
* [ENVIRONMENT_VARIABLE] (`port=<n>;token=<t>[;overlay=<dir>]`) **and** the app was built for it: one of
15+
* its installed payload roots ([PythonPayload.installed]) holds [ENABLE_MARKER]. Then the agent's Python
16+
* ([DEV_AGENT_SOURCE], embedded from `src/commonMain/python/python_multiplatform_devagent.py`) is loaded as
17+
* the module [MODULE] and listens on 127.0.0.1 with that token, which must be at least 32 random bytes
18+
* (`secrets.token_urlsafe(32)`). A refused or malformed request is reported on stderr and does not stop the app.
19+
*
20+
* The marker is the build-time switch. This library is one artifact for an app's debug and release builds
21+
* alike, so a constant in it cannot tell them apart; the app's build can, and toolchain stages the marker
22+
* into a debug variant's payload only. Without it, a release build opens no port even if something starts
23+
* its process with the variable set: "a release launch never sets it" is the launcher's promise, which this
24+
* runtime does not rely on.
25+
*/
26+
internal object PythonDevAgent {
27+
const val ENVIRONMENT_VARIABLE = "PYTHON_MULTIPLATFORM_DEV_AGENT"
28+
const val MODULE = "python_multiplatform_devagent"
29+
const val ENABLE_MARKER = "python_multiplatform_devagent.enable"
30+
31+
private const val PY_FILE_INPUT: Int = 257
32+
private var loaded = false
33+
private var requested = false
34+
35+
fun startIfRequested() {
36+
if (requested || readEnvVar(ENVIRONMENT_VARIABLE).isNullOrBlank()) return
37+
requested = true
38+
if (enabledRoot() == null) {
39+
println(
40+
"WARNING: python-multiplatform dev agent not started: $ENVIRONMENT_VARIABLE is set, but no " +
41+
"payload root holds $ENABLE_MARKER, so this is not a build made for it (a release build).",
42+
)
43+
return
44+
}
45+
try {
46+
load()
47+
Python3.exec("__import__(\"$MODULE\").start_from_env()")
48+
} catch (e: Exception) {
49+
println("WARNING: python-multiplatform dev agent not started: ${e.message}")
50+
}
51+
}
52+
53+
/** The installed payload root that holds [ENABLE_MARKER], if any. */
54+
private fun enabledRoot(): String? =
55+
PythonPayload.installed.firstOrNull { pathIsAccessible("$it/$ENABLE_MARKER") }
56+
57+
/** Loads [DEV_AGENT_SOURCE] as the module [MODULE] (once). */
58+
fun load() {
59+
if (loaded) return
60+
Python3.withPython {
61+
val code = Py_CompileString(DEV_AGENT_SOURCE, "<python-multiplatform:devagent>", PY_FILE_INPUT)
62+
?: throw pyErrorOrGeneric("Failed to compile the dev agent")
63+
try {
64+
val module = PyImport_ExecCodeModule(MODULE, code)
65+
?: throw pyErrorOrGeneric("Failed to load the dev agent")
66+
Py_DecRef(module)
67+
} finally {
68+
Py_DecRef(code)
69+
}
70+
}
71+
loaded = true
72+
}
73+
}

‎python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package python.multiplatform.ffi
22

33
import python.multiplatform.BuildConfig
44
import python.multiplatform.env.PythonHomeCheck
5+
import python.multiplatform.env.PythonDevAgent
56
import python.multiplatform.env.PythonPayload
67
import python.multiplatform.env.applyPackagedPythonHome
78
import python.multiplatform.ffi.exceptions.PyException
@@ -51,6 +52,7 @@ object Python3 {
5152
// does its work at most once per process, so arriving here repeatedly is free.
5253
PythonPayload.installStagedRootsOnStartup()
5354
installBuiltinCompiledOnce()
55+
PythonDevAgent.startIfRequested()
5456
return
5557
}
5658
// Runs once per process, before the call whose own failure mode is either an uncatchable
@@ -94,6 +96,9 @@ object Python3 {
9496
// is a separate mechanism entirely -- see PythonPayload for the whole reasoning, and for
9597
// the `autoInstall` switch that turns this off.
9698
PythonPayload.installStagedRootsOnStartup()
99+
// The dev agent (SPEC L-12): only when a debug launch asked for it, after the payload is on
100+
// sys.path so its overlay can go in front of it.
101+
PythonDevAgent.startIfRequested()
97102
}
98103

99104
private var builtinCompiledInstalled = false

0 commit comments

Comments
 (0)