Dev agent: a debug-only channel from a notebook or hot reload into the running interpreter (#233)

When the launcher sets PYTHON_MULTIPLATFORM_DEV_AGENT (port, token,
optional overlay), Python3.initialize() loads the embedded stdlib-only
agent and it listens on 127.0.0.1. After a hello with the token: execute
in __main__ with streamed output, the last value and errors; complete;
push .py files into an overlay first on sys.path and re-import the loaded
ones through typedpython.importgate when present, keeping the old module
on failure. The Python file is the source of truth; generateDevAgentSource
embeds it as a Kotlin constant. SPEC L-12.

Author: b-re-w

.github/workflows/desktop.yml

@@ -42,6 +42,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-konan-
 
+      # The dev agent's protocol (SPEC L-12, #233): stdlib-only, so the runner's own python3 runs it.
+      - name: Dev agent protocol tests
+        working-directory: python-multiplatform/src/test/python
+        run: python3 -m unittest -v test_devagent
+
       - name: Run Desktop Tests
         # Using --console=plain so logs are readable in GitHub Actions
         run: ./gradlew :python-multiplatform:desktopTest --console=plain

docs/SPEC.md

@@ -135,6 +135,20 @@ fail, then implement.
   source directories, file selection, a staged fake tree); `PM/iosSimulatorArm64Test/.../env/IosPythonHomeTest.kt`
   (resolution order; the simulator test task's `PYTHONHOME` still wins). The installed-app run on the
   simulator is a manual check, `docs/platforms/ios-app-bundle.md`.
+- **L-12** The dev agent (#233; toolchain `docs/design/live-connection.md`, option B): when the launcher sets
+  `PYTHON_MULTIPLATFORM_DEV_AGENT` to `port=<n>;token=<t>[;overlay=<dir>]` (a debug launch; a release build
+  never does), `Python3.initialize()` loads the embedded agent (`src/commonMain/python/
+  python_multiplatform_devagent.py`, generated into Kotlin by `generateDevAgentSource`) as the module
+  `python_multiplatform_devagent` and it listens on 127.0.0.1 only. A connection's first message must be
+  `hello` with the token. Then, one JSON object per line: `execute` runs code in `__main__`'s persistent
+  namespace, streams stdout and stderr, returns the last expression's value and reports errors with their
+  traceback; `complete` completes from that namespace; `push` writes `.py` files into a writable overlay first
+  on `sys.path` and re-imports the loaded ones, children first, through `typedpython.importgate` when the app
+  has it (N-7b), keeping the old module when a re-import is refused or fails. A malformed request is printed
+  and does not stop the app. `Status: partial`: desktop
+  (`PM/desktopTest/.../env/PythonDevAgentTest.kt`: the embedded agent answers over loopback, and a child JVM started with the variable has an agent on that port while one started without it has none; the protocol, `python-multiplatform/src/test/python/
+  test_devagent.py`, CI `Dev agent protocol tests`); nothing sets the variable on Android or iOS yet
+  (toolchain's M2, M3); no interrupt; `execute` runs on the agent's thread (no main-thread dispatch yet).
 
 ## 2. Low-level C API (downcall surface)
 

python-multiplatform/build.gradle.kts

@@ -719,6 +719,41 @@ val generatePythonProperties by tasks.registering {
     }
 }
 
+/**
+ * The dev agent (SPEC L-12, #233) is Python, kept as a real file -- `src/commonMain/python/
+ * python_multiplatform_devagent.py`, which `src/test/python/test_devagent.py` tests with plain CPython -- and
+ * embedded here as a Kotlin string constant, the way the runtime's other helper sources are, so every platform
+ * carries it with no resource or packaging step.
+ */
+val generateDevAgentSource by tasks.registering {
+    val source = layout.projectDirectory.file("src/commonMain/python/python_multiplatform_devagent.py")
+    val outputDir = layout.buildDirectory.dir("generated/devagent/kotlin")
+    inputs.file(source)
+    outputs.dir(outputDir)
+    doLast {
+        val escaped = buildString {
+            source.asFile.readText().forEach { c ->
+                when (c) {
+                    '\\' -> append("\\\\")
+                    '"' -> append("\\\"")
+                    '$' -> append("\\$")
+                    '\n' -> append("\\n")
+                    '\r' -> {}
+                    else -> append(c)
+                }
+            }
+        }
+        val file = outputDir.get().asFile.resolve("python/multiplatform/env/DevAgentSource.kt")
+        file.parentFile.mkdirs()
+        file.writeText(
+            "// Generated by :python-multiplatform:generateDevAgentSource from\n" +
+                "// src/commonMain/python/python_multiplatform_devagent.py. Do not edit.\n" +
+                "package python.multiplatform.env\n\n" +
+                "internal const val DEV_AGENT_SOURCE: String = \"" + escaped + "\"\n",
+        )
+    }
+}
+
 val androidBuildDir = "$projectDir/build/android"
 
 // =================================================================================================
@@ -1303,7 +1338,9 @@ kotlin {
 
     sourceSets {
         val jvmMain by creating
-        val commonMain by getting
+        val commonMain by getting {
+            kotlin.srcDir(generateDevAgentSource)
+        }
         val commonTest by getting {
             dependencies {
                 implementation(libs.kotlin.test)

python-multiplatform/src/commonMain/kotlin/python/multiplatform/env/PythonDevAgent.kt

@@ -0,0 +1,54 @@
+package python.multiplatform.env
+
+import python.multiplatform.ffi.Python3
+import python.native.ffi.PyImport_ExecCodeModule
+import python.native.ffi.Py_CompileString
+import python.native.ffi.Py_DecRef
+import python.multiplatform.ffi.pyErrorOrGeneric
+
+/**
+ * The dev agent (SPEC L-12, #233): a debug-only channel from a notebook or a hot reload into this
+ * interpreter, designed in toolchain's `docs/design/live-connection.md` (option B).
+ *
+ * [Python3.initialize] calls [startIfRequested]. It does nothing unless the launcher set
+ * [ENVIRONMENT_VARIABLE] (`port=<n>;token=<t>[;overlay=<dir>]`), which a debug launch does and a release
+ * build never does; then the agent's Python ([DEV_AGENT_SOURCE], embedded from
+ * `src/commonMain/python/python_multiplatform_devagent.py`) is loaded as the module [MODULE] and listens on
+ * 127.0.0.1 with that token. A malformed request is reported on stderr and does not stop the app.
+ */
+internal object PythonDevAgent {
+    const val ENVIRONMENT_VARIABLE = "PYTHON_MULTIPLATFORM_DEV_AGENT"
+    const val MODULE = "python_multiplatform_devagent"
+
+    private const val PY_FILE_INPUT: Int = 257
+    private var loaded = false
+    private var requested = false
+
+    fun startIfRequested() {
+        if (requested || readEnvVar(ENVIRONMENT_VARIABLE).isNullOrBlank()) return
+        requested = true
+        try {
+            load()
+            Python3.exec("__import__(\"$MODULE\").start_from_env()")
+        } catch (e: Exception) {
+            println("WARNING: python-multiplatform dev agent not started: ${e.message}")
+        }
+    }
+
+    /** Loads [DEV_AGENT_SOURCE] as the module [MODULE] (once). */
+    fun load() {
+        if (loaded) return
+        Python3.withPython {
+            val code = Py_CompileString(DEV_AGENT_SOURCE, "<python-multiplatform:devagent>", PY_FILE_INPUT)
+                ?: throw pyErrorOrGeneric("Failed to compile the dev agent")
+            try {
+                val module = PyImport_ExecCodeModule(MODULE, code)
+                    ?: throw pyErrorOrGeneric("Failed to load the dev agent")
+                Py_DecRef(module)
+            } finally {
+                Py_DecRef(code)
+            }
+        }
+        loaded = true
+    }
+}

python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt

@@ -2,6 +2,7 @@ package python.multiplatform.ffi
 
 import python.multiplatform.BuildConfig
 import python.multiplatform.env.PythonHomeCheck
+import python.multiplatform.env.PythonDevAgent
 import python.multiplatform.env.PythonPayload
 import python.multiplatform.env.applyPackagedPythonHome
 import python.multiplatform.ffi.exceptions.PyException
@@ -51,6 +52,7 @@ object Python3 {
             // does its work at most once per process, so arriving here repeatedly is free.
             PythonPayload.installStagedRootsOnStartup()
             installBuiltinCompiledOnce()
+            PythonDevAgent.startIfRequested()
             return
         }
         // Runs once per process, before the call whose own failure mode is either an uncatchable
@@ -94,6 +96,9 @@ object Python3 {
         // is a separate mechanism entirely -- see PythonPayload for the whole reasoning, and for
         // the `autoInstall` switch that turns this off.
         PythonPayload.installStagedRootsOnStartup()
+        // The dev agent (SPEC L-12): only when a debug launch asked for it, after the payload is on
+        // sys.path so its overlay can go in front of it.
+        PythonDevAgent.startIfRequested()
     }
 
     private var builtinCompiledInstalled = false