MCP client as a new subproject

.github/workflows/ci.yml

@@ -26,9 +26,13 @@ jobs:
         uses: actions/setup-java@v5
         with:
           distribution: 'temurin'
-          java-version: 11
+          java-version: 21
           cache: 'sbt'
       - uses: sbt/setup-sbt@3e125ece5c3e5248e18da9ed8d2cce3d335ec8dd # v1, specifically v1.1.14
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '24'
       - name: Install scala-cli
         uses: VirtusLab/scala-cli-setup@68bd9c30954d20e6cb6ddaf01b3b38336f25df4b # main, specifically v1.10.1
         with:
@@ -37,10 +41,15 @@ jobs:
         run: sbt -v scalafmtCheckAll
       - name: Compile
         run: sbt -v compile
-      - name: Verify that examples compile using Scala CLI
-        run: sbt -v "project examples" verifyExamplesCompileUsingScalaCli  
+        # TODO bring this step back after first release with new project structure (client + server)
+#      - name: Verify that examples compile using Scala CLI
+#        run: sbt -v "project examples" verifyExamplesCompileUsingScalaCli
       - name: Test
         run: sbt -v test
+      - name: Client conformance
+        run: sbt -v "clientConformance/conformance client --suite core"
+      - name: Server conformance
+        run: sbt -v "serverConformance/conformance server"
       - uses: actions/upload-artifact@v5  # upload test results
         if: success() || failure()        # run this step even if previous step failed
         with:

build.sbt

@@ -1,11 +1,13 @@
-import com.softwaremill.SbtSoftwareMillCommon.commonSmlBuildSettings
 import com.softwaremill.Publish.{ossPublishSettings, updateDocs}
+import com.softwaremill.SbtSoftwareMillCommon.commonSmlBuildSettings
 import com.softwaremill.UpdateVersionInDocs
 
-// Version constants
 val scalaTestV = "3.2.20"
 val circeV = "0.14.15"
+val slf4jV = "2.0.18"
+val logbackV = "1.5.32"
 val tapirV = "1.13.19"
+val sttpClientV = "4.0.23"
 
 lazy val verifyExamplesCompileUsingScalaCli = taskKey[Unit]("Verify that each example compiles using Scala CLI")
 
@@ -19,32 +21,57 @@ lazy val commonSettings = commonSmlBuildSettings ++ ossPublishSettings ++ Seq(
     }
   }.value,
   Test / scalacOptions += "-Wconf:msg=unused value of type org.scalatest.Assertion:s",
-  Test / scalacOptions += "-Wconf:msg=unused value of type org.scalatest.compatible.Assertion:s"
+  Test / scalacOptions += "-Wconf:msg=unused value of type org.scalatest.compatible.Assertion:s",
+  scalacOptions ++= Seq("-Wunused:all", "-Werror")
 )
 
 val scalaTest = "org.scalatest" %% "scalatest" % scalaTestV % Test
 
-lazy val rootProject = (project in file("."))
+lazy val root = (project in file("."))
   .settings(commonSettings: _*)
   .settings(publishArtifact := false, name := "chimp")
-  .aggregate(core, examples)
+  .aggregate(core, server, client, examples, serverConformance, clientConformance)
+
+val conformance = inputKey[Unit]("Run the MCP conformance harness via npx, extra args are passed through")
 
 lazy val core: Project = (project in file("core"))
   .settings(commonSettings: _*)
   .settings(
-    name := "core",
+    name := "chimp-core",
     libraryDependencies ++= Seq(
       scalaTest,
       "io.circe" %% "circe-core" % circeV,
       "io.circe" %% "circe-generic" % circeV,
       "io.circe" %% "circe-parser" % circeV,
+      "org.slf4j" % "slf4j-api" % slf4jV,
+      "com.networknt" % "json-schema-validator" % "3.0.2" % Test
+    )
+  )
+
+lazy val server: Project = (project in file("server"))
+  .settings(commonSettings: _*)
+  .settings(
+    name := "chimp-server",
+    libraryDependencies ++= Seq(
+      scalaTest,
       "com.softwaremill.sttp.tapir" %% "tapir-core" % tapirV,
       "com.softwaremill.sttp.tapir" %% "tapir-json-circe" % tapirV,
       "com.softwaremill.sttp.tapir" %% "tapir-apispec-docs" % tapirV,
-      "com.softwaremill.sttp.apispec" %% "jsonschema-circe" % "0.11.10",
-      "org.slf4j" % "slf4j-api" % "2.0.18"
+      "com.softwaremill.sttp.apispec" %% "jsonschema-circe" % "0.11.10"
+    )
+  )
+  .dependsOn(core)
+
+lazy val client: Project = (project in file("client"))
+  .settings(commonSettings: _*)
+  .settings(
+    name := "chimp-client",
+    libraryDependencies ++= Seq(
+      scalaTest,
+      "com.softwaremill.sttp.client4" %% "core" % sttpClientV
     )
   )
+  .dependsOn(core)
 
 lazy val examples = (project in file("examples"))
   .settings(commonSettings: _*)
@@ -55,8 +82,127 @@ lazy val examples = (project in file("examples"))
       "com.softwaremill.sttp.client4" %% "core" % "4.0.23",
       "com.softwaremill.sttp.tapir" %% "tapir-netty-server-sync" % tapirV,
       "com.softwaremill.sttp.tapir" %% "tapir-zio-http-server" % tapirV,
-      "ch.qos.logback" % "logback-classic" % "1.5.32"
+      "ch.qos.logback" % "logback-classic" % logbackV
     ),
     verifyExamplesCompileUsingScalaCli := VerifyExamplesCompileUsingScalaCli(sLog.value, sourceDirectory.value)
   )
-  .dependsOn(core)
+  .dependsOn(server, client)
+
+import sbtassembly.AssemblyPlugin.autoImport.*
+
+lazy val assemblySettings = Seq(
+  assembly / assemblyMergeStrategy := {
+    case PathList("META-INF", "MANIFEST.MF")     => MergeStrategy.discard
+    case PathList("META-INF", "INDEX.LIST")      => MergeStrategy.discard
+    case PathList("META-INF", "DEPENDENCIES")    => MergeStrategy.discard
+    case PathList("META-INF", "services", _ @_*) => MergeStrategy.concat
+    case PathList("META-INF", xs @ _*) if xs.lastOption.exists(s => s.endsWith(".SF") || s.endsWith(".DSA") || s.endsWith(".RSA")) =>
+      MergeStrategy.discard
+    case PathList("META-INF", _ @_*)   => MergeStrategy.first
+    case PathList("module-info.class") => MergeStrategy.discard
+    case _                             => MergeStrategy.first
+  }
+)
+
+lazy val serverConformance = (project in file("server-conformance"))
+  .enablePlugins(AssemblyPlugin)
+  .settings(commonSettings: _*)
+  .settings(assemblySettings: _*)
+  .settings(
+    publishArtifact := false,
+    name := "server-conformance",
+    Compile / mainClass := Some("chimp.conformance.server.Main"),
+    assembly / assemblyJarName := "chimp-server-conformance.jar",
+    libraryDependencies ++= Seq(
+      "com.softwaremill.sttp.tapir" %% "tapir-netty-server-sync" % tapirV,
+      "ch.qos.logback" % "logback-classic" % logbackV
+    ),
+    conformance := {
+      import complete.DefaultParsers.*
+
+      import scala.sys.process.*
+      val args = spaceDelimited("<args>").parsed.toList
+      val jar = assembly.value
+      val rootDir = (LocalRootProject / baseDirectory).value
+      val baseline = (rootDir / "conformance-baseline.yml").getAbsolutePath
+      val log = streams.value.log
+
+      val urlPromise = scala.concurrent.Promise[String]()
+      val pb = new java.lang.ProcessBuilder("java", "-jar", jar.getAbsolutePath).redirectErrorStream(false)
+      val proc = pb.start()
+      val readerThread = new Thread(() => {
+        val reader = new java.io.BufferedReader(new java.io.InputStreamReader(proc.getInputStream, "UTF-8"))
+        try {
+          val line = reader.readLine()
+          if (line != null && line.startsWith("http")) urlPromise.trySuccess(line.trim)
+          else urlPromise.tryFailure(new RuntimeException(s"Server did not print a URL; first line was: $line"))
+          var more: String = reader.readLine()
+          while (more != null) {
+            log.info(s"[server] $more")
+            more = reader.readLine()
+          }
+        } catch {
+          case t: Throwable => urlPromise.tryFailure(t)
+        }
+      })
+      readerThread.setDaemon(true)
+      readerThread.start()
+
+      val errReaderThread = new Thread(() => {
+        val reader = new java.io.BufferedReader(new java.io.InputStreamReader(proc.getErrorStream, "UTF-8"))
+        try {
+          var line: String = reader.readLine()
+          while (line != null) {
+            log.warn(s"[server-stderr] $line")
+            line = reader.readLine()
+          }
+        } catch {
+          case _: Throwable => ()
+        }
+      })
+      errReaderThread.setDaemon(true)
+      errReaderThread.start()
+
+      try {
+        val url = scala.concurrent.Await.result(urlPromise.future, scala.concurrent.duration.Duration("15s"))
+        log.info(s"Server started at $url")
+        val cmd = List("npx", "@modelcontextprotocol/conformance") ++ args ++
+          List("--url", url, "--expected-failures", baseline)
+        val rc = Process(cmd, rootDir).!
+        if (rc != 0) sys.error(s"conformance harness exited with code $rc")
+      } finally {
+        proc.destroy()
+        if (!proc.waitFor(2, java.util.concurrent.TimeUnit.SECONDS)) proc.destroyForcibly()
+      }
+    }
+  )
+  .dependsOn(server)
+
+lazy val clientConformance = (project in file("client-conformance"))
+  .enablePlugins(AssemblyPlugin)
+  .settings(commonSettings: _*)
+  .settings(assemblySettings: _*)
+  .settings(
+    publishArtifact := false,
+    name := "client-conformance",
+    Compile / mainClass := Some("chimp.conformance.client.Main"),
+    assembly / assemblyJarName := "chimp-client-conformance.jar",
+    libraryDependencies ++= Seq(
+      "ch.qos.logback" % "logback-classic" % "1.5.32"
+    ),
+    conformance := {
+      import complete.DefaultParsers.*
+
+      import scala.sys.process.*
+      val args = spaceDelimited("<args>").parsed.toList
+      val _ = assembly.value
+      val baseDir = baseDirectory.value
+      val rootDir = (LocalRootProject / baseDirectory).value
+      val wrapper = (baseDir / "bin" / "chimp-conformance-client").getAbsolutePath
+      val cmd = List("npx", "@modelcontextprotocol/conformance") ++ args ++
+        List("--command", wrapper, "--expected-failures", (rootDir / "conformance-baseline.yml").getAbsolutePath)
+      val rc = Process(cmd, rootDir).!
+      if (rc != 0) sys.error(s"conformance harness exited with code $rc")
+    }
+  )
+  .dependsOn(client)

client-conformance/README.md

@@ -0,0 +1,45 @@
+# client-conformance
+
+Runs chimp's MCP client against the
+official [MCP conformance test suite](https://github.com/modelcontextprotocol/conformance).
+
+## What it does
+
+The conformance harness is the inverse of a server: it **starts a test MCP server**, spawns the binary configured
+via `--command` (this fat-jar wrapped in [`bin/chimp-conformance-client`](bin/chimp-conformance-client)), and passes the
+test-server URL as the last argument. The client process reads the `MCP_CONFORMANCE_SCENARIO` env var to decide what
+protocol exchange to drive, talks to the harness's server, and exits 0 on success.
+
+This subproject is the binary the harness invokes. `Main.scala` dispatches on the scenario name and uses `chimp-client`
+to drive the protocol.
+
+## How to run
+
+Using sbt task (assembles a client fat jar and runs test suite in one step):
+
+```bash
+sbt 'clientConformance/conformance client --suite core'
+sbt 'clientConformance/conformance client --scenario initialize'
+sbt 'clientConformance/conformance client --scenario tools_call'
+```
+
+The `@modelcontextprotocol/conformance` will be installed using npm, it must be available on the PATH.
+
+## Adding a scenario
+
+Add a case in `Main.scala` matching on the scenario name (the value of `MCP_CONFORMANCE_SCENARIO`), drive the protocol
+with `McpClient`, return exit code 0 on success. Once it passes, remove the entry from the baseline file (see below).
+
+## The baseline file
+
+[`conformance-baseline.yml`](../conformance-baseline.yml) lists scenarios that are known to fail today. The harness uses
+it like this:
+
+| Scenario result | In baseline? | Exit code | Meaning                               |
+|-----------------|--------------|-----------|---------------------------------------|
+| Fails           | Yes          | 0         | Expected failure — keep working on it |
+| Fails           | No           | 1         | Regression — CI fails                 |
+| Passes          | Yes          | 1         | Stale baseline — remove the entry     |
+| Passes          | No           | 0         | Normal pass                           |
+
+So the file shrinks as the SDK matures.

client-conformance/bin/chimp-conformance-client

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euo pipefail
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+JAR="${DIR}/../target/scala-3.3.7/chimp-client-conformance.jar"
+if [[ ! -f "${JAR}" ]]; then
+  echo "Fat jar not found at ${JAR}. Run 'sbt clientConformance/assembly' first." >&2
+  exit 127
+fi
+exec java -jar "${JAR}" "$@"

client-conformance/src/main/resources/logback.xml

@@ -0,0 +1,11 @@
+<configuration>
+  <appender name="STDERR" class="ch.qos.logback.core.ConsoleAppender">
+    <target>System.err</target>
+    <encoder>
+      <pattern>%d{HH:mm:ss.SSS} %-5level %logger{36} - %msg%n</pattern>
+    </encoder>
+  </appender>
+  <root level="warn">
+    <appender-ref ref="STDERR" />
+  </root>
+</configuration>

client-conformance/src/main/scala/chimp/conformance/client/Main.scala

@@ -0,0 +1,66 @@
+package chimp.conformance.client
+
+import chimp.client.McpClient
+import chimp.client.transport.HttpTransport
+import chimp.protocol.*
+import io.circe.Json
+import sttp.client4.DefaultSyncBackend
+import sttp.model.Uri
+import sttp.shared.Identity
+
+object Main:
+
+  private val clientInfo = Implementation(name = "chimp-conformance-client", version = "0.1.0")
+
+  def main(args: Array[String]): Unit =
+    if args.isEmpty then
+      System.err.println("Usage: chimp-conformance-client <serverUrl>")
+      sys.exit(2)
+
+    val serverUrl = Uri.parse(args.last) match
+      case Right(url) => url
+      case Left(e)    => System.err.println(s"Invalid server URL: $e"); sys.exit(2)
+
+    val scenario = sys.env.getOrElse("MCP_CONFORMANCE_SCENARIO", "")
+    val protocolVersion: ProtocolVersion = sys.env
+      .get("MCP_CONFORMANCE_PROTOCOL_VERSION")
+      .flatMap(ProtocolVersion.from)
+      .getOrElse(ProtocolVersion.Latest)
+
+    val backend = DefaultSyncBackend()
+    val transport = HttpTransport[Identity](backend, serverUrl, protocolVersion)
+
+    val rc: Int =
+      try
+        scenario match
+          case "initialize" =>
+            val client = McpClient[Identity](transport, clientInfo, protocolVersion = protocolVersion)
+            val _ = client.initialize()
+            client.close()
+            0
+
+          case "tools_call" =>
+            val client = McpClient[Identity](transport, clientInfo, protocolVersion = protocolVersion)
+            val _ = client.initialize()
+            val _ = client.callTool(
+              "add_numbers",
+              Json.obj("a" -> Json.fromInt(2), "b" -> Json.fromInt(3))
+            )
+            client.close()
+            0
+
+          case s if s == "elicitation-sep1034-client-defaults" || s == "sse-retry" || s.startsWith("auth/") =>
+            2
+
+          case other =>
+            System.err.println(s"Scenario not implemented: $other")
+            3
+      catch
+        case t: Throwable =>
+          t.printStackTrace()
+          1
+      finally
+        try backend.close()
+        catch case _: Throwable => ()
+
+    sys.exit(rc)