diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 1294746..d025b2b 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,7 +9,7 @@ "source": { "source": "npm", "package": "@copilotkit/aimock", - "version": "^1.11.0" + "version": "^1.13.0" }, "description": "Fixture authoring skill for @copilotkit/aimock — match fields, response types, embeddings, structured output, sequential responses, streaming physics, agent loop patterns, gotchas, and debugging" } diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index f01d5ff..9fff0b8 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "llmock", - "version": "1.11.0", + "version": "1.13.0", "description": "Fixture authoring guidance for @copilotkit/aimock", "author": { "name": "CopilotKit" diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml index d344fd5..4d5822d 100644 --- a/.github/workflows/publish-release.yml +++ b/.github/workflows/publish-release.yml @@ -49,6 +49,10 @@ jobs: fs.writeFileSync('package.json', JSON.stringify(pkg, null, 2) + '\n'); " + - name: Strip video URLs from README for npm + if: steps.check.outputs.published == 'false' + run: sed -i '/^https:\/\/github.com\/user-attachments\//d' README.md + - name: Build and publish if: steps.check.outputs.published == 'false' run: pnpm build && npm publish --access public diff --git a/CHANGELOG.md b/CHANGELOG.md index 7397b00..94559d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # @copilotkit/aimock +## 1.13.0 + +### Minor Changes + +- Add GitHub Action for one-line CI setup — `uses: CopilotKit/aimock@v1` with fixtures, config, port, args, and health check (#102) +- Wire fixture converters into CLI — `npx aimock convert vidaimock` and `npx aimock convert mockllm` as first-class subcommands (#102) +- Add 30 npm keywords for search discoverability (#102) +- Add fixture gallery with 11 examples covering all mock types, plus browsable docs page at /examples (#102) +- Add vitest and jest plugins for zero-config testing — `import { useAimock } from "@copilotkit/aimock/vitest"` (#102) +- Strip video URLs from README for npm publishing (#102) + ## 1.12.0 ### Minor Changes diff --git a/README.md b/README.md index 0e6ed07..db1565e 100644 --- a/README.md +++ b/README.md @@ -51,8 +51,23 @@ Run them all on one port with `npx aimock --config aimock.json`, or use the prog - **[WebSocket APIs](https://aimock.copilotkit.dev/websocket)** — OpenAI Realtime, Responses WS, Gemini Live - **[Prometheus Metrics](https://aimock.copilotkit.dev/metrics)** — Request counts, latencies, fixture match rates - **[Docker + Helm](https://aimock.copilotkit.dev/docker)** — Container image and Helm chart for CI/CD +- **[Vitest & Jest Plugins](https://aimock.copilotkit.dev/test-plugins)** — Zero-config `useAimock()` with auto lifecycle and env patching - **Zero dependencies** — Everything from Node.js builtins +## GitHub Action + +```yaml +- uses: CopilotKit/aimock@v1 + with: + fixtures: ./test/fixtures + +- run: npm test + env: + OPENAI_BASE_URL: http://127.0.0.1:4010/v1 +``` + +See the [GitHub Action docs](https://aimock.copilotkit.dev/github-action) for all inputs and examples. + ## CLI ```bash @@ -65,6 +80,10 @@ npx aimock --config aimock.json # Record mode: proxy to real APIs, save fixtures npx aimock --record --provider-openai https://api.openai.com +# Convert fixtures from other tools +npx aimock convert vidaimock ./templates/ ./fixtures/ +npx aimock convert mockllm ./config.yaml ./fixtures/ + # Docker docker run -d -p 4010:4010 -v ./fixtures:/fixtures ghcr.io/copilotkit/aimock -f /fixtures ``` @@ -75,7 +94,7 @@ Step-by-step migration guides: [MSW](https://aimock.copilotkit.dev/migrate-from- ## Documentation -**[https://aimock.copilotkit.dev](https://aimock.copilotkit.dev)** +**[https://aimock.copilotkit.dev](https://aimock.copilotkit.dev)** · [Example fixtures](https://aimock.copilotkit.dev/examples) ## Real-World Usage diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..ffcf69f --- /dev/null +++ b/action.yml @@ -0,0 +1,91 @@ +name: "aimock" +description: "Start an aimock server for AI application testing — LLM APIs, MCP, A2A, AG-UI, vector DBs" +branding: + icon: "server" + color: "green" + +inputs: + fixtures: + description: "Path to fixture files or directory" + required: false + default: "./fixtures" + config: + description: "Path to aimock JSON config file (overrides fixtures)" + required: false + port: + description: "Port to listen on" + required: false + default: "4010" + host: + description: "Host to bind to" + required: false + default: "127.0.0.1" + version: + description: "aimock version to install (default: latest)" + required: false + default: "latest" + args: + description: "Additional CLI arguments (e.g., --strict --record --provider-openai https://api.openai.com)" + required: false + default: "" + wait-timeout: + description: "Max seconds to wait for health check (default: 30)" + required: false + default: "30" + +outputs: + url: + description: "The aimock server URL (e.g., http://127.0.0.1:4010)" + value: ${{ steps.start.outputs.url }} + +runs: + using: "composite" + steps: + - name: Install aimock + shell: bash + run: npm install -g @copilotkit/aimock@${{ inputs.version }} + + - name: Start aimock + id: start + shell: bash + run: | + URL="http://${{ inputs.host }}:${{ inputs.port }}" + echo "url=${URL}" >> $GITHUB_OUTPUT + + if [ -n "${{ inputs.config }}" ]; then + aimock --config "${{ inputs.config }}" \ + --port ${{ inputs.port }} \ + --host ${{ inputs.host }} \ + ${{ inputs.args }} & + else + llmock --fixtures "${{ inputs.fixtures }}" \ + --port ${{ inputs.port }} \ + --host ${{ inputs.host }} \ + ${{ inputs.args }} & + fi + + echo $! > /tmp/aimock.pid + echo "Started aimock (PID: $(cat /tmp/aimock.pid))" + + - name: Wait for health check + shell: bash + run: | + URL="http://${{ inputs.host }}:${{ inputs.port }}/health" + TIMEOUT=${{ inputs.wait-timeout }} + ELAPSED=0 + + echo "Waiting for ${URL} ..." + while [ $ELAPSED -lt $TIMEOUT ]; do + if curl -sf "$URL" > /dev/null 2>&1; then + echo "aimock is ready at http://${{ inputs.host }}:${{ inputs.port }}" + exit 0 + fi + sleep 1 + ELAPSED=$((ELAPSED + 1)) + done + + echo "::error::aimock failed to start within ${TIMEOUT}s" + if [ -f /tmp/aimock.pid ]; then + kill "$(cat /tmp/aimock.pid)" 2>/dev/null || true + fi + exit 1 diff --git a/charts/aimock/Chart.yaml b/charts/aimock/Chart.yaml index 1d2e733..faf508a 100644 --- a/charts/aimock/Chart.yaml +++ b/charts/aimock/Chart.yaml @@ -3,4 +3,4 @@ name: aimock description: Mock infrastructure for AI application testing (OpenAI, Anthropic, Gemini, MCP, A2A, vector) type: application version: 0.1.0 -appVersion: "1.12.0" +appVersion: "1.13.0" diff --git a/docs/aimock-cli/index.html b/docs/aimock-cli/index.html index d15e72d..dd237de 100644 --- a/docs/aimock-cli/index.html +++ b/docs/aimock-cli/index.html @@ -280,6 +280,59 @@

Docker Usage

+

Fixture Converters

+

Convert fixtures from other mock tools to aimock format.

+ +

Usage

+
+
Convert fixtures shell
+
npx aimock convert <format> <input> [output]
+
+ +

Supported Formats

+ + + + + + + + + + + + + + + + + + + + +
FormatSourceDescription
vidaimockVidaiMock Tera templates + Converts .tera / .json / .txt templates to + aimock fixture JSON +
mockllmmock-llm YAML config + Converts mock-llm YAML configs to aimock fixture JSON. Also extracts MCP tools if + present. +
+ +

Examples

+
+
+ Converter examples shell +
+
# Convert a directory of VidaiMock templates
+$ npx aimock convert vidaimock ./templates/ ./fixtures/converted.json
+
+# Convert a mock-llm YAML config
+$ npx aimock convert mockllm ./config.yaml ./fixtures/converted.json
+
+# Print to stdout (omit output path)
+$ npx aimock convert vidaimock ./templates/
+
+

Docker Compose

diff --git a/docs/docs/index.html b/docs/docs/index.html index 5097d3c..c867408 100644 --- a/docs/docs/index.html +++ b/docs/docs/index.html @@ -305,8 +305,8 @@

The Suite

Chaos testing, metrics, drift detection, Docker

diff --git a/docs/examples/index.html b/docs/examples/index.html new file mode 100644 index 0000000..59b6b5d --- /dev/null +++ b/docs/examples/index.html @@ -0,0 +1,514 @@ + + + + + + Example Fixtures — aimock + + + + + + + + + +
+ + +
+

Example Fixtures

+

+ Ready-to-use fixture examples for every mock type. Copy any example to get started + quickly, or use fixtures/examples/full-suite.json as a complete config + template. +

+ + + +

LLM Fixtures

+ +

Embeddings

+

Matching on inputText and returning a vector.

+
+
+ fixtures/examples/llm/embeddings.json json +
+
{
+  "fixtures": [
+    {
+      "match": { "inputText": "hello world" },
+      "response": {
+        "embedding": [0.0023064255, -0.009327292, 0.015797347]
+      }
+    }
+  ]
+}
+
+ +

Streaming Physics

+

+ Realistic streaming timing with ttft, tps, and + jitter. +

+
+
+ fixtures/examples/llm/streaming-physics.json json +
+
{
+  "fixtures": [
+    {
+      "match": { "userMessage": "explain gravity" },
+      "response": {
+        "content": "Gravity is a fundamental force of nature that attracts objects with mass toward one another. It keeps planets in orbit around the sun and holds galaxies together."
+      },
+      "streamingProfile": {
+        "ttft": 200,
+        "tps": 40,
+        "jitter": 0.1
+      }
+    }
+  ]
+}
+
+ +

Error Injection

+

Rate limit error response for any message.

+
+
+ fixtures/examples/llm/error-injection.json json +
+
{
+  "fixtures": [
+    {
+      "match": { "userMessage": ".*" },
+      "response": {
+        "error": {
+          "message": "Rate limit exceeded. Please retry after 30 seconds.",
+          "type": "rate_limit_error",
+          "code": "rate_limit_exceeded"
+        },
+        "status": 429
+      }
+    }
+  ]
+}
+
+ +

Sequential Responses

+

Stateful multi-turn responses using sequenceIndex.

+
+
+ fixtures/examples/llm/sequential-responses.json json +
+
{
+  "fixtures": [
+    {
+      "match": { "userMessage": "tell me a joke", "sequenceIndex": 0 },
+      "response": {
+        "content": "Why did the programmer quit his job? Because he didn't get arrays!"
+      }
+    },
+    {
+      "match": { "userMessage": "tell me a joke", "sequenceIndex": 1 },
+      "response": { "content": "Why do Java developers wear glasses? Because they can't C#!" }
+    },
+    {
+      "match": { "userMessage": "tell me a joke", "sequenceIndex": 2 },
+      "response": {
+        "content": "A SQL query walks into a bar, sees two tables, and asks: 'Can I join you?'"
+      }
+    }
+  ]
+}
+
+ + + +

Protocol Configs

+ +

MCP

+

Tools and resources for the Model Context Protocol mock.

+
+
+ fixtures/examples/mcp/mcp-config.json json +
+
{
+  "mcp": {
+    "tools": [
+      {
+        "name": "search",
+        "description": "Search the web",
+        "inputSchema": {
+          "type": "object",
+          "properties": {
+            "query": { "type": "string" }
+          },
+          "required": ["query"]
+        },
+        "result": "No results found for the given query."
+      }
+    ],
+    "resources": [
+      {
+        "uri": "file:///readme",
+        "name": "README",
+        "mimeType": "text/plain",
+        "text": "# My Project\n\nThis is the project README."
+      }
+    ]
+  }
+}
+
+ +

A2A

+

Agent registration and streaming tasks for the Agent-to-Agent protocol mock.

+
+
+ fixtures/examples/a2a/a2a-config.json json +
+
{
+  "a2a": {
+    "agents": [
+      {
+        "name": "research-agent",
+        "description": "An agent that researches topics and returns summaries",
+        "version": "1.0.0",
+        "skills": [
+          {
+            "id": "web-research",
+            "name": "Web Research",
+            "description": "Search the web and summarize findings",
+            "tags": ["research", "search"]
+          }
+        ],
+        "capabilities": { "streaming": true },
+        "messages": [
+          {
+            "pattern": "research",
+            "parts": [{ "text": "Here is a summary of my research findings on the topic." }]
+          }
+        ],
+        "streamingTasks": [
+          {
+            "pattern": "deep-research",
+            "events": [
+              { "type": "status", "state": "TASK_STATE_WORKING" },
+              {
+                "type": "artifact",
+                "name": "research-report",
+                "parts": [{ "text": "## Research Report\n\nFindings from deep research..." }],
+                "lastChunk": true
+              },
+              { "type": "status", "state": "TASK_STATE_COMPLETED" }
+            ],
+            "delayMs": 100
+          }
+        ]
+      }
+    ]
+  }
+}
+
+ +

AG-UI

+

Text response and tool call event stream for the AG-UI protocol mock.

+
+
+ fixtures/examples/agui/agui-text-response.json json +
+
{
+  "agui": {
+    "fixtures": [
+      {
+        "match": { "message": "hello" },
+        "text": "Hi! How can I help you today?"
+      },
+      {
+        "match": { "message": "search" },
+        "events": [
+          { "type": "RUN_STARTED", "threadId": "t1", "runId": "r1" },
+          { "type": "TOOL_CALL_START", "toolCallId": "tc1", "toolCallName": "web_search" },
+          {
+            "type": "TOOL_CALL_ARGS",
+            "toolCallId": "tc1",
+            "delta": "{\"query\": \"latest news\"}"
+          },
+          { "type": "TOOL_CALL_END", "toolCallId": "tc1" },
+          {
+            "type": "TEXT_MESSAGE_START",
+            "messageId": "m1",
+            "role": "assistant"
+          },
+          {
+            "type": "TEXT_MESSAGE_CONTENT",
+            "messageId": "m1",
+            "delta": "Here are the latest results..."
+          },
+          { "type": "TEXT_MESSAGE_END", "messageId": "m1" },
+          { "type": "RUN_FINISHED", "threadId": "t1", "runId": "r1" }
+        ]
+      }
+    ]
+  }
+}
+
+ +

Vector

+

Collection with vectors and query results for the vector database mock.

+
+
+ fixtures/examples/vector/vector-config.json json +
+
{
+  "vector": {
+    "collections": [
+      {
+        "name": "documents",
+        "dimension": 3,
+        "vectors": [
+          {
+            "id": "doc-1",
+            "values": [0.1, 0.2, 0.3],
+            "metadata": { "title": "Getting Started", "category": "tutorial" }
+          },
+          {
+            "id": "doc-2",
+            "values": [0.4, 0.5, 0.6],
+            "metadata": { "title": "API Reference", "category": "reference" }
+          }
+        ],
+        "queryResults": [
+          {
+            "id": "doc-1",
+            "score": 0.95,
+            "metadata": { "title": "Getting Started", "category": "tutorial" }
+          },
+          {
+            "id": "doc-2",
+            "score": 0.82,
+            "metadata": { "title": "API Reference", "category": "reference" }
+          }
+        ]
+      }
+    ]
+  }
+}
+
+ + + +

Testing & Operations

+ +

Chaos Testing

+

Configure drop, malformed, and disconnect rates for chaos testing.

+
+
+ fixtures/examples/chaos/chaos-config.json json +
+
{
+  "llm": {
+    "fixtures": "fixtures/example-greeting.json",
+    "chaos": {
+      "dropRate": 0.1,
+      "malformedRate": 0.05,
+      "disconnectRate": 0.02
+    }
+  }
+}
+
+ +

Record & Replay

+

Provider URLs for proxy mode to record live API traffic.

+
+
+ fixtures/examples/record-replay/record-config.json json +
+
{
+  "llm": {
+    "record": {
+      "providers": {
+        "openai": "https://api.openai.com/v1",
+        "anthropic": "https://api.anthropic.com"
+      },
+      "fixturePath": "./recorded-fixtures"
+    }
+  }
+}
+
+ + + +

Full Suite

+ +

Complete Config

+

A complete configuration running all mocks on one port.

+
+
+ fixtures/examples/full-suite.json json +
+
{
+  "port": 4000,
+  "host": "127.0.0.1",
+  "metrics": true,
+  "strict": true,
+
+  "llm": {
+    "fixtures": "fixtures/example-greeting.json",
+    "chaos": {
+      "dropRate": 0.01,
+      "malformedRate": 0.005,
+      "disconnectRate": 0.002
+    }
+  },
+
+  "mcp": {
+    "serverInfo": { "name": "full-suite-mcp", "version": "1.0.0" },
+    "tools": [
+      {
+        "name": "search",
+        "description": "Search the knowledge base",
+        "inputSchema": {
+          "type": "object",
+          "properties": {
+            "query": { "type": "string" },
+            "limit": { "type": "number" }
+          },
+          "required": ["query"]
+        },
+        "result": "Found 3 results for your query."
+      }
+    ],
+    "resources": [
+      {
+        "uri": "file:///config",
+        "name": "Configuration",
+        "mimeType": "application/json",
+        "text": "{\"version\": \"1.0\", \"environment\": \"test\"}"
+      }
+    ],
+    "prompts": [
+      {
+        "name": "summarize",
+        "description": "Summarize a document",
+        "arguments": [{ "name": "text", "description": "The text to summarize", "required": true }],
+        "result": {
+          "messages": [
+            {
+              "role": "assistant",
+              "content": { "type": "text", "text": "Here is a summary of the provided text." }
+            }
+          ]
+        }
+      }
+    ]
+  },
+
+  "a2a": {
+    "agents": [
+      {
+        "name": "assistant",
+        "description": "A general-purpose assistant agent",
+        "version": "1.0.0",
+        "skills": [{ "id": "qa", "name": "Q&A", "description": "Answer questions" }],
+        "capabilities": { "streaming": true },
+        "messages": [
+          {
+            "pattern": ".*",
+            "parts": [{ "text": "I can help you with that." }]
+          }
+        ]
+      }
+    ]
+  },
+
+  "agui": {
+    "fixtures": [
+      {
+        "match": { "message": "hello" },
+        "text": "Hello from the full-suite mock!"
+      },
+      {
+        "match": { "toolName": "get_data" },
+        "events": [
+          { "type": "RUN_STARTED", "threadId": "t1", "runId": "r1" },
+          { "type": "TOOL_CALL_START", "toolCallId": "tc1", "toolCallName": "get_data" },
+          { "type": "TOOL_CALL_ARGS", "toolCallId": "tc1", "delta": "{}" },
+          { "type": "TOOL_CALL_END", "toolCallId": "tc1" },
+          { "type": "RUN_FINISHED", "threadId": "t1", "runId": "r1" }
+        ]
+      }
+    ]
+  },
+
+  "vector": {
+    "collections": [
+      {
+        "name": "knowledge-base",
+        "dimension": 384,
+        "queryResults": [
+          {
+            "id": "kb-001",
+            "score": 0.97,
+            "metadata": { "source": "docs", "title": "Quick Start Guide" }
+          }
+        ]
+      }
+    ]
+  },
+
+  "services": {
+    "search": true,
+    "rerank": true,
+    "moderate": true
+  }
+}
+
+
+ +
+ + + + + diff --git a/docs/github-action/index.html b/docs/github-action/index.html new file mode 100644 index 0000000..dfbedcd --- /dev/null +++ b/docs/github-action/index.html @@ -0,0 +1,212 @@ + + + + + + GitHub Action — aimock + + + + + + + + + +
+ + +
+

GitHub Action

+

+ One-line CI setup. Start an aimock server in GitHub Actions with a single + uses: step. +

+ +

Quick Start

+
+
workflow.yml yaml
+
- uses: CopilotKit/aimock@v1
+  with:
+    fixtures: ./test/fixtures
+
+- run: pnpm test
+  env:
+    OPENAI_BASE_URL: http://127.0.0.1:4010/v1
+
+ +

Inputs

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
InputDefaultDescription
fixtures./fixturesPath to fixture files or directory
configPath to aimock JSON config file (overrides fixtures)
port4010Port to listen on
host127.0.0.1Host to bind to
versionlatestaimock version to install
argsAdditional CLI arguments
wait-timeout30Max seconds to wait for health check
+ +

Outputs

+ + + + + + + + + + + + + +
OutputDescription
urlThe aimock server URL (e.g., http://127.0.0.1:4010)
+ +

Examples

+ +

Basic with fixtures

+
+
workflow.yml yaml
+
steps:
+  - uses: actions/checkout@v4
+  - uses: CopilotKit/aimock@v1
+    with:
+      fixtures: ./fixtures
+  - run: npm test
+    env:
+      OPENAI_BASE_URL: http://127.0.0.1:4010/v1
+
+ +

Full suite with config

+
+
workflow.yml yaml
+
steps:
+  - uses: actions/checkout@v4
+  - uses: CopilotKit/aimock@v1
+    with:
+      config: ./aimock.json
+      args: --strict
+  - run: npm test
+
+ +

Record mode (proxy to real APIs)

+
+
workflow.yml yaml
+
steps:
+  - uses: actions/checkout@v4
+  - uses: CopilotKit/aimock@v1
+    with:
+      fixtures: ./fixtures
+      args: --record --provider-openai https://api.openai.com
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+  - run: npm test
+
+ +

Using the URL output

+
+
workflow.yml yaml
+
steps:
+  - uses: CopilotKit/aimock@v1
+    id: mock
+    with:
+      fixtures: ./fixtures
+  - run: echo "Mock server at ${{ steps.mock.outputs.url }}"
+
+ +

How It Works

+
    +
  1. Installs @copilotkit/aimock via npm
  2. +
  3. Starts the server in the background
  4. +
  5. + Polls /health until the server is ready (up to + wait-timeout seconds) +
  6. +
  7. Exports the URL as a step output
  8. +
  9. The server runs for the duration of the job and is cleaned up automatically
  10. +
+
+ +
+ + + + + diff --git a/docs/index.html b/docs/index.html index 406061d..444ea5f 100644 --- a/docs/index.html +++ b/docs/index.html @@ -1415,6 +1415,15 @@

Multimedia APIs

+
🧪
+

Vitest & Jest Plugins

+

+ Zero-config useAimock() with auto server lifecycle, env var patching, and + match count reset. +

+
+ +
📊

Drift Detection

Fixtures stay accurate as providers evolve. Fixes ship before your tests break.

@@ -1699,6 +1708,22 @@

How aimock compares

+ + GitHub Action + Built-in ✓ + + + + + + + Vitest / Jest plugins + Built-in ✓ + + + + + Vector DB mocking Built-in ✓ @@ -1789,7 +1814,8 @@

Built for production

fixture-driven responses - in the codebase. + in the codebase. Browse the + example fixtures to get started.

diff --git a/docs/sidebar.js b/docs/sidebar.js index 3159755..305ee9b 100644 --- a/docs/sidebar.js +++ b/docs/sidebar.js @@ -9,6 +9,7 @@ { label: "Record & Replay", href: "/record-replay" }, { label: "Quick Start: LLM", href: "/chat-completions" }, { label: "Quick Start: aimock", href: "/aimock-cli" }, + { label: "Examples", href: "/examples" }, ], }, { @@ -65,6 +66,8 @@ links: [ { label: "aimock CLI & Config", href: "/aimock-cli" }, { label: "Docker & Helm", href: "/docker" }, + { label: "GitHub Action", href: "/github-action" }, + { label: "Test Plugins", href: "/test-plugins" }, { label: "Drift Detection", href: "/drift-detection" }, ], }, diff --git a/docs/style.css b/docs/style.css index 1209062..aff06be 100644 --- a/docs/style.css +++ b/docs/style.css @@ -154,7 +154,7 @@ body::before { top: 57px; /* nav height */ left: 0; width: var(--sidebar-width); - height: calc(100vh - 57px - 50px); + height: calc(100vh - 57px); overflow-y: auto; background: var(--bg-surface); border-right: 1px solid var(--border); diff --git a/docs/test-plugins/index.html b/docs/test-plugins/index.html new file mode 100644 index 0000000..9ee9b52 --- /dev/null +++ b/docs/test-plugins/index.html @@ -0,0 +1,224 @@ + + + + + + Test Framework Plugins — aimock + + + + + + + + + +
+ + +
+

Test Framework Plugins

+

+ Zero-config integration for vitest and jest. Import useAimock, write tests + — the server lifecycle, env vars, and cleanup are handled automatically. +

+ +

Vitest

+
+
test/app.test.ts ts
+
import { useAimock } from "@copilotkit/aimock/vitest";
+
+const mock = useAimock({ fixtures: "./fixtures" });
+
+it("responds to hello", async () => {
+  // OPENAI_BASE_URL is already set
+  const res = await myApp.chat("hello");
+  expect(res).toBe("Hi there!");
+});
+
+ +

Jest

+
+
test/app.test.ts ts
+
import { useAimock } from "@copilotkit/aimock/jest";
+
+const mock = useAimock({ fixtures: "./fixtures" });
+
+it("responds to hello", async () => {
+  const res = await myApp.chat("hello");
+  expect(res).toBe("Hi there!");
+});
+
+ +

Options

+

Both plugins accept the same UseAimockOptions object:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
OptionTypeDefaultDescription
fixturesstringPath to fixture file or directory
patchEnvbooleantrueAuto-set OPENAI_BASE_URL and ANTHROPIC_BASE_URL
portnumber0 (random)Port to listen on
hoststring127.0.0.1Host to bind to
logLevelstringsilentLog verbosity
+ +

The Handle

+

+ The getter function returned by useAimock() returns an + AimockHandle: +

+ + + + + + + + + + + + + + + + + + + + +
PropertyTypeDescription
llmLLMockThe LLMock instance — add fixtures programmatically
urlstringThe server URL (e.g., http://127.0.0.1:4010)
+ +

Programmatic fixture registration

+
+
test/custom.test.ts ts
+
const mock = useAimock();
+
+it("custom fixture", async () => {
+  mock().llm.onMessage("custom", { content: "Custom response" });
+  // ...
+});
+
+ +

Lifecycle

+

Both plugins register framework hooks to manage the server automatically:

+
    +
  1. + beforeAll — starts the server, loads fixtures from + the fixtures path, and patches OPENAI_BASE_URL / + ANTHROPIC_BASE_URL environment variables +
  2. +
  3. + beforeEach — resets match counts (sequential fixture + counters return to zero, but fixtures themselves are preserved) +
  4. +
  5. + afterAll — stops the server and restores the + original environment variables +
  6. +
+ +

Without Plugins (Manual)

+

+ For comparison, here is the equivalent manual setup. The plugins above handle all of this + for you: +

+
+
test/manual.test.ts ts
+
import { LLMock } from "@copilotkit/aimock";
+
+let mock: LLMock;
+
+beforeAll(async () => {
+  mock = new LLMock();
+  mock.onMessage("hello", { content: "Hi!" });
+  await mock.start();
+  process.env.OPENAI_BASE_URL = `${mock.url}/v1`;
+});
+
+afterAll(async () => {
+  await mock.stop();
+  delete process.env.OPENAI_BASE_URL;
+});
+
+
+ +
+ + + + + diff --git a/fixtures/examples/a2a/a2a-config.json b/fixtures/examples/a2a/a2a-config.json new file mode 100644 index 0000000..816a765 --- /dev/null +++ b/fixtures/examples/a2a/a2a-config.json @@ -0,0 +1,42 @@ +{ + "a2a": { + "agents": [ + { + "name": "research-agent", + "description": "An agent that researches topics and returns summaries", + "version": "1.0.0", + "skills": [ + { + "id": "web-research", + "name": "Web Research", + "description": "Search the web and summarize findings", + "tags": ["research", "search"] + } + ], + "capabilities": { "streaming": true }, + "messages": [ + { + "pattern": "research", + "parts": [{ "text": "Here is a summary of my research findings on the topic." }] + } + ], + "streamingTasks": [ + { + "pattern": "deep-research", + "events": [ + { "type": "status", "state": "TASK_STATE_WORKING" }, + { + "type": "artifact", + "name": "research-report", + "parts": [{ "text": "## Research Report\n\nFindings from deep research..." }], + "lastChunk": true + }, + { "type": "status", "state": "TASK_STATE_COMPLETED" } + ], + "delayMs": 100 + } + ] + } + ] + } +} diff --git a/fixtures/examples/agui/agui-text-response.json b/fixtures/examples/agui/agui-text-response.json new file mode 100644 index 0000000..55ede05 --- /dev/null +++ b/fixtures/examples/agui/agui-text-response.json @@ -0,0 +1,35 @@ +{ + "agui": { + "fixtures": [ + { + "match": { "message": "hello" }, + "text": "Hi! How can I help you today?" + }, + { + "match": { "message": "search" }, + "events": [ + { "type": "RUN_STARTED", "threadId": "t1", "runId": "r1" }, + { "type": "TOOL_CALL_START", "toolCallId": "tc1", "toolCallName": "web_search" }, + { + "type": "TOOL_CALL_ARGS", + "toolCallId": "tc1", + "delta": "{\"query\": \"latest news\"}" + }, + { "type": "TOOL_CALL_END", "toolCallId": "tc1" }, + { + "type": "TEXT_MESSAGE_START", + "messageId": "m1", + "role": "assistant" + }, + { + "type": "TEXT_MESSAGE_CONTENT", + "messageId": "m1", + "delta": "Here are the latest results..." + }, + { "type": "TEXT_MESSAGE_END", "messageId": "m1" }, + { "type": "RUN_FINISHED", "threadId": "t1", "runId": "r1" } + ] + } + ] + } +} diff --git a/fixtures/examples/chaos/chaos-config.json b/fixtures/examples/chaos/chaos-config.json new file mode 100644 index 0000000..d00814a --- /dev/null +++ b/fixtures/examples/chaos/chaos-config.json @@ -0,0 +1,10 @@ +{ + "llm": { + "fixtures": "fixtures/example-greeting.json", + "chaos": { + "dropRate": 0.1, + "malformedRate": 0.05, + "disconnectRate": 0.02 + } + } +} diff --git a/fixtures/examples/full-suite.json b/fixtures/examples/full-suite.json new file mode 100644 index 0000000..32e9284 --- /dev/null +++ b/fixtures/examples/full-suite.json @@ -0,0 +1,116 @@ +{ + "port": 4000, + "host": "127.0.0.1", + "metrics": true, + "strict": true, + + "llm": { + "fixtures": "fixtures/example-greeting.json", + "chaos": { + "dropRate": 0.01, + "malformedRate": 0.005, + "disconnectRate": 0.002 + } + }, + + "mcp": { + "serverInfo": { "name": "full-suite-mcp", "version": "1.0.0" }, + "tools": [ + { + "name": "search", + "description": "Search the knowledge base", + "inputSchema": { + "type": "object", + "properties": { + "query": { "type": "string" }, + "limit": { "type": "number" } + }, + "required": ["query"] + }, + "result": "Found 3 results for your query." + } + ], + "resources": [ + { + "uri": "file:///config", + "name": "Configuration", + "mimeType": "application/json", + "text": "{\"version\": \"1.0\", \"environment\": \"test\"}" + } + ], + "prompts": [ + { + "name": "summarize", + "description": "Summarize a document", + "arguments": [{ "name": "text", "description": "The text to summarize", "required": true }], + "result": { + "messages": [ + { + "role": "assistant", + "content": { "type": "text", "text": "Here is a summary of the provided text." } + } + ] + } + } + ] + }, + + "a2a": { + "agents": [ + { + "name": "assistant", + "description": "A general-purpose assistant agent", + "version": "1.0.0", + "skills": [{ "id": "qa", "name": "Q&A", "description": "Answer questions" }], + "capabilities": { "streaming": true }, + "messages": [ + { + "pattern": ".*", + "parts": [{ "text": "I can help you with that." }] + } + ] + } + ] + }, + + "agui": { + "fixtures": [ + { + "match": { "message": "hello" }, + "text": "Hello from the full-suite mock!" + }, + { + "match": { "toolName": "get_data" }, + "events": [ + { "type": "RUN_STARTED", "threadId": "t1", "runId": "r1" }, + { "type": "TOOL_CALL_START", "toolCallId": "tc1", "toolCallName": "get_data" }, + { "type": "TOOL_CALL_ARGS", "toolCallId": "tc1", "delta": "{}" }, + { "type": "TOOL_CALL_END", "toolCallId": "tc1" }, + { "type": "RUN_FINISHED", "threadId": "t1", "runId": "r1" } + ] + } + ] + }, + + "vector": { + "collections": [ + { + "name": "knowledge-base", + "dimension": 384, + "queryResults": [ + { + "id": "kb-001", + "score": 0.97, + "metadata": { "source": "docs", "title": "Quick Start Guide" } + } + ] + } + ] + }, + + "services": { + "search": true, + "rerank": true, + "moderate": true + } +} diff --git a/fixtures/examples/llm/embeddings.json b/fixtures/examples/llm/embeddings.json new file mode 100644 index 0000000..1b61bfe --- /dev/null +++ b/fixtures/examples/llm/embeddings.json @@ -0,0 +1,10 @@ +{ + "fixtures": [ + { + "match": { "inputText": "hello world" }, + "response": { + "embedding": [0.0023064255, -0.009327292, 0.015797347] + } + } + ] +} diff --git a/fixtures/examples/llm/error-injection.json b/fixtures/examples/llm/error-injection.json new file mode 100644 index 0000000..6b1ce54 --- /dev/null +++ b/fixtures/examples/llm/error-injection.json @@ -0,0 +1,15 @@ +{ + "fixtures": [ + { + "match": { "userMessage": ".*" }, + "response": { + "error": { + "message": "Rate limit exceeded. Please retry after 30 seconds.", + "type": "rate_limit_error", + "code": "rate_limit_exceeded" + }, + "status": 429 + } + } + ] +} diff --git a/fixtures/examples/llm/sequential-responses.json b/fixtures/examples/llm/sequential-responses.json new file mode 100644 index 0000000..221d0be --- /dev/null +++ b/fixtures/examples/llm/sequential-responses.json @@ -0,0 +1,20 @@ +{ + "fixtures": [ + { + "match": { "userMessage": "tell me a joke", "sequenceIndex": 0 }, + "response": { + "content": "Why did the programmer quit his job? Because he didn't get arrays!" + } + }, + { + "match": { "userMessage": "tell me a joke", "sequenceIndex": 1 }, + "response": { "content": "Why do Java developers wear glasses? Because they can't C#!" } + }, + { + "match": { "userMessage": "tell me a joke", "sequenceIndex": 2 }, + "response": { + "content": "A SQL query walks into a bar, sees two tables, and asks: 'Can I join you?'" + } + } + ] +} diff --git a/fixtures/examples/llm/streaming-physics.json b/fixtures/examples/llm/streaming-physics.json new file mode 100644 index 0000000..82ddaf9 --- /dev/null +++ b/fixtures/examples/llm/streaming-physics.json @@ -0,0 +1,15 @@ +{ + "fixtures": [ + { + "match": { "userMessage": "explain gravity" }, + "response": { + "content": "Gravity is a fundamental force of nature that attracts objects with mass toward one another. It keeps planets in orbit around the sun and holds galaxies together." + }, + "streamingProfile": { + "ttft": 200, + "tps": 40, + "jitter": 0.1 + } + } + ] +} diff --git a/fixtures/examples/mcp/mcp-config.json b/fixtures/examples/mcp/mcp-config.json new file mode 100644 index 0000000..dee9f0b --- /dev/null +++ b/fixtures/examples/mcp/mcp-config.json @@ -0,0 +1,26 @@ +{ + "mcp": { + "tools": [ + { + "name": "search", + "description": "Search the web", + "inputSchema": { + "type": "object", + "properties": { + "query": { "type": "string" } + }, + "required": ["query"] + }, + "result": "No results found for the given query." + } + ], + "resources": [ + { + "uri": "file:///readme", + "name": "README", + "mimeType": "text/plain", + "text": "# My Project\n\nThis is the project README." + } + ] + } +} diff --git a/fixtures/examples/record-replay/record-config.json b/fixtures/examples/record-replay/record-config.json new file mode 100644 index 0000000..7409b96 --- /dev/null +++ b/fixtures/examples/record-replay/record-config.json @@ -0,0 +1,11 @@ +{ + "llm": { + "record": { + "providers": { + "openai": "https://api.openai.com/v1", + "anthropic": "https://api.anthropic.com" + }, + "fixturePath": "./recorded-fixtures" + } + } +} diff --git a/fixtures/examples/vector/vector-config.json b/fixtures/examples/vector/vector-config.json new file mode 100644 index 0000000..3babb07 --- /dev/null +++ b/fixtures/examples/vector/vector-config.json @@ -0,0 +1,34 @@ +{ + "vector": { + "collections": [ + { + "name": "documents", + "dimension": 3, + "vectors": [ + { + "id": "doc-1", + "values": [0.1, 0.2, 0.3], + "metadata": { "title": "Getting Started", "category": "tutorial" } + }, + { + "id": "doc-2", + "values": [0.4, 0.5, 0.6], + "metadata": { "title": "API Reference", "category": "reference" } + } + ], + "queryResults": [ + { + "id": "doc-1", + "score": 0.95, + "metadata": { "title": "Getting Started", "category": "tutorial" } + }, + { + "id": "doc-2", + "score": 0.82, + "metadata": { "title": "API Reference", "category": "reference" } + } + ] + } + ] + } +} diff --git a/package.json b/package.json index 6ff5c19..cb374dc 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,40 @@ { "name": "@copilotkit/aimock", - "version": "1.12.0", - "description": "Mock infrastructure for AI application testing — LLM APIs, MCP tools, A2A agents, AG-UI event streams, vector databases, search, and more. Zero dependencies.", + "version": "1.13.0", + "description": "Mock infrastructure for AI application testing — LLM APIs, image generation, text-to-speech, transcription, video generation, MCP tools, A2A agents, AG-UI event streams, vector databases, search, rerank, and moderation. One package, one port, zero dependencies.", "license": "MIT", + "keywords": [ + "mock", + "llm", + "openai", + "anthropic", + "claude", + "gemini", + "bedrock", + "azure-openai", + "ollama", + "cohere", + "vertex-ai", + "mcp", + "a2a", + "ag-ui", + "vector", + "pinecone", + "qdrant", + "chromadb", + "testing", + "fixtures", + "sse", + "streaming", + "websocket", + "record-replay", + "drift-detection", + "ai-testing", + "api-mock", + "http-mock", + "embeddings", + "copilotkit" + ], "repository": { "type": "git", "url": "https://github.com/CopilotKit/aimock" @@ -52,6 +84,26 @@ "types": "./dist/vector-stub.d.cts", "default": "./dist/vector-stub.cjs" } + }, + "./vitest": { + "import": { + "types": "./dist/vitest.d.ts", + "default": "./dist/vitest.js" + }, + "require": { + "types": "./dist/vitest.d.cts", + "default": "./dist/vitest.cjs" + } + }, + "./jest": { + "import": { + "types": "./dist/jest.d.ts", + "default": "./dist/jest.js" + }, + "require": { + "types": "./dist/jest.d.cts", + "default": "./dist/jest.cjs" + } } }, "main": "./dist/index.cjs", @@ -67,6 +119,12 @@ ], "vector": [ "./dist/vector-stub.d.ts" + ], + "vitest": [ + "./dist/vitest.d.ts" + ], + "jest": [ + "./dist/jest.d.ts" ] } }, diff --git a/scripts/convert-mockllm.ts b/scripts/convert-mockllm.ts index 23692c0..53d8a04 100644 --- a/scripts/convert-mockllm.ts +++ b/scripts/convert-mockllm.ts @@ -1,432 +1,30 @@ #!/usr/bin/env tsx /** - * mock-llm (dwmkerr) -> aimock fixture converter - * - * Parses mock-llm YAML config files and produces aimock fixture JSON. + * mock-llm (dwmkerr) -> aimock fixture converter (standalone script) * * Usage: * npx tsx scripts/convert-mockllm.ts [output.json] * - * If output is omitted, prints to stdout. + * Core logic lives in src/convert-mockllm.ts — this script is a thin CLI + * wrapper. Prefer `npx aimock convert mockllm` instead. */ import { readFileSync, writeFileSync } from "node:fs"; import { resolve } from "node:path"; - -// --------------------------------------------------------------------------- -// Minimal YAML parser -// --------------------------------------------------------------------------- -// Handles the subset used by mock-llm configs: indented maps, arrays with -// `-` prefix, quoted/unquoted strings, numbers, booleans, and null. -// Does NOT handle: anchors, aliases, multi-line scalars, flow collections, -// tags, or other advanced YAML features. - -interface YamlLine { - indent: number; - raw: string; - content: string; // trimmed, without trailing comment - isArrayItem: boolean; - arrayItemContent: string; // content after "- " -} - -function tokenizeYamlLines(input: string): YamlLine[] { - const lines: YamlLine[] = []; - for (const raw of input.split("\n")) { - // Skip blank lines and full-line comments - const trimmed = raw.trimStart(); - if (trimmed === "" || trimmed.startsWith("#")) continue; - - const indent = raw.length - raw.trimStart().length; - // Strip trailing comments (but not inside quoted strings) - const content = stripTrailingComment(trimmed); - const isArrayItem = content.startsWith("- "); - const arrayItemContent = isArrayItem ? content.slice(2).trim() : ""; - - lines.push({ indent, raw, content, isArrayItem, arrayItemContent }); - } - return lines; -} - -function stripTrailingComment(s: string): string { - // Naive: find # not inside quotes - let inSingle = false; - let inDouble = false; - for (let i = 0; i < s.length; i++) { - const ch = s[i]; - if (ch === "'" && !inDouble) inSingle = !inSingle; - if (ch === '"' && !inSingle) inDouble = !inDouble; - if (ch === "#" && !inSingle && !inDouble && i > 0 && s[i - 1] === " ") { - return s.slice(0, i).trimEnd(); - } - } - return s; -} - -function parseScalar(value: string): unknown { - if (value === "" || value === "~" || value === "null") return null; - if (value === "true") return true; - if (value === "false") return false; - - // Quoted string - if ( - (value.startsWith('"') && value.endsWith('"')) || - (value.startsWith("'") && value.endsWith("'")) - ) { - return value.slice(1, -1); - } - - // Number - const num = Number(value); - if (!Number.isNaN(num) && value !== "") return num; - - // Unquoted string - return value; -} - -export function parseSimpleYaml(input: string): unknown { - const lines = tokenizeYamlLines(input); - if (lines.length === 0) return null; - - const result = parseBlock(lines, 0, 0); - return result.value; -} - -interface ParseResult { - value: unknown; - nextIndex: number; -} - -function parseBlock(lines: YamlLine[], startIndex: number, minIndent: number): ParseResult { - if (startIndex >= lines.length) { - return { value: null, nextIndex: startIndex }; - } - - const line = lines[startIndex]; - - // Determine if this block is an array or a map - if (line.isArrayItem && line.indent >= minIndent) { - return parseArray(lines, startIndex, line.indent); - } - - // Map - if (line.content.includes(":")) { - return parseMap(lines, startIndex, line.indent); - } - - // Single scalar - return { value: parseScalar(line.content), nextIndex: startIndex + 1 }; -} - -function parseArray(lines: YamlLine[], startIndex: number, baseIndent: number): ParseResult { - const arr: unknown[] = []; - let i = startIndex; - - while (i < lines.length) { - const line = lines[i]; - if (line.indent < baseIndent) break; - if (line.indent > baseIndent) break; // shouldn't happen at array level - if (!line.isArrayItem) break; - - const itemContent = line.arrayItemContent; - - if (itemContent === "") { - // Array item with nested block on next lines - const nested = parseBlock(lines, i + 1, baseIndent + 1); - arr.push(nested.value); - i = nested.nextIndex; - } else if (itemContent.includes(":")) { - // Inline map start: "- key: value" possibly with more keys below - // Parse as a map, treating the "- " offset as extra indent - const inlineMap = parseArrayItemMap(lines, i, baseIndent); - arr.push(inlineMap.value); - i = inlineMap.nextIndex; - } else { - // Simple scalar array item - arr.push(parseScalar(itemContent)); - i++; - } - } - - return { value: arr, nextIndex: i }; -} - -function parseArrayItemMap( - lines: YamlLine[], - startIndex: number, - arrayIndent: number, -): ParseResult { - // First line is "- key: value", subsequent lines at indent > arrayIndent are part of this map - const map: Record = {}; - const firstLine = lines[startIndex]; - const firstContent = firstLine.arrayItemContent; - - // Parse the first key: value from the array item line - const colonIdx = findColon(firstContent); - if (colonIdx === -1) { - return { value: parseScalar(firstContent), nextIndex: startIndex + 1 }; - } - - const key = firstContent.slice(0, colonIdx).trim(); - const valueStr = firstContent.slice(colonIdx + 1).trim(); - - if (valueStr === "") { - // Value is a nested block - const nested = parseBlock(lines, startIndex + 1, arrayIndent + 2); - map[key] = nested.value; - let i = nested.nextIndex; - - // Continue reading sibling keys at the array-item's content indent - const siblingIndent = arrayIndent + 2; - while (i < lines.length && lines[i].indent >= siblingIndent && !lines[i].isArrayItem) { - if (lines[i].indent === siblingIndent || lines[i].indent > siblingIndent) { - // Only parse if at exactly sibling indent and is a map key - if (lines[i].indent === siblingIndent && lines[i].content.includes(":")) { - const mapResult = parseMapEntries(lines, i, siblingIndent, map); - i = mapResult.nextIndex; - } else { - break; - } - } - } - - return { value: map, nextIndex: i }; - } else { - map[key] = parseScalar(valueStr); - } - - // Read additional keys at indent > arrayIndent (the " key: value" lines after "- first: val") - let i = startIndex + 1; - const contentIndent = arrayIndent + 2; // "- " adds 2 to effective indent - - while (i < lines.length) { - const line = lines[i]; - if (line.indent < contentIndent) break; - if (line.isArrayItem && line.indent <= arrayIndent) break; - - if (line.indent === contentIndent && !line.isArrayItem && line.content.includes(":")) { - const colonPos = findColon(line.content); - if (colonPos === -1) break; - const k = line.content.slice(0, colonPos).trim(); - const v = line.content.slice(colonPos + 1).trim(); - - if (v === "") { - const nested = parseBlock(lines, i + 1, contentIndent + 1); - map[k] = nested.value; - i = nested.nextIndex; - } else { - map[k] = parseScalar(v); - i++; - } - } else if (line.indent === contentIndent && line.isArrayItem) { - // This is a new array item at the same level -- not part of this map - break; - } else if (line.indent > contentIndent) { - // Skip nested content already consumed - i++; - } else { - break; - } - } - - return { value: map, nextIndex: i }; -} - -function parseMap(lines: YamlLine[], startIndex: number, baseIndent: number): ParseResult { - const map: Record = {}; - const result = parseMapEntries(lines, startIndex, baseIndent, map); - return { value: map, nextIndex: result.nextIndex }; -} - -function parseMapEntries( - lines: YamlLine[], - startIndex: number, - baseIndent: number, - map: Record, -): ParseResult { - let i = startIndex; - - while (i < lines.length) { - const line = lines[i]; - if (line.indent < baseIndent) break; - if (line.indent > baseIndent) { - // Shouldn't happen at map level if properly structured -- skip - i++; - continue; - } - if (line.isArrayItem) break; - - const colonIdx = findColon(line.content); - if (colonIdx === -1) { - // Not a map entry - break; - } - - const key = line.content.slice(0, colonIdx).trim(); - const valueStr = line.content.slice(colonIdx + 1).trim(); - - if (valueStr === "") { - // Value is a nested block on subsequent lines - const nested = parseBlock(lines, i + 1, baseIndent + 1); - map[key] = nested.value; - i = nested.nextIndex; - } else { - map[key] = parseScalar(valueStr); - i++; - } - } - - return { value: map, nextIndex: i }; -} - -function findColon(s: string): number { - let inSingle = false; - let inDouble = false; - for (let i = 0; i < s.length; i++) { - const ch = s[i]; - if (ch === "'" && !inDouble) inSingle = !inSingle; - if (ch === '"' && !inSingle) inDouble = !inDouble; - if (ch === ":" && !inSingle && !inDouble) { - // Must be followed by space, end of line, or nothing - if (i === s.length - 1 || s[i + 1] === " ") { - return i; - } - } - } - return -1; -} - -// --------------------------------------------------------------------------- -// mock-llm config types -// --------------------------------------------------------------------------- - -export interface MockLLMRoute { - path: string; - method?: string; - match?: { - body?: { - messages?: Array<{ role: string; content: string }>; - }; - }; - response: Record; -} - -export interface MockLLMTool { - name: string; - description?: string; - parameters?: Record; -} - -export interface MockLLMConfig { - routes?: MockLLMRoute[]; - mcp?: { - tools?: MockLLMTool[]; - }; -} - -// --------------------------------------------------------------------------- -// aimock output types -// --------------------------------------------------------------------------- - -export interface AimockFixture { - match?: { userMessage?: string }; - response: { content?: string; toolCalls?: Array<{ name: string; arguments: string }> }; - _comment?: string; -} - -export interface AimockMCPTool { - name: string; - description?: string; - inputSchema?: Record; -} - -export interface ConvertResult { - fixtures: AimockFixture[]; - mcpTools?: AimockMCPTool[]; -} - -// --------------------------------------------------------------------------- -// Converter -// --------------------------------------------------------------------------- - -export function convertConfig(config: MockLLMConfig): ConvertResult { - const fixtures: AimockFixture[] = []; - - if (config.routes) { - for (const route of config.routes) { - const fixture = convertRoute(route); - if (fixture) { - fixtures.push(fixture); - } - } - } - - const result: ConvertResult = { fixtures }; - - if (config.mcp?.tools && config.mcp.tools.length > 0) { - result.mcpTools = config.mcp.tools.map(convertMCPTool); - } - - return result; -} - -function convertRoute(route: MockLLMRoute): AimockFixture | null { - // Extract content from response.choices[0].message.content - const content = extractResponseContent(route.response); - if (content === null) return null; - - const fixture: AimockFixture = { - match: {}, - response: { content }, - }; - - // Extract match criteria from match.body.messages - const userMessage = extractUserMessage(route); - if (userMessage) { - fixture.match = { userMessage }; - } else { - // Use path as a comment/identifier when no match criteria - fixture._comment = `${route.method ?? "POST"} ${route.path}`; - } - - return fixture; -} - -function extractResponseContent(response: Record): string | null { - const choices = response.choices as Array> | undefined; - if (!Array.isArray(choices) || choices.length === 0) return null; - - const firstChoice = choices[0]; - const message = firstChoice.message as Record | undefined; - if (!message) return null; - - const content = message.content; - if (typeof content !== "string") return null; - - return content; -} - -function extractUserMessage(route: MockLLMRoute): string | null { - const messages = route.match?.body?.messages; - if (!Array.isArray(messages) || messages.length === 0) return null; - - // Find the last user message - for (let i = messages.length - 1; i >= 0; i--) { - if (messages[i].role === "user") { - return messages[i].content; - } - } - - // Fall back to last message content regardless of role - return messages[messages.length - 1].content ?? null; -} - -function convertMCPTool(tool: MockLLMTool): AimockMCPTool { - const result: AimockMCPTool = { name: tool.name }; - if (tool.description) result.description = tool.description; - if (tool.parameters) result.inputSchema = tool.parameters; - return result; -} +import { parseSimpleYaml, convertConfig, type MockLLMConfig } from "../src/convert-mockllm.js"; + +// Re-export everything the test files import from here +export { + parseSimpleYaml, + convertConfig, + type MockLLMRoute, + type MockLLMTool, + type MockLLMConfig, + type AimockFixture, + type AimockMCPTool, + type ConvertResult, +} from "../src/convert-mockllm.js"; // --------------------------------------------------------------------------- // CLI diff --git a/scripts/convert-vidaimock.ts b/scripts/convert-vidaimock.ts index a900db7..b9adb1d 100644 --- a/scripts/convert-vidaimock.ts +++ b/scripts/convert-vidaimock.ts @@ -1,195 +1,33 @@ #!/usr/bin/env tsx /** - * VidaiMock -> aimock Fixture Converter - * - * Reads VidaiMock Tera template files (single file or directory) and produces - * aimock-compatible fixture JSON. + * VidaiMock -> aimock Fixture Converter (standalone script) * * Usage: * npx tsx scripts/convert-vidaimock.ts [output-path] * - * - If is a directory, every .tera / .json / .txt file inside it - * is treated as a VidaiMock response template. - * - If is a single file, only that file is converted. - * - [output-path] defaults to stdout when omitted; pass a path to write JSON - * to a file instead. - */ - -import { readFileSync, writeFileSync, readdirSync, statSync } from "node:fs"; -import { resolve, basename, extname } from "node:path"; - -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- - -export interface AimockFixture { - match: { userMessage: string }; - response: { content: string }; -} - -export interface AimockFixtureFile { - fixtures: AimockFixture[]; -} - -// --------------------------------------------------------------------------- -// Tera template stripping -// --------------------------------------------------------------------------- - -/** - * Strip Tera template syntax and extract a usable response content string. - * - * Strategy: - * 1. If the template looks like JSON, try to pull out the nested - * `choices[].message.content` value (the most common VidaiMock pattern). - * 2. Otherwise fall back to stripping all Tera delimiters and returning the - * remaining text with placeholder variable names. - */ -export function stripTeraTemplate(raw: string): string { - const trimmed = raw.trim(); - - // --- Attempt JSON extraction first ----------------------------------- - const contentValue = extractJsonContent(trimmed); - if (contentValue !== null) return contentValue; - - // --- Fallback: strip Tera syntax ------------------------------------- - let text = trimmed; - - // Remove comment blocks {# ... #} - text = text.replace(/\{#[\s\S]*?#\}/g, ""); - - // Remove block tags {% ... %} - text = text.replace(/\{%[\s\S]*?%\}/g, ""); - - // Replace expression tags {{ expr }} with the expression name - text = text.replace(/\{\{\s*([\w.]+)\s*\}\}/g, "[$1]"); - - // Collapse excessive whitespace but preserve intentional newlines - text = text - .split("\n") - .map((l) => l.trim()) - .filter((l) => l.length > 0) - .join("\n"); - - return text; -} - -/** - * Try to parse the template as JSON (after substituting Tera expressions with - * dummy strings) and pull out `choices[0].message.content`. - */ -function extractJsonContent(raw: string): string | null { - try { - // Step 1: remove Tera comments and blocks - let substituted = raw.replace(/\{#[\s\S]*?#\}/g, "").replace(/\{%[\s\S]*?%\}/g, ""); - - // Step 2: Replace Tera expressions inside existing JSON strings. - // Pattern: the expression is already within quotes, e.g. "foo-{{ bar }}-baz" - // We replace the {{ ... }} with the placeholder without adding extra quotes. - substituted = substituted.replace( - /"([^"]*?)\{\{\s*([\w.]+)\s*\}\}([^"]*?)"/g, - (_, before, varName, after) => `"${before}[${varName}]${after}"`, - ); - - // Step 3: Replace standalone Tera expressions (not inside quotes), - // e.g. a bare `{{ content }}` used as a JSON value — wrap with quotes. - substituted = substituted.replace(/\{\{\s*([\w.]+)\s*\}\}/g, '"[$1]"'); - - const parsed = JSON.parse(substituted); - - if ( - parsed && - Array.isArray(parsed.choices) && - parsed.choices.length > 0 && - parsed.choices[0]?.message?.content !== undefined - ) { - return String(parsed.choices[0].message.content); - } - } catch { - // Not valid JSON even after substitution — fall through - } - return null; -} - -// --------------------------------------------------------------------------- -// Filename -> match derivation -// --------------------------------------------------------------------------- - -/** - * Derive a `userMessage` match string from the template filename. - * - * Examples: - * "greeting.tera" -> "greeting" - * "tell_me_a_joke.json" -> "tell me a joke" - * "003-weather.txt" -> "weather" + * Core logic lives in src/convert-vidaimock.ts — this script is a thin CLI + * wrapper. Prefer `npx aimock convert vidaimock` instead. */ -export function deriveMatchFromFilename(filename: string): string { - let name = basename(filename, extname(filename)); - - // Strip leading numeric prefixes like "003-" - name = name.replace(/^\d+[-_]/, ""); - - // Replace underscores / hyphens with spaces - name = name.replace(/[-_]+/g, " "); - - return name.trim(); -} - -// --------------------------------------------------------------------------- -// File / directory conversion -// --------------------------------------------------------------------------- -const TEMPLATE_EXTENSIONS = new Set([ - ".tera", - ".json", - ".txt", - ".html", - ".jinja", - ".jinja2", - ".j2", -]); - -export function convertFile(filePath: string): AimockFixture | null { - try { - const raw = readFileSync(filePath, "utf-8"); - const content = stripTeraTemplate(raw); - if (!content) return null; - - const match = deriveMatchFromFilename(filePath); - return { match: { userMessage: match }, response: { content } }; - } catch { - // Unreadable / binary file — skip gracefully - return null; - } -} - -export function convertDirectory(dirPath: string): AimockFixture[] { - const fixtures: AimockFixture[] = []; - - let entries: string[]; - try { - entries = readdirSync(dirPath); - } catch { - return fixtures; - } - - for (const entry of entries.sort()) { - const fullPath = resolve(dirPath, entry); - try { - if (!statSync(fullPath).isFile()) continue; - } catch { - continue; - } - - const ext = extname(entry).toLowerCase(); - if (!TEMPLATE_EXTENSIONS.has(ext)) continue; - - const fixture = convertFile(fullPath); - if (fixture) fixtures.push(fixture); - } - - return fixtures; -} +import { writeFileSync, statSync } from "node:fs"; +import { resolve } from "node:path"; +import { + convertFile, + convertDirectory, + type AimockFixture, + type AimockFixtureFile, +} from "../src/convert-vidaimock.js"; + +// Re-export everything the test files import from here +export { + stripTeraTemplate, + deriveMatchFromFilename, + convertFile, + convertDirectory, + type AimockFixture, + type AimockFixtureFile, +} from "../src/convert-vidaimock.js"; // --------------------------------------------------------------------------- // CLI entry point diff --git a/scripts/update-competitive-matrix.ts b/scripts/update-competitive-matrix.ts index 43e852e..f0ed191 100644 --- a/scripts/update-competitive-matrix.ts +++ b/scripts/update-competitive-matrix.ts @@ -140,6 +140,20 @@ const FEATURE_RULES: FeatureRule[] = [ rowLabel: "AG-UI event mocking", keywords: ["ag-ui", "agui", "agent-ui", "copilotkit.*frontend", "event stream mock"], }, + { + rowLabel: "GitHub Action", + keywords: ["github.*action", "action.yml", "uses:.*mock", "ci.*action"], + }, + { + rowLabel: "Vitest / Jest plugins", + keywords: [ + "vitest.*plugin", + "jest.*plugin", + "useAimock", + "useMock.*test", + "test.*framework.*integrat", + ], + }, ]; /** Maps competitor display names to their migration page paths (relative to docs/) */ diff --git a/src/aimock-cli.ts b/src/aimock-cli.ts index 77e8b6c..7c07d98 100644 --- a/src/aimock-cli.ts +++ b/src/aimock-cli.ts @@ -2,15 +2,21 @@ import { parseArgs } from "node:util"; import { resolve } from "node:path"; import { loadConfig, startFromConfig } from "./config-loader.js"; +import { runConvertCli, type ConvertCliDeps } from "./convert.js"; const HELP = ` Usage: aimock [options] + aimock convert [output] Options: -c, --config Path to aimock config JSON file (required) -p, --port Port override (default: from config or 0) -h, --host Host override (default: from config or 127.0.0.1) --help Show this help message + +Subcommands: + convert Convert third-party mock configs to aimock format + Run "aimock convert --help" for details `.trim(); export interface AimockCliDeps { @@ -21,6 +27,7 @@ export interface AimockCliDeps { loadConfigFn?: typeof loadConfig; startFromConfigFn?: typeof startFromConfig; onReady?: (ctx: { shutdown: () => void }) => void; + convertDeps?: Partial; } export function runAimockCli(deps: AimockCliDeps = {}): void { @@ -32,6 +39,18 @@ export function runAimockCli(deps: AimockCliDeps = {}): void { const loadConfigFn = deps.loadConfigFn ?? loadConfig; const startFromConfigFn = deps.startFromConfigFn ?? startFromConfig; + // Intercept "convert" subcommand before parseArgs (which uses strict mode) + if (argv[0] === "convert") { + runConvertCli({ + argv: argv.slice(1), + log, + logError, + exit, + ...deps.convertDeps, + }); + return; + } + let values; try { ({ values } = parseArgs({ diff --git a/src/convert-mockllm.ts b/src/convert-mockllm.ts new file mode 100644 index 0000000..cf64b81 --- /dev/null +++ b/src/convert-mockllm.ts @@ -0,0 +1,420 @@ +/** + * mock-llm (dwmkerr) -> aimock fixture converter + * + * Core conversion logic. Used by both the CLI (`aimock convert mockllm`) + * and the standalone script (`scripts/convert-mockllm.ts`). + */ + +// --------------------------------------------------------------------------- +// Minimal YAML parser +// --------------------------------------------------------------------------- +// Handles the subset used by mock-llm configs: indented maps, arrays with +// `-` prefix, quoted/unquoted strings, numbers, booleans, and null. +// Does NOT handle: anchors, aliases, multi-line scalars, flow collections, +// tags, or other advanced YAML features. + +interface YamlLine { + indent: number; + raw: string; + content: string; // trimmed, without trailing comment + isArrayItem: boolean; + arrayItemContent: string; // content after "- " +} + +function tokenizeYamlLines(input: string): YamlLine[] { + const lines: YamlLine[] = []; + for (const raw of input.split("\n")) { + // Skip blank lines and full-line comments + const trimmed = raw.trimStart(); + if (trimmed === "" || trimmed.startsWith("#")) continue; + + const indent = raw.length - raw.trimStart().length; + // Strip trailing comments (but not inside quoted strings) + const content = stripTrailingComment(trimmed); + const isArrayItem = content.startsWith("- "); + const arrayItemContent = isArrayItem ? content.slice(2).trim() : ""; + + lines.push({ indent, raw, content, isArrayItem, arrayItemContent }); + } + return lines; +} + +function stripTrailingComment(s: string): string { + // Naive: find # not inside quotes + let inSingle = false; + let inDouble = false; + for (let i = 0; i < s.length; i++) { + const ch = s[i]; + if (ch === "'" && !inDouble) inSingle = !inSingle; + if (ch === '"' && !inSingle) inDouble = !inDouble; + if (ch === "#" && !inSingle && !inDouble && i > 0 && s[i - 1] === " ") { + return s.slice(0, i).trimEnd(); + } + } + return s; +} + +function parseScalar(value: string): unknown { + if (value === "" || value === "~" || value === "null") return null; + if (value === "true") return true; + if (value === "false") return false; + + // Quoted string + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + return value.slice(1, -1); + } + + // Number + const num = Number(value); + if (!Number.isNaN(num) && value !== "") return num; + + // Unquoted string + return value; +} + +export function parseSimpleYaml(input: string): unknown { + const lines = tokenizeYamlLines(input); + if (lines.length === 0) return null; + + const result = parseBlock(lines, 0, 0); + return result.value; +} + +interface ParseResult { + value: unknown; + nextIndex: number; +} + +function parseBlock(lines: YamlLine[], startIndex: number, minIndent: number): ParseResult { + if (startIndex >= lines.length) { + return { value: null, nextIndex: startIndex }; + } + + const line = lines[startIndex]; + + // Determine if this block is an array or a map + if (line.isArrayItem && line.indent >= minIndent) { + return parseArray(lines, startIndex, line.indent); + } + + // Map + if (line.content.includes(":")) { + return parseMap(lines, startIndex, line.indent); + } + + // Single scalar + return { value: parseScalar(line.content), nextIndex: startIndex + 1 }; +} + +function parseArray(lines: YamlLine[], startIndex: number, baseIndent: number): ParseResult { + const arr: unknown[] = []; + let i = startIndex; + + while (i < lines.length) { + const line = lines[i]; + if (line.indent < baseIndent) break; + if (line.indent > baseIndent) break; // shouldn't happen at array level + if (!line.isArrayItem) break; + + const itemContent = line.arrayItemContent; + + if (itemContent === "") { + // Array item with nested block on next lines + const nested = parseBlock(lines, i + 1, baseIndent + 1); + arr.push(nested.value); + i = nested.nextIndex; + } else if (itemContent.includes(":")) { + // Inline map start: "- key: value" possibly with more keys below + // Parse as a map, treating the "- " offset as extra indent + const inlineMap = parseArrayItemMap(lines, i, baseIndent); + arr.push(inlineMap.value); + i = inlineMap.nextIndex; + } else { + // Simple scalar array item + arr.push(parseScalar(itemContent)); + i++; + } + } + + return { value: arr, nextIndex: i }; +} + +function parseArrayItemMap( + lines: YamlLine[], + startIndex: number, + arrayIndent: number, +): ParseResult { + // First line is "- key: value", subsequent lines at indent > arrayIndent are part of this map + const map: Record = {}; + const firstLine = lines[startIndex]; + const firstContent = firstLine.arrayItemContent; + + // Parse the first key: value from the array item line + const colonIdx = findColon(firstContent); + if (colonIdx === -1) { + return { value: parseScalar(firstContent), nextIndex: startIndex + 1 }; + } + + const key = firstContent.slice(0, colonIdx).trim(); + const valueStr = firstContent.slice(colonIdx + 1).trim(); + + if (valueStr === "") { + // Value is a nested block + const nested = parseBlock(lines, startIndex + 1, arrayIndent + 2); + map[key] = nested.value; + let i = nested.nextIndex; + + // Continue reading sibling keys at the array-item's content indent + const siblingIndent = arrayIndent + 2; + while (i < lines.length && lines[i].indent >= siblingIndent && !lines[i].isArrayItem) { + if (lines[i].indent === siblingIndent || lines[i].indent > siblingIndent) { + // Only parse if at exactly sibling indent and is a map key + if (lines[i].indent === siblingIndent && lines[i].content.includes(":")) { + const mapResult = parseMapEntries(lines, i, siblingIndent, map); + i = mapResult.nextIndex; + } else { + break; + } + } + } + + return { value: map, nextIndex: i }; + } else { + map[key] = parseScalar(valueStr); + } + + // Read additional keys at indent > arrayIndent (the " key: value" lines after "- first: val") + let i = startIndex + 1; + const contentIndent = arrayIndent + 2; // "- " adds 2 to effective indent + + while (i < lines.length) { + const line = lines[i]; + if (line.indent < contentIndent) break; + if (line.isArrayItem && line.indent <= arrayIndent) break; + + if (line.indent === contentIndent && !line.isArrayItem && line.content.includes(":")) { + const colonPos = findColon(line.content); + if (colonPos === -1) break; + const k = line.content.slice(0, colonPos).trim(); + const v = line.content.slice(colonPos + 1).trim(); + + if (v === "") { + const nested = parseBlock(lines, i + 1, contentIndent + 1); + map[k] = nested.value; + i = nested.nextIndex; + } else { + map[k] = parseScalar(v); + i++; + } + } else if (line.indent === contentIndent && line.isArrayItem) { + // This is a new array item at the same level -- not part of this map + break; + } else if (line.indent > contentIndent) { + // Skip nested content already consumed + i++; + } else { + break; + } + } + + return { value: map, nextIndex: i }; +} + +function parseMap(lines: YamlLine[], startIndex: number, baseIndent: number): ParseResult { + const map: Record = {}; + const result = parseMapEntries(lines, startIndex, baseIndent, map); + return { value: map, nextIndex: result.nextIndex }; +} + +function parseMapEntries( + lines: YamlLine[], + startIndex: number, + baseIndent: number, + map: Record, +): ParseResult { + let i = startIndex; + + while (i < lines.length) { + const line = lines[i]; + if (line.indent < baseIndent) break; + if (line.indent > baseIndent) { + // Shouldn't happen at map level if properly structured -- skip + i++; + continue; + } + if (line.isArrayItem) break; + + const colonIdx = findColon(line.content); + if (colonIdx === -1) { + // Not a map entry + break; + } + + const key = line.content.slice(0, colonIdx).trim(); + const valueStr = line.content.slice(colonIdx + 1).trim(); + + if (valueStr === "") { + // Value is a nested block on subsequent lines + const nested = parseBlock(lines, i + 1, baseIndent + 1); + map[key] = nested.value; + i = nested.nextIndex; + } else { + map[key] = parseScalar(valueStr); + i++; + } + } + + return { value: map, nextIndex: i }; +} + +function findColon(s: string): number { + let inSingle = false; + let inDouble = false; + for (let i = 0; i < s.length; i++) { + const ch = s[i]; + if (ch === "'" && !inDouble) inSingle = !inSingle; + if (ch === '"' && !inSingle) inDouble = !inDouble; + if (ch === ":" && !inSingle && !inDouble) { + // Must be followed by space, end of line, or nothing + if (i === s.length - 1 || s[i + 1] === " ") { + return i; + } + } + } + return -1; +} + +// --------------------------------------------------------------------------- +// mock-llm config types +// --------------------------------------------------------------------------- + +export interface MockLLMRoute { + path: string; + method?: string; + match?: { + body?: { + messages?: Array<{ role: string; content: string }>; + }; + }; + response: Record; +} + +export interface MockLLMTool { + name: string; + description?: string; + parameters?: Record; +} + +export interface MockLLMConfig { + routes?: MockLLMRoute[]; + mcp?: { + tools?: MockLLMTool[]; + }; +} + +// --------------------------------------------------------------------------- +// aimock output types +// --------------------------------------------------------------------------- + +export interface AimockFixture { + match?: { userMessage?: string }; + response: { content?: string; toolCalls?: Array<{ name: string; arguments: string }> }; + _comment?: string; +} + +export interface AimockMCPTool { + name: string; + description?: string; + inputSchema?: Record; +} + +export interface ConvertResult { + fixtures: AimockFixture[]; + mcpTools?: AimockMCPTool[]; +} + +// --------------------------------------------------------------------------- +// Converter +// --------------------------------------------------------------------------- + +export function convertConfig(config: MockLLMConfig): ConvertResult { + const fixtures: AimockFixture[] = []; + + if (config.routes) { + for (const route of config.routes) { + const fixture = convertRoute(route); + if (fixture) { + fixtures.push(fixture); + } + } + } + + const result: ConvertResult = { fixtures }; + + if (config.mcp?.tools && config.mcp.tools.length > 0) { + result.mcpTools = config.mcp.tools.map(convertMCPTool); + } + + return result; +} + +function convertRoute(route: MockLLMRoute): AimockFixture | null { + // Extract content from response.choices[0].message.content + const content = extractResponseContent(route.response); + if (content === null) return null; + + const fixture: AimockFixture = { + match: {}, + response: { content }, + }; + + // Extract match criteria from match.body.messages + const userMessage = extractUserMessage(route); + if (userMessage) { + fixture.match = { userMessage }; + } else { + // Use path as a comment/identifier when no match criteria + fixture._comment = `${route.method ?? "POST"} ${route.path}`; + } + + return fixture; +} + +function extractResponseContent(response: Record): string | null { + const choices = response.choices as Array> | undefined; + if (!Array.isArray(choices) || choices.length === 0) return null; + + const firstChoice = choices[0]; + const message = firstChoice.message as Record | undefined; + if (!message) return null; + + const content = message.content; + if (typeof content !== "string") return null; + + return content; +} + +function extractUserMessage(route: MockLLMRoute): string | null { + const messages = route.match?.body?.messages; + if (!Array.isArray(messages) || messages.length === 0) return null; + + // Find the last user message + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i].role === "user") { + return messages[i].content; + } + } + + // Fall back to last message content regardless of role + return messages[messages.length - 1].content ?? null; +} + +function convertMCPTool(tool: MockLLMTool): AimockMCPTool { + const result: AimockMCPTool = { name: tool.name }; + if (tool.description) result.description = tool.description; + if (tool.parameters) result.inputSchema = tool.parameters; + return result; +} diff --git a/src/convert-vidaimock.ts b/src/convert-vidaimock.ts new file mode 100644 index 0000000..df69b3d --- /dev/null +++ b/src/convert-vidaimock.ts @@ -0,0 +1,184 @@ +/** + * VidaiMock -> aimock Fixture Converter + * + * Core conversion logic. Used by both the CLI (`aimock convert vidaimock`) + * and the standalone script (`scripts/convert-vidaimock.ts`). + */ + +import { readFileSync, readdirSync, statSync } from "node:fs"; +import { resolve, basename, extname } from "node:path"; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface AimockFixture { + match: { userMessage: string }; + response: { content: string }; +} + +export interface AimockFixtureFile { + fixtures: AimockFixture[]; +} + +// --------------------------------------------------------------------------- +// Tera template stripping +// --------------------------------------------------------------------------- + +/** + * Strip Tera template syntax and extract a usable response content string. + * + * Strategy: + * 1. If the template looks like JSON, try to pull out the nested + * `choices[].message.content` value (the most common VidaiMock pattern). + * 2. Otherwise fall back to stripping all Tera delimiters and returning the + * remaining text with placeholder variable names. + */ +export function stripTeraTemplate(raw: string): string { + const trimmed = raw.trim(); + + // --- Attempt JSON extraction first ----------------------------------- + const contentValue = extractJsonContent(trimmed); + if (contentValue !== null) return contentValue; + + // --- Fallback: strip Tera syntax ------------------------------------- + let text = trimmed; + + // Remove comment blocks {# ... #} + text = text.replace(/\{#[\s\S]*?#\}/g, ""); + + // Remove block tags {% ... %} + text = text.replace(/\{%[\s\S]*?%\}/g, ""); + + // Replace expression tags {{ expr }} with the expression name + text = text.replace(/\{\{\s*([\w.]+)\s*\}\}/g, "[$1]"); + + // Collapse excessive whitespace but preserve intentional newlines + text = text + .split("\n") + .map((l) => l.trim()) + .filter((l) => l.length > 0) + .join("\n"); + + return text; +} + +/** + * Try to parse the template as JSON (after substituting Tera expressions with + * dummy strings) and pull out `choices[0].message.content`. + */ +function extractJsonContent(raw: string): string | null { + try { + // Step 1: remove Tera comments and blocks + let substituted = raw.replace(/\{#[\s\S]*?#\}/g, "").replace(/\{%[\s\S]*?%\}/g, ""); + + // Step 2: Replace Tera expressions inside existing JSON strings. + // Pattern: the expression is already within quotes, e.g. "foo-{{ bar }}-baz" + // We replace the {{ ... }} with the placeholder without adding extra quotes. + substituted = substituted.replace( + /"([^"]*?)\{\{\s*([\w.]+)\s*\}\}([^"]*?)"/g, + (_, before, varName, after) => `"${before}[${varName}]${after}"`, + ); + + // Step 3: Replace standalone Tera expressions (not inside quotes), + // e.g. a bare `{{ content }}` used as a JSON value — wrap with quotes. + substituted = substituted.replace(/\{\{\s*([\w.]+)\s*\}\}/g, '"[$1]"'); + + const parsed = JSON.parse(substituted); + + if ( + parsed && + Array.isArray(parsed.choices) && + parsed.choices.length > 0 && + parsed.choices[0]?.message?.content !== undefined + ) { + return String(parsed.choices[0].message.content); + } + } catch { + // Not valid JSON even after substitution — fall through + } + return null; +} + +// --------------------------------------------------------------------------- +// Filename -> match derivation +// --------------------------------------------------------------------------- + +/** + * Derive a `userMessage` match string from the template filename. + * + * Examples: + * "greeting.tera" -> "greeting" + * "tell_me_a_joke.json" -> "tell me a joke" + * "003-weather.txt" -> "weather" + */ +export function deriveMatchFromFilename(filename: string): string { + let name = basename(filename, extname(filename)); + + // Strip leading numeric prefixes like "003-" + name = name.replace(/^\d+[-_]/, ""); + + // Replace underscores / hyphens with spaces + name = name.replace(/[-_]+/g, " "); + + return name.trim(); +} + +// --------------------------------------------------------------------------- +// File / directory conversion +// --------------------------------------------------------------------------- + +const TEMPLATE_EXTENSIONS = new Set([ + ".tera", + ".json", + ".txt", + ".html", + ".jinja", + ".jinja2", + ".j2", +]); + +export function convertFile(filePath: string): AimockFixture | null { + let raw: string; + try { + raw = readFileSync(filePath, "utf-8"); + } catch { + // Unreadable / binary file — skip gracefully + return null; + } + + const content = stripTeraTemplate(raw); + if (!content) return null; + + const match = deriveMatchFromFilename(filePath); + return { match: { userMessage: match }, response: { content } }; +} + +export function convertDirectory(dirPath: string): AimockFixture[] { + const fixtures: AimockFixture[] = []; + + let entries: string[]; + try { + entries = readdirSync(dirPath); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") return fixtures; + throw err; // permission errors, etc. should propagate + } + + for (const entry of entries.sort()) { + const fullPath = resolve(dirPath, entry); + try { + if (!statSync(fullPath).isFile()) continue; + } catch { + continue; + } + + const ext = extname(entry).toLowerCase(); + if (!TEMPLATE_EXTENSIONS.has(ext)) continue; + + const fixture = convertFile(fullPath); + if (fixture) fixtures.push(fixture); + } + + return fixtures; +} diff --git a/src/convert.ts b/src/convert.ts new file mode 100644 index 0000000..c354ddc --- /dev/null +++ b/src/convert.ts @@ -0,0 +1,194 @@ +/** + * CLI dispatcher for `aimock convert` subcommands. + * + * Delegates to the converter modules in src/convert-vidaimock.ts and + * src/convert-mockllm.ts. + */ + +import { readFileSync, writeFileSync, statSync } from "node:fs"; +import { resolve } from "node:path"; +import { convertFile, convertDirectory, type AimockFixtureFile } from "./convert-vidaimock.js"; +import { parseSimpleYaml, convertConfig, type MockLLMConfig } from "./convert-mockllm.js"; + +const CONVERT_HELP = ` +Usage: aimock convert [output] + +Formats: + vidaimock Convert VidaiMock Tera templates to aimock JSON + mockllm Convert mock-llm YAML config to aimock JSON + +Examples: + aimock convert vidaimock ./templates/ ./fixtures/ + aimock convert mockllm ./config.yaml ./fixtures/converted.json +`.trim(); + +export interface ConvertCliDeps { + argv: string[]; + log: (msg: string) => void; + logError: (msg: string) => void; + exit: (code: number) => void; +} + +export function runConvertCli(deps: ConvertCliDeps): void { + const { argv, log, logError, exit } = deps; + + if (argv.length === 0 || argv[0] === "--help" || argv[0] === "-h") { + if (argv.length === 0) { + logError(CONVERT_HELP); + exit(1); + } else { + log(CONVERT_HELP); + exit(0); + } + return; + } + + const format = argv[0]; + const inputArg = argv[1]; + const outputArg = argv[2]; + + if (!inputArg) { + logError(`Error: missing argument.\n\n${CONVERT_HELP}`); + exit(1); + return; + } + + switch (format) { + case "vidaimock": + runVidaimockConvert(inputArg, outputArg, { log, logError, exit }); + break; + case "mockllm": + runMockllmConvert(inputArg, outputArg, { log, logError, exit }); + break; + default: + logError(`Error: unknown format "${format}".\n\n${CONVERT_HELP}`); + exit(1); + } +} + +// --------------------------------------------------------------------------- +// VidaiMock converter +// --------------------------------------------------------------------------- + +function runVidaimockConvert( + inputArg: string, + outputArg: string | undefined, + io: { log: (msg: string) => void; logError: (msg: string) => void; exit: (code: number) => void }, +): void { + const inputPath = resolve(inputArg); + const outputPath = outputArg ? resolve(outputArg) : null; + + let fixtures: AimockFixtureFile["fixtures"]; + + try { + const stat = statSync(inputPath); + if (stat.isDirectory()) { + fixtures = convertDirectory(inputPath); + } else { + const single = convertFile(inputPath); + fixtures = single ? [single] : []; + } + } catch (err) { + io.logError(`Error reading input path: ${inputPath}`); + io.logError(err instanceof Error ? err.message : String(err)); + io.exit(1); + return; + } + + if (fixtures.length === 0) { + io.logError("No fixtures produced — check that the input contains valid VidaiMock templates."); + io.exit(1); + return; + } + + const output: AimockFixtureFile = { fixtures }; + const json = JSON.stringify(output, null, 2) + "\n"; + + if (outputPath) { + try { + writeFileSync(outputPath, json, "utf-8"); + } catch (err) { + io.logError(`Error writing output: ${(err as Error).message}`); + io.exit(1); + return; + } + io.log(`Wrote ${fixtures.length} fixture(s) to ${outputPath}`); + } else { + io.log(json.trimEnd()); + } +} + +// --------------------------------------------------------------------------- +// mock-llm converter +// --------------------------------------------------------------------------- + +function runMockllmConvert( + inputArg: string, + outputArg: string | undefined, + io: { log: (msg: string) => void; logError: (msg: string) => void; exit: (code: number) => void }, +): void { + const inputPath = resolve(inputArg); + const outputPath = outputArg ? resolve(outputArg) : null; + + let yamlContent: string; + try { + yamlContent = readFileSync(inputPath, "utf-8"); + } catch (err) { + io.logError(`Error reading input file: ${(err as Error).message}`); + io.exit(1); + return; + } + + const parsed = parseSimpleYaml(yamlContent) as MockLLMConfig | null; + if (!parsed || typeof parsed !== "object") { + io.logError("Error: could not parse YAML config"); + io.exit(1); + return; + } + + const result = convertConfig(parsed); + const fixtureJson = JSON.stringify({ fixtures: result.fixtures }, null, 2); + + if (outputPath) { + try { + writeFileSync(outputPath, fixtureJson + "\n", "utf-8"); + } catch (err) { + io.logError(`Error writing output: ${(err as Error).message}`); + io.exit(1); + return; + } + io.log(`Wrote fixtures to ${outputPath}`); + + if (result.mcpTools) { + const configPath = outputPath.endsWith(".json") + ? outputPath.replace(/\.json$/, ".aimock.json") + : outputPath + ".aimock.json"; + const aimockConfig = { + llm: { fixtures: outputPath }, + mcp: { + tools: result.mcpTools.map((t) => ({ + name: t.name, + description: t.description ?? "", + inputSchema: t.inputSchema ?? {}, + result: `Mock result for ${t.name}`, + })), + }, + }; + try { + writeFileSync(configPath, JSON.stringify(aimockConfig, null, 2) + "\n", "utf-8"); + } catch (err) { + io.logError(`Error writing config: ${(err as Error).message}`); + io.exit(1); + return; + } + io.log(`Wrote aimock config with MCP tools to ${configPath}`); + } + } else { + io.log(fixtureJson); + + if (result.mcpTools) { + io.log("\n--- MCP Tools (aimock config format) ---"); + io.log(JSON.stringify({ mcp: { tools: result.mcpTools } }, null, 2)); + } + } +} diff --git a/src/jest.ts b/src/jest.ts new file mode 100644 index 0000000..c508feb --- /dev/null +++ b/src/jest.ts @@ -0,0 +1,116 @@ +/** + * Jest integration for aimock. + * + * Usage: + * import { useAimock } from "@copilotkit/aimock/jest"; + * + * const mock = useAimock({ fixtures: "./fixtures" }); + * + * it("responds", async () => { + * const res = await fetch(`${mock().url}/v1/chat/completions`, { ... }); + * }); + */ + +/* eslint-disable no-var */ +// Jest globals — available at runtime in jest test files +declare var beforeAll: (fn: () => Promise | void, timeout?: number) => void; +declare var afterAll: (fn: () => Promise | void, timeout?: number) => void; +declare var beforeEach: (fn: () => Promise | void, timeout?: number) => void; +/* eslint-enable no-var */ + +import { LLMock } from "./llmock.js"; +import { loadFixtureFile, loadFixturesFromDir } from "./fixture-loader.js"; +import type { Fixture, MockServerOptions } from "./types.js"; +import { statSync } from "node:fs"; +import { resolve } from "node:path"; + +export interface UseAimockOptions extends MockServerOptions { + /** Path to fixture file or directory. Loaded automatically on start. */ + fixtures?: string; + /** If true, sets process.env.OPENAI_BASE_URL to the mock URL + /v1. */ + patchEnv?: boolean; +} + +export interface AimockHandle { + /** The LLMock instance. */ + readonly llm: LLMock; + /** The server URL (e.g., http://127.0.0.1:4010). */ + readonly url: string; +} + +/** + * Start an aimock server for the duration of the test suite. + * + * - `beforeAll`: starts the server and optionally loads fixtures + * - `beforeEach`: resets fixture match counts (not fixtures themselves) + * - `afterAll`: stops the server + * + * Returns a getter function — call it inside tests to access the handle. + * + * NOTE: Jest globals (beforeAll, afterAll, beforeEach) must be available + * in the test environment. This works with the default jest configuration. + */ +export function useAimock(options: UseAimockOptions = {}): () => AimockHandle { + let handle: AimockHandle | null = null; + + beforeAll(async () => { + const { fixtures: fixturePath, patchEnv, ...serverOpts } = options; + const llm = new LLMock(serverOpts); + + if (fixturePath) { + const resolved = resolve(fixturePath); + const loadedFixtures = loadFixtures(resolved); + for (const f of loadedFixtures) { + llm.addFixture(f); + } + } + + const url = await llm.start(); + + if (patchEnv !== false) { + process.env.OPENAI_BASE_URL = `${url}/v1`; + process.env.ANTHROPIC_BASE_URL = `${url}/v1`; + } + + handle = { llm, url }; + }); + + beforeEach(() => { + if (handle) { + handle.llm.resetMatchCounts(); + } + }); + + afterAll(async () => { + if (handle) { + if (options.patchEnv !== false) { + delete process.env.OPENAI_BASE_URL; + delete process.env.ANTHROPIC_BASE_URL; + } + await handle.llm.stop(); + handle = null; + } + }); + + return () => { + if (!handle) { + throw new Error("useAimock(): server not started — are you calling this inside a test?"); + } + return handle; + }; +} + +function loadFixtures(fixturePath: string): Fixture[] { + try { + const stat = statSync(fixturePath); + if (stat.isDirectory()) { + return loadFixturesFromDir(fixturePath); + } + return loadFixtureFile(fixturePath); + } catch { + return []; + } +} + +export { LLMock } from "./llmock.js"; +export type { MockServerOptions, Fixture } from "./types.js"; diff --git a/src/vitest.ts b/src/vitest.ts new file mode 100644 index 0000000..72a1b7c --- /dev/null +++ b/src/vitest.ts @@ -0,0 +1,107 @@ +/** + * Vitest integration for aimock. + * + * Usage: + * import { useAimock } from "@copilotkit/aimock/vitest"; + * + * const mock = useAimock({ fixtures: "./fixtures" }); + * + * it("responds", async () => { + * const res = await fetch(`${mock().url}/v1/chat/completions`, { ... }); + * }); + */ + +import { beforeAll, afterAll, beforeEach } from "vitest"; +import { LLMock } from "./llmock.js"; +import { loadFixtureFile, loadFixturesFromDir } from "./fixture-loader.js"; +import type { Fixture, MockServerOptions } from "./types.js"; +import { statSync } from "node:fs"; +import { resolve } from "node:path"; + +export interface UseAimockOptions extends MockServerOptions { + /** Path to fixture file or directory. Loaded automatically on start. */ + fixtures?: string; + /** If true, sets process.env.OPENAI_BASE_URL to the mock URL + /v1. */ + patchEnv?: boolean; +} + +export interface AimockHandle { + /** The LLMock instance. */ + readonly llm: LLMock; + /** The server URL (e.g., http://127.0.0.1:4010). */ + readonly url: string; +} + +/** + * Start an aimock server for the duration of the test suite. + * + * - `beforeAll`: starts the server and optionally loads fixtures + * - `beforeEach`: resets fixture match counts (not fixtures themselves) + * - `afterAll`: stops the server + * + * Returns a getter function — call it inside tests to access the handle. + */ +export function useAimock(options: UseAimockOptions = {}): () => AimockHandle { + let handle: AimockHandle | null = null; + + beforeAll(async () => { + const { fixtures: fixturePath, patchEnv, ...serverOpts } = options; + const llm = new LLMock(serverOpts); + + if (fixturePath) { + const resolved = resolve(fixturePath); + const loadedFixtures = loadFixtures(resolved); + for (const f of loadedFixtures) { + llm.addFixture(f); + } + } + + const url = await llm.start(); + + if (patchEnv !== false) { + process.env.OPENAI_BASE_URL = `${url}/v1`; + process.env.ANTHROPIC_BASE_URL = `${url}/v1`; + } + + handle = { llm, url }; + }); + + beforeEach(() => { + if (handle) { + handle.llm.resetMatchCounts(); + } + }); + + afterAll(async () => { + if (handle) { + if (options.patchEnv !== false) { + delete process.env.OPENAI_BASE_URL; + delete process.env.ANTHROPIC_BASE_URL; + } + await handle.llm.stop(); + handle = null; + } + }); + + return () => { + if (!handle) { + throw new Error("useAimock(): server not started — are you calling this inside a test?"); + } + return handle; + }; +} + +function loadFixtures(fixturePath: string): Fixture[] { + try { + const stat = statSync(fixturePath); + if (stat.isDirectory()) { + return loadFixturesFromDir(fixturePath); + } + return loadFixtureFile(fixturePath); + } catch { + return []; + } +} + +export { LLMock } from "./llmock.js"; +export type { MockServerOptions, Fixture } from "./types.js"; diff --git a/tsdown.config.ts b/tsdown.config.ts index 5698d8a..712829d 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -7,6 +7,8 @@ export default defineConfig({ "src/mcp-stub.ts", "src/a2a-stub.ts", "src/vector-stub.ts", + "src/vitest.ts", + "src/jest.ts", ], format: ["esm", "cjs"], dts: true,