From d76924179d189491db979346e6c235e3f603dd6c Mon Sep 17 00:00:00 2001 From: Scarlett Attensil Date: Tue, 19 May 2026 11:27:12 -0700 Subject: [PATCH 01/10] Rename aiconfig-* skills to agentcontrol-* namespace Config-specific skills get the agentcontrol-configs- prefix; project/ tools and dangling sdk/context/segments cross-refs get plain agentcontrol-. Updates skill directories, SKILL.md frontmatter, skills.json, README catalog, and all cross-references. --- README.md | 20 ++--- skills.json | 48 +++++------ .../SKILL.md | 2 +- .../SKILL.md | 14 ++-- .../references/anthropic-tracking.md | 0 .../references/bedrock-tracking.md | 0 .../references/gemini-tracking.md | 2 +- .../references/langchain-tracking.md | 2 +- .../references/metrics-api.md | 0 .../references/openai-tracking.md | 0 .../references/strands-tracking.md | 2 +- .../references/streaming-tracking.md | 0 .../README.md | 10 +-- .../SKILL.md | 18 ++--- .../SKILL.md | 14 ++-- .../README.md | 20 ++--- .../SKILL.md | 80 +++++++++---------- .../references/agent-graph-reference.md | 0 .../references/agent-mode-frameworks.md | 12 +-- .../references/before-after-examples.md | 0 .../references/fallback-defaults-pattern.md | 2 +- .../references/phase-1-analysis-checklist.md | 6 +- .../references/sdk-ai-tracker-patterns.md | 8 +- .../README.md | 12 +-- .../SKILL.md | 10 +-- .../SKILL.md | 2 +- .../README.md | 14 ++-- .../SKILL.md | 16 ++-- .../README.md | 8 +- .../SKILL.md | 6 +- .../README.md | 10 +-- .../SKILL.md | 6 +- .../README.md | 4 +- .../SKILL.md | 16 ++-- .../references/admin-tooling.md | 0 .../references/env-config.md | 2 +- .../references/go-setup.md | 0 .../references/iac-automation.md | 0 .../references/multi-language-setup.md | 0 .../references/nodejs-setup.md | 0 .../references/project-cloning.md | 0 .../references/python-setup.md | 0 .../references/quick-start.md | 0 .../README.md | 8 +- .../SKILL.md | 18 ++--- 45 files changed, 196 insertions(+), 196 deletions(-) rename skills/ai-configs/{aiconfig-agent-graphs => agentcontrol-configs-agent-graphs}/SKILL.md (99%) rename skills/ai-configs/{aiconfig-ai-metrics => agentcontrol-configs-ai-metrics}/SKILL.md (92%) rename skills/ai-configs/{aiconfig-ai-metrics => agentcontrol-configs-ai-metrics}/references/anthropic-tracking.md (100%) rename skills/ai-configs/{aiconfig-ai-metrics => agentcontrol-configs-ai-metrics}/references/bedrock-tracking.md (100%) rename skills/ai-configs/{aiconfig-ai-metrics => agentcontrol-configs-ai-metrics}/references/gemini-tracking.md (99%) rename skills/ai-configs/{aiconfig-ai-metrics => agentcontrol-configs-ai-metrics}/references/langchain-tracking.md (97%) rename skills/ai-configs/{aiconfig-ai-metrics => agentcontrol-configs-ai-metrics}/references/metrics-api.md (100%) rename skills/ai-configs/{aiconfig-ai-metrics => agentcontrol-configs-ai-metrics}/references/openai-tracking.md (100%) rename skills/ai-configs/{aiconfig-ai-metrics => agentcontrol-configs-ai-metrics}/references/strands-tracking.md (93%) rename skills/ai-configs/{aiconfig-ai-metrics => agentcontrol-configs-ai-metrics}/references/streaming-tracking.md (100%) rename skills/ai-configs/{aiconfig-create => agentcontrol-configs-create}/README.md (73%) rename skills/ai-configs/{aiconfig-create => agentcontrol-configs-create}/SKILL.md (90%) rename skills/ai-configs/{aiconfig-custom-metrics => agentcontrol-configs-custom-metrics}/SKILL.md (97%) rename skills/ai-configs/{aiconfig-migrate => agentcontrol-configs-migrate}/README.md (75%) rename skills/ai-configs/{aiconfig-migrate => agentcontrol-configs-migrate}/SKILL.md (86%) rename skills/ai-configs/{aiconfig-migrate => agentcontrol-configs-migrate}/references/agent-graph-reference.md (100%) rename skills/ai-configs/{aiconfig-migrate => agentcontrol-configs-migrate}/references/agent-mode-frameworks.md (98%) rename skills/ai-configs/{aiconfig-migrate => agentcontrol-configs-migrate}/references/before-after-examples.md (100%) rename skills/ai-configs/{aiconfig-migrate => agentcontrol-configs-migrate}/references/fallback-defaults-pattern.md (99%) rename skills/ai-configs/{aiconfig-migrate => agentcontrol-configs-migrate}/references/phase-1-analysis-checklist.md (97%) rename skills/ai-configs/{aiconfig-migrate => agentcontrol-configs-migrate}/references/sdk-ai-tracker-patterns.md (97%) rename skills/ai-configs/{aiconfig-online-evals => agentcontrol-configs-online-evals}/README.md (67%) rename skills/ai-configs/{aiconfig-online-evals => agentcontrol-configs-online-evals}/SKILL.md (98%) rename skills/ai-configs/{aiconfig-snippets => agentcontrol-configs-snippets}/SKILL.md (99%) rename skills/ai-configs/{aiconfig-targeting => agentcontrol-configs-targeting}/README.md (63%) rename skills/ai-configs/{aiconfig-targeting => agentcontrol-configs-targeting}/SKILL.md (97%) rename skills/ai-configs/{aiconfig-update => agentcontrol-configs-update}/README.md (78%) rename skills/ai-configs/{aiconfig-update => agentcontrol-configs-update}/SKILL.md (95%) rename skills/ai-configs/{aiconfig-variations => agentcontrol-configs-variations}/README.md (71%) rename skills/ai-configs/{aiconfig-variations => agentcontrol-configs-variations}/SKILL.md (97%) rename skills/ai-configs/{aiconfig-projects => agentcontrol-projects}/README.md (93%) rename skills/ai-configs/{aiconfig-projects => agentcontrol-projects}/SKILL.md (94%) rename skills/ai-configs/{aiconfig-projects => agentcontrol-projects}/references/admin-tooling.md (100%) rename skills/ai-configs/{aiconfig-projects => agentcontrol-projects}/references/env-config.md (99%) rename skills/ai-configs/{aiconfig-projects => agentcontrol-projects}/references/go-setup.md (100%) rename skills/ai-configs/{aiconfig-projects => agentcontrol-projects}/references/iac-automation.md (100%) rename skills/ai-configs/{aiconfig-projects => agentcontrol-projects}/references/multi-language-setup.md (100%) rename skills/ai-configs/{aiconfig-projects => agentcontrol-projects}/references/nodejs-setup.md (100%) rename skills/ai-configs/{aiconfig-projects => agentcontrol-projects}/references/project-cloning.md (100%) rename skills/ai-configs/{aiconfig-projects => agentcontrol-projects}/references/python-setup.md (100%) rename skills/ai-configs/{aiconfig-projects => agentcontrol-projects}/references/quick-start.md (100%) rename skills/ai-configs/{aiconfig-tools => agentcontrol-tools}/README.md (76%) rename skills/ai-configs/{aiconfig-tools => agentcontrol-tools}/SKILL.md (91%) diff --git a/README.md b/README.md index 1fcf9eb..af9ea8d 100644 --- a/README.md +++ b/README.md @@ -22,16 +22,16 @@ Agent Skills are modular, text-based playbooks that teach an agent how to perfor | Skill | Description | |-------|-------------| -| `ai-configs/aiconfig-create` | Create AI Configs with variations for agent or completion mode | -| `ai-configs/aiconfig-migrate` | Migrate an app with hardcoded LLM prompts to AI Configs in five stages (extract, wrap, tools, tracking, evals) | -| `ai-configs/aiconfig-update` | Update and delete AI Configs, manage lifecycle | -| `ai-configs/aiconfig-variations` | Manage AI Config variations for A/B testing | -| `ai-configs/aiconfig-tools` | Create and attach tools for function calling | -| `ai-configs/aiconfig-projects` | Create and manage projects to organize AI Configs | -| `ai-configs/aiconfig-online-evals` | Attach LLM-as-a-judge evaluators to AI Configs | -| `ai-configs/aiconfig-targeting` | Configure targeting rules for AI Config rollouts | -| `ai-configs/aiconfig-snippets` | Create and manage reusable prompt snippets across AI Configs | -| `ai-configs/aiconfig-agent-graphs` | Create and manage multi-agent graphs with routing and handoffs | +| `ai-configs/agentcontrol-configs-create` | Create AI Configs with variations for agent or completion mode | +| `ai-configs/agentcontrol-configs-migrate` | Migrate an app with hardcoded LLM prompts to AI Configs in five stages (extract, wrap, tools, tracking, evals) | +| `ai-configs/agentcontrol-configs-update` | Update and delete AI Configs, manage lifecycle | +| `ai-configs/agentcontrol-configs-variations` | Manage AI Config variations for A/B testing | +| `ai-configs/agentcontrol-tools` | Create and attach tools for function calling | +| `ai-configs/agentcontrol-projects` | Create and manage projects to organize AI Configs | +| `ai-configs/agentcontrol-configs-online-evals` | Attach LLM-as-a-judge evaluators to AI Configs | +| `ai-configs/agentcontrol-configs-targeting` | Configure targeting rules for AI Config rollouts | +| `ai-configs/agentcontrol-configs-snippets` | Create and manage reusable prompt snippets across AI Configs | +| `ai-configs/agentcontrol-configs-agent-graphs` | Create and manage multi-agent graphs with routing and handoffs | ### Experiments diff --git a/skills.json b/skills.json index 1812d1c..d94f7de 100644 --- a/skills.json +++ b/skills.json @@ -1,94 +1,94 @@ { "skills": [ { - "name": "aiconfig-agent-graphs", + "name": "agentcontrol-configs-agent-graphs", "description": "Create and manage agent graphs \u2014 directed graphs of AI Configs connected by edges with handoff logic. Use when building multi-agent workflows where configs route to each other.", - "path": "skills/ai-configs/aiconfig-agent-graphs", + "path": "skills/ai-configs/agentcontrol-configs-agent-graphs", "version": "0.1.0", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "aiconfig-ai-metrics", + "name": "agentcontrol-configs-ai-metrics", "description": "Instrument an existing codebase with LaunchDarkly AI Config tracking. Walks the four-tier ladder (managed runner \u2192 provider package \u2192 custom extractor + trackMetricsOf \u2192 raw manual) and picks the lowest-ceremony option that still captures duration, tokens, and success/error.", - "path": "skills/ai-configs/aiconfig-ai-metrics", + "path": "skills/ai-configs/agentcontrol-configs-ai-metrics", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the LaunchDarkly server-side AI SDK (`launchdarkly-server-sdk-ai>=0.20.0` for Python or `@launchdarkly/server-sdk-ai>=0.20.0` for Node) and an existing AI Config." }, { - "name": "aiconfig-create", + "name": "agentcontrol-configs-create", "description": "Create and configure AI Configs in LaunchDarkly. Helps you choose between agent vs completion mode, create the config, add variations with models and prompts, and verify the setup.", - "path": "skills/ai-configs/aiconfig-create", + "path": "skills/ai-configs/agentcontrol-configs-create", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "aiconfig-custom-metrics", + "name": "agentcontrol-configs-custom-metrics", "description": "Create, track, retrieve, update, and delete custom business metrics for AI Configs. Covers full lifecycle: define metric kinds via API, emit events via SDK, and query results.", - "path": "skills/ai-configs/aiconfig-custom-metrics", + "path": "skills/ai-configs/agentcontrol-configs-custom-metrics", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the LaunchDarkly server SDK and a LaunchDarkly API token with the `writer` role for metric management." }, { - "name": "aiconfig-migrate", + "name": "agentcontrol-configs-migrate", "description": "Migrate an application with hardcoded LLM prompts to a full LaunchDarkly AI Configs implementation in five stages: audit the code, wrap the call, move the tools, add tracking, attach evaluators. Use when the user wants to externalize model/prompt configuration, move from direct provider calls (OpenAI, Anthropic, Bedrock, Gemini, Strands) to a managed AI Config, or stage a full hardcoded-to-LaunchDarkly migration.", - "path": "skills/ai-configs/aiconfig-migrate", + "path": "skills/ai-configs/agentcontrol-configs-migrate", "version": "0.1.0", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "aiconfig-online-evals", + "name": "agentcontrol-configs-online-evals", "description": "Attach judges to AI Config variations for automatic LLM-as-a-judge evaluation. Create custom judges, configure sampling rates, and monitor quality scores.", - "path": "skills/ai-configs/aiconfig-online-evals", + "path": "skills/ai-configs/agentcontrol-configs-online-evals", "version": "0.1.0", "compatibility": "Requires LaunchDarkly API access token with ai-configs:write permission. SDK versions Python v0.20.0+ or Node.js v0.20.0+ for automatic metric recording and the consolidated `track_judge_result` / `trackJudgeResult` API." }, { - "name": "aiconfig-projects", + "name": "agentcontrol-projects", "description": "Guide for setting up LaunchDarkly projects in your codebase. Helps you assess your stack, choose the right approach, and integrate project management that makes sense for your architecture.", - "path": "skills/ai-configs/aiconfig-projects", + "path": "skills/ai-configs/agentcontrol-projects", "version": "0.4.0", "compatibility": "Requires LaunchDarkly API access token with projects:write permission or LaunchDarkly MCP server." }, { - "name": "aiconfig-snippets", + "name": "agentcontrol-configs-snippets", "description": "Create and manage prompt snippets \u2014 reusable text blocks referenced inside AI Config variation prompts. Keeps common instructions, personas, and guardrails consistent across multiple configs.", - "path": "skills/ai-configs/aiconfig-snippets", + "path": "skills/ai-configs/agentcontrol-configs-snippets", "version": "0.1.0", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "aiconfig-targeting", + "name": "agentcontrol-configs-targeting", "description": "Configure AI Config targeting rules to control which variations serve to different users. Enable percentage rollouts, attribute-based rules, segment targeting, and guarded rollouts.", - "path": "skills/ai-configs/aiconfig-targeting", + "path": "skills/ai-configs/agentcontrol-configs-targeting", "version": "0.1.0", "compatibility": "Requires LaunchDarkly API access token with ai-configs:write permission." }, { - "name": "aiconfig-tools", + "name": "agentcontrol-tools", "description": "Give your AI agents capabilities through tools (function calling). Helps you identify what your AI needs to do, create tool definitions, and attach them to AI Config variations.", - "path": "skills/ai-configs/aiconfig-tools", + "path": "skills/ai-configs/agentcontrol-tools", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "aiconfig-update", + "name": "agentcontrol-configs-update", "description": "Update, archive, and delete LaunchDarkly AI Configs and their variations. Use when you need to modify config properties, change model parameters, update instructions or messages, archive unused configs, or permanently remove them.", - "path": "skills/ai-configs/aiconfig-update", + "path": "skills/ai-configs/agentcontrol-configs-update", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "aiconfig-variations", + "name": "agentcontrol-configs-variations", "description": "Experiment with AI configurations by creating and managing variations. Helps you test different models, prompts, and parameters to find what works best through systematic experimentation.", - "path": "skills/ai-configs/aiconfig-variations", + "path": "skills/ai-configs/agentcontrol-configs-variations", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" diff --git a/skills/ai-configs/aiconfig-agent-graphs/SKILL.md b/skills/ai-configs/agentcontrol-configs-agent-graphs/SKILL.md similarity index 99% rename from skills/ai-configs/aiconfig-agent-graphs/SKILL.md rename to skills/ai-configs/agentcontrol-configs-agent-graphs/SKILL.md index 1916f6c..8510615 100644 --- a/skills/ai-configs/aiconfig-agent-graphs/SKILL.md +++ b/skills/ai-configs/agentcontrol-configs-agent-graphs/SKILL.md @@ -1,5 +1,5 @@ --- -name: aiconfig-agent-graphs +name: agentcontrol-configs-agent-graphs description: "Create and manage agent graphs — directed graphs of AI Configs connected by edges with handoff logic. Use when building multi-agent workflows where configs route to each other." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server diff --git a/skills/ai-configs/aiconfig-ai-metrics/SKILL.md b/skills/ai-configs/agentcontrol-configs-ai-metrics/SKILL.md similarity index 92% rename from skills/ai-configs/aiconfig-ai-metrics/SKILL.md rename to skills/ai-configs/agentcontrol-configs-ai-metrics/SKILL.md index 5d4aadc..52623db 100644 --- a/skills/ai-configs/aiconfig-ai-metrics/SKILL.md +++ b/skills/ai-configs/agentcontrol-configs-ai-metrics/SKILL.md @@ -1,5 +1,5 @@ --- -name: aiconfig-ai-metrics +name: agentcontrol-configs-ai-metrics description: "Instrument an existing codebase with LaunchDarkly AI Config tracking. Walks the four-tier ladder (managed runner → provider package → custom extractor + trackMetricsOf → raw manual) and picks the lowest-ceremony option that still captures duration, tokens, and success/error." license: Apache-2.0 compatibility: Requires the LaunchDarkly server-side AI SDK (`launchdarkly-server-sdk-ai>=0.20.0` for Python or `@launchdarkly/server-sdk-ai>=0.20.0` for Node) and an existing AI Config. @@ -38,12 +38,12 @@ Before picking a tier, find the provider call and answer these questions: - [ ] **Provider?** OpenAI, Anthropic, Bedrock, Gemini, Azure, custom HTTP? → cross-reference with the package availability matrix below. - [ ] **Streaming?** If yes, you'll need TTFT tracking, which means Tier 4 for the TTFT part even if the rest is Tier 2. - [ ] **Language?** Python or Node? Provider-package coverage differs between them. -- [ ] **Already using an AI Config?** If not, route to `aiconfig-create` first — tracking requires a tracker, which is obtained by calling `create_tracker()` / `createTracker()` on the config object returned by `completion_config()` / `completionConfig()` / `createModel()`. +- [ ] **Already using an AI Config?** If not, route to `agentcontrol-configs-create` first — tracking requires a tracker, which is obtained by calling `create_tracker()` / `createTracker()` on the config object returned by `completion_config()` / `completionConfig()` / `createModel()`. - [ ] **On the current SDK API?** If the call site uses `aiclient.config(...)` / `aiClient.config(...)` or constructs an `AIConfig(...)` / `LDAIConfig` default, it's on the pre-0.20 surface. Migrate it as part of this work before adding tracking: - `aiclient.config(...)` → `aiclient.completion_config(...)` for one-shot/chat or `aiclient.agent_config(...)` for agent mode (mirror the call signature). Node is the same with camelCase. - `AIConfig(...)` default → `AICompletionConfigDefault(...)` or `AIAgentConfigDefault(...)` (Node: `LDAICompletionConfigDefault` / `LDAIAgentConfigDefault`). `AIConfig` is the base class the SDK returns; it isn't a valid default-value constructor — the typed `*Default` variants are. - If the result was being tuple-unpacked (`config, tracker = aiclient.config(...)`), drop the unpack — the new methods return a single config object. Obtain the tracker via `config.create_tracker()` / `aiConfig.createTracker()`. - - For deeper rewrites (call sites with hardcoded model/prompt as well), hand off to `aiconfig-migrate` instead of doing the full migration here. + - For deeper rewrites (call sites with hardcoded model/prompt as well), hand off to `agentcontrol-configs-migrate` instead of doing the full migration here. ### 2. Look up your Tier-2 option @@ -103,7 +103,7 @@ Obtain a tracker via the factory on the config object: `tracker = config.create_ ## Related skills -- `aiconfig-create` — prerequisite if the app doesn't have an AI Config yet -- `aiconfig-custom-metrics` — business metrics (conversion, resolution, retention) layered on top of the AI metrics this skill captures -- `aiconfig-online-evals` — automatic quality scoring (LLM-as-judge) on sampled live requests; complementary to the metrics here -- `aiconfig-migrate` — Stage 4 of the hardcoded-to-AI-Configs migration delegates to this skill +- `agentcontrol-configs-create` — prerequisite if the app doesn't have an AI Config yet +- `agentcontrol-configs-custom-metrics` — business metrics (conversion, resolution, retention) layered on top of the AI metrics this skill captures +- `agentcontrol-configs-online-evals` — automatic quality scoring (LLM-as-judge) on sampled live requests; complementary to the metrics here +- `agentcontrol-configs-migrate` — Stage 4 of the hardcoded-to-AI-Configs migration delegates to this skill diff --git a/skills/ai-configs/aiconfig-ai-metrics/references/anthropic-tracking.md b/skills/ai-configs/agentcontrol-configs-ai-metrics/references/anthropic-tracking.md similarity index 100% rename from skills/ai-configs/aiconfig-ai-metrics/references/anthropic-tracking.md rename to skills/ai-configs/agentcontrol-configs-ai-metrics/references/anthropic-tracking.md diff --git a/skills/ai-configs/aiconfig-ai-metrics/references/bedrock-tracking.md b/skills/ai-configs/agentcontrol-configs-ai-metrics/references/bedrock-tracking.md similarity index 100% rename from skills/ai-configs/aiconfig-ai-metrics/references/bedrock-tracking.md rename to skills/ai-configs/agentcontrol-configs-ai-metrics/references/bedrock-tracking.md diff --git a/skills/ai-configs/aiconfig-ai-metrics/references/gemini-tracking.md b/skills/ai-configs/agentcontrol-configs-ai-metrics/references/gemini-tracking.md similarity index 99% rename from skills/ai-configs/aiconfig-ai-metrics/references/gemini-tracking.md rename to skills/ai-configs/agentcontrol-configs-ai-metrics/references/gemini-tracking.md index c2ede4b..5fa6941 100644 --- a/skills/ai-configs/aiconfig-ai-metrics/references/gemini-tracking.md +++ b/skills/ai-configs/agentcontrol-configs-ai-metrics/references/gemini-tracking.md @@ -191,7 +191,7 @@ gemini_tools = [ ] if ld_tools else [] ``` -Tool handlers stay in your application code — LaunchDarkly stores the schema, your application owns the behavior. For the full agent loop pattern (`MAX_STEPS`, `functionCalls` handling, `tracker.track_tool_call`), see the agent-mode section of `aiconfig-tools`. +Tool handlers stay in your application code — LaunchDarkly stores the schema, your application owns the behavior. For the full agent loop pattern (`MAX_STEPS`, `functionCalls` handling, `tracker.track_tool_call`), see the agent-mode section of `agentcontrol-tools`. ## Tier 2 option — route via LangChain diff --git a/skills/ai-configs/aiconfig-ai-metrics/references/langchain-tracking.md b/skills/ai-configs/agentcontrol-configs-ai-metrics/references/langchain-tracking.md similarity index 97% rename from skills/ai-configs/aiconfig-ai-metrics/references/langchain-tracking.md rename to skills/ai-configs/agentcontrol-configs-ai-metrics/references/langchain-tracking.md index c67fc74..51329e7 100644 --- a/skills/ai-configs/aiconfig-ai-metrics/references/langchain-tracking.md +++ b/skills/ai-configs/agentcontrol-configs-ai-metrics/references/langchain-tracking.md @@ -252,7 +252,7 @@ for (const msg of result.messages ?? []) { ## Binding AI-Config-attached tools with `build_structured_tools` -If the variation has tools attached (via `/aiconfig-tools`), use `build_structured_tools` rather than hand-rolling a `TOOL_REGISTRY` / `resolve_tools` / `ALL_TOOLS` shape. The helper reads `ai_config.model.parameters.tools`, picks the matching entries from your `{name: callable}` registry, wraps them as LangChain `StructuredTool` instances, and preserves the LD tool key as the `StructuredTool.name` (so `ToolNode(...)` lookup works without a second mapping). +If the variation has tools attached (via `/agentcontrol-tools`), use `build_structured_tools` rather than hand-rolling a `TOOL_REGISTRY` / `resolve_tools` / `ALL_TOOLS` shape. The helper reads `ai_config.model.parameters.tools`, picks the matching entries from your `{name: callable}` registry, wraps them as LangChain `StructuredTool` instances, and preserves the LD tool key as the `StructuredTool.name` (so `ToolNode(...)` lookup works without a second mapping). ```python # tools.py — implementations only; no manual schema, no resolve_tools() diff --git a/skills/ai-configs/aiconfig-ai-metrics/references/metrics-api.md b/skills/ai-configs/agentcontrol-configs-ai-metrics/references/metrics-api.md similarity index 100% rename from skills/ai-configs/aiconfig-ai-metrics/references/metrics-api.md rename to skills/ai-configs/agentcontrol-configs-ai-metrics/references/metrics-api.md diff --git a/skills/ai-configs/aiconfig-ai-metrics/references/openai-tracking.md b/skills/ai-configs/agentcontrol-configs-ai-metrics/references/openai-tracking.md similarity index 100% rename from skills/ai-configs/aiconfig-ai-metrics/references/openai-tracking.md rename to skills/ai-configs/agentcontrol-configs-ai-metrics/references/openai-tracking.md diff --git a/skills/ai-configs/aiconfig-ai-metrics/references/strands-tracking.md b/skills/ai-configs/agentcontrol-configs-ai-metrics/references/strands-tracking.md similarity index 93% rename from skills/ai-configs/aiconfig-ai-metrics/references/strands-tracking.md rename to skills/ai-configs/agentcontrol-configs-ai-metrics/references/strands-tracking.md index 95135e0..9490a96 100644 --- a/skills/ai-configs/aiconfig-ai-metrics/references/strands-tracking.md +++ b/skills/ai-configs/agentcontrol-configs-ai-metrics/references/strands-tracking.md @@ -79,7 +79,7 @@ Pick the style that matches the rest of the codebase — the two variants record ## Provider dispatch stays in your code -Strands model classes are provider-specific (`AnthropicModel`, `OpenAIModel`, `BedrockModel`). To serve more than one provider from a single AI Config key, dispatch on `agent_config.provider.name` before constructing the `Agent`. See [agent-mode-frameworks.md § Strands Agent](../../aiconfig-migrate/references/agent-mode-frameworks.md) for the `create_strands_model` dispatcher, including the rule that `parameters.tools` must be dropped before being passed into the Strands model class (tools flow through the `Agent` constructor, not through model params). +Strands model classes are provider-specific (`AnthropicModel`, `OpenAIModel`, `BedrockModel`). To serve more than one provider from a single AI Config key, dispatch on `agent_config.provider.name` before constructing the `Agent`. See [agent-mode-frameworks.md § Strands Agent](../../agentcontrol-configs-migrate/references/agent-mode-frameworks.md) for the `create_strands_model` dispatcher, including the rule that `parameters.tools` must be dropped before being passed into the Strands model class (tools flow through the `Agent` constructor, not through model params). ## Always flush before exit diff --git a/skills/ai-configs/aiconfig-ai-metrics/references/streaming-tracking.md b/skills/ai-configs/agentcontrol-configs-ai-metrics/references/streaming-tracking.md similarity index 100% rename from skills/ai-configs/aiconfig-ai-metrics/references/streaming-tracking.md rename to skills/ai-configs/agentcontrol-configs-ai-metrics/references/streaming-tracking.md diff --git a/skills/ai-configs/aiconfig-create/README.md b/skills/ai-configs/agentcontrol-configs-create/README.md similarity index 73% rename from skills/ai-configs/aiconfig-create/README.md rename to skills/ai-configs/agentcontrol-configs-create/README.md index 0a46092..901271e 100644 --- a/skills/ai-configs/aiconfig-create/README.md +++ b/skills/ai-configs/agentcontrol-configs-create/README.md @@ -12,7 +12,7 @@ This skill teaches agents how to: ## Installation (Local) -Copy `skills/ai-configs/aiconfig-create/` into your agent client's skills path. +Copy `skills/ai-configs/agentcontrol-configs-create/` into your agent client's skills path. ## Prerequisites @@ -31,16 +31,16 @@ Set up an AI config for content generation using Claude ## Structure ``` -aiconfig-create/ +agentcontrol-configs-create/ ├── SKILL.md └── README.md ``` ## Related -- [AI Config Projects](../aiconfig-projects/): Create projects first -- [AI Config Tools](../aiconfig-tools/): Add tools after creating config -- [AI Config Variations](../aiconfig-variations/): Add more variations for experimentation +- [AI Config Projects](../agentcontrol-projects/): Create projects first +- [AI Config Tools](../agentcontrol-tools/): Add tools after creating config +- [AI Config Variations](../agentcontrol-configs-variations/): Add more variations for experimentation - [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) ## License diff --git a/skills/ai-configs/aiconfig-create/SKILL.md b/skills/ai-configs/agentcontrol-configs-create/SKILL.md similarity index 90% rename from skills/ai-configs/aiconfig-create/SKILL.md rename to skills/ai-configs/agentcontrol-configs-create/SKILL.md index 0815a89..43a5c76 100644 --- a/skills/ai-configs/aiconfig-create/SKILL.md +++ b/skills/ai-configs/agentcontrol-configs-create/SKILL.md @@ -1,5 +1,5 @@ --- -name: aiconfig-create +name: agentcontrol-configs-create description: "Create and configure AI Configs in LaunchDarkly. Helps you choose between agent vs completion mode, create the config, add variations with models and prompts, and verify the setup." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server @@ -12,7 +12,7 @@ metadata: You're using a skill that will guide you through creating an AI Config in LaunchDarkly. Your job is to understand the use case, choose the right mode, create the config and its variations, and verify everything is set up correctly. -> **⚠️ This skill creates a config — it does not make it servable.** A freshly-created AI Config has its **fallthrough pointing at an auto-generated disabled variation**, not at the variation you just created. The SDK will return `ai_config.enabled=False` on every evaluation until you flip targeting on and point the fallthrough at your new variation. This is not a bug — it's the default state. **You must run `/aiconfig-targeting` (or the equivalent REST / CLI call shown in Step 5) before verifying against the SDK**, or verification will look like the LD-served path is broken when it isn't. The single most common failure mode users hit with this skill is skipping the targeting step and spending time debugging `enabled=False` in their application code. +> **⚠️ This skill creates a config — it does not make it servable.** A freshly-created AI Config has its **fallthrough pointing at an auto-generated disabled variation**, not at the variation you just created. The SDK will return `ai_config.enabled=False` on every evaluation until you flip targeting on and point the fallthrough at your new variation. This is not a bug — it's the default state. **You must run `/agentcontrol-configs-targeting` (or the equivalent REST / CLI call shown in Step 5) before verifying against the SDK**, or verification will look like the LD-served path is broken when it isn't. The single most common failure mode users hit with this skill is skipping the targeting step and spending time debugging `enabled=False` in their application code. ## Prerequisites @@ -147,7 +147,7 @@ Print this checklist verbatim to the user after Step 4, then wait for confirmati > > 🔴 **The SDK will return `enabled=False` until you flip targeting on.** The fallthrough is currently pointing at an auto-generated disabled variation, not at the `{variationKey}` you just created. > -> **Next step — run `/aiconfig-targeting`** with these inputs: +> **Next step — run `/agentcontrol-configs-targeting`** with these inputs: > - Project key: `{projectKey}` > - Config key: `{configKey}` > - Environment key: the env whose SDK key is in your `.env` (usually `test` or `production`) @@ -159,7 +159,7 @@ Print this checklist verbatim to the user after Step 4, then wait for confirmati #### Direct shortcut if the user wants to flip targeting without invoking the sibling skill -`aiconfig-targeting` is the canonical path — it handles percentage rollouts, targeted rules, and variation-ID lookups. But for the simplest case ("promote the new variation to fallthrough in one environment"), you can run the underlying semantic PATCH yourself once you know the new variation's `_id`. +`agentcontrol-configs-targeting` is the canonical path — it handles percentage rollouts, targeted rules, and variation-ID lookups. But for the simplest case ("promote the new variation to fallthrough in one environment"), you can run the underlying semantic PATCH yourself once you know the new variation's `_id`. Get the variation ID (use `get-ai-config` MCP, or): ```bash @@ -203,7 +203,7 @@ The `create-ai-config-variation` tool validates this format and rejects invalid |-----------|--------| | Config already exists | Ask if user wants to update instead | | Variation shows "NO MODEL" | Use `update-ai-config-variation` to set modelConfigKey | -| Need to attach tools | Create tools first (`aiconfig-tools` skill), then update the variation | +| Need to attach tools | Create tools first (`agentcontrol-tools` skill), then update the variation | ## What NOT to Do @@ -213,10 +213,10 @@ The `create-ai-config-variation` tool validates this format and rejects invalid - Don't forget modelConfigKey (models won't show in the UI) - Don't omit `modelName` from the initial variation call. It is required at create time; setting it via a follow-up PATCH is a workaround for a bug, not the intended flow. The PATCH field is also `modelName`, not `name`. - Don't drop to raw `curl` + `jq` for verification. Use `get-ai-config` (MCP) — it returns a typed object and avoids brittle `jq` filters that break on response-shape variation. -- Don't consider the workflow complete until the user has been told to run `aiconfig-targeting`. A created variation that isn't promoted to fallthrough returns `enabled=False` to every consumer. +- Don't consider the workflow complete until the user has been told to run `agentcontrol-configs-targeting`. A created variation that isn't promoted to fallthrough returns `enabled=False` to every consumer. ## Related Skills -- `aiconfig-tools` -- Create tools before attaching -- `aiconfig-variations` -- Add more variations for experimentation -- `aiconfig-update` -- Modify configs based on learnings +- `agentcontrol-tools` -- Create tools before attaching +- `agentcontrol-configs-variations` -- Add more variations for experimentation +- `agentcontrol-configs-update` -- Modify configs based on learnings diff --git a/skills/ai-configs/aiconfig-custom-metrics/SKILL.md b/skills/ai-configs/agentcontrol-configs-custom-metrics/SKILL.md similarity index 97% rename from skills/ai-configs/aiconfig-custom-metrics/SKILL.md rename to skills/ai-configs/agentcontrol-configs-custom-metrics/SKILL.md index 741183f..b39e397 100644 --- a/skills/ai-configs/aiconfig-custom-metrics/SKILL.md +++ b/skills/ai-configs/agentcontrol-configs-custom-metrics/SKILL.md @@ -1,5 +1,5 @@ --- -name: aiconfig-custom-metrics +name: agentcontrol-configs-custom-metrics description: "Create, track, retrieve, update, and delete custom business metrics for AI Configs. Covers full lifecycle: define metric kinds via API, emit events via SDK, and query results." license: Apache-2.0 compatibility: Requires the LaunchDarkly server SDK and a LaunchDarkly API token with the `writer` role for metric management. @@ -14,9 +14,9 @@ Full lifecycle management of custom business metrics: create metric definitions ## Prerequisites -- LaunchDarkly SDK initialized (see `aiconfig-sdk`) +- LaunchDarkly SDK initialized (see `agentcontrol-sdk`) - LaunchDarkly API token with `writer` role for metric management -- Understanding of built-in AI metrics (see `aiconfig-ai-metrics`) +- Understanding of built-in AI metrics (see `agentcontrol-configs-ai-metrics`) ## API Key Detection @@ -149,7 +149,7 @@ from ldclient import Context from ldclient.config import Config import ldclient -# Initialize (see aiconfig-sdk for details) +# Initialize (see agentcontrol-sdk for details) ldclient.set_config(Config("your-sdk-key")) ld_client = ldclient.get() @@ -492,9 +492,9 @@ Custom metrics appear in: ## Related Skills -- `aiconfig-sdk` - SDK setup -- `aiconfig-ai-metrics` - Built-in AI metrics (tokens, duration, cost) -- `aiconfig-online-evals` - Quality metrics via judges +- `agentcontrol-sdk` - SDK setup +- `agentcontrol-configs-ai-metrics` - Built-in AI metrics (tokens, duration, cost) +- `agentcontrol-configs-online-evals` - Quality metrics via judges ## References diff --git a/skills/ai-configs/aiconfig-migrate/README.md b/skills/ai-configs/agentcontrol-configs-migrate/README.md similarity index 75% rename from skills/ai-configs/aiconfig-migrate/README.md rename to skills/ai-configs/agentcontrol-configs-migrate/README.md index 6849861..ed61dd0 100644 --- a/skills/ai-configs/aiconfig-migrate/README.md +++ b/skills/ai-configs/agentcontrol-configs-migrate/README.md @@ -9,14 +9,14 @@ This skill orchestrates the full migration journey from hardcoded `openai.chat.c The five stages: 1. **Extract** hardcoded model names, prompts, and parameters (read-only) -2. **Wrap** the call site in `completion_config` / `completionConfig` with a safe fallback — delegates the config creation to `aiconfig-create` -3. **Tools** — move function-calling schemas into LaunchDarkly — delegates to `aiconfig-tools` +2. **Wrap** the call site in `completion_config` / `completionConfig` with a safe fallback — delegates the config creation to `agentcontrol-configs-create` +3. **Tools** — move function-calling schemas into LaunchDarkly — delegates to `agentcontrol-tools` 4. **Tracking** — wire `track_duration`, `track_tokens`, `track_success`/`track_error`, optional `track_feedback` — inline, with a reference doc covering every SDK method in Python and Node side by side -5. **Evals** — attach judges for LLM-as-a-judge scoring — delegates to `aiconfig-online-evals` +5. **Evals** — attach judges for LLM-as-a-judge scoring — delegates to `agentcontrol-configs-online-evals` ## Installation (Local) -Copy `skills/ai-configs/aiconfig-migrate/` into your agent client's skills path. +Copy `skills/ai-configs/agentcontrol-configs-migrate/` into your agent client's skills path. ## Prerequisites @@ -41,7 +41,7 @@ Wire up the AI tracker and attach accuracy + relevance judges to our existing co ## Structure ``` -aiconfig-migrate/ +agentcontrol-configs-migrate/ ├── SKILL.md ├── README.md └── references/ @@ -55,11 +55,11 @@ aiconfig-migrate/ ## Related -- [AI Config Create](../aiconfig-create/): Delegated to by Stage 2 (wrap) -- [AI Config Tools](../aiconfig-tools/): Delegated to by Stage 3 (tools) -- [AI Config Online Evals](../aiconfig-online-evals/): Delegated to by Stage 5 (evals) -- [AI Config Variations](../aiconfig-variations/): Next step after migration for A/B testing -- [AI Config Targeting](../aiconfig-targeting/): Next step after migration for rollout control +- [AI Config Create](../agentcontrol-configs-create/): Delegated to by Stage 2 (wrap) +- [AI Config Tools](../agentcontrol-tools/): Delegated to by Stage 3 (tools) +- [AI Config Online Evals](../agentcontrol-configs-online-evals/): Delegated to by Stage 5 (evals) +- [AI Config Variations](../agentcontrol-configs-variations/): Next step after migration for A/B testing +- [AI Config Targeting](../agentcontrol-configs-targeting/): Next step after migration for rollout control - [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) ## License diff --git a/skills/ai-configs/aiconfig-migrate/SKILL.md b/skills/ai-configs/agentcontrol-configs-migrate/SKILL.md similarity index 86% rename from skills/ai-configs/aiconfig-migrate/SKILL.md rename to skills/ai-configs/agentcontrol-configs-migrate/SKILL.md index 296f466..f23a609 100644 --- a/skills/ai-configs/aiconfig-migrate/SKILL.md +++ b/skills/ai-configs/agentcontrol-configs-migrate/SKILL.md @@ -1,5 +1,5 @@ --- -name: aiconfig-migrate +name: agentcontrol-configs-migrate description: "Migrate an application with hardcoded LLM prompts to a full LaunchDarkly AI Configs implementation in five stages: audit the code, wrap the call, move the tools, add tracking, attach evaluators. Use when the user wants to externalize model/prompt configuration, move from direct provider calls (OpenAI, Anthropic, Bedrock, Gemini, Strands) to a managed AI Config, or stage a full hardcoded-to-LaunchDarkly migration." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server @@ -22,7 +22,7 @@ You're using a skill that will guide you through migrating an application from h > > 1. **Tracker in the wrong scope.** For an agent with a loop, mint `create_tracker()` once per user turn in a `setup_run` entry node — not inside `call_model`. Per-iteration factory calls produce N `runId`s and trip the at-most-once guards. See [agent-mode-frameworks.md § Custom `StateGraph`](references/agent-mode-frameworks.md). > 2. **`load_chat_model` wrapper reuse.** Templates like `langchain-ai/react-agent` ship a `load_chat_model(f"{provider}/{name}")` helper that wraps `init_chat_model(...)` and silently drops every variation parameter. **Delete it** (don't just avoid using it) and replace call sites with `create_langchain_model(ai_config)`. -> 3. **Fallthrough not flipped after `/aiconfig-create`.** A freshly-created AI Config's fallthrough points at an auto-generated disabled variation, so the SDK returns `enabled=False` until `/aiconfig-targeting` runs. Flip it before Stage 2 verification. +> 3. **Fallthrough not flipped after `/agentcontrol-configs-create`.** A freshly-created AI Config's fallthrough points at an auto-generated disabled variation, so the SDK returns `enabled=False` until `/agentcontrol-configs-targeting` runs. Flip it before Stage 2 verification. ## Coverage — which shapes are well-trodden vs require extrapolation @@ -30,16 +30,16 @@ The skill is optimized for Python and Node.js / TypeScript; other languages are | Shape | Python | Node.js | Reference | |-------|--------|---------|-----------| -| One-shot completion (direct OpenAI / Anthropic / Bedrock / Gemini call) | ✅ Worked example | ✅ Worked example | [before-after-examples.md](references/before-after-examples.md), per-provider docs in `aiconfig-ai-metrics/references/` | -| Chat loop via managed runner (`ManagedModel`) | ✅ Tier 1 pattern | ✅ Tier 1 pattern | [aiconfig-ai-metrics SKILL.md](../aiconfig-ai-metrics/SKILL.md) | -| LangChain single-call | ✅ Worked example | ✅ Worked example | [langchain-tracking.md](../aiconfig-ai-metrics/references/langchain-tracking.md) | +| One-shot completion (direct OpenAI / Anthropic / Bedrock / Gemini call) | ✅ Worked example | ✅ Worked example | [before-after-examples.md](references/before-after-examples.md), per-provider docs in `agentcontrol-configs-ai-metrics/references/` | +| Chat loop via managed runner (`ManagedModel`) | ✅ Tier 1 pattern | ✅ Tier 1 pattern | [agentcontrol-configs-ai-metrics SKILL.md](../agentcontrol-configs-ai-metrics/SKILL.md) | +| LangChain single-call | ✅ Worked example | ✅ Worked example | [langchain-tracking.md](../agentcontrol-configs-ai-metrics/references/langchain-tracking.md) | | LangGraph prebuilt agent (Python `langchain.agents.create_agent`, Node `createReactAgent`) | ✅ Worked example | ✅ Worked example | [agent-mode-frameworks.md § LangGraph](references/agent-mode-frameworks.md) | | LangGraph custom `StateGraph` with run-scoped tracker (setup_run + call_model + finalize) | ✅ Deep worked example | ⚠️ Mentioned — translate from Python | [agent-mode-frameworks.md § Custom `StateGraph`](references/agent-mode-frameworks.md) | | CrewAI `Agent` | ✅ Worked example | — (not a Node framework) | [agent-mode-frameworks.md § CrewAI](references/agent-mode-frameworks.md) | | Strands `Agent` | ✅ Worked example | ⚠️ BedrockModel + OpenAIModel only (no Anthropic) | [agent-mode-frameworks.md § Strands](references/agent-mode-frameworks.md) | | Custom ReAct loop (hand-rolled, any framework or none) | ✅ Worked example | ⚠️ Apply framework-agnostic invariants; translate from Python | [agent-mode-frameworks.md § Custom ReAct loop](references/agent-mode-frameworks.md) | -| Vercel AI SDK (`generateText` / `streamText`) | — (not a Python framework) | ⚠️ Provider package exists; no worked example in skill | `aiconfig-ai-metrics` provider-package matrix | -| Streaming (SSE / WebSocket) | ⚠️ Delegated to `aiconfig-ai-metrics` streaming doc | ⚠️ Same — use `trackStreamMetricsOf` + manual TTFT | [streaming-tracking.md](../aiconfig-ai-metrics/references/streaming-tracking.md) | +| Vercel AI SDK (`generateText` / `streamText`) | — (not a Python framework) | ⚠️ Provider package exists; no worked example in skill | `agentcontrol-configs-ai-metrics` provider-package matrix | +| Streaming (SSE / WebSocket) | ⚠️ Delegated to `agentcontrol-configs-ai-metrics` streaming doc | ⚠️ Same — use `trackStreamMetricsOf` + manual TTFT | [streaming-tracking.md](../agentcontrol-configs-ai-metrics/references/streaming-tracking.md) | | Multi-agent graph (supervisor + workers) | ⚠️ Out of main scope; see reference | ⚠️ Out of main scope; see reference | [agent-graph-reference.md](references/agent-graph-reference.md) | | Non-LangGraph agent frameworks (Pydantic AI, DSPy, AutoGen, Haystack, LlamaIndex agents, Semantic Kernel) | ⚠️ Apply the three invariants; no framework-specific example | ⚠️ Same | [agent-mode-frameworks.md § Framework-agnostic invariants](references/agent-mode-frameworks.md) | | Go, Ruby, .NET | ℹ️ Install commands only | ℹ️ Install commands only | [phase-1-analysis-checklist.md § SDK routing table](references/phase-1-analysis-checklist.md) | @@ -67,11 +67,11 @@ If a CHANGELOG entry post-dates this skill and changes an API you're about to us **Hand-off model.** This skill does **not** auto-invoke other skills. At each stage that needs a LaunchDarkly write, this skill prepares the inputs (config key, mode, model, prompt, tool schemas, judge keys) and then **tells the user to run the next slash-command themselves**. After the user finishes that sibling skill, return to the next step here. Treat the "Delegate" lines below as next-step instructions, not auto-handoffs. **Sibling skills the user runs at each stage:** -- `aiconfig-projects` — pre-Stage 2, only if no project exists yet -- `aiconfig-create` — Stage 2 (creates the AI Config and first variation) -- `aiconfig-tools` — Stage 3 (creates tool definitions and attaches them) -- `aiconfig-targeting` — between Stage 2 and Stage 4 (promotes the new variation to fallthrough so the SDK actually serves it) -- `aiconfig-online-evals` — Stage 5 (attaches judges, creates custom judges) +- `agentcontrol-projects` — pre-Stage 2, only if no project exists yet +- `agentcontrol-configs-create` — Stage 2 (creates the AI Config and first variation) +- `agentcontrol-tools` — Stage 3 (creates tool definitions and attaches them) +- `agentcontrol-configs-targeting` — between Stage 2 and Stage 4 (promotes the new variation to fallthrough so the SDK actually serves it) +- `agentcontrol-configs-online-evals` — Stage 5 (attaches judges, creates custom judges) ## Core Principles @@ -79,7 +79,7 @@ If a CHANGELOG entry post-dates this skill and changes an API you're about to us 2. **Replace config, not business logic.** The SDK call is a drop-in for the place where the model, parameters, and prompt are *defined* — not for the provider call itself. OpenAI/Anthropic/Bedrock calls stay where they are. 3. **Fallback mirrors current behavior.** The fallback passed to `completion_config` / `agent_config` must preserve the hardcoded values you removed, so the app is unchanged if LaunchDarkly is unreachable. 4. **Stages are ordered.** Wrap before you add tools. Add tools before you track. Track before you add evals. Skipping ahead produces configs without traffic, metrics without context, and judges with nothing to score. -5. **Hand off to focused skills, manually.** Each stage that needs a LaunchDarkly write tells the user to run a sibling slash-command (`/aiconfig-create`, `/aiconfig-tools`, `/aiconfig-targeting`, `/aiconfig-online-evals`) and waits for them to come back. This skill does **not** auto-invoke other skills. +5. **Hand off to focused skills, manually.** Each stage that needs a LaunchDarkly write tells the user to run a sibling slash-command (`/agentcontrol-configs-create`, `/agentcontrol-tools`, `/agentcontrol-configs-targeting`, `/agentcontrol-configs-online-evals`) and waits for them to come back. This skill does **not** auto-invoke other skills. ## Workflow @@ -160,7 +160,7 @@ This is the first stage that writes code. It has nine sub-steps. - OpenAI: `launchdarkly-server-sdk-ai-openai>=0.4.0` (Python) / `@launchdarkly/server-sdk-ai-openai@^0.5.5` (Node) - LangChain / LangGraph: `launchdarkly-server-sdk-ai-langchain>=0.5.0` (Python) / `@launchdarkly/server-sdk-ai-langchain@^0.5.5` (Node) - Vercel AI SDK (Node only): `@launchdarkly/server-sdk-ai-vercel@^0.5.5` - - Anthropic, Gemini, Bedrock — no provider package published; use Tier-3 custom extractor (see `aiconfig-ai-metrics`) + - Anthropic, Gemini, Bedrock — no provider package published; use Tier-3 custom extractor (see `agentcontrol-configs-ai-metrics`) 3. **Initialize `LDAIClient` once at startup.** Reuse any existing `LDClient` — do not create a second base client. Place the initialization in the same module that owns existing app config. @@ -207,11 +207,11 @@ This is the first stage that writes code. It has nine sub-steps. const aiClient = initAi(ldClient); ``` -4. **Hand off to `aiconfig-create`.** Print the extracted model, prompt/instructions, parameters, and mode from the Stage 1 manifest, then tell the user: *"Run `/aiconfig-create` with these inputs, then come back here."* Supply the config key you want the code to call (e.g. `chat-assistant`). Do not attempt to auto-invoke the sibling skill — wait for the user to finish it before continuing. +4. **Hand off to `agentcontrol-configs-create`.** Print the extracted model, prompt/instructions, parameters, and mode from the Stage 1 manifest, then tell the user: *"Run `/agentcontrol-configs-create` with these inputs, then come back here."* Supply the config key you want the code to call (e.g. `chat-assistant`). Do not attempt to auto-invoke the sibling skill — wait for the user to finish it before continuing. - **After `aiconfig-create` finishes, the user must also run `/aiconfig-targeting` to promote the new variation to fallthrough.** A freshly created variation returns `enabled=False` to every consumer until targeting is updated. Skip this and Stage 2 verification (sub-step 9 below) will silently take the fallback path on every request. + **After `agentcontrol-configs-create` finishes, the user must also run `/agentcontrol-configs-targeting` to promote the new variation to fallthrough.** A freshly created variation returns `enabled=False` to every consumer until targeting is updated. Skip this and Stage 2 verification (sub-step 9 below) will silently take the fallback path on every request. -5. **Rewrite template placeholders to Mustache syntax.** If the hardcoded prompt interpolates runtime values with Python `.format()`, f-strings, JS template literals, or any other non-Mustache syntax (e.g. `{system_time}`, `${userName}`, `%(topic)s`), rewrite every placeholder to `{{ variable }}` Mustache form. Do this in **both** the file you're about to send to `/aiconfig-create` *and* the fallback string you'll write in sub-step 6. The AI SDK interpolates variables through a Mustache renderer on the LD-served path *and* the fallback path using the fourth-argument `variables` dict to `completion_config(...)` / `completionConfig(...)`. Leaving a Python-style `{system_time}` literal in the fallback ships a silent regression when LaunchDarkly is unreachable — the renderer won't match the single-brace form and the literal `{system_time}` goes to the provider as part of the prompt. +5. **Rewrite template placeholders to Mustache syntax.** If the hardcoded prompt interpolates runtime values with Python `.format()`, f-strings, JS template literals, or any other non-Mustache syntax (e.g. `{system_time}`, `${userName}`, `%(topic)s`), rewrite every placeholder to `{{ variable }}` Mustache form. Do this in **both** the file you're about to send to `/agentcontrol-configs-create` *and* the fallback string you'll write in sub-step 6. The AI SDK interpolates variables through a Mustache renderer on the LD-served path *and* the fallback path using the fourth-argument `variables` dict to `completion_config(...)` / `completionConfig(...)`. Leaving a Python-style `{system_time}` literal in the fallback ships a silent regression when LaunchDarkly is unreachable — the renderer won't match the single-brace form and the literal `{system_time}` goes to the provider as part of the prompt. **Before:** ```python @@ -318,7 +318,7 @@ This is the first stage that writes code. It has nine sub-steps. 9. **Verify.** Run the app with a valid `LD_SDK_KEY`; confirm the call succeeds and the response matches pre-migration output. Then temporarily set `LD_SDK_KEY=sdk-invalid` (or unset it) and confirm the fallback path runs without error. Both paths must work before moving to Stage 3. -Delegate: **`aiconfig-create`** (sub-step 4). +Delegate: **`agentcontrol-configs-create`** (sub-step 4). ### Step 3: Move tools into the config (Stage 3) @@ -338,7 +338,7 @@ Skip this step if the audited app has no function calling / tools. Otherwise: For LangChain/LangGraph tools defined with `@tool`, extract the schema via `tool.args_schema.model_json_schema()` (or the equivalent Pydantic `model_json_schema()` call). For plain async callables used as tools (common in custom StateGraph shapes), LangChain infers the schema from the function signature at bind time — extract it via `StructuredTool.from_function(fn).args_schema.model_json_schema()`. Do not hand-write the schema. -2. **Hand off to `aiconfig-tools`.** Print the extracted tool names, descriptions, and schemas, then tell the user: *"Run `/aiconfig-tools` with these tools and the variation key, then come back here."* The sibling skill creates tool definitions (`create-ai-tool`) and attaches them to the variation (`update-ai-config-variation`). Wait for the user to finish before proceeding to sub-step 3. Do not auto-invoke. +2. **Hand off to `agentcontrol-tools`.** Print the extracted tool names, descriptions, and schemas, then tell the user: *"Run `/agentcontrol-tools` with these tools and the variation key, then come back here."* The sibling skill creates tool definitions (`create-ai-tool`) and attaches them to the variation (`update-ai-config-variation`). Wait for the user to finish before proceeding to sub-step 3. Do not auto-invoke. 3. **Replace the hardcoded tools array at the call site** with a read from `config.tools` (or the SDK equivalent for your language). Load the actual implementation functions dynamically from the tool names — see [agent-mode-frameworks.md](references/agent-mode-frameworks.md) for the dynamic-tool-factory pattern from the devrel agents tutorial. @@ -346,13 +346,13 @@ Skip this step if the audited app has no function calling / tools. Otherwise: 4. **Verify.** Run the app; confirm the tool flows still execute correctly. `get-ai-config` (via the delegate) confirms the tools are attached server-side. -Delegate: **`aiconfig-tools`** (sub-step 2). +Delegate: **`agentcontrol-tools`** (sub-step 2). ### Step 4: Instrument the tracker (Stage 4) -Delegate: **`aiconfig-ai-metrics`** wires the per-request `tracker.track_*` calls (duration, tokens, success/error, feedback) around the provider call. Use **`aiconfig-custom-metrics`** alongside it if the app needs business metrics beyond the built-in AI ones. Note: do not confuse this with `launchdarkly-metric-instrument`, which is for `ldClient.track()` feature metrics — a different API. See [sdk-ai-tracker-patterns.md](references/sdk-ai-tracker-patterns.md) for the full per-method Python + Node matrix that the delegate skill draws on. +Delegate: **`agentcontrol-configs-ai-metrics`** wires the per-request `tracker.track_*` calls (duration, tokens, success/error, feedback) around the provider call. Use **`agentcontrol-configs-custom-metrics`** alongside it if the app needs business metrics beyond the built-in AI ones. Note: do not confuse this with `launchdarkly-metric-instrument`, which is for `ldClient.track()` feature metrics — a different API. See [sdk-ai-tracker-patterns.md](references/sdk-ai-tracker-patterns.md) for the full per-method Python + Node matrix that the delegate skill draws on. -Hand off: print the AI Config key, variation key, provider, and whether the call is streaming, then tell the user: *"Run `/aiconfig-ai-metrics` with these inputs, then come back here."* Do not auto-invoke. Return here for sub-step 5 (verify) once they're done. +Hand off: print the AI Config key, variation key, provider, and whether the call is streaming, then tell the user: *"Run `/agentcontrol-configs-ai-metrics` with these inputs, then come back here."* Do not auto-invoke. Return here for sub-step 5 (verify) once they're done. 1. **Create the tracker.** Obtain a per-execution tracker via the factory on the config returned in Stage 2: `tracker = config.create_tracker()` (Python) or `const tracker = aiConfig.createTracker();` (Node). Call the factory **once per user turn** and reuse the returned `tracker` for every tracking call in that turn — each call mints a fresh `runId` that tags every event emitted from the turn so they can be correlated via exported events or downstream queries. (The Monitoring tab aggregates today; run-level grouping is a downstream concern — but the `runId` is also what the SDK's at-most-once guards are keyed on, so minting a new one mid-turn breaks the guard semantics regardless of where the events end up.) @@ -404,7 +404,7 @@ Hand off: print the AI Config key, variation key, provider, and whether the call ); ``` - For Anthropic direct, Bedrock (no provider package), Gemini, and custom HTTP, write a small extractor returning `LDAIMetrics` — see the delegate skill's [anthropic-tracking.md](../aiconfig-ai-metrics/references/anthropic-tracking.md), [bedrock-tracking.md](../aiconfig-ai-metrics/references/bedrock-tracking.md), and [gemini-tracking.md](../aiconfig-ai-metrics/references/gemini-tracking.md). LangChain single-node and LangGraph go through the `launchdarkly-server-sdk-ai-langchain` / `@launchdarkly/server-sdk-ai-langchain` provider package. Build the model with `create_langchain_model(config)` (Python) / `createLangChainModel(config)` (Node) — both forward all variation parameters — and track with `get_ai_metrics_from_response` / `getAIMetricsFromResponse`. See [langchain-tracking.md](../aiconfig-ai-metrics/references/langchain-tracking.md). + For Anthropic direct, Bedrock (no provider package), Gemini, and custom HTTP, write a small extractor returning `LDAIMetrics` — see the delegate skill's [anthropic-tracking.md](../agentcontrol-configs-ai-metrics/references/anthropic-tracking.md), [bedrock-tracking.md](../agentcontrol-configs-ai-metrics/references/bedrock-tracking.md), and [gemini-tracking.md](../agentcontrol-configs-ai-metrics/references/gemini-tracking.md). LangChain single-node and LangGraph go through the `launchdarkly-server-sdk-ai-langchain` / `@launchdarkly/server-sdk-ai-langchain` provider package. Build the model with `create_langchain_model(config)` (Python) / `createLangChainModel(config)` (Node) — both forward all variation parameters — and track with `get_ai_metrics_from_response` / `getAIMetricsFromResponse`. See [langchain-tracking.md](../agentcontrol-configs-ai-metrics/references/langchain-tracking.md). 4. **Wire feedback tracking if the app has thumbs-up/down UI.** Both SDKs expose `trackFeedback` with a `{kind}` argument. @@ -442,11 +442,11 @@ Hand off: print the AI Config key, variation key, provider, and whether the call - Run the `default` variation (or whichever variation mirrors the pre-migration hardcoded behavior) against the dataset first — this is the baseline. - Clone it into a second variation pointing at a **different model family** (e.g., if the baseline is `anthropic/claude-sonnet-4-5`, clone to `openai/gpt-4o` or `openai/gpt-4o-mini`). The comparison is most informative across families, not across siblings. - Attach the built-in **Accuracy** judge with a pass threshold of **0.85**, and run both variations against the same dataset. - - Promote the winner to fallthrough via `/aiconfig-targeting` only if it beats the baseline on Accuracy and does not regress on Relevance or Toxicity. + - Promote the winner to fallthrough via `/agentcontrol-configs-targeting` only if it beats the baseline on Accuracy and does not regress on Relevance or Toxicity. Write this shape into the project's `datasets/README.md` (or equivalent) so the comparison pattern is reproducible after the migration ships. -3. **Hand off to `aiconfig-online-evals`** — only for UI-attached judges (completion mode) or to create custom judge AI Configs that will be referenced by the programmatic path. Tell the user: *"Run `/aiconfig-online-evals` with these inputs, then come back here."* Do not auto-invoke. Pass: +3. **Hand off to `agentcontrol-configs-online-evals`** — only for UI-attached judges (completion mode) or to create custom judge AI Configs that will be referenced by the programmatic path. Tell the user: *"Run `/agentcontrol-configs-online-evals` with these inputs, then come back here."* Do not auto-invoke. Pass: - The parent AI Config key and variation key - A list of built-in judges (Accuracy, Relevance, Toxicity) or custom judge keys to create/attach - Target environment @@ -480,14 +480,14 @@ Hand off: print the AI Config key, variation key, provider, and whether the call - **`sampling_rate` is a parameter on `evaluate()`**, not on `create_judge`. It defaults to `1.0` (evaluate every call). For live paths, pass something lower (0.1–0.25) to control cost. - **`evaluate()` returns a `JudgeResult`** (never `None`). Check `result.sampled` to know whether the evaluation actually ran, and call `track_judge_result(result)`. Node uses `trackJudgeResult(result)` and `LDJudgeResult` with the same `sampled` field. - **Ask the user which judge AI Config key to use.** LaunchDarkly ships three built-in judges — Accuracy, Relevance, Toxicity — but the actual AI Config **keys** for the built-ins are not canonical SDK constants and aren't documented. Have the user open **AI Configs > Library** in the LD UI and copy the key of the judge they want to reference, or create a custom judge AI Config via `aiconfig-create` first. + **Ask the user which judge AI Config key to use.** LaunchDarkly ships three built-in judges — Accuracy, Relevance, Toxicity — but the actual AI Config **keys** for the built-ins are not canonical SDK constants and aren't documented. Have the user open **AI Configs > Library** in the LD UI and copy the key of the judge they want to reference, or create a custom judge AI Config via `agentcontrol-configs-create` first. 5. **Verify.** - **UI-attached auto judges:** trigger a request in staging, open the Monitoring tab → "Evaluator metrics" dropdown. Scores appear within 1–2 minutes at the configured sampling rate. - **Programmatic direct-judge:** hit the wrapped endpoint and confirm `track_judge_result` lands on the parent config's Monitoring tab. - **Offline eval:** run the dataset through the LD Playground, compare baseline vs new-variation scores side by side. No runtime wiring required. -Delegate: **`aiconfig-online-evals`** (sub-step 3, optional — only for UI-attached judges or custom-judge creation; offline eval doesn't delegate). +Delegate: **`agentcontrol-configs-online-evals`** (sub-step 3, optional — only for UI-attached judges or custom-judge creation; offline eval doesn't delegate). ## Edge Cases @@ -498,9 +498,9 @@ Delegate: **`aiconfig-online-evals`** (sub-step 3, optional — only for UI-atta | Retry wrapper around the provider call | The tracker is minted once at the top of the user turn; the retry loop is inside that scope. Every retry attempt shares the same `runId`. Tracker calls (`track_duration` / `track_tokens` / `track_success` / `track_error`) live *outside* the retry body — one call at the end of the turn, on the success path or the final-failure path | | App has no tools — Stage 3 skipped | Move directly from Stage 2 verification to Stage 4 (tracking) | | Mode mismatch: user said agent, audit shows one-shot chat | Choose completion mode unless the app uses a LangGraph prebuilt agent (`langchain.agents.create_agent` in Python or `createReactAgent` in Node), CrewAI `Agent`, Strands `Agent`, or a similar goal-driven framework | -| App uses Strands Agents (Python) | Agent mode. Build a `create_strands_model` dispatcher keyed on `agent_config.provider.name` that returns `AnthropicModel(model_id=..., max_tokens=...)` or `OpenAIModel(model_id=..., params=...)`. Drop `parameters.tools` before passing params to the model class — Strands receives tools via `Agent(tools=[...])`. Tracking is Tier 3: wrap `invoke_async` with `tracker.track_duration_of(...)` and record tokens from `result.metrics.accumulated_usage`. See [agent-mode-frameworks.md § Strands Agent](references/agent-mode-frameworks.md) and [strands-tracking.md](../aiconfig-ai-metrics/references/strands-tracking.md) | +| App uses Strands Agents (Python) | Agent mode. Build a `create_strands_model` dispatcher keyed on `agent_config.provider.name` that returns `AnthropicModel(model_id=..., max_tokens=...)` or `OpenAIModel(model_id=..., params=...)`. Drop `parameters.tools` before passing params to the model class — Strands receives tools via `Agent(tools=[...])`. Tracking is Tier 3: wrap `invoke_async` with `tracker.track_duration_of(...)` and record tokens from `result.metrics.accumulated_usage`. See [agent-mode-frameworks.md § Strands Agent](references/agent-mode-frameworks.md) and [strands-tracking.md](../agentcontrol-configs-ai-metrics/references/strands-tracking.md) | | Strands app on TypeScript | TS SDK ships `BedrockModel` and `OpenAIModel` only — cannot serve Anthropic-backed variations. Use the Python SDK if multi-provider variations are required | -| TypeScript app using Anthropic SDK | No `trackAnthropicMetrics` helper exists. Use Tier 3: `trackMetricsOf` with a small custom extractor that reads `response.usage.input_tokens` / `response.usage.output_tokens` and returns `LDAIMetrics`. See [anthropic-tracking.md](../aiconfig-ai-metrics/references/anthropic-tracking.md) in the `aiconfig-ai-metrics` skill for the exact extractor | +| TypeScript app using Anthropic SDK | No `trackAnthropicMetrics` helper exists. Use Tier 3: `trackMetricsOf` with a small custom extractor that reads `response.usage.input_tokens` / `response.usage.output_tokens` and returns `LDAIMetrics`. See [anthropic-tracking.md](../agentcontrol-configs-ai-metrics/references/anthropic-tracking.md) in the `agentcontrol-configs-ai-metrics` skill for the exact extractor | | Fallback would silently crash because `LD_SDK_KEY` is missing | Log a startup warning; proceed with the fallback. Never raise at import time | | Multi-agent graph (supervisor + workers) | Stop after migrating a single agent. Agent Graph Definitions are available in **both** SDKs — Python via `launchdarkly-server-sdk-ai.agent_graph` and Node via the graph API in `@launchdarkly/server-sdk-ai`. Read [agent-graph-reference.md](references/agent-graph-reference.md) for the graph-level migration path — it is deliberately out of this skill's main scope | | Single-agent (ReAct, tool loop) + agent mode | Default to offline eval via the LD Playground + Datasets for Stage 5. UI-attached judges are completion-only today, and programmatic direct-judge adds per-call cost that is usually not worth it until after the migration is live and stable. Point at the [Offline Evals guide](https://docs.launchdarkly.com/guides/ai-configs/offline-evaluations) | @@ -525,17 +525,17 @@ These are ordered by how likely they are to show up as a first-run failure. The - **If the repo already contains a `load_chat_model(f"{provider}/{name}")` helper, delete it — don't just avoid using it.** This exact shape ships with `langchain-ai/react-agent` and is copied into dozens of derivative repos; look for `utils.load_chat_model`, `utils.build_model`, or any one-arg `init_chat_model` wrapper that splits a `"provider/model"` string. Re-using it is the first-run failure mode: every variation parameter (temperature, max_tokens, top_p, stop sequences) silently drops on the floor because `init_chat_model` only receives the name and provider. `create_langchain_model(ai_config)` is a one-for-one replacement that forwards the whole `model.parameters` dict. Replace every call site, then delete the wrapper file-side so the next reader can't reach for it. - **Same rule applies to hand-rolled `resolve_tools` / `TOOL_REGISTRY` / `ALL_TOOLS` helpers.** If the template already has a `resolve_tools(tool_keys)` or an `ALL_TOOLS` module-level list, import `build_structured_tools` from `ldai_langchain.langchain_helper` and delete the hand-rolled version. `build_structured_tools(ai_config, TOOL_REGISTRY_DICT)` reads `ai_config.model.parameters.tools` and wraps the matching callables as LangChain `StructuredTool`s with the LD tool key as the `StructuredTool.name` — so `ToolNode` lookup works without a second mapping. Don't leave both in the repo. -- Don't put app-scoped knobs directly in `model.parameters`. `create_langchain_model` forwards every key in `parameters` to the provider SDK via `init_chat_model`, so a `max_search_results` / `retry_budget` / `feature_toggle` entry will crash the provider with an unexpected-keyword-argument error. The correct home is `model.custom`, which the provider helpers ignore and the app reads via `ai_config.model.get_custom("key")`. The MCP `update-ai-config-variation` tool does not currently expose top-level `custom`, so pick one of two paths: (a) PATCH the variation via the REST API to set `model.custom` directly, or (b) set it via MCP inside `parameters.custom` (as a nested dict) and use a defensive accessor that reads both locations. Full walk-through with code samples in [langchain-tracking.md § MCP caveat](../aiconfig-ai-metrics/references/langchain-tracking.md). +- Don't put app-scoped knobs directly in `model.parameters`. `create_langchain_model` forwards every key in `parameters` to the provider SDK via `init_chat_model`, so a `max_search_results` / `retry_budget` / `feature_toggle` entry will crash the provider with an unexpected-keyword-argument error. The correct home is `model.custom`, which the provider helpers ignore and the app reads via `ai_config.model.get_custom("key")`. The MCP `update-ai-config-variation` tool does not currently expose top-level `custom`, so pick one of two paths: (a) PATCH the variation via the REST API to set `model.custom` directly, or (b) set it via MCP inside `parameters.custom` (as a nested dict) and use a defensive accessor that reads both locations. Full walk-through with code samples in [langchain-tracking.md § MCP caveat](../agentcontrol-configs-ai-metrics/references/langchain-tracking.md). - Don't re-encode tool schemas inside the fallback. When LaunchDarkly is unreachable the fallback should run without tools (or with whatever minimal provider-bound parameters the app needs to keep operating). Building a `_FALLBACK_TOOLS` array that duplicates the AI Config's tool schema re-introduces the hardcoded config the migration was supposed to move out of code. - Don't import `LaunchDarklyCallbackHandler` from `ldai.langchain` — neither the class nor the dotted module path exists. The Python LangChain helper package is `ldai_langchain` (top-level module, underscore). Use `create_langchain_model(config)` + `track_metrics_of_async(get_ai_metrics_from_response, lambda: llm.ainvoke(messages))` as the canonical pattern. ### Stage / handoff discipline - Don't skip Step 1 even when the user says "just wrap it." Without the audit, the fallback will drift from the hardcoded behavior. -- Don't delegate to `aiconfig-create` before extracting the prompt and model — the delegate needs them as inputs. -- Don't try to attach tools during initial `setup-ai-config`. Tool attachment is a separate step owned by `aiconfig-tools`. -- Don't claim you "delegated to `aiconfig-create`" or any other sibling skill. This skill does not auto-invoke. At each handoff, print the inputs and tell the user to run the sibling slash-command, then wait. Anything else misleads the user about what just happened. -- Don't skip the `/aiconfig-targeting` step between Stage 2 and Stage 4. A freshly created variation returns `enabled=False` until targeting promotes it to fallthrough — Stage 2 verification will silently take the fallback path on every request. +- Don't delegate to `agentcontrol-configs-create` before extracting the prompt and model — the delegate needs them as inputs. +- Don't try to attach tools during initial `setup-ai-config`. Tool attachment is a separate step owned by `agentcontrol-tools`. +- Don't claim you "delegated to `agentcontrol-configs-create`" or any other sibling skill. This skill does not auto-invoke. At each handoff, print the inputs and tell the user to run the sibling slash-command, then wait. Anything else misleads the user about what just happened. +- Don't skip the `/agentcontrol-configs-targeting` step between Stage 2 and Stage 4. A freshly created variation returns `enabled=False` until targeting promotes it to fallthrough — Stage 2 verification will silently take the fallback path on every request. - Don't attempt a multi-agent graph migration in one pass. Migrate a single agent first; use [agent-graph-reference.md](references/agent-graph-reference.md) as the next-step read. ### Stage 5 evaluations @@ -554,12 +554,12 @@ These are ordered by how likely they are to show up as a first-run failure. The ## Related Skills -- `aiconfig-create` — called by Stage 2 to create the config -- `aiconfig-tools` — called by Stage 3 to create and attach tool definitions -- `aiconfig-online-evals` — called by Stage 5 to attach judges -- `aiconfig-variations` — add variations for A/B testing after migration is complete -- `aiconfig-targeting` — roll out new variations to users after migration is complete -- `aiconfig-update` — modify config properties as your app evolves +- `agentcontrol-configs-create` — called by Stage 2 to create the config +- `agentcontrol-tools` — called by Stage 3 to create and attach tool definitions +- `agentcontrol-configs-online-evals` — called by Stage 5 to attach judges +- `agentcontrol-configs-variations` — add variations for A/B testing after migration is complete +- `agentcontrol-configs-targeting` — roll out new variations to users after migration is complete +- `agentcontrol-configs-update` — modify config properties as your app evolves - `launchdarkly-metric-instrument` — for `ldClient.track()` feature metrics (NOT for AI tracker calls) ## References diff --git a/skills/ai-configs/aiconfig-migrate/references/agent-graph-reference.md b/skills/ai-configs/agentcontrol-configs-migrate/references/agent-graph-reference.md similarity index 100% rename from skills/ai-configs/aiconfig-migrate/references/agent-graph-reference.md rename to skills/ai-configs/agentcontrol-configs-migrate/references/agent-graph-reference.md diff --git a/skills/ai-configs/aiconfig-migrate/references/agent-mode-frameworks.md b/skills/ai-configs/agentcontrol-configs-migrate/references/agent-mode-frameworks.md similarity index 98% rename from skills/ai-configs/aiconfig-migrate/references/agent-mode-frameworks.md rename to skills/ai-configs/agentcontrol-configs-migrate/references/agent-mode-frameworks.md index b986441..d511330 100644 --- a/skills/ai-configs/aiconfig-migrate/references/agent-mode-frameworks.md +++ b/skills/ai-configs/agentcontrol-configs-migrate/references/agent-mode-frameworks.md @@ -16,9 +16,9 @@ Completion mode is the default and covers direct provider calls (OpenAI, Anthrop Agent mode returns an `instructions` string. Completion mode returns a `messages` array. Both modes support tools, parameters, and the same tracker — the only difference is the input shape the SDK returns to you. -**Caveat:** judges cannot be attached to agent-mode variations via the LaunchDarkly UI. Agent mode evaluations must go through the programmatic judge API (`create_judge(...).evaluate(input, output)`). See `aiconfig-online-evals` for the programmatic path. +**Caveat:** judges cannot be attached to agent-mode variations via the LaunchDarkly UI. Agent mode evaluations must go through the programmatic judge API (`create_judge(...).evaluate(input, output)`). See `agentcontrol-configs-online-evals` for the programmatic path. -**Model construction for LangChain / LangGraph.** When the framework runs on top of LangChain (which includes LangGraph's prebuilt agent and most custom graphs), build the chat model with `create_langchain_model(ai_config)` (Python) or `createLangChainModel(aiConfig)` (Node). These helpers forward every variation parameter (`temperature`, `max_tokens`, `top_p`, …) and handle LaunchDarkly→LangChain provider-name mapping internally. Do not hand-roll `init_chat_model(model=..., model_provider=...)` — it silently drops every variation parameter. See [langchain-tracking.md](../../aiconfig-ai-metrics/references/langchain-tracking.md) for the canonical single-model and LangGraph patterns, including the SDK helpers `sum_token_usage_from_messages` / `get_tool_calls_from_response` (Python, `ldai_langchain`) used inside the `track_metrics_of_async` / `trackMetricsOf` extractor. +**Model construction for LangChain / LangGraph.** When the framework runs on top of LangChain (which includes LangGraph's prebuilt agent and most custom graphs), build the chat model with `create_langchain_model(ai_config)` (Python) or `createLangChainModel(aiConfig)` (Node). These helpers forward every variation parameter (`temperature`, `max_tokens`, `top_p`, …) and handle LaunchDarkly→LangChain provider-name mapping internally. Do not hand-roll `init_chat_model(model=..., model_provider=...)` — it silently drops every variation parameter. See [langchain-tracking.md](../../agentcontrol-configs-ai-metrics/references/langchain-tracking.md) for the canonical single-model and LangGraph patterns, including the SDK helpers `sum_token_usage_from_messages` / `get_tool_calls_from_response` (Python, `ldai_langchain`) used inside the `track_metrics_of_async` / `trackMetricsOf` extractor. ## Framework-agnostic invariants for the run-scoped pattern @@ -291,7 +291,7 @@ async def run_turn(ai_client, user_id: str, user_input: str): **Key points:** - `system_prompt=agent_config.instructions` — the instructions string replaces the hardcoded system prompt. - `create_strands_model(agent_config)` is the provider-dispatch seam. Add a branch per provider the variation can serve. -- The tracker is Tier 3: `tracker.track_duration_of(...)` + an explicit `track_tokens` call fed by `track_strands_metrics`. See [strands-tracking.md](../../aiconfig-ai-metrics/references/strands-tracking.md) for the single-call `track_metrics_of_async` variant and the per-field breakdown of `accumulated_usage`. +- The tracker is Tier 3: `tracker.track_duration_of(...)` + an explicit `track_tokens` call fed by `track_strands_metrics`. See [strands-tracking.md](../../agentcontrol-configs-ai-metrics/references/strands-tracking.md) for the single-call `track_metrics_of_async` variant and the per-field breakdown of `accumulated_usage`. - Always `ldclient.get().flush()` before process exit in short-lived scripts — trailing events can otherwise be lost. **TypeScript caveat.** The Strands TypeScript SDK ships `BedrockModel` and `OpenAIModel` only — it cannot run Anthropic-backed variations. If the app needs to serve both OpenAI and Anthropic from a single AI Config, use the Python SDK. @@ -799,12 +799,12 @@ agent = create_agent( **What this gives you:** - Toggle a tool on/off by editing the AI Config in LaunchDarkly — no redeploy needed to remove a tool from production -- Roll out a new tool to 5% of users by editing targeting rules (combined with `aiconfig-targeting`) +- Roll out a new tool to 5% of users by editing targeting rules (combined with `agentcontrol-configs-targeting`) - Keep the actual tool implementation code in the repo; only metadata lives in LaunchDarkly ### Extracting schemas from existing hardcoded tools -If your tools are defined with LangChain `@tool` or Pydantic `BaseModel` schemas, extract the JSON schema programmatically to pass to `aiconfig-tools` during Stage 3: +If your tools are defined with LangChain `@tool` or Pydantic `BaseModel` schemas, extract the JSON schema programmatically to pass to `agentcontrol-tools` during Stage 3: ```python from my_tools import search_kb # a @tool-decorated function @@ -813,7 +813,7 @@ schema = search_kb.args_schema.model_json_schema() # schema is a dict ready to pass as the tool's parameters field ``` -Do not hand-write the schema — LangChain already generated it from the function signature, and Pydantic will keep it in sync. The `aiconfig-tools` delegate accepts raw JSON Schema for the parameters field. +Do not hand-write the schema — LangChain already generated it from the function signature, and Pydantic will keep it in sync. The `agentcontrol-tools` delegate accepts raw JSON Schema for the parameters field. ### Dynamic schemas from LaunchDarkly diff --git a/skills/ai-configs/aiconfig-migrate/references/before-after-examples.md b/skills/ai-configs/agentcontrol-configs-migrate/references/before-after-examples.md similarity index 100% rename from skills/ai-configs/aiconfig-migrate/references/before-after-examples.md rename to skills/ai-configs/agentcontrol-configs-migrate/references/before-after-examples.md diff --git a/skills/ai-configs/aiconfig-migrate/references/fallback-defaults-pattern.md b/skills/ai-configs/agentcontrol-configs-migrate/references/fallback-defaults-pattern.md similarity index 99% rename from skills/ai-configs/aiconfig-migrate/references/fallback-defaults-pattern.md rename to skills/ai-configs/agentcontrol-configs-migrate/references/fallback-defaults-pattern.md index 1e6f58b..155c306 100644 --- a/skills/ai-configs/aiconfig-migrate/references/fallback-defaults-pattern.md +++ b/skills/ai-configs/agentcontrol-configs-migrate/references/fallback-defaults-pattern.md @@ -305,7 +305,7 @@ config = ai_client.agent_config( The SDK runs both paths through the same Mustache renderer. Leaving a Python-style `{var}` literal in the fallback ships a silent regression: LaunchDarkly serves correctly-interpolated output; the fallback ships the unrendered literal, or (worse) a value frozen at import time. -Same rule applies to JS template literals and any other non-Mustache scheme — rewrite to `{{ variable }}` before handing off to `/aiconfig-create`. +Same rule applies to JS template literals and any other non-Mustache scheme — rewrite to `{{ variable }}` before handing off to `/agentcontrol-configs-create`. ## Critical rules (apply to all three patterns) diff --git a/skills/ai-configs/aiconfig-migrate/references/phase-1-analysis-checklist.md b/skills/ai-configs/agentcontrol-configs-migrate/references/phase-1-analysis-checklist.md similarity index 97% rename from skills/ai-configs/aiconfig-migrate/references/phase-1-analysis-checklist.md rename to skills/ai-configs/agentcontrol-configs-migrate/references/phase-1-analysis-checklist.md index aadb437..006e6e9 100644 --- a/skills/ai-configs/aiconfig-migrate/references/phase-1-analysis-checklist.md +++ b/skills/ai-configs/agentcontrol-configs-migrate/references/phase-1-analysis-checklist.md @@ -151,7 +151,7 @@ Feeds into Stage 2 (install + wrap). Quoted from the `ai-configs-relaunch-guides | LangChain / LangGraph | `@launchdarkly/server-sdk-ai-langchain` | `createLangChainModel(config)` (forwards all variation parameters and handles provider-name mapping) + `getAIMetricsFromResponse` with `trackMetricsOf` | | Vercel AI SDK | `@launchdarkly/server-sdk-ai-vercel` | `getAIMetricsFromResponse` + `trackMetricsOf`, or `VercelRunnerFactory.createVercelModel(aiConfig)` for a managed runner | -Python currently ships helper packages for OpenAI (`ldai_openai`) and LangChain (`ldai_langchain`). The LangChain Python package exposes `create_langchain_model(config)` (builds a LangChain chat model from the AI Config, forwarding every variation parameter and mapping LD provider names to LangChain equivalents), `convert_messages_to_langchain`, and `get_ai_metrics_from_response` — the same package covers LangGraph. Use `create_langchain_model(config)` + `track_metrics_of_async(get_ai_metrics_from_response, lambda: llm.ainvoke(messages))` as the canonical single-call pattern. See [langchain-tracking.md](../../aiconfig-ai-metrics/references/langchain-tracking.md) for both LangChain and LangGraph patterns and [sdk-ai-tracker-patterns.md](sdk-ai-tracker-patterns.md) for the full tracker-method matrix. +Python currently ships helper packages for OpenAI (`ldai_openai`) and LangChain (`ldai_langchain`). The LangChain Python package exposes `create_langchain_model(config)` (builds a LangChain chat model from the AI Config, forwarding every variation parameter and mapping LD provider names to LangChain equivalents), `convert_messages_to_langchain`, and `get_ai_metrics_from_response` — the same package covers LangGraph. Use `create_langchain_model(config)` + `track_metrics_of_async(get_ai_metrics_from_response, lambda: llm.ainvoke(messages))` as the canonical single-call pattern. See [langchain-tracking.md](../../agentcontrol-configs-ai-metrics/references/langchain-tracking.md) for both LangChain and LangGraph patterns and [sdk-ai-tracker-patterns.md](sdk-ai-tracker-patterns.md) for the full tracker-method matrix. ## Phase 1 output format @@ -183,9 +183,9 @@ Coverage totals: N hardcoded code targets · M externalized prompt f Proposed plan: Stage 1 (Audit): Read-only manifest of hardcoded targets; flag placeholders for Mustache rewrite and knobs for model.custom Stage 2 (Wrap): Install SDK, create AI Config 'chat-assistant', inline fallback mirrors current values (Mustache syntax), rewrite the call site - Stage 3 (Tools): Skipped (no function calling) / Attach 2 tools via aiconfig-tools + Stage 3 (Tools): Skipped (no function calling) / Attach 2 tools via agentcontrol-tools Stage 4 (Tracking): Inline tracker wiring (track_duration + track_tokens + track_success/error) — run-scoped tracker for agent loops - Stage 5 (Evals): Attach built-in 'accuracy' judge at 0.25 sampling via aiconfig-online-evals + Stage 5 (Evals): Attach built-in 'accuracy' judge at 0.25 sampling via agentcontrol-configs-online-evals ``` ## STOP diff --git a/skills/ai-configs/aiconfig-migrate/references/sdk-ai-tracker-patterns.md b/skills/ai-configs/agentcontrol-configs-migrate/references/sdk-ai-tracker-patterns.md similarity index 97% rename from skills/ai-configs/aiconfig-migrate/references/sdk-ai-tracker-patterns.md rename to skills/ai-configs/agentcontrol-configs-migrate/references/sdk-ai-tracker-patterns.md index f39d813..c663754 100644 --- a/skills/ai-configs/aiconfig-migrate/references/sdk-ai-tracker-patterns.md +++ b/skills/ai-configs/agentcontrol-configs-migrate/references/sdk-ai-tracker-patterns.md @@ -297,7 +297,7 @@ response = tracker.track_metrics_of( | Helper | Signature | Tier | Notes | |--------|-----------|------|-------| | `trackMetricsOf(extractor, func)` | `await tracker.trackMetricsOf((result) => extractor(result), async () => ...)` | **2 / 3** | **Canonical generic wrapper.** `extractor` maps provider response → `LDAIMetrics`. Use a provider package's bare `getAIMetricsFromResponse` for Tier 2 (`@launchdarkly/server-sdk-ai-openai`, `-langchain`, `-vercel`) or a small custom function for Tier 3. | -| `trackStreamMetricsOf(extractor, streamCreator)` | `tracker.trackStreamMetricsOf(async (chunks) => extractor(chunks), () => createStream())` | 2 / 3 | Stream variant. Does **not** capture TTFT automatically — if you need TTFT, use the manual pattern in [streaming-tracking.md](../../aiconfig-ai-metrics/references/streaming-tracking.md). | +| `trackStreamMetricsOf(extractor, streamCreator)` | `tracker.trackStreamMetricsOf(async (chunks) => extractor(chunks), () => createStream())` | 2 / 3 | Stream variant. Does **not** capture TTFT automatically — if you need TTFT, use the manual pattern in [streaming-tracking.md](../../agentcontrol-configs-ai-metrics/references/streaming-tracking.md). | | `trackDurationOf(func)` | `await tracker.trackDurationOf(async () => ...)` | 4 | Wraps an async callable; captures duration only. Pair with explicit `trackTokens` + `trackSuccess`. | Example — OpenAI via `trackMetricsOf` + the provider package: ```typescript @@ -339,7 +339,7 @@ The managed runner handles message history, provider dispatch (via the installed ### Anthropic has no provider package today -Neither `@launchdarkly/server-sdk-ai-anthropic` nor `launchdarkly-server-sdk-ai-anthropic` exists as of this writing. For Anthropic direct calls, write a custom extractor and pass it to `track_metrics_of` / `trackMetricsOf` — see the Python example above or the full walk-through in [anthropic-tracking.md](../../aiconfig-ai-metrics/references/anthropic-tracking.md). If the app is open to LangChain, routing Anthropic through `ChatAnthropic` and the LangChain provider package recovers Tier 2 with zero extractor code. +Neither `@launchdarkly/server-sdk-ai-anthropic` nor `launchdarkly-server-sdk-ai-anthropic` exists as of this writing. For Anthropic direct calls, write a custom extractor and pass it to `track_metrics_of` / `trackMetricsOf` — see the Python example above or the full walk-through in [anthropic-tracking.md](../../agentcontrol-configs-ai-metrics/references/anthropic-tracking.md). If the app is open to LangChain, routing Anthropic through `ChatAnthropic` and the LangChain provider package recovers Tier 2 with zero extractor code. ## Tier decision table @@ -352,7 +352,7 @@ Neither `@launchdarkly/server-sdk-ai-anthropic` nor `launchdarkly-server-sdk-ai- | Anthropic direct SDK | **3** | Custom extractor reading `response.usage.input_tokens` / `output_tokens` | | Bedrock Converse (no provider package) | **3** | Custom extractor reading `response.usage.inputTokens` / `outputTokens` (or route via LangChain for Tier 2) | | Gemini / Google GenAI, Cohere, custom HTTP | **3** | Custom extractor | -| Streaming response with TTFT required | **4** | Manual `trackTimeToFirstToken` + `trackDuration` + `trackTokens` + `trackSuccess` — see [streaming-tracking.md](../../aiconfig-ai-metrics/references/streaming-tracking.md) | +| Streaming response with TTFT required | **4** | Manual `trackTimeToFirstToken` + `trackDuration` + `trackTokens` + `trackSuccess` — see [streaming-tracking.md](../../agentcontrol-configs-ai-metrics/references/streaming-tracking.md) | | Streaming response without TTFT (Node) | **2 / 3** | `trackStreamMetricsOf(extractor, streamFn)` | ## Streaming responses @@ -450,7 +450,7 @@ Run the checklist in order. Each step rules out one cause. ## Common gotchas -- **`model.parameters` vs `model.custom`.** `create_langchain_model` (Python) / `createLangChainModel` (Node) forwards every key in `model.parameters` to the provider SDK. App-scoped knobs (search result limits, retry budgets, feature toggles) **must** live in `model.custom` or the provider will crash at runtime with an unexpected-keyword-argument error. Read them with `ai_config.model.get_custom("key")`. Full walk-through with the MCP/REST-API caveat in [langchain-tracking.md § `model.parameters` vs `model.custom`](../../aiconfig-ai-metrics/references/langchain-tracking.md). +- **`model.parameters` vs `model.custom`.** `create_langchain_model` (Python) / `createLangChainModel` (Node) forwards every key in `model.parameters` to the provider SDK. App-scoped knobs (search result limits, retry budgets, feature toggles) **must** live in `model.custom` or the provider will crash at runtime with an unexpected-keyword-argument error. Read them with `ai_config.model.get_custom("key")`. Full walk-through with the MCP/REST-API caveat in [langchain-tracking.md § `model.parameters` vs `model.custom`](../../agentcontrol-configs-ai-metrics/references/langchain-tracking.md). - **`track_tokens` token shape.** The Python `TokenUsage` dataclass requires `total` to be set — it is not derived. Compute `total = input + output` if the provider doesn't return one. - **`track_feedback` lifecycle.** The feedback call must be made on a tracker bound to the same `runId` that produced the response. If the thumbs-up comes in a later process, use the cross-process resumption pattern above — do **not** call `create_tracker()` again in the consumer, because that mints a *new* `runId`. - **OpenAI streaming tokens.** OpenAI only emits `usage` in the final chunk when `stream_options={"include_usage": True}` is passed. Without that flag, you have to tokenize manually — `tiktoken` for OpenAI models. diff --git a/skills/ai-configs/aiconfig-online-evals/README.md b/skills/ai-configs/agentcontrol-configs-online-evals/README.md similarity index 67% rename from skills/ai-configs/aiconfig-online-evals/README.md rename to skills/ai-configs/agentcontrol-configs-online-evals/README.md index f577c79..0f623d3 100644 --- a/skills/ai-configs/aiconfig-online-evals/README.md +++ b/skills/ai-configs/agentcontrol-configs-online-evals/README.md @@ -12,12 +12,12 @@ This skill teaches agents how to: ## Installation (Local) -Copy `skills/ai-configs/aiconfig-online-evals/` into your agent client's skills path. +Copy `skills/ai-configs/agentcontrol-configs-online-evals/` into your agent client's skills path. ## Prerequisites - LaunchDarkly API access token with `ai-configs:write` permission -- Existing AI Config with variations (use `aiconfig-create` skill) +- Existing AI Config with variations (use `agentcontrol-configs-create` skill) - For custom judges: understanding of LLM-as-a-judge methodology ## Usage @@ -33,16 +33,16 @@ Create a custom judge that checks for scope creep in code changes ## Structure ``` -aiconfig-online-evals/ +agentcontrol-configs-online-evals/ ├── SKILL.md └── README.md ``` ## Related -- [AI Config Create](../aiconfig-create/) - Create AI Configs first -- [AI Config Targeting](../aiconfig-targeting/) - Enable targeting on judges -- [AI Config Variations](../aiconfig-variations/) - Manage variations +- [AI Config Create](../agentcontrol-configs-create/) - Create AI Configs first +- [AI Config Targeting](../agentcontrol-configs-targeting/) - Enable targeting on judges +- [AI Config Variations](../agentcontrol-configs-variations/) - Manage variations - [Online Evaluations Docs](https://docs.launchdarkly.com/home/ai-configs/online-evaluations) ## License diff --git a/skills/ai-configs/aiconfig-online-evals/SKILL.md b/skills/ai-configs/agentcontrol-configs-online-evals/SKILL.md similarity index 98% rename from skills/ai-configs/aiconfig-online-evals/SKILL.md rename to skills/ai-configs/agentcontrol-configs-online-evals/SKILL.md index 051b37a..35fd1cf 100644 --- a/skills/ai-configs/aiconfig-online-evals/SKILL.md +++ b/skills/ai-configs/agentcontrol-configs-online-evals/SKILL.md @@ -1,5 +1,5 @@ --- -name: aiconfig-online-evals +name: agentcontrol-configs-online-evals description: Attach judges to AI Config variations for automatic LLM-as-a-judge evaluation. Create custom judges, configure sampling rates, and monitor quality scores. compatibility: Requires LaunchDarkly API access token with ai-configs:write permission. SDK versions Python v0.20.0+ or Node.js v0.20.0+ for automatic metric recording and the consolidated `track_judge_result` / `trackJudgeResult` API. metadata: @@ -15,7 +15,7 @@ Attach judges to AI Config variations for automatic quality scoring using LLM-as - LaunchDarkly account with AI Configs enabled - API access token with write permissions -- Existing AI Config with variations (use `aiconfig-create` skill) +- Existing AI Config with variations (use `agentcontrol-configs-create` skill) - For automatic metric recording and the consolidated judge-result API: Python AI SDK v0.20.0+ or Node.js AI SDK v0.20.0+ ## API Key Detection @@ -437,9 +437,9 @@ After attaching judges: ## Related Skills -- `aiconfig-create` - Create AI Configs and judges -- `aiconfig-targeting` - Configure targeting rules -- `aiconfig-variations` - Manage variations +- `agentcontrol-configs-create` - Create AI Configs and judges +- `agentcontrol-configs-targeting` - Configure targeting rules +- `agentcontrol-configs-variations` - Manage variations ## References diff --git a/skills/ai-configs/aiconfig-snippets/SKILL.md b/skills/ai-configs/agentcontrol-configs-snippets/SKILL.md similarity index 99% rename from skills/ai-configs/aiconfig-snippets/SKILL.md rename to skills/ai-configs/agentcontrol-configs-snippets/SKILL.md index db536b1..be0436a 100644 --- a/skills/ai-configs/aiconfig-snippets/SKILL.md +++ b/skills/ai-configs/agentcontrol-configs-snippets/SKILL.md @@ -1,5 +1,5 @@ --- -name: aiconfig-snippets +name: agentcontrol-configs-snippets description: "Create and manage prompt snippets — reusable text blocks referenced inside AI Config variation prompts. Keeps common instructions, personas, and guardrails consistent across multiple configs." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server diff --git a/skills/ai-configs/aiconfig-targeting/README.md b/skills/ai-configs/agentcontrol-configs-targeting/README.md similarity index 63% rename from skills/ai-configs/aiconfig-targeting/README.md rename to skills/ai-configs/agentcontrol-configs-targeting/README.md index 0d831f4..e599dc6 100644 --- a/skills/ai-configs/aiconfig-targeting/README.md +++ b/skills/ai-configs/agentcontrol-configs-targeting/README.md @@ -13,13 +13,13 @@ This skill teaches agents how to: ## Installation (Local) -Copy `skills/ai-configs/aiconfig-targeting/` into your agent client's skills path. +Copy `skills/ai-configs/agentcontrol-configs-targeting/` into your agent client's skills path. ## Prerequisites - LaunchDarkly API access token with `ai-configs:write` permission -- Existing AI Config with variations (use `aiconfig-create` skill) -- Understanding of contexts (see `aiconfig-context-basic` skill) +- Existing AI Config with variations (use `agentcontrol-configs-create` skill) +- Understanding of contexts (see `agentcontrol-context-basic` skill) ## Usage @@ -34,16 +34,16 @@ Add a percentage rollout: 60% to variation A, 40% to variation B for premium use ## Structure ``` -aiconfig-targeting/ +agentcontrol-configs-targeting/ ├── SKILL.md └── README.md ``` ## Related -- [AI Config Create](../aiconfig-create/) - Create AI Configs first -- [AI Config Variations](../aiconfig-variations/) - Create variations to target -- [AI Config Online Evals](../aiconfig-online-evals/) - Attach judges +- [AI Config Create](../agentcontrol-configs-create/) - Create AI Configs first +- [AI Config Variations](../agentcontrol-configs-variations/) - Create variations to target +- [AI Config Online Evals](../agentcontrol-configs-online-evals/) - Attach judges - [Targeting Docs](https://docs.launchdarkly.com/home/ai-configs/target) ## License diff --git a/skills/ai-configs/aiconfig-targeting/SKILL.md b/skills/ai-configs/agentcontrol-configs-targeting/SKILL.md similarity index 97% rename from skills/ai-configs/aiconfig-targeting/SKILL.md rename to skills/ai-configs/agentcontrol-configs-targeting/SKILL.md index 4058fd7..4ea5154 100644 --- a/skills/ai-configs/aiconfig-targeting/SKILL.md +++ b/skills/ai-configs/agentcontrol-configs-targeting/SKILL.md @@ -1,5 +1,5 @@ --- -name: aiconfig-targeting +name: agentcontrol-configs-targeting description: Configure AI Config targeting rules to control which variations serve to different users. Enable percentage rollouts, attribute-based rules, segment targeting, and guarded rollouts. compatibility: Requires LaunchDarkly API access token with ai-configs:write permission. metadata: @@ -16,7 +16,7 @@ Configure targeting rules for AI Configs to control which variations serve to di - LaunchDarkly account with AI Configs enabled - API access token with write permissions - Project key and environment key -- Existing AI Config with variations (use `aiconfig-create` skill) +- Existing AI Config with variations (use `agentcontrol-configs-create` skill) ## API Key Detection @@ -488,16 +488,16 @@ After configuring targeting: ``` https://app.launchdarkly.com/projects/{projectKey}/ai-configs/{configKey} ``` -2. **Monitor performance** with `aiconfig-ai-metrics` -3. **Attach judges** with `aiconfig-online-evals` +2. **Monitor performance** with `agentcontrol-configs-ai-metrics` +3. **Attach judges** with `agentcontrol-configs-online-evals` 4. **Set up guarded rollouts** for automatic regression detection ## Related Skills -- `aiconfig-create` - Create AI Configs with variations -- `aiconfig-variations` - Manage variations -- `aiconfig-online-evals` - Attach judges -- `aiconfig-segments` - Create segments for targeting +- `agentcontrol-configs-create` - Create AI Configs with variations +- `agentcontrol-configs-variations` - Manage variations +- `agentcontrol-configs-online-evals` - Attach judges +- `agentcontrol-segments` - Create segments for targeting ## References diff --git a/skills/ai-configs/aiconfig-update/README.md b/skills/ai-configs/agentcontrol-configs-update/README.md similarity index 78% rename from skills/ai-configs/aiconfig-update/README.md rename to skills/ai-configs/agentcontrol-configs-update/README.md index 7754356..ca74615 100644 --- a/skills/ai-configs/aiconfig-update/README.md +++ b/skills/ai-configs/agentcontrol-configs-update/README.md @@ -13,7 +13,7 @@ This skill teaches agents how to: ## Installation (Local) -Copy `skills/ai-configs/aiconfig-update/` into your agent client's skills path. +Copy `skills/ai-configs/agentcontrol-configs-update/` into your agent client's skills path. ## Prerequisites @@ -36,15 +36,15 @@ Archive the old chatbot config ## Structure ``` -aiconfig-update/ +agentcontrol-configs-update/ ├── SKILL.md └── README.md ``` ## Related -- [AI Config Create](../aiconfig-create/): Create configs -- [AI Config Variations](../aiconfig-variations/): Add or test variations +- [AI Config Create](../agentcontrol-configs-create/): Create configs +- [AI Config Variations](../agentcontrol-configs-variations/): Add or test variations - [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) ## License diff --git a/skills/ai-configs/aiconfig-update/SKILL.md b/skills/ai-configs/agentcontrol-configs-update/SKILL.md similarity index 95% rename from skills/ai-configs/aiconfig-update/SKILL.md rename to skills/ai-configs/agentcontrol-configs-update/SKILL.md index 7b43620..4cf3f40 100644 --- a/skills/ai-configs/aiconfig-update/SKILL.md +++ b/skills/ai-configs/agentcontrol-configs-update/SKILL.md @@ -1,5 +1,5 @@ --- -name: aiconfig-update +name: agentcontrol-configs-update description: "Update, archive, and delete LaunchDarkly AI Configs and their variations. Use when you need to modify config properties, change model parameters, update instructions or messages, archive unused configs, or permanently remove them." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server @@ -90,5 +90,5 @@ Use `get-ai-config` to confirm the response shows your updated values. ## Related Skills -- `aiconfig-variations` -- Create variations to test changes side-by-side -- `aiconfig-tools` -- Update tool attachments +- `agentcontrol-configs-variations` -- Create variations to test changes side-by-side +- `agentcontrol-tools` -- Update tool attachments diff --git a/skills/ai-configs/aiconfig-variations/README.md b/skills/ai-configs/agentcontrol-configs-variations/README.md similarity index 71% rename from skills/ai-configs/aiconfig-variations/README.md rename to skills/ai-configs/agentcontrol-configs-variations/README.md index fe91011..6a7f0b7 100644 --- a/skills/ai-configs/aiconfig-variations/README.md +++ b/skills/ai-configs/agentcontrol-configs-variations/README.md @@ -11,7 +11,7 @@ This skill teaches agents how to: ## Installation (Local) -Copy `skills/ai-configs/aiconfig-variations/` into your agent client's skills path. +Copy `skills/ai-configs/agentcontrol-configs-variations/` into your agent client's skills path. ## Prerequisites @@ -30,16 +30,16 @@ Create variations to compare Claude vs GPT-4 for our agent ## Structure ``` -aiconfig-variations/ +agentcontrol-configs-variations/ ├── SKILL.md └── README.md ``` ## Related -- [AI Config Create](../aiconfig-create/): Create the config first -- [AI Config Update](../aiconfig-update/): Modify existing variations -- [AI Config Tools](../aiconfig-tools/): Attach tools to variations +- [AI Config Create](../agentcontrol-configs-create/): Create the config first +- [AI Config Update](../agentcontrol-configs-update/): Modify existing variations +- [AI Config Tools](../agentcontrol-tools/): Attach tools to variations - [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) ## License diff --git a/skills/ai-configs/aiconfig-variations/SKILL.md b/skills/ai-configs/agentcontrol-configs-variations/SKILL.md similarity index 97% rename from skills/ai-configs/aiconfig-variations/SKILL.md rename to skills/ai-configs/agentcontrol-configs-variations/SKILL.md index 03ea46c..c028296 100644 --- a/skills/ai-configs/aiconfig-variations/SKILL.md +++ b/skills/ai-configs/agentcontrol-configs-variations/SKILL.md @@ -1,5 +1,5 @@ --- -name: aiconfig-variations +name: agentcontrol-configs-variations description: "Experiment with AI configurations by creating and managing variations. Helps you test different models, prompts, and parameters to find what works best through systematic experimentation." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server @@ -106,5 +106,5 @@ When the user wants to try a different model, prompt, or parameters, **always cr ## Related Skills -- `aiconfig-create` -- Create the initial config -- `aiconfig-update` -- Refine based on learnings +- `agentcontrol-configs-create` -- Create the initial config +- `agentcontrol-configs-update` -- Refine based on learnings diff --git a/skills/ai-configs/aiconfig-projects/README.md b/skills/ai-configs/agentcontrol-projects/README.md similarity index 93% rename from skills/ai-configs/aiconfig-projects/README.md rename to skills/ai-configs/agentcontrol-projects/README.md index e48531f..a6c1679 100644 --- a/skills/ai-configs/aiconfig-projects/README.md +++ b/skills/ai-configs/agentcontrol-projects/README.md @@ -17,7 +17,7 @@ For now, install by placing this skill directory where your agent client loads s Examples: -- **Generic**: copy `skills/ai-configs/aiconfig-projects/` into your client's skills path +- **Generic**: copy `skills/ai-configs/agentcontrol-projects/` into your client's skills path ## Prerequisites @@ -44,7 +44,7 @@ Add LaunchDarkly project management to this codebase ## Structure ``` -aiconfig-projects/ +agentcontrol-projects/ ├── SKILL.md ├── README.md └── references/ diff --git a/skills/ai-configs/aiconfig-projects/SKILL.md b/skills/ai-configs/agentcontrol-projects/SKILL.md similarity index 94% rename from skills/ai-configs/aiconfig-projects/SKILL.md rename to skills/ai-configs/agentcontrol-projects/SKILL.md index 38f327d..21f0d75 100644 --- a/skills/ai-configs/aiconfig-projects/SKILL.md +++ b/skills/ai-configs/agentcontrol-projects/SKILL.md @@ -1,5 +1,5 @@ --- -name: aiconfig-projects +name: agentcontrol-projects description: Guide for setting up LaunchDarkly projects in your codebase. Helps you assess your stack, choose the right approach, and integrate project management that makes sense for your architecture. compatibility: Requires LaunchDarkly API access token with projects:write permission or LaunchDarkly MCP server. metadata: @@ -221,16 +221,16 @@ ai-apac → Asia-Pacific Region After setting up projects: -1. **Create AI Configs** - Use the `aiconfig-create` skill -2. **Set up SDK Integration** - Use the `aiconfig-sdk` skill -3. **Configure Targeting** - Use the `aiconfig-targeting` skill +1. **Create AI Configs** - Use the `agentcontrol-configs-create` skill +2. **Set up SDK Integration** - Use the `agentcontrol-sdk` skill +3. **Configure Targeting** - Use the `agentcontrol-configs-targeting` skill ## Related Skills -- `aiconfig-create` - Create AI Configs in projects -- `aiconfig-sdk` - Integrate SDK in your application -- `aiconfig-targeting` - Configure AI Config targeting -- `aiconfig-variations` - Manage config variations +- `agentcontrol-configs-create` - Create AI Configs in projects +- `agentcontrol-sdk` - Integrate SDK in your application +- `agentcontrol-configs-targeting` - Configure AI Config targeting +- `agentcontrol-configs-variations` - Manage config variations ## References diff --git a/skills/ai-configs/aiconfig-projects/references/admin-tooling.md b/skills/ai-configs/agentcontrol-projects/references/admin-tooling.md similarity index 100% rename from skills/ai-configs/aiconfig-projects/references/admin-tooling.md rename to skills/ai-configs/agentcontrol-projects/references/admin-tooling.md diff --git a/skills/ai-configs/aiconfig-projects/references/env-config.md b/skills/ai-configs/agentcontrol-projects/references/env-config.md similarity index 99% rename from skills/ai-configs/aiconfig-projects/references/env-config.md rename to skills/ai-configs/agentcontrol-projects/references/env-config.md index 337ca7f..21033e9 100644 --- a/skills/ai-configs/aiconfig-projects/references/env-config.md +++ b/skills/ai-configs/agentcontrol-projects/references/env-config.md @@ -363,6 +363,6 @@ def verify_sdk_key(sdk_key: str): ## Next Steps -- [Integrate SDK in your application](../aiconfig-sdk/SKILL.md) +- [Integrate SDK in your application](../agentcontrol-sdk/SKILL.md) - [Set up project cloning](project-cloning.md) - [Build automation scripts](iac-automation.md) diff --git a/skills/ai-configs/aiconfig-projects/references/go-setup.md b/skills/ai-configs/agentcontrol-projects/references/go-setup.md similarity index 100% rename from skills/ai-configs/aiconfig-projects/references/go-setup.md rename to skills/ai-configs/agentcontrol-projects/references/go-setup.md diff --git a/skills/ai-configs/aiconfig-projects/references/iac-automation.md b/skills/ai-configs/agentcontrol-projects/references/iac-automation.md similarity index 100% rename from skills/ai-configs/aiconfig-projects/references/iac-automation.md rename to skills/ai-configs/agentcontrol-projects/references/iac-automation.md diff --git a/skills/ai-configs/aiconfig-projects/references/multi-language-setup.md b/skills/ai-configs/agentcontrol-projects/references/multi-language-setup.md similarity index 100% rename from skills/ai-configs/aiconfig-projects/references/multi-language-setup.md rename to skills/ai-configs/agentcontrol-projects/references/multi-language-setup.md diff --git a/skills/ai-configs/aiconfig-projects/references/nodejs-setup.md b/skills/ai-configs/agentcontrol-projects/references/nodejs-setup.md similarity index 100% rename from skills/ai-configs/aiconfig-projects/references/nodejs-setup.md rename to skills/ai-configs/agentcontrol-projects/references/nodejs-setup.md diff --git a/skills/ai-configs/aiconfig-projects/references/project-cloning.md b/skills/ai-configs/agentcontrol-projects/references/project-cloning.md similarity index 100% rename from skills/ai-configs/aiconfig-projects/references/project-cloning.md rename to skills/ai-configs/agentcontrol-projects/references/project-cloning.md diff --git a/skills/ai-configs/aiconfig-projects/references/python-setup.md b/skills/ai-configs/agentcontrol-projects/references/python-setup.md similarity index 100% rename from skills/ai-configs/aiconfig-projects/references/python-setup.md rename to skills/ai-configs/agentcontrol-projects/references/python-setup.md diff --git a/skills/ai-configs/aiconfig-projects/references/quick-start.md b/skills/ai-configs/agentcontrol-projects/references/quick-start.md similarity index 100% rename from skills/ai-configs/aiconfig-projects/references/quick-start.md rename to skills/ai-configs/agentcontrol-projects/references/quick-start.md diff --git a/skills/ai-configs/aiconfig-tools/README.md b/skills/ai-configs/agentcontrol-tools/README.md similarity index 76% rename from skills/ai-configs/aiconfig-tools/README.md rename to skills/ai-configs/agentcontrol-tools/README.md index 7b573ba..8b1fe28 100644 --- a/skills/ai-configs/aiconfig-tools/README.md +++ b/skills/ai-configs/agentcontrol-tools/README.md @@ -12,7 +12,7 @@ This skill teaches agents how to: ## Installation (Local) -Copy `skills/ai-configs/aiconfig-tools/` into your agent client's skills path. +Copy `skills/ai-configs/agentcontrol-tools/` into your agent client's skills path. ## Prerequisites @@ -31,15 +31,15 @@ Create tools for the content assistant to call our API ## Structure ``` -aiconfig-tools/ +agentcontrol-tools/ ├── SKILL.md └── README.md ``` ## Related -- [AI Config Create](../aiconfig-create/): Create the config before adding tools -- [AI Config Variations](../aiconfig-variations/): Manage variations that tools attach to +- [AI Config Create](../agentcontrol-configs-create/): Create the config before adding tools +- [AI Config Variations](../agentcontrol-configs-variations/): Manage variations that tools attach to - [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) ## License diff --git a/skills/ai-configs/aiconfig-tools/SKILL.md b/skills/ai-configs/agentcontrol-tools/SKILL.md similarity index 91% rename from skills/ai-configs/aiconfig-tools/SKILL.md rename to skills/ai-configs/agentcontrol-tools/SKILL.md index 3bce3fa..79d7660 100644 --- a/skills/ai-configs/aiconfig-tools/SKILL.md +++ b/skills/ai-configs/agentcontrol-tools/SKILL.md @@ -1,5 +1,5 @@ --- -name: aiconfig-tools +name: agentcontrol-tools description: "Give your AI agents capabilities through tools (function calling). Helps you identify what your AI needs to do, create tool definitions, and attach them to AI Config variations." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server @@ -211,13 +211,13 @@ for _ in range(MAX_STEPS): messages.append({"role": "user", "content": tool_results}) ``` -Per-provider tool-call payload shapes live in the `aiconfig-ai-metrics` references: +Per-provider tool-call payload shapes live in the `agentcontrol-configs-ai-metrics` references: -- [openai-tracking.md](../aiconfig-ai-metrics/references/openai-tracking.md) — Chat Completions + Responses API -- [anthropic-tracking.md](../aiconfig-ai-metrics/references/anthropic-tracking.md) — `tool_use` blocks and `tool_result` payloads -- [bedrock-tracking.md](../aiconfig-ai-metrics/references/bedrock-tracking.md) — `toolUse` / `toolResult` Converse format -- [gemini-tracking.md](../aiconfig-ai-metrics/references/gemini-tracking.md) — `functionCalls` / `functionResponse` parts -- [langchain-tracking.md](../aiconfig-ai-metrics/references/langchain-tracking.md) — LangGraph tool loop inherits from `create_react_agent` +- [openai-tracking.md](../agentcontrol-configs-ai-metrics/references/openai-tracking.md) — Chat Completions + Responses API +- [anthropic-tracking.md](../agentcontrol-configs-ai-metrics/references/anthropic-tracking.md) — `tool_use` blocks and `tool_result` payloads +- [bedrock-tracking.md](../agentcontrol-configs-ai-metrics/references/bedrock-tracking.md) — `toolUse` / `toolResult` Converse format +- [gemini-tracking.md](../agentcontrol-configs-ai-metrics/references/gemini-tracking.md) — `functionCalls` / `functionResponse` parts +- [langchain-tracking.md](../agentcontrol-configs-ai-metrics/references/langchain-tracking.md) — LangGraph tool loop inherits from `create_react_agent` ## Orchestrator Note @@ -240,5 +240,5 @@ LangGraph, CrewAI, and AutoGen often generate schemas from function definitions. ## Related Skills -- `aiconfig-create` -- Create config before attaching tools -- `aiconfig-variations` -- Manage variations with different tool sets +- `agentcontrol-configs-create` -- Create config before attaching tools +- `agentcontrol-configs-variations` -- Manage variations with different tool sets From 5fb03e90c67152a9dc01095201be9c1705591a8d Mon Sep 17 00:00:00 2001 From: Scarlett Attensil Date: Tue, 19 May 2026 11:37:34 -0700 Subject: [PATCH 02/10] Drop agentcontrol- prefix; use configs-* naming Per review: remove the agentcontrol- prefix entirely and prefix every AI Config skill with configs-. Renames directories, SKILL.md frontmatter, skills.json, README catalog, and all cross-references; also tweaks the agent-graphs description (AI Configs -> configs). --- README.md | 20 ++--- skills.json | 48 +++++------ .../SKILL.md | 4 +- .../SKILL.md | 14 ++-- .../references/anthropic-tracking.md | 0 .../references/bedrock-tracking.md | 0 .../references/gemini-tracking.md | 2 +- .../references/langchain-tracking.md | 2 +- .../references/metrics-api.md | 0 .../references/openai-tracking.md | 0 .../references/strands-tracking.md | 2 +- .../references/streaming-tracking.md | 0 .../README.md | 10 +-- .../SKILL.md | 18 ++--- .../SKILL.md | 14 ++-- .../README.md | 20 ++--- .../SKILL.md | 80 +++++++++---------- .../references/agent-graph-reference.md | 0 .../references/agent-mode-frameworks.md | 12 +-- .../references/before-after-examples.md | 0 .../references/fallback-defaults-pattern.md | 2 +- .../references/phase-1-analysis-checklist.md | 6 +- .../references/sdk-ai-tracker-patterns.md | 8 +- .../README.md | 12 +-- .../SKILL.md | 10 +-- .../README.md | 4 +- .../SKILL.md | 16 ++-- .../references/admin-tooling.md | 0 .../references/env-config.md | 2 +- .../references/go-setup.md | 0 .../references/iac-automation.md | 0 .../references/multi-language-setup.md | 0 .../references/nodejs-setup.md | 0 .../references/project-cloning.md | 0 .../references/python-setup.md | 0 .../references/quick-start.md | 0 .../SKILL.md | 2 +- .../README.md | 14 ++-- .../SKILL.md | 16 ++-- .../README.md | 8 +- .../SKILL.md | 18 ++--- .../README.md | 8 +- .../SKILL.md | 6 +- .../README.md | 10 +-- .../SKILL.md | 6 +- 45 files changed, 197 insertions(+), 197 deletions(-) rename skills/ai-configs/{agentcontrol-configs-agent-graphs => configs-agent-graphs}/SKILL.md (97%) rename skills/ai-configs/{agentcontrol-configs-ai-metrics => configs-ai-metrics}/SKILL.md (92%) rename skills/ai-configs/{agentcontrol-configs-ai-metrics => configs-ai-metrics}/references/anthropic-tracking.md (100%) rename skills/ai-configs/{agentcontrol-configs-ai-metrics => configs-ai-metrics}/references/bedrock-tracking.md (100%) rename skills/ai-configs/{agentcontrol-configs-ai-metrics => configs-ai-metrics}/references/gemini-tracking.md (99%) rename skills/ai-configs/{agentcontrol-configs-ai-metrics => configs-ai-metrics}/references/langchain-tracking.md (97%) rename skills/ai-configs/{agentcontrol-configs-ai-metrics => configs-ai-metrics}/references/metrics-api.md (100%) rename skills/ai-configs/{agentcontrol-configs-ai-metrics => configs-ai-metrics}/references/openai-tracking.md (100%) rename skills/ai-configs/{agentcontrol-configs-ai-metrics => configs-ai-metrics}/references/strands-tracking.md (93%) rename skills/ai-configs/{agentcontrol-configs-ai-metrics => configs-ai-metrics}/references/streaming-tracking.md (100%) rename skills/ai-configs/{agentcontrol-configs-create => configs-create}/README.md (73%) rename skills/ai-configs/{agentcontrol-configs-create => configs-create}/SKILL.md (90%) rename skills/ai-configs/{agentcontrol-configs-custom-metrics => configs-custom-metrics}/SKILL.md (97%) rename skills/ai-configs/{agentcontrol-configs-migrate => configs-migrate}/README.md (75%) rename skills/ai-configs/{agentcontrol-configs-migrate => configs-migrate}/SKILL.md (86%) rename skills/ai-configs/{agentcontrol-configs-migrate => configs-migrate}/references/agent-graph-reference.md (100%) rename skills/ai-configs/{agentcontrol-configs-migrate => configs-migrate}/references/agent-mode-frameworks.md (98%) rename skills/ai-configs/{agentcontrol-configs-migrate => configs-migrate}/references/before-after-examples.md (100%) rename skills/ai-configs/{agentcontrol-configs-migrate => configs-migrate}/references/fallback-defaults-pattern.md (99%) rename skills/ai-configs/{agentcontrol-configs-migrate => configs-migrate}/references/phase-1-analysis-checklist.md (97%) rename skills/ai-configs/{agentcontrol-configs-migrate => configs-migrate}/references/sdk-ai-tracker-patterns.md (97%) rename skills/ai-configs/{agentcontrol-configs-online-evals => configs-online-evals}/README.md (67%) rename skills/ai-configs/{agentcontrol-configs-online-evals => configs-online-evals}/SKILL.md (98%) rename skills/ai-configs/{agentcontrol-projects => configs-projects}/README.md (93%) rename skills/ai-configs/{agentcontrol-projects => configs-projects}/SKILL.md (94%) rename skills/ai-configs/{agentcontrol-projects => configs-projects}/references/admin-tooling.md (100%) rename skills/ai-configs/{agentcontrol-projects => configs-projects}/references/env-config.md (99%) rename skills/ai-configs/{agentcontrol-projects => configs-projects}/references/go-setup.md (100%) rename skills/ai-configs/{agentcontrol-projects => configs-projects}/references/iac-automation.md (100%) rename skills/ai-configs/{agentcontrol-projects => configs-projects}/references/multi-language-setup.md (100%) rename skills/ai-configs/{agentcontrol-projects => configs-projects}/references/nodejs-setup.md (100%) rename skills/ai-configs/{agentcontrol-projects => configs-projects}/references/project-cloning.md (100%) rename skills/ai-configs/{agentcontrol-projects => configs-projects}/references/python-setup.md (100%) rename skills/ai-configs/{agentcontrol-projects => configs-projects}/references/quick-start.md (100%) rename skills/ai-configs/{agentcontrol-configs-snippets => configs-snippets}/SKILL.md (99%) rename skills/ai-configs/{agentcontrol-configs-targeting => configs-targeting}/README.md (63%) rename skills/ai-configs/{agentcontrol-configs-targeting => configs-targeting}/SKILL.md (97%) rename skills/ai-configs/{agentcontrol-tools => configs-tools}/README.md (76%) rename skills/ai-configs/{agentcontrol-tools => configs-tools}/SKILL.md (91%) rename skills/ai-configs/{agentcontrol-configs-update => configs-update}/README.md (78%) rename skills/ai-configs/{agentcontrol-configs-update => configs-update}/SKILL.md (95%) rename skills/ai-configs/{agentcontrol-configs-variations => configs-variations}/README.md (71%) rename skills/ai-configs/{agentcontrol-configs-variations => configs-variations}/SKILL.md (97%) diff --git a/README.md b/README.md index af9ea8d..8f2b952 100644 --- a/README.md +++ b/README.md @@ -22,16 +22,16 @@ Agent Skills are modular, text-based playbooks that teach an agent how to perfor | Skill | Description | |-------|-------------| -| `ai-configs/agentcontrol-configs-create` | Create AI Configs with variations for agent or completion mode | -| `ai-configs/agentcontrol-configs-migrate` | Migrate an app with hardcoded LLM prompts to AI Configs in five stages (extract, wrap, tools, tracking, evals) | -| `ai-configs/agentcontrol-configs-update` | Update and delete AI Configs, manage lifecycle | -| `ai-configs/agentcontrol-configs-variations` | Manage AI Config variations for A/B testing | -| `ai-configs/agentcontrol-tools` | Create and attach tools for function calling | -| `ai-configs/agentcontrol-projects` | Create and manage projects to organize AI Configs | -| `ai-configs/agentcontrol-configs-online-evals` | Attach LLM-as-a-judge evaluators to AI Configs | -| `ai-configs/agentcontrol-configs-targeting` | Configure targeting rules for AI Config rollouts | -| `ai-configs/agentcontrol-configs-snippets` | Create and manage reusable prompt snippets across AI Configs | -| `ai-configs/agentcontrol-configs-agent-graphs` | Create and manage multi-agent graphs with routing and handoffs | +| `ai-configs/configs-create` | Create AI Configs with variations for agent or completion mode | +| `ai-configs/configs-migrate` | Migrate an app with hardcoded LLM prompts to AI Configs in five stages (extract, wrap, tools, tracking, evals) | +| `ai-configs/configs-update` | Update and delete AI Configs, manage lifecycle | +| `ai-configs/configs-variations` | Manage AI Config variations for A/B testing | +| `ai-configs/configs-tools` | Create and attach tools for function calling | +| `ai-configs/configs-projects` | Create and manage projects to organize AI Configs | +| `ai-configs/configs-online-evals` | Attach LLM-as-a-judge evaluators to AI Configs | +| `ai-configs/configs-targeting` | Configure targeting rules for AI Config rollouts | +| `ai-configs/configs-snippets` | Create and manage reusable prompt snippets across AI Configs | +| `ai-configs/configs-agent-graphs` | Create and manage multi-agent graphs with routing and handoffs | ### Experiments diff --git a/skills.json b/skills.json index d94f7de..67b298c 100644 --- a/skills.json +++ b/skills.json @@ -1,94 +1,94 @@ { "skills": [ { - "name": "agentcontrol-configs-agent-graphs", + "name": "configs-agent-graphs", "description": "Create and manage agent graphs \u2014 directed graphs of AI Configs connected by edges with handoff logic. Use when building multi-agent workflows where configs route to each other.", - "path": "skills/ai-configs/agentcontrol-configs-agent-graphs", + "path": "skills/ai-configs/configs-agent-graphs", "version": "0.1.0", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "agentcontrol-configs-ai-metrics", + "name": "configs-ai-metrics", "description": "Instrument an existing codebase with LaunchDarkly AI Config tracking. Walks the four-tier ladder (managed runner \u2192 provider package \u2192 custom extractor + trackMetricsOf \u2192 raw manual) and picks the lowest-ceremony option that still captures duration, tokens, and success/error.", - "path": "skills/ai-configs/agentcontrol-configs-ai-metrics", + "path": "skills/ai-configs/configs-ai-metrics", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the LaunchDarkly server-side AI SDK (`launchdarkly-server-sdk-ai>=0.20.0` for Python or `@launchdarkly/server-sdk-ai>=0.20.0` for Node) and an existing AI Config." }, { - "name": "agentcontrol-configs-create", + "name": "configs-create", "description": "Create and configure AI Configs in LaunchDarkly. Helps you choose between agent vs completion mode, create the config, add variations with models and prompts, and verify the setup.", - "path": "skills/ai-configs/agentcontrol-configs-create", + "path": "skills/ai-configs/configs-create", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "agentcontrol-configs-custom-metrics", + "name": "configs-custom-metrics", "description": "Create, track, retrieve, update, and delete custom business metrics for AI Configs. Covers full lifecycle: define metric kinds via API, emit events via SDK, and query results.", - "path": "skills/ai-configs/agentcontrol-configs-custom-metrics", + "path": "skills/ai-configs/configs-custom-metrics", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the LaunchDarkly server SDK and a LaunchDarkly API token with the `writer` role for metric management." }, { - "name": "agentcontrol-configs-migrate", + "name": "configs-migrate", "description": "Migrate an application with hardcoded LLM prompts to a full LaunchDarkly AI Configs implementation in five stages: audit the code, wrap the call, move the tools, add tracking, attach evaluators. Use when the user wants to externalize model/prompt configuration, move from direct provider calls (OpenAI, Anthropic, Bedrock, Gemini, Strands) to a managed AI Config, or stage a full hardcoded-to-LaunchDarkly migration.", - "path": "skills/ai-configs/agentcontrol-configs-migrate", + "path": "skills/ai-configs/configs-migrate", "version": "0.1.0", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "agentcontrol-configs-online-evals", + "name": "configs-online-evals", "description": "Attach judges to AI Config variations for automatic LLM-as-a-judge evaluation. Create custom judges, configure sampling rates, and monitor quality scores.", - "path": "skills/ai-configs/agentcontrol-configs-online-evals", + "path": "skills/ai-configs/configs-online-evals", "version": "0.1.0", "compatibility": "Requires LaunchDarkly API access token with ai-configs:write permission. SDK versions Python v0.20.0+ or Node.js v0.20.0+ for automatic metric recording and the consolidated `track_judge_result` / `trackJudgeResult` API." }, { - "name": "agentcontrol-projects", + "name": "configs-projects", "description": "Guide for setting up LaunchDarkly projects in your codebase. Helps you assess your stack, choose the right approach, and integrate project management that makes sense for your architecture.", - "path": "skills/ai-configs/agentcontrol-projects", + "path": "skills/ai-configs/configs-projects", "version": "0.4.0", "compatibility": "Requires LaunchDarkly API access token with projects:write permission or LaunchDarkly MCP server." }, { - "name": "agentcontrol-configs-snippets", + "name": "configs-snippets", "description": "Create and manage prompt snippets \u2014 reusable text blocks referenced inside AI Config variation prompts. Keeps common instructions, personas, and guardrails consistent across multiple configs.", - "path": "skills/ai-configs/agentcontrol-configs-snippets", + "path": "skills/ai-configs/configs-snippets", "version": "0.1.0", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "agentcontrol-configs-targeting", + "name": "configs-targeting", "description": "Configure AI Config targeting rules to control which variations serve to different users. Enable percentage rollouts, attribute-based rules, segment targeting, and guarded rollouts.", - "path": "skills/ai-configs/agentcontrol-configs-targeting", + "path": "skills/ai-configs/configs-targeting", "version": "0.1.0", "compatibility": "Requires LaunchDarkly API access token with ai-configs:write permission." }, { - "name": "agentcontrol-tools", + "name": "configs-tools", "description": "Give your AI agents capabilities through tools (function calling). Helps you identify what your AI needs to do, create tool definitions, and attach them to AI Config variations.", - "path": "skills/ai-configs/agentcontrol-tools", + "path": "skills/ai-configs/configs-tools", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "agentcontrol-configs-update", + "name": "configs-update", "description": "Update, archive, and delete LaunchDarkly AI Configs and their variations. Use when you need to modify config properties, change model parameters, update instructions or messages, archive unused configs, or permanently remove them.", - "path": "skills/ai-configs/agentcontrol-configs-update", + "path": "skills/ai-configs/configs-update", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "agentcontrol-configs-variations", + "name": "configs-variations", "description": "Experiment with AI configurations by creating and managing variations. Helps you test different models, prompts, and parameters to find what works best through systematic experimentation.", - "path": "skills/ai-configs/agentcontrol-configs-variations", + "path": "skills/ai-configs/configs-variations", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" diff --git a/skills/ai-configs/agentcontrol-configs-agent-graphs/SKILL.md b/skills/ai-configs/configs-agent-graphs/SKILL.md similarity index 97% rename from skills/ai-configs/agentcontrol-configs-agent-graphs/SKILL.md rename to skills/ai-configs/configs-agent-graphs/SKILL.md index 8510615..d5176fe 100644 --- a/skills/ai-configs/agentcontrol-configs-agent-graphs/SKILL.md +++ b/skills/ai-configs/configs-agent-graphs/SKILL.md @@ -1,6 +1,6 @@ --- -name: agentcontrol-configs-agent-graphs -description: "Create and manage agent graphs — directed graphs of AI Configs connected by edges with handoff logic. Use when building multi-agent workflows where configs route to each other." +name: configs-agent-graphs +description: "Create and manage agent graphs — directed graphs of configs connected by edges with handoff logic. Use when building multi-agent workflows where configs route to each other." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server metadata: diff --git a/skills/ai-configs/agentcontrol-configs-ai-metrics/SKILL.md b/skills/ai-configs/configs-ai-metrics/SKILL.md similarity index 92% rename from skills/ai-configs/agentcontrol-configs-ai-metrics/SKILL.md rename to skills/ai-configs/configs-ai-metrics/SKILL.md index 52623db..96170a3 100644 --- a/skills/ai-configs/agentcontrol-configs-ai-metrics/SKILL.md +++ b/skills/ai-configs/configs-ai-metrics/SKILL.md @@ -1,5 +1,5 @@ --- -name: agentcontrol-configs-ai-metrics +name: configs-ai-metrics description: "Instrument an existing codebase with LaunchDarkly AI Config tracking. Walks the four-tier ladder (managed runner → provider package → custom extractor + trackMetricsOf → raw manual) and picks the lowest-ceremony option that still captures duration, tokens, and success/error." license: Apache-2.0 compatibility: Requires the LaunchDarkly server-side AI SDK (`launchdarkly-server-sdk-ai>=0.20.0` for Python or `@launchdarkly/server-sdk-ai>=0.20.0` for Node) and an existing AI Config. @@ -38,12 +38,12 @@ Before picking a tier, find the provider call and answer these questions: - [ ] **Provider?** OpenAI, Anthropic, Bedrock, Gemini, Azure, custom HTTP? → cross-reference with the package availability matrix below. - [ ] **Streaming?** If yes, you'll need TTFT tracking, which means Tier 4 for the TTFT part even if the rest is Tier 2. - [ ] **Language?** Python or Node? Provider-package coverage differs between them. -- [ ] **Already using an AI Config?** If not, route to `agentcontrol-configs-create` first — tracking requires a tracker, which is obtained by calling `create_tracker()` / `createTracker()` on the config object returned by `completion_config()` / `completionConfig()` / `createModel()`. +- [ ] **Already using an AI Config?** If not, route to `configs-create` first — tracking requires a tracker, which is obtained by calling `create_tracker()` / `createTracker()` on the config object returned by `completion_config()` / `completionConfig()` / `createModel()`. - [ ] **On the current SDK API?** If the call site uses `aiclient.config(...)` / `aiClient.config(...)` or constructs an `AIConfig(...)` / `LDAIConfig` default, it's on the pre-0.20 surface. Migrate it as part of this work before adding tracking: - `aiclient.config(...)` → `aiclient.completion_config(...)` for one-shot/chat or `aiclient.agent_config(...)` for agent mode (mirror the call signature). Node is the same with camelCase. - `AIConfig(...)` default → `AICompletionConfigDefault(...)` or `AIAgentConfigDefault(...)` (Node: `LDAICompletionConfigDefault` / `LDAIAgentConfigDefault`). `AIConfig` is the base class the SDK returns; it isn't a valid default-value constructor — the typed `*Default` variants are. - If the result was being tuple-unpacked (`config, tracker = aiclient.config(...)`), drop the unpack — the new methods return a single config object. Obtain the tracker via `config.create_tracker()` / `aiConfig.createTracker()`. - - For deeper rewrites (call sites with hardcoded model/prompt as well), hand off to `agentcontrol-configs-migrate` instead of doing the full migration here. + - For deeper rewrites (call sites with hardcoded model/prompt as well), hand off to `configs-migrate` instead of doing the full migration here. ### 2. Look up your Tier-2 option @@ -103,7 +103,7 @@ Obtain a tracker via the factory on the config object: `tracker = config.create_ ## Related skills -- `agentcontrol-configs-create` — prerequisite if the app doesn't have an AI Config yet -- `agentcontrol-configs-custom-metrics` — business metrics (conversion, resolution, retention) layered on top of the AI metrics this skill captures -- `agentcontrol-configs-online-evals` — automatic quality scoring (LLM-as-judge) on sampled live requests; complementary to the metrics here -- `agentcontrol-configs-migrate` — Stage 4 of the hardcoded-to-AI-Configs migration delegates to this skill +- `configs-create` — prerequisite if the app doesn't have an AI Config yet +- `configs-custom-metrics` — business metrics (conversion, resolution, retention) layered on top of the AI metrics this skill captures +- `configs-online-evals` — automatic quality scoring (LLM-as-judge) on sampled live requests; complementary to the metrics here +- `configs-migrate` — Stage 4 of the hardcoded-to-AI-Configs migration delegates to this skill diff --git a/skills/ai-configs/agentcontrol-configs-ai-metrics/references/anthropic-tracking.md b/skills/ai-configs/configs-ai-metrics/references/anthropic-tracking.md similarity index 100% rename from skills/ai-configs/agentcontrol-configs-ai-metrics/references/anthropic-tracking.md rename to skills/ai-configs/configs-ai-metrics/references/anthropic-tracking.md diff --git a/skills/ai-configs/agentcontrol-configs-ai-metrics/references/bedrock-tracking.md b/skills/ai-configs/configs-ai-metrics/references/bedrock-tracking.md similarity index 100% rename from skills/ai-configs/agentcontrol-configs-ai-metrics/references/bedrock-tracking.md rename to skills/ai-configs/configs-ai-metrics/references/bedrock-tracking.md diff --git a/skills/ai-configs/agentcontrol-configs-ai-metrics/references/gemini-tracking.md b/skills/ai-configs/configs-ai-metrics/references/gemini-tracking.md similarity index 99% rename from skills/ai-configs/agentcontrol-configs-ai-metrics/references/gemini-tracking.md rename to skills/ai-configs/configs-ai-metrics/references/gemini-tracking.md index 5fa6941..9b14542 100644 --- a/skills/ai-configs/agentcontrol-configs-ai-metrics/references/gemini-tracking.md +++ b/skills/ai-configs/configs-ai-metrics/references/gemini-tracking.md @@ -191,7 +191,7 @@ gemini_tools = [ ] if ld_tools else [] ``` -Tool handlers stay in your application code — LaunchDarkly stores the schema, your application owns the behavior. For the full agent loop pattern (`MAX_STEPS`, `functionCalls` handling, `tracker.track_tool_call`), see the agent-mode section of `agentcontrol-tools`. +Tool handlers stay in your application code — LaunchDarkly stores the schema, your application owns the behavior. For the full agent loop pattern (`MAX_STEPS`, `functionCalls` handling, `tracker.track_tool_call`), see the agent-mode section of `configs-tools`. ## Tier 2 option — route via LangChain diff --git a/skills/ai-configs/agentcontrol-configs-ai-metrics/references/langchain-tracking.md b/skills/ai-configs/configs-ai-metrics/references/langchain-tracking.md similarity index 97% rename from skills/ai-configs/agentcontrol-configs-ai-metrics/references/langchain-tracking.md rename to skills/ai-configs/configs-ai-metrics/references/langchain-tracking.md index 51329e7..c6e4e67 100644 --- a/skills/ai-configs/agentcontrol-configs-ai-metrics/references/langchain-tracking.md +++ b/skills/ai-configs/configs-ai-metrics/references/langchain-tracking.md @@ -252,7 +252,7 @@ for (const msg of result.messages ?? []) { ## Binding AI-Config-attached tools with `build_structured_tools` -If the variation has tools attached (via `/agentcontrol-tools`), use `build_structured_tools` rather than hand-rolling a `TOOL_REGISTRY` / `resolve_tools` / `ALL_TOOLS` shape. The helper reads `ai_config.model.parameters.tools`, picks the matching entries from your `{name: callable}` registry, wraps them as LangChain `StructuredTool` instances, and preserves the LD tool key as the `StructuredTool.name` (so `ToolNode(...)` lookup works without a second mapping). +If the variation has tools attached (via `/configs-tools`), use `build_structured_tools` rather than hand-rolling a `TOOL_REGISTRY` / `resolve_tools` / `ALL_TOOLS` shape. The helper reads `ai_config.model.parameters.tools`, picks the matching entries from your `{name: callable}` registry, wraps them as LangChain `StructuredTool` instances, and preserves the LD tool key as the `StructuredTool.name` (so `ToolNode(...)` lookup works without a second mapping). ```python # tools.py — implementations only; no manual schema, no resolve_tools() diff --git a/skills/ai-configs/agentcontrol-configs-ai-metrics/references/metrics-api.md b/skills/ai-configs/configs-ai-metrics/references/metrics-api.md similarity index 100% rename from skills/ai-configs/agentcontrol-configs-ai-metrics/references/metrics-api.md rename to skills/ai-configs/configs-ai-metrics/references/metrics-api.md diff --git a/skills/ai-configs/agentcontrol-configs-ai-metrics/references/openai-tracking.md b/skills/ai-configs/configs-ai-metrics/references/openai-tracking.md similarity index 100% rename from skills/ai-configs/agentcontrol-configs-ai-metrics/references/openai-tracking.md rename to skills/ai-configs/configs-ai-metrics/references/openai-tracking.md diff --git a/skills/ai-configs/agentcontrol-configs-ai-metrics/references/strands-tracking.md b/skills/ai-configs/configs-ai-metrics/references/strands-tracking.md similarity index 93% rename from skills/ai-configs/agentcontrol-configs-ai-metrics/references/strands-tracking.md rename to skills/ai-configs/configs-ai-metrics/references/strands-tracking.md index 9490a96..1741254 100644 --- a/skills/ai-configs/agentcontrol-configs-ai-metrics/references/strands-tracking.md +++ b/skills/ai-configs/configs-ai-metrics/references/strands-tracking.md @@ -79,7 +79,7 @@ Pick the style that matches the rest of the codebase — the two variants record ## Provider dispatch stays in your code -Strands model classes are provider-specific (`AnthropicModel`, `OpenAIModel`, `BedrockModel`). To serve more than one provider from a single AI Config key, dispatch on `agent_config.provider.name` before constructing the `Agent`. See [agent-mode-frameworks.md § Strands Agent](../../agentcontrol-configs-migrate/references/agent-mode-frameworks.md) for the `create_strands_model` dispatcher, including the rule that `parameters.tools` must be dropped before being passed into the Strands model class (tools flow through the `Agent` constructor, not through model params). +Strands model classes are provider-specific (`AnthropicModel`, `OpenAIModel`, `BedrockModel`). To serve more than one provider from a single AI Config key, dispatch on `agent_config.provider.name` before constructing the `Agent`. See [agent-mode-frameworks.md § Strands Agent](../../configs-migrate/references/agent-mode-frameworks.md) for the `create_strands_model` dispatcher, including the rule that `parameters.tools` must be dropped before being passed into the Strands model class (tools flow through the `Agent` constructor, not through model params). ## Always flush before exit diff --git a/skills/ai-configs/agentcontrol-configs-ai-metrics/references/streaming-tracking.md b/skills/ai-configs/configs-ai-metrics/references/streaming-tracking.md similarity index 100% rename from skills/ai-configs/agentcontrol-configs-ai-metrics/references/streaming-tracking.md rename to skills/ai-configs/configs-ai-metrics/references/streaming-tracking.md diff --git a/skills/ai-configs/agentcontrol-configs-create/README.md b/skills/ai-configs/configs-create/README.md similarity index 73% rename from skills/ai-configs/agentcontrol-configs-create/README.md rename to skills/ai-configs/configs-create/README.md index 901271e..264e703 100644 --- a/skills/ai-configs/agentcontrol-configs-create/README.md +++ b/skills/ai-configs/configs-create/README.md @@ -12,7 +12,7 @@ This skill teaches agents how to: ## Installation (Local) -Copy `skills/ai-configs/agentcontrol-configs-create/` into your agent client's skills path. +Copy `skills/ai-configs/configs-create/` into your agent client's skills path. ## Prerequisites @@ -31,16 +31,16 @@ Set up an AI config for content generation using Claude ## Structure ``` -agentcontrol-configs-create/ +configs-create/ ├── SKILL.md └── README.md ``` ## Related -- [AI Config Projects](../agentcontrol-projects/): Create projects first -- [AI Config Tools](../agentcontrol-tools/): Add tools after creating config -- [AI Config Variations](../agentcontrol-configs-variations/): Add more variations for experimentation +- [AI Config Projects](../configs-projects/): Create projects first +- [AI Config Tools](../configs-tools/): Add tools after creating config +- [AI Config Variations](../configs-variations/): Add more variations for experimentation - [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) ## License diff --git a/skills/ai-configs/agentcontrol-configs-create/SKILL.md b/skills/ai-configs/configs-create/SKILL.md similarity index 90% rename from skills/ai-configs/agentcontrol-configs-create/SKILL.md rename to skills/ai-configs/configs-create/SKILL.md index 43a5c76..e2b1a4f 100644 --- a/skills/ai-configs/agentcontrol-configs-create/SKILL.md +++ b/skills/ai-configs/configs-create/SKILL.md @@ -1,5 +1,5 @@ --- -name: agentcontrol-configs-create +name: configs-create description: "Create and configure AI Configs in LaunchDarkly. Helps you choose between agent vs completion mode, create the config, add variations with models and prompts, and verify the setup." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server @@ -12,7 +12,7 @@ metadata: You're using a skill that will guide you through creating an AI Config in LaunchDarkly. Your job is to understand the use case, choose the right mode, create the config and its variations, and verify everything is set up correctly. -> **⚠️ This skill creates a config — it does not make it servable.** A freshly-created AI Config has its **fallthrough pointing at an auto-generated disabled variation**, not at the variation you just created. The SDK will return `ai_config.enabled=False` on every evaluation until you flip targeting on and point the fallthrough at your new variation. This is not a bug — it's the default state. **You must run `/agentcontrol-configs-targeting` (or the equivalent REST / CLI call shown in Step 5) before verifying against the SDK**, or verification will look like the LD-served path is broken when it isn't. The single most common failure mode users hit with this skill is skipping the targeting step and spending time debugging `enabled=False` in their application code. +> **⚠️ This skill creates a config — it does not make it servable.** A freshly-created AI Config has its **fallthrough pointing at an auto-generated disabled variation**, not at the variation you just created. The SDK will return `ai_config.enabled=False` on every evaluation until you flip targeting on and point the fallthrough at your new variation. This is not a bug — it's the default state. **You must run `/configs-targeting` (or the equivalent REST / CLI call shown in Step 5) before verifying against the SDK**, or verification will look like the LD-served path is broken when it isn't. The single most common failure mode users hit with this skill is skipping the targeting step and spending time debugging `enabled=False` in their application code. ## Prerequisites @@ -147,7 +147,7 @@ Print this checklist verbatim to the user after Step 4, then wait for confirmati > > 🔴 **The SDK will return `enabled=False` until you flip targeting on.** The fallthrough is currently pointing at an auto-generated disabled variation, not at the `{variationKey}` you just created. > -> **Next step — run `/agentcontrol-configs-targeting`** with these inputs: +> **Next step — run `/configs-targeting`** with these inputs: > - Project key: `{projectKey}` > - Config key: `{configKey}` > - Environment key: the env whose SDK key is in your `.env` (usually `test` or `production`) @@ -159,7 +159,7 @@ Print this checklist verbatim to the user after Step 4, then wait for confirmati #### Direct shortcut if the user wants to flip targeting without invoking the sibling skill -`agentcontrol-configs-targeting` is the canonical path — it handles percentage rollouts, targeted rules, and variation-ID lookups. But for the simplest case ("promote the new variation to fallthrough in one environment"), you can run the underlying semantic PATCH yourself once you know the new variation's `_id`. +`configs-targeting` is the canonical path — it handles percentage rollouts, targeted rules, and variation-ID lookups. But for the simplest case ("promote the new variation to fallthrough in one environment"), you can run the underlying semantic PATCH yourself once you know the new variation's `_id`. Get the variation ID (use `get-ai-config` MCP, or): ```bash @@ -203,7 +203,7 @@ The `create-ai-config-variation` tool validates this format and rejects invalid |-----------|--------| | Config already exists | Ask if user wants to update instead | | Variation shows "NO MODEL" | Use `update-ai-config-variation` to set modelConfigKey | -| Need to attach tools | Create tools first (`agentcontrol-tools` skill), then update the variation | +| Need to attach tools | Create tools first (`configs-tools` skill), then update the variation | ## What NOT to Do @@ -213,10 +213,10 @@ The `create-ai-config-variation` tool validates this format and rejects invalid - Don't forget modelConfigKey (models won't show in the UI) - Don't omit `modelName` from the initial variation call. It is required at create time; setting it via a follow-up PATCH is a workaround for a bug, not the intended flow. The PATCH field is also `modelName`, not `name`. - Don't drop to raw `curl` + `jq` for verification. Use `get-ai-config` (MCP) — it returns a typed object and avoids brittle `jq` filters that break on response-shape variation. -- Don't consider the workflow complete until the user has been told to run `agentcontrol-configs-targeting`. A created variation that isn't promoted to fallthrough returns `enabled=False` to every consumer. +- Don't consider the workflow complete until the user has been told to run `configs-targeting`. A created variation that isn't promoted to fallthrough returns `enabled=False` to every consumer. ## Related Skills -- `agentcontrol-tools` -- Create tools before attaching -- `agentcontrol-configs-variations` -- Add more variations for experimentation -- `agentcontrol-configs-update` -- Modify configs based on learnings +- `configs-tools` -- Create tools before attaching +- `configs-variations` -- Add more variations for experimentation +- `configs-update` -- Modify configs based on learnings diff --git a/skills/ai-configs/agentcontrol-configs-custom-metrics/SKILL.md b/skills/ai-configs/configs-custom-metrics/SKILL.md similarity index 97% rename from skills/ai-configs/agentcontrol-configs-custom-metrics/SKILL.md rename to skills/ai-configs/configs-custom-metrics/SKILL.md index b39e397..b787dcd 100644 --- a/skills/ai-configs/agentcontrol-configs-custom-metrics/SKILL.md +++ b/skills/ai-configs/configs-custom-metrics/SKILL.md @@ -1,5 +1,5 @@ --- -name: agentcontrol-configs-custom-metrics +name: configs-custom-metrics description: "Create, track, retrieve, update, and delete custom business metrics for AI Configs. Covers full lifecycle: define metric kinds via API, emit events via SDK, and query results." license: Apache-2.0 compatibility: Requires the LaunchDarkly server SDK and a LaunchDarkly API token with the `writer` role for metric management. @@ -14,9 +14,9 @@ Full lifecycle management of custom business metrics: create metric definitions ## Prerequisites -- LaunchDarkly SDK initialized (see `agentcontrol-sdk`) +- LaunchDarkly SDK initialized (see `configs-sdk`) - LaunchDarkly API token with `writer` role for metric management -- Understanding of built-in AI metrics (see `agentcontrol-configs-ai-metrics`) +- Understanding of built-in AI metrics (see `configs-ai-metrics`) ## API Key Detection @@ -149,7 +149,7 @@ from ldclient import Context from ldclient.config import Config import ldclient -# Initialize (see agentcontrol-sdk for details) +# Initialize (see configs-sdk for details) ldclient.set_config(Config("your-sdk-key")) ld_client = ldclient.get() @@ -492,9 +492,9 @@ Custom metrics appear in: ## Related Skills -- `agentcontrol-sdk` - SDK setup -- `agentcontrol-configs-ai-metrics` - Built-in AI metrics (tokens, duration, cost) -- `agentcontrol-configs-online-evals` - Quality metrics via judges +- `configs-sdk` - SDK setup +- `configs-ai-metrics` - Built-in AI metrics (tokens, duration, cost) +- `configs-online-evals` - Quality metrics via judges ## References diff --git a/skills/ai-configs/agentcontrol-configs-migrate/README.md b/skills/ai-configs/configs-migrate/README.md similarity index 75% rename from skills/ai-configs/agentcontrol-configs-migrate/README.md rename to skills/ai-configs/configs-migrate/README.md index ed61dd0..34053c8 100644 --- a/skills/ai-configs/agentcontrol-configs-migrate/README.md +++ b/skills/ai-configs/configs-migrate/README.md @@ -9,14 +9,14 @@ This skill orchestrates the full migration journey from hardcoded `openai.chat.c The five stages: 1. **Extract** hardcoded model names, prompts, and parameters (read-only) -2. **Wrap** the call site in `completion_config` / `completionConfig` with a safe fallback — delegates the config creation to `agentcontrol-configs-create` -3. **Tools** — move function-calling schemas into LaunchDarkly — delegates to `agentcontrol-tools` +2. **Wrap** the call site in `completion_config` / `completionConfig` with a safe fallback — delegates the config creation to `configs-create` +3. **Tools** — move function-calling schemas into LaunchDarkly — delegates to `configs-tools` 4. **Tracking** — wire `track_duration`, `track_tokens`, `track_success`/`track_error`, optional `track_feedback` — inline, with a reference doc covering every SDK method in Python and Node side by side -5. **Evals** — attach judges for LLM-as-a-judge scoring — delegates to `agentcontrol-configs-online-evals` +5. **Evals** — attach judges for LLM-as-a-judge scoring — delegates to `configs-online-evals` ## Installation (Local) -Copy `skills/ai-configs/agentcontrol-configs-migrate/` into your agent client's skills path. +Copy `skills/ai-configs/configs-migrate/` into your agent client's skills path. ## Prerequisites @@ -41,7 +41,7 @@ Wire up the AI tracker and attach accuracy + relevance judges to our existing co ## Structure ``` -agentcontrol-configs-migrate/ +configs-migrate/ ├── SKILL.md ├── README.md └── references/ @@ -55,11 +55,11 @@ agentcontrol-configs-migrate/ ## Related -- [AI Config Create](../agentcontrol-configs-create/): Delegated to by Stage 2 (wrap) -- [AI Config Tools](../agentcontrol-tools/): Delegated to by Stage 3 (tools) -- [AI Config Online Evals](../agentcontrol-configs-online-evals/): Delegated to by Stage 5 (evals) -- [AI Config Variations](../agentcontrol-configs-variations/): Next step after migration for A/B testing -- [AI Config Targeting](../agentcontrol-configs-targeting/): Next step after migration for rollout control +- [AI Config Create](../configs-create/): Delegated to by Stage 2 (wrap) +- [AI Config Tools](../configs-tools/): Delegated to by Stage 3 (tools) +- [AI Config Online Evals](../configs-online-evals/): Delegated to by Stage 5 (evals) +- [AI Config Variations](../configs-variations/): Next step after migration for A/B testing +- [AI Config Targeting](../configs-targeting/): Next step after migration for rollout control - [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) ## License diff --git a/skills/ai-configs/agentcontrol-configs-migrate/SKILL.md b/skills/ai-configs/configs-migrate/SKILL.md similarity index 86% rename from skills/ai-configs/agentcontrol-configs-migrate/SKILL.md rename to skills/ai-configs/configs-migrate/SKILL.md index f23a609..5a64b4d 100644 --- a/skills/ai-configs/agentcontrol-configs-migrate/SKILL.md +++ b/skills/ai-configs/configs-migrate/SKILL.md @@ -1,5 +1,5 @@ --- -name: agentcontrol-configs-migrate +name: configs-migrate description: "Migrate an application with hardcoded LLM prompts to a full LaunchDarkly AI Configs implementation in five stages: audit the code, wrap the call, move the tools, add tracking, attach evaluators. Use when the user wants to externalize model/prompt configuration, move from direct provider calls (OpenAI, Anthropic, Bedrock, Gemini, Strands) to a managed AI Config, or stage a full hardcoded-to-LaunchDarkly migration." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server @@ -22,7 +22,7 @@ You're using a skill that will guide you through migrating an application from h > > 1. **Tracker in the wrong scope.** For an agent with a loop, mint `create_tracker()` once per user turn in a `setup_run` entry node — not inside `call_model`. Per-iteration factory calls produce N `runId`s and trip the at-most-once guards. See [agent-mode-frameworks.md § Custom `StateGraph`](references/agent-mode-frameworks.md). > 2. **`load_chat_model` wrapper reuse.** Templates like `langchain-ai/react-agent` ship a `load_chat_model(f"{provider}/{name}")` helper that wraps `init_chat_model(...)` and silently drops every variation parameter. **Delete it** (don't just avoid using it) and replace call sites with `create_langchain_model(ai_config)`. -> 3. **Fallthrough not flipped after `/agentcontrol-configs-create`.** A freshly-created AI Config's fallthrough points at an auto-generated disabled variation, so the SDK returns `enabled=False` until `/agentcontrol-configs-targeting` runs. Flip it before Stage 2 verification. +> 3. **Fallthrough not flipped after `/configs-create`.** A freshly-created AI Config's fallthrough points at an auto-generated disabled variation, so the SDK returns `enabled=False` until `/configs-targeting` runs. Flip it before Stage 2 verification. ## Coverage — which shapes are well-trodden vs require extrapolation @@ -30,16 +30,16 @@ The skill is optimized for Python and Node.js / TypeScript; other languages are | Shape | Python | Node.js | Reference | |-------|--------|---------|-----------| -| One-shot completion (direct OpenAI / Anthropic / Bedrock / Gemini call) | ✅ Worked example | ✅ Worked example | [before-after-examples.md](references/before-after-examples.md), per-provider docs in `agentcontrol-configs-ai-metrics/references/` | -| Chat loop via managed runner (`ManagedModel`) | ✅ Tier 1 pattern | ✅ Tier 1 pattern | [agentcontrol-configs-ai-metrics SKILL.md](../agentcontrol-configs-ai-metrics/SKILL.md) | -| LangChain single-call | ✅ Worked example | ✅ Worked example | [langchain-tracking.md](../agentcontrol-configs-ai-metrics/references/langchain-tracking.md) | +| One-shot completion (direct OpenAI / Anthropic / Bedrock / Gemini call) | ✅ Worked example | ✅ Worked example | [before-after-examples.md](references/before-after-examples.md), per-provider docs in `configs-ai-metrics/references/` | +| Chat loop via managed runner (`ManagedModel`) | ✅ Tier 1 pattern | ✅ Tier 1 pattern | [configs-ai-metrics SKILL.md](../configs-ai-metrics/SKILL.md) | +| LangChain single-call | ✅ Worked example | ✅ Worked example | [langchain-tracking.md](../configs-ai-metrics/references/langchain-tracking.md) | | LangGraph prebuilt agent (Python `langchain.agents.create_agent`, Node `createReactAgent`) | ✅ Worked example | ✅ Worked example | [agent-mode-frameworks.md § LangGraph](references/agent-mode-frameworks.md) | | LangGraph custom `StateGraph` with run-scoped tracker (setup_run + call_model + finalize) | ✅ Deep worked example | ⚠️ Mentioned — translate from Python | [agent-mode-frameworks.md § Custom `StateGraph`](references/agent-mode-frameworks.md) | | CrewAI `Agent` | ✅ Worked example | — (not a Node framework) | [agent-mode-frameworks.md § CrewAI](references/agent-mode-frameworks.md) | | Strands `Agent` | ✅ Worked example | ⚠️ BedrockModel + OpenAIModel only (no Anthropic) | [agent-mode-frameworks.md § Strands](references/agent-mode-frameworks.md) | | Custom ReAct loop (hand-rolled, any framework or none) | ✅ Worked example | ⚠️ Apply framework-agnostic invariants; translate from Python | [agent-mode-frameworks.md § Custom ReAct loop](references/agent-mode-frameworks.md) | -| Vercel AI SDK (`generateText` / `streamText`) | — (not a Python framework) | ⚠️ Provider package exists; no worked example in skill | `agentcontrol-configs-ai-metrics` provider-package matrix | -| Streaming (SSE / WebSocket) | ⚠️ Delegated to `agentcontrol-configs-ai-metrics` streaming doc | ⚠️ Same — use `trackStreamMetricsOf` + manual TTFT | [streaming-tracking.md](../agentcontrol-configs-ai-metrics/references/streaming-tracking.md) | +| Vercel AI SDK (`generateText` / `streamText`) | — (not a Python framework) | ⚠️ Provider package exists; no worked example in skill | `configs-ai-metrics` provider-package matrix | +| Streaming (SSE / WebSocket) | ⚠️ Delegated to `configs-ai-metrics` streaming doc | ⚠️ Same — use `trackStreamMetricsOf` + manual TTFT | [streaming-tracking.md](../configs-ai-metrics/references/streaming-tracking.md) | | Multi-agent graph (supervisor + workers) | ⚠️ Out of main scope; see reference | ⚠️ Out of main scope; see reference | [agent-graph-reference.md](references/agent-graph-reference.md) | | Non-LangGraph agent frameworks (Pydantic AI, DSPy, AutoGen, Haystack, LlamaIndex agents, Semantic Kernel) | ⚠️ Apply the three invariants; no framework-specific example | ⚠️ Same | [agent-mode-frameworks.md § Framework-agnostic invariants](references/agent-mode-frameworks.md) | | Go, Ruby, .NET | ℹ️ Install commands only | ℹ️ Install commands only | [phase-1-analysis-checklist.md § SDK routing table](references/phase-1-analysis-checklist.md) | @@ -67,11 +67,11 @@ If a CHANGELOG entry post-dates this skill and changes an API you're about to us **Hand-off model.** This skill does **not** auto-invoke other skills. At each stage that needs a LaunchDarkly write, this skill prepares the inputs (config key, mode, model, prompt, tool schemas, judge keys) and then **tells the user to run the next slash-command themselves**. After the user finishes that sibling skill, return to the next step here. Treat the "Delegate" lines below as next-step instructions, not auto-handoffs. **Sibling skills the user runs at each stage:** -- `agentcontrol-projects` — pre-Stage 2, only if no project exists yet -- `agentcontrol-configs-create` — Stage 2 (creates the AI Config and first variation) -- `agentcontrol-tools` — Stage 3 (creates tool definitions and attaches them) -- `agentcontrol-configs-targeting` — between Stage 2 and Stage 4 (promotes the new variation to fallthrough so the SDK actually serves it) -- `agentcontrol-configs-online-evals` — Stage 5 (attaches judges, creates custom judges) +- `configs-projects` — pre-Stage 2, only if no project exists yet +- `configs-create` — Stage 2 (creates the AI Config and first variation) +- `configs-tools` — Stage 3 (creates tool definitions and attaches them) +- `configs-targeting` — between Stage 2 and Stage 4 (promotes the new variation to fallthrough so the SDK actually serves it) +- `configs-online-evals` — Stage 5 (attaches judges, creates custom judges) ## Core Principles @@ -79,7 +79,7 @@ If a CHANGELOG entry post-dates this skill and changes an API you're about to us 2. **Replace config, not business logic.** The SDK call is a drop-in for the place where the model, parameters, and prompt are *defined* — not for the provider call itself. OpenAI/Anthropic/Bedrock calls stay where they are. 3. **Fallback mirrors current behavior.** The fallback passed to `completion_config` / `agent_config` must preserve the hardcoded values you removed, so the app is unchanged if LaunchDarkly is unreachable. 4. **Stages are ordered.** Wrap before you add tools. Add tools before you track. Track before you add evals. Skipping ahead produces configs without traffic, metrics without context, and judges with nothing to score. -5. **Hand off to focused skills, manually.** Each stage that needs a LaunchDarkly write tells the user to run a sibling slash-command (`/agentcontrol-configs-create`, `/agentcontrol-tools`, `/agentcontrol-configs-targeting`, `/agentcontrol-configs-online-evals`) and waits for them to come back. This skill does **not** auto-invoke other skills. +5. **Hand off to focused skills, manually.** Each stage that needs a LaunchDarkly write tells the user to run a sibling slash-command (`/configs-create`, `/configs-tools`, `/configs-targeting`, `/configs-online-evals`) and waits for them to come back. This skill does **not** auto-invoke other skills. ## Workflow @@ -160,7 +160,7 @@ This is the first stage that writes code. It has nine sub-steps. - OpenAI: `launchdarkly-server-sdk-ai-openai>=0.4.0` (Python) / `@launchdarkly/server-sdk-ai-openai@^0.5.5` (Node) - LangChain / LangGraph: `launchdarkly-server-sdk-ai-langchain>=0.5.0` (Python) / `@launchdarkly/server-sdk-ai-langchain@^0.5.5` (Node) - Vercel AI SDK (Node only): `@launchdarkly/server-sdk-ai-vercel@^0.5.5` - - Anthropic, Gemini, Bedrock — no provider package published; use Tier-3 custom extractor (see `agentcontrol-configs-ai-metrics`) + - Anthropic, Gemini, Bedrock — no provider package published; use Tier-3 custom extractor (see `configs-ai-metrics`) 3. **Initialize `LDAIClient` once at startup.** Reuse any existing `LDClient` — do not create a second base client. Place the initialization in the same module that owns existing app config. @@ -207,11 +207,11 @@ This is the first stage that writes code. It has nine sub-steps. const aiClient = initAi(ldClient); ``` -4. **Hand off to `agentcontrol-configs-create`.** Print the extracted model, prompt/instructions, parameters, and mode from the Stage 1 manifest, then tell the user: *"Run `/agentcontrol-configs-create` with these inputs, then come back here."* Supply the config key you want the code to call (e.g. `chat-assistant`). Do not attempt to auto-invoke the sibling skill — wait for the user to finish it before continuing. +4. **Hand off to `configs-create`.** Print the extracted model, prompt/instructions, parameters, and mode from the Stage 1 manifest, then tell the user: *"Run `/configs-create` with these inputs, then come back here."* Supply the config key you want the code to call (e.g. `chat-assistant`). Do not attempt to auto-invoke the sibling skill — wait for the user to finish it before continuing. - **After `agentcontrol-configs-create` finishes, the user must also run `/agentcontrol-configs-targeting` to promote the new variation to fallthrough.** A freshly created variation returns `enabled=False` to every consumer until targeting is updated. Skip this and Stage 2 verification (sub-step 9 below) will silently take the fallback path on every request. + **After `configs-create` finishes, the user must also run `/configs-targeting` to promote the new variation to fallthrough.** A freshly created variation returns `enabled=False` to every consumer until targeting is updated. Skip this and Stage 2 verification (sub-step 9 below) will silently take the fallback path on every request. -5. **Rewrite template placeholders to Mustache syntax.** If the hardcoded prompt interpolates runtime values with Python `.format()`, f-strings, JS template literals, or any other non-Mustache syntax (e.g. `{system_time}`, `${userName}`, `%(topic)s`), rewrite every placeholder to `{{ variable }}` Mustache form. Do this in **both** the file you're about to send to `/agentcontrol-configs-create` *and* the fallback string you'll write in sub-step 6. The AI SDK interpolates variables through a Mustache renderer on the LD-served path *and* the fallback path using the fourth-argument `variables` dict to `completion_config(...)` / `completionConfig(...)`. Leaving a Python-style `{system_time}` literal in the fallback ships a silent regression when LaunchDarkly is unreachable — the renderer won't match the single-brace form and the literal `{system_time}` goes to the provider as part of the prompt. +5. **Rewrite template placeholders to Mustache syntax.** If the hardcoded prompt interpolates runtime values with Python `.format()`, f-strings, JS template literals, or any other non-Mustache syntax (e.g. `{system_time}`, `${userName}`, `%(topic)s`), rewrite every placeholder to `{{ variable }}` Mustache form. Do this in **both** the file you're about to send to `/configs-create` *and* the fallback string you'll write in sub-step 6. The AI SDK interpolates variables through a Mustache renderer on the LD-served path *and* the fallback path using the fourth-argument `variables` dict to `completion_config(...)` / `completionConfig(...)`. Leaving a Python-style `{system_time}` literal in the fallback ships a silent regression when LaunchDarkly is unreachable — the renderer won't match the single-brace form and the literal `{system_time}` goes to the provider as part of the prompt. **Before:** ```python @@ -318,7 +318,7 @@ This is the first stage that writes code. It has nine sub-steps. 9. **Verify.** Run the app with a valid `LD_SDK_KEY`; confirm the call succeeds and the response matches pre-migration output. Then temporarily set `LD_SDK_KEY=sdk-invalid` (or unset it) and confirm the fallback path runs without error. Both paths must work before moving to Stage 3. -Delegate: **`agentcontrol-configs-create`** (sub-step 4). +Delegate: **`configs-create`** (sub-step 4). ### Step 3: Move tools into the config (Stage 3) @@ -338,7 +338,7 @@ Skip this step if the audited app has no function calling / tools. Otherwise: For LangChain/LangGraph tools defined with `@tool`, extract the schema via `tool.args_schema.model_json_schema()` (or the equivalent Pydantic `model_json_schema()` call). For plain async callables used as tools (common in custom StateGraph shapes), LangChain infers the schema from the function signature at bind time — extract it via `StructuredTool.from_function(fn).args_schema.model_json_schema()`. Do not hand-write the schema. -2. **Hand off to `agentcontrol-tools`.** Print the extracted tool names, descriptions, and schemas, then tell the user: *"Run `/agentcontrol-tools` with these tools and the variation key, then come back here."* The sibling skill creates tool definitions (`create-ai-tool`) and attaches them to the variation (`update-ai-config-variation`). Wait for the user to finish before proceeding to sub-step 3. Do not auto-invoke. +2. **Hand off to `configs-tools`.** Print the extracted tool names, descriptions, and schemas, then tell the user: *"Run `/configs-tools` with these tools and the variation key, then come back here."* The sibling skill creates tool definitions (`create-ai-tool`) and attaches them to the variation (`update-ai-config-variation`). Wait for the user to finish before proceeding to sub-step 3. Do not auto-invoke. 3. **Replace the hardcoded tools array at the call site** with a read from `config.tools` (or the SDK equivalent for your language). Load the actual implementation functions dynamically from the tool names — see [agent-mode-frameworks.md](references/agent-mode-frameworks.md) for the dynamic-tool-factory pattern from the devrel agents tutorial. @@ -346,13 +346,13 @@ Skip this step if the audited app has no function calling / tools. Otherwise: 4. **Verify.** Run the app; confirm the tool flows still execute correctly. `get-ai-config` (via the delegate) confirms the tools are attached server-side. -Delegate: **`agentcontrol-tools`** (sub-step 2). +Delegate: **`configs-tools`** (sub-step 2). ### Step 4: Instrument the tracker (Stage 4) -Delegate: **`agentcontrol-configs-ai-metrics`** wires the per-request `tracker.track_*` calls (duration, tokens, success/error, feedback) around the provider call. Use **`agentcontrol-configs-custom-metrics`** alongside it if the app needs business metrics beyond the built-in AI ones. Note: do not confuse this with `launchdarkly-metric-instrument`, which is for `ldClient.track()` feature metrics — a different API. See [sdk-ai-tracker-patterns.md](references/sdk-ai-tracker-patterns.md) for the full per-method Python + Node matrix that the delegate skill draws on. +Delegate: **`configs-ai-metrics`** wires the per-request `tracker.track_*` calls (duration, tokens, success/error, feedback) around the provider call. Use **`configs-custom-metrics`** alongside it if the app needs business metrics beyond the built-in AI ones. Note: do not confuse this with `launchdarkly-metric-instrument`, which is for `ldClient.track()` feature metrics — a different API. See [sdk-ai-tracker-patterns.md](references/sdk-ai-tracker-patterns.md) for the full per-method Python + Node matrix that the delegate skill draws on. -Hand off: print the AI Config key, variation key, provider, and whether the call is streaming, then tell the user: *"Run `/agentcontrol-configs-ai-metrics` with these inputs, then come back here."* Do not auto-invoke. Return here for sub-step 5 (verify) once they're done. +Hand off: print the AI Config key, variation key, provider, and whether the call is streaming, then tell the user: *"Run `/configs-ai-metrics` with these inputs, then come back here."* Do not auto-invoke. Return here for sub-step 5 (verify) once they're done. 1. **Create the tracker.** Obtain a per-execution tracker via the factory on the config returned in Stage 2: `tracker = config.create_tracker()` (Python) or `const tracker = aiConfig.createTracker();` (Node). Call the factory **once per user turn** and reuse the returned `tracker` for every tracking call in that turn — each call mints a fresh `runId` that tags every event emitted from the turn so they can be correlated via exported events or downstream queries. (The Monitoring tab aggregates today; run-level grouping is a downstream concern — but the `runId` is also what the SDK's at-most-once guards are keyed on, so minting a new one mid-turn breaks the guard semantics regardless of where the events end up.) @@ -404,7 +404,7 @@ Hand off: print the AI Config key, variation key, provider, and whether the call ); ``` - For Anthropic direct, Bedrock (no provider package), Gemini, and custom HTTP, write a small extractor returning `LDAIMetrics` — see the delegate skill's [anthropic-tracking.md](../agentcontrol-configs-ai-metrics/references/anthropic-tracking.md), [bedrock-tracking.md](../agentcontrol-configs-ai-metrics/references/bedrock-tracking.md), and [gemini-tracking.md](../agentcontrol-configs-ai-metrics/references/gemini-tracking.md). LangChain single-node and LangGraph go through the `launchdarkly-server-sdk-ai-langchain` / `@launchdarkly/server-sdk-ai-langchain` provider package. Build the model with `create_langchain_model(config)` (Python) / `createLangChainModel(config)` (Node) — both forward all variation parameters — and track with `get_ai_metrics_from_response` / `getAIMetricsFromResponse`. See [langchain-tracking.md](../agentcontrol-configs-ai-metrics/references/langchain-tracking.md). + For Anthropic direct, Bedrock (no provider package), Gemini, and custom HTTP, write a small extractor returning `LDAIMetrics` — see the delegate skill's [anthropic-tracking.md](../configs-ai-metrics/references/anthropic-tracking.md), [bedrock-tracking.md](../configs-ai-metrics/references/bedrock-tracking.md), and [gemini-tracking.md](../configs-ai-metrics/references/gemini-tracking.md). LangChain single-node and LangGraph go through the `launchdarkly-server-sdk-ai-langchain` / `@launchdarkly/server-sdk-ai-langchain` provider package. Build the model with `create_langchain_model(config)` (Python) / `createLangChainModel(config)` (Node) — both forward all variation parameters — and track with `get_ai_metrics_from_response` / `getAIMetricsFromResponse`. See [langchain-tracking.md](../configs-ai-metrics/references/langchain-tracking.md). 4. **Wire feedback tracking if the app has thumbs-up/down UI.** Both SDKs expose `trackFeedback` with a `{kind}` argument. @@ -442,11 +442,11 @@ Hand off: print the AI Config key, variation key, provider, and whether the call - Run the `default` variation (or whichever variation mirrors the pre-migration hardcoded behavior) against the dataset first — this is the baseline. - Clone it into a second variation pointing at a **different model family** (e.g., if the baseline is `anthropic/claude-sonnet-4-5`, clone to `openai/gpt-4o` or `openai/gpt-4o-mini`). The comparison is most informative across families, not across siblings. - Attach the built-in **Accuracy** judge with a pass threshold of **0.85**, and run both variations against the same dataset. - - Promote the winner to fallthrough via `/agentcontrol-configs-targeting` only if it beats the baseline on Accuracy and does not regress on Relevance or Toxicity. + - Promote the winner to fallthrough via `/configs-targeting` only if it beats the baseline on Accuracy and does not regress on Relevance or Toxicity. Write this shape into the project's `datasets/README.md` (or equivalent) so the comparison pattern is reproducible after the migration ships. -3. **Hand off to `agentcontrol-configs-online-evals`** — only for UI-attached judges (completion mode) or to create custom judge AI Configs that will be referenced by the programmatic path. Tell the user: *"Run `/agentcontrol-configs-online-evals` with these inputs, then come back here."* Do not auto-invoke. Pass: +3. **Hand off to `configs-online-evals`** — only for UI-attached judges (completion mode) or to create custom judge AI Configs that will be referenced by the programmatic path. Tell the user: *"Run `/configs-online-evals` with these inputs, then come back here."* Do not auto-invoke. Pass: - The parent AI Config key and variation key - A list of built-in judges (Accuracy, Relevance, Toxicity) or custom judge keys to create/attach - Target environment @@ -480,14 +480,14 @@ Hand off: print the AI Config key, variation key, provider, and whether the call - **`sampling_rate` is a parameter on `evaluate()`**, not on `create_judge`. It defaults to `1.0` (evaluate every call). For live paths, pass something lower (0.1–0.25) to control cost. - **`evaluate()` returns a `JudgeResult`** (never `None`). Check `result.sampled` to know whether the evaluation actually ran, and call `track_judge_result(result)`. Node uses `trackJudgeResult(result)` and `LDJudgeResult` with the same `sampled` field. - **Ask the user which judge AI Config key to use.** LaunchDarkly ships three built-in judges — Accuracy, Relevance, Toxicity — but the actual AI Config **keys** for the built-ins are not canonical SDK constants and aren't documented. Have the user open **AI Configs > Library** in the LD UI and copy the key of the judge they want to reference, or create a custom judge AI Config via `agentcontrol-configs-create` first. + **Ask the user which judge AI Config key to use.** LaunchDarkly ships three built-in judges — Accuracy, Relevance, Toxicity — but the actual AI Config **keys** for the built-ins are not canonical SDK constants and aren't documented. Have the user open **AI Configs > Library** in the LD UI and copy the key of the judge they want to reference, or create a custom judge AI Config via `configs-create` first. 5. **Verify.** - **UI-attached auto judges:** trigger a request in staging, open the Monitoring tab → "Evaluator metrics" dropdown. Scores appear within 1–2 minutes at the configured sampling rate. - **Programmatic direct-judge:** hit the wrapped endpoint and confirm `track_judge_result` lands on the parent config's Monitoring tab. - **Offline eval:** run the dataset through the LD Playground, compare baseline vs new-variation scores side by side. No runtime wiring required. -Delegate: **`agentcontrol-configs-online-evals`** (sub-step 3, optional — only for UI-attached judges or custom-judge creation; offline eval doesn't delegate). +Delegate: **`configs-online-evals`** (sub-step 3, optional — only for UI-attached judges or custom-judge creation; offline eval doesn't delegate). ## Edge Cases @@ -498,9 +498,9 @@ Delegate: **`agentcontrol-configs-online-evals`** (sub-step 3, optional — only | Retry wrapper around the provider call | The tracker is minted once at the top of the user turn; the retry loop is inside that scope. Every retry attempt shares the same `runId`. Tracker calls (`track_duration` / `track_tokens` / `track_success` / `track_error`) live *outside* the retry body — one call at the end of the turn, on the success path or the final-failure path | | App has no tools — Stage 3 skipped | Move directly from Stage 2 verification to Stage 4 (tracking) | | Mode mismatch: user said agent, audit shows one-shot chat | Choose completion mode unless the app uses a LangGraph prebuilt agent (`langchain.agents.create_agent` in Python or `createReactAgent` in Node), CrewAI `Agent`, Strands `Agent`, or a similar goal-driven framework | -| App uses Strands Agents (Python) | Agent mode. Build a `create_strands_model` dispatcher keyed on `agent_config.provider.name` that returns `AnthropicModel(model_id=..., max_tokens=...)` or `OpenAIModel(model_id=..., params=...)`. Drop `parameters.tools` before passing params to the model class — Strands receives tools via `Agent(tools=[...])`. Tracking is Tier 3: wrap `invoke_async` with `tracker.track_duration_of(...)` and record tokens from `result.metrics.accumulated_usage`. See [agent-mode-frameworks.md § Strands Agent](references/agent-mode-frameworks.md) and [strands-tracking.md](../agentcontrol-configs-ai-metrics/references/strands-tracking.md) | +| App uses Strands Agents (Python) | Agent mode. Build a `create_strands_model` dispatcher keyed on `agent_config.provider.name` that returns `AnthropicModel(model_id=..., max_tokens=...)` or `OpenAIModel(model_id=..., params=...)`. Drop `parameters.tools` before passing params to the model class — Strands receives tools via `Agent(tools=[...])`. Tracking is Tier 3: wrap `invoke_async` with `tracker.track_duration_of(...)` and record tokens from `result.metrics.accumulated_usage`. See [agent-mode-frameworks.md § Strands Agent](references/agent-mode-frameworks.md) and [strands-tracking.md](../configs-ai-metrics/references/strands-tracking.md) | | Strands app on TypeScript | TS SDK ships `BedrockModel` and `OpenAIModel` only — cannot serve Anthropic-backed variations. Use the Python SDK if multi-provider variations are required | -| TypeScript app using Anthropic SDK | No `trackAnthropicMetrics` helper exists. Use Tier 3: `trackMetricsOf` with a small custom extractor that reads `response.usage.input_tokens` / `response.usage.output_tokens` and returns `LDAIMetrics`. See [anthropic-tracking.md](../agentcontrol-configs-ai-metrics/references/anthropic-tracking.md) in the `agentcontrol-configs-ai-metrics` skill for the exact extractor | +| TypeScript app using Anthropic SDK | No `trackAnthropicMetrics` helper exists. Use Tier 3: `trackMetricsOf` with a small custom extractor that reads `response.usage.input_tokens` / `response.usage.output_tokens` and returns `LDAIMetrics`. See [anthropic-tracking.md](../configs-ai-metrics/references/anthropic-tracking.md) in the `configs-ai-metrics` skill for the exact extractor | | Fallback would silently crash because `LD_SDK_KEY` is missing | Log a startup warning; proceed with the fallback. Never raise at import time | | Multi-agent graph (supervisor + workers) | Stop after migrating a single agent. Agent Graph Definitions are available in **both** SDKs — Python via `launchdarkly-server-sdk-ai.agent_graph` and Node via the graph API in `@launchdarkly/server-sdk-ai`. Read [agent-graph-reference.md](references/agent-graph-reference.md) for the graph-level migration path — it is deliberately out of this skill's main scope | | Single-agent (ReAct, tool loop) + agent mode | Default to offline eval via the LD Playground + Datasets for Stage 5. UI-attached judges are completion-only today, and programmatic direct-judge adds per-call cost that is usually not worth it until after the migration is live and stable. Point at the [Offline Evals guide](https://docs.launchdarkly.com/guides/ai-configs/offline-evaluations) | @@ -525,17 +525,17 @@ These are ordered by how likely they are to show up as a first-run failure. The - **If the repo already contains a `load_chat_model(f"{provider}/{name}")` helper, delete it — don't just avoid using it.** This exact shape ships with `langchain-ai/react-agent` and is copied into dozens of derivative repos; look for `utils.load_chat_model`, `utils.build_model`, or any one-arg `init_chat_model` wrapper that splits a `"provider/model"` string. Re-using it is the first-run failure mode: every variation parameter (temperature, max_tokens, top_p, stop sequences) silently drops on the floor because `init_chat_model` only receives the name and provider. `create_langchain_model(ai_config)` is a one-for-one replacement that forwards the whole `model.parameters` dict. Replace every call site, then delete the wrapper file-side so the next reader can't reach for it. - **Same rule applies to hand-rolled `resolve_tools` / `TOOL_REGISTRY` / `ALL_TOOLS` helpers.** If the template already has a `resolve_tools(tool_keys)` or an `ALL_TOOLS` module-level list, import `build_structured_tools` from `ldai_langchain.langchain_helper` and delete the hand-rolled version. `build_structured_tools(ai_config, TOOL_REGISTRY_DICT)` reads `ai_config.model.parameters.tools` and wraps the matching callables as LangChain `StructuredTool`s with the LD tool key as the `StructuredTool.name` — so `ToolNode` lookup works without a second mapping. Don't leave both in the repo. -- Don't put app-scoped knobs directly in `model.parameters`. `create_langchain_model` forwards every key in `parameters` to the provider SDK via `init_chat_model`, so a `max_search_results` / `retry_budget` / `feature_toggle` entry will crash the provider with an unexpected-keyword-argument error. The correct home is `model.custom`, which the provider helpers ignore and the app reads via `ai_config.model.get_custom("key")`. The MCP `update-ai-config-variation` tool does not currently expose top-level `custom`, so pick one of two paths: (a) PATCH the variation via the REST API to set `model.custom` directly, or (b) set it via MCP inside `parameters.custom` (as a nested dict) and use a defensive accessor that reads both locations. Full walk-through with code samples in [langchain-tracking.md § MCP caveat](../agentcontrol-configs-ai-metrics/references/langchain-tracking.md). +- Don't put app-scoped knobs directly in `model.parameters`. `create_langchain_model` forwards every key in `parameters` to the provider SDK via `init_chat_model`, so a `max_search_results` / `retry_budget` / `feature_toggle` entry will crash the provider with an unexpected-keyword-argument error. The correct home is `model.custom`, which the provider helpers ignore and the app reads via `ai_config.model.get_custom("key")`. The MCP `update-ai-config-variation` tool does not currently expose top-level `custom`, so pick one of two paths: (a) PATCH the variation via the REST API to set `model.custom` directly, or (b) set it via MCP inside `parameters.custom` (as a nested dict) and use a defensive accessor that reads both locations. Full walk-through with code samples in [langchain-tracking.md § MCP caveat](../configs-ai-metrics/references/langchain-tracking.md). - Don't re-encode tool schemas inside the fallback. When LaunchDarkly is unreachable the fallback should run without tools (or with whatever minimal provider-bound parameters the app needs to keep operating). Building a `_FALLBACK_TOOLS` array that duplicates the AI Config's tool schema re-introduces the hardcoded config the migration was supposed to move out of code. - Don't import `LaunchDarklyCallbackHandler` from `ldai.langchain` — neither the class nor the dotted module path exists. The Python LangChain helper package is `ldai_langchain` (top-level module, underscore). Use `create_langchain_model(config)` + `track_metrics_of_async(get_ai_metrics_from_response, lambda: llm.ainvoke(messages))` as the canonical pattern. ### Stage / handoff discipline - Don't skip Step 1 even when the user says "just wrap it." Without the audit, the fallback will drift from the hardcoded behavior. -- Don't delegate to `agentcontrol-configs-create` before extracting the prompt and model — the delegate needs them as inputs. -- Don't try to attach tools during initial `setup-ai-config`. Tool attachment is a separate step owned by `agentcontrol-tools`. -- Don't claim you "delegated to `agentcontrol-configs-create`" or any other sibling skill. This skill does not auto-invoke. At each handoff, print the inputs and tell the user to run the sibling slash-command, then wait. Anything else misleads the user about what just happened. -- Don't skip the `/agentcontrol-configs-targeting` step between Stage 2 and Stage 4. A freshly created variation returns `enabled=False` until targeting promotes it to fallthrough — Stage 2 verification will silently take the fallback path on every request. +- Don't delegate to `configs-create` before extracting the prompt and model — the delegate needs them as inputs. +- Don't try to attach tools during initial `setup-ai-config`. Tool attachment is a separate step owned by `configs-tools`. +- Don't claim you "delegated to `configs-create`" or any other sibling skill. This skill does not auto-invoke. At each handoff, print the inputs and tell the user to run the sibling slash-command, then wait. Anything else misleads the user about what just happened. +- Don't skip the `/configs-targeting` step between Stage 2 and Stage 4. A freshly created variation returns `enabled=False` until targeting promotes it to fallthrough — Stage 2 verification will silently take the fallback path on every request. - Don't attempt a multi-agent graph migration in one pass. Migrate a single agent first; use [agent-graph-reference.md](references/agent-graph-reference.md) as the next-step read. ### Stage 5 evaluations @@ -554,12 +554,12 @@ These are ordered by how likely they are to show up as a first-run failure. The ## Related Skills -- `agentcontrol-configs-create` — called by Stage 2 to create the config -- `agentcontrol-tools` — called by Stage 3 to create and attach tool definitions -- `agentcontrol-configs-online-evals` — called by Stage 5 to attach judges -- `agentcontrol-configs-variations` — add variations for A/B testing after migration is complete -- `agentcontrol-configs-targeting` — roll out new variations to users after migration is complete -- `agentcontrol-configs-update` — modify config properties as your app evolves +- `configs-create` — called by Stage 2 to create the config +- `configs-tools` — called by Stage 3 to create and attach tool definitions +- `configs-online-evals` — called by Stage 5 to attach judges +- `configs-variations` — add variations for A/B testing after migration is complete +- `configs-targeting` — roll out new variations to users after migration is complete +- `configs-update` — modify config properties as your app evolves - `launchdarkly-metric-instrument` — for `ldClient.track()` feature metrics (NOT for AI tracker calls) ## References diff --git a/skills/ai-configs/agentcontrol-configs-migrate/references/agent-graph-reference.md b/skills/ai-configs/configs-migrate/references/agent-graph-reference.md similarity index 100% rename from skills/ai-configs/agentcontrol-configs-migrate/references/agent-graph-reference.md rename to skills/ai-configs/configs-migrate/references/agent-graph-reference.md diff --git a/skills/ai-configs/agentcontrol-configs-migrate/references/agent-mode-frameworks.md b/skills/ai-configs/configs-migrate/references/agent-mode-frameworks.md similarity index 98% rename from skills/ai-configs/agentcontrol-configs-migrate/references/agent-mode-frameworks.md rename to skills/ai-configs/configs-migrate/references/agent-mode-frameworks.md index d511330..859e191 100644 --- a/skills/ai-configs/agentcontrol-configs-migrate/references/agent-mode-frameworks.md +++ b/skills/ai-configs/configs-migrate/references/agent-mode-frameworks.md @@ -16,9 +16,9 @@ Completion mode is the default and covers direct provider calls (OpenAI, Anthrop Agent mode returns an `instructions` string. Completion mode returns a `messages` array. Both modes support tools, parameters, and the same tracker — the only difference is the input shape the SDK returns to you. -**Caveat:** judges cannot be attached to agent-mode variations via the LaunchDarkly UI. Agent mode evaluations must go through the programmatic judge API (`create_judge(...).evaluate(input, output)`). See `agentcontrol-configs-online-evals` for the programmatic path. +**Caveat:** judges cannot be attached to agent-mode variations via the LaunchDarkly UI. Agent mode evaluations must go through the programmatic judge API (`create_judge(...).evaluate(input, output)`). See `configs-online-evals` for the programmatic path. -**Model construction for LangChain / LangGraph.** When the framework runs on top of LangChain (which includes LangGraph's prebuilt agent and most custom graphs), build the chat model with `create_langchain_model(ai_config)` (Python) or `createLangChainModel(aiConfig)` (Node). These helpers forward every variation parameter (`temperature`, `max_tokens`, `top_p`, …) and handle LaunchDarkly→LangChain provider-name mapping internally. Do not hand-roll `init_chat_model(model=..., model_provider=...)` — it silently drops every variation parameter. See [langchain-tracking.md](../../agentcontrol-configs-ai-metrics/references/langchain-tracking.md) for the canonical single-model and LangGraph patterns, including the SDK helpers `sum_token_usage_from_messages` / `get_tool_calls_from_response` (Python, `ldai_langchain`) used inside the `track_metrics_of_async` / `trackMetricsOf` extractor. +**Model construction for LangChain / LangGraph.** When the framework runs on top of LangChain (which includes LangGraph's prebuilt agent and most custom graphs), build the chat model with `create_langchain_model(ai_config)` (Python) or `createLangChainModel(aiConfig)` (Node). These helpers forward every variation parameter (`temperature`, `max_tokens`, `top_p`, …) and handle LaunchDarkly→LangChain provider-name mapping internally. Do not hand-roll `init_chat_model(model=..., model_provider=...)` — it silently drops every variation parameter. See [langchain-tracking.md](../../configs-ai-metrics/references/langchain-tracking.md) for the canonical single-model and LangGraph patterns, including the SDK helpers `sum_token_usage_from_messages` / `get_tool_calls_from_response` (Python, `ldai_langchain`) used inside the `track_metrics_of_async` / `trackMetricsOf` extractor. ## Framework-agnostic invariants for the run-scoped pattern @@ -291,7 +291,7 @@ async def run_turn(ai_client, user_id: str, user_input: str): **Key points:** - `system_prompt=agent_config.instructions` — the instructions string replaces the hardcoded system prompt. - `create_strands_model(agent_config)` is the provider-dispatch seam. Add a branch per provider the variation can serve. -- The tracker is Tier 3: `tracker.track_duration_of(...)` + an explicit `track_tokens` call fed by `track_strands_metrics`. See [strands-tracking.md](../../agentcontrol-configs-ai-metrics/references/strands-tracking.md) for the single-call `track_metrics_of_async` variant and the per-field breakdown of `accumulated_usage`. +- The tracker is Tier 3: `tracker.track_duration_of(...)` + an explicit `track_tokens` call fed by `track_strands_metrics`. See [strands-tracking.md](../../configs-ai-metrics/references/strands-tracking.md) for the single-call `track_metrics_of_async` variant and the per-field breakdown of `accumulated_usage`. - Always `ldclient.get().flush()` before process exit in short-lived scripts — trailing events can otherwise be lost. **TypeScript caveat.** The Strands TypeScript SDK ships `BedrockModel` and `OpenAIModel` only — it cannot run Anthropic-backed variations. If the app needs to serve both OpenAI and Anthropic from a single AI Config, use the Python SDK. @@ -799,12 +799,12 @@ agent = create_agent( **What this gives you:** - Toggle a tool on/off by editing the AI Config in LaunchDarkly — no redeploy needed to remove a tool from production -- Roll out a new tool to 5% of users by editing targeting rules (combined with `agentcontrol-configs-targeting`) +- Roll out a new tool to 5% of users by editing targeting rules (combined with `configs-targeting`) - Keep the actual tool implementation code in the repo; only metadata lives in LaunchDarkly ### Extracting schemas from existing hardcoded tools -If your tools are defined with LangChain `@tool` or Pydantic `BaseModel` schemas, extract the JSON schema programmatically to pass to `agentcontrol-tools` during Stage 3: +If your tools are defined with LangChain `@tool` or Pydantic `BaseModel` schemas, extract the JSON schema programmatically to pass to `configs-tools` during Stage 3: ```python from my_tools import search_kb # a @tool-decorated function @@ -813,7 +813,7 @@ schema = search_kb.args_schema.model_json_schema() # schema is a dict ready to pass as the tool's parameters field ``` -Do not hand-write the schema — LangChain already generated it from the function signature, and Pydantic will keep it in sync. The `agentcontrol-tools` delegate accepts raw JSON Schema for the parameters field. +Do not hand-write the schema — LangChain already generated it from the function signature, and Pydantic will keep it in sync. The `configs-tools` delegate accepts raw JSON Schema for the parameters field. ### Dynamic schemas from LaunchDarkly diff --git a/skills/ai-configs/agentcontrol-configs-migrate/references/before-after-examples.md b/skills/ai-configs/configs-migrate/references/before-after-examples.md similarity index 100% rename from skills/ai-configs/agentcontrol-configs-migrate/references/before-after-examples.md rename to skills/ai-configs/configs-migrate/references/before-after-examples.md diff --git a/skills/ai-configs/agentcontrol-configs-migrate/references/fallback-defaults-pattern.md b/skills/ai-configs/configs-migrate/references/fallback-defaults-pattern.md similarity index 99% rename from skills/ai-configs/agentcontrol-configs-migrate/references/fallback-defaults-pattern.md rename to skills/ai-configs/configs-migrate/references/fallback-defaults-pattern.md index 155c306..4a02575 100644 --- a/skills/ai-configs/agentcontrol-configs-migrate/references/fallback-defaults-pattern.md +++ b/skills/ai-configs/configs-migrate/references/fallback-defaults-pattern.md @@ -305,7 +305,7 @@ config = ai_client.agent_config( The SDK runs both paths through the same Mustache renderer. Leaving a Python-style `{var}` literal in the fallback ships a silent regression: LaunchDarkly serves correctly-interpolated output; the fallback ships the unrendered literal, or (worse) a value frozen at import time. -Same rule applies to JS template literals and any other non-Mustache scheme — rewrite to `{{ variable }}` before handing off to `/agentcontrol-configs-create`. +Same rule applies to JS template literals and any other non-Mustache scheme — rewrite to `{{ variable }}` before handing off to `/configs-create`. ## Critical rules (apply to all three patterns) diff --git a/skills/ai-configs/agentcontrol-configs-migrate/references/phase-1-analysis-checklist.md b/skills/ai-configs/configs-migrate/references/phase-1-analysis-checklist.md similarity index 97% rename from skills/ai-configs/agentcontrol-configs-migrate/references/phase-1-analysis-checklist.md rename to skills/ai-configs/configs-migrate/references/phase-1-analysis-checklist.md index 006e6e9..8f86d9d 100644 --- a/skills/ai-configs/agentcontrol-configs-migrate/references/phase-1-analysis-checklist.md +++ b/skills/ai-configs/configs-migrate/references/phase-1-analysis-checklist.md @@ -151,7 +151,7 @@ Feeds into Stage 2 (install + wrap). Quoted from the `ai-configs-relaunch-guides | LangChain / LangGraph | `@launchdarkly/server-sdk-ai-langchain` | `createLangChainModel(config)` (forwards all variation parameters and handles provider-name mapping) + `getAIMetricsFromResponse` with `trackMetricsOf` | | Vercel AI SDK | `@launchdarkly/server-sdk-ai-vercel` | `getAIMetricsFromResponse` + `trackMetricsOf`, or `VercelRunnerFactory.createVercelModel(aiConfig)` for a managed runner | -Python currently ships helper packages for OpenAI (`ldai_openai`) and LangChain (`ldai_langchain`). The LangChain Python package exposes `create_langchain_model(config)` (builds a LangChain chat model from the AI Config, forwarding every variation parameter and mapping LD provider names to LangChain equivalents), `convert_messages_to_langchain`, and `get_ai_metrics_from_response` — the same package covers LangGraph. Use `create_langchain_model(config)` + `track_metrics_of_async(get_ai_metrics_from_response, lambda: llm.ainvoke(messages))` as the canonical single-call pattern. See [langchain-tracking.md](../../agentcontrol-configs-ai-metrics/references/langchain-tracking.md) for both LangChain and LangGraph patterns and [sdk-ai-tracker-patterns.md](sdk-ai-tracker-patterns.md) for the full tracker-method matrix. +Python currently ships helper packages for OpenAI (`ldai_openai`) and LangChain (`ldai_langchain`). The LangChain Python package exposes `create_langchain_model(config)` (builds a LangChain chat model from the AI Config, forwarding every variation parameter and mapping LD provider names to LangChain equivalents), `convert_messages_to_langchain`, and `get_ai_metrics_from_response` — the same package covers LangGraph. Use `create_langchain_model(config)` + `track_metrics_of_async(get_ai_metrics_from_response, lambda: llm.ainvoke(messages))` as the canonical single-call pattern. See [langchain-tracking.md](../../configs-ai-metrics/references/langchain-tracking.md) for both LangChain and LangGraph patterns and [sdk-ai-tracker-patterns.md](sdk-ai-tracker-patterns.md) for the full tracker-method matrix. ## Phase 1 output format @@ -183,9 +183,9 @@ Coverage totals: N hardcoded code targets · M externalized prompt f Proposed plan: Stage 1 (Audit): Read-only manifest of hardcoded targets; flag placeholders for Mustache rewrite and knobs for model.custom Stage 2 (Wrap): Install SDK, create AI Config 'chat-assistant', inline fallback mirrors current values (Mustache syntax), rewrite the call site - Stage 3 (Tools): Skipped (no function calling) / Attach 2 tools via agentcontrol-tools + Stage 3 (Tools): Skipped (no function calling) / Attach 2 tools via configs-tools Stage 4 (Tracking): Inline tracker wiring (track_duration + track_tokens + track_success/error) — run-scoped tracker for agent loops - Stage 5 (Evals): Attach built-in 'accuracy' judge at 0.25 sampling via agentcontrol-configs-online-evals + Stage 5 (Evals): Attach built-in 'accuracy' judge at 0.25 sampling via configs-online-evals ``` ## STOP diff --git a/skills/ai-configs/agentcontrol-configs-migrate/references/sdk-ai-tracker-patterns.md b/skills/ai-configs/configs-migrate/references/sdk-ai-tracker-patterns.md similarity index 97% rename from skills/ai-configs/agentcontrol-configs-migrate/references/sdk-ai-tracker-patterns.md rename to skills/ai-configs/configs-migrate/references/sdk-ai-tracker-patterns.md index c663754..c755d4a 100644 --- a/skills/ai-configs/agentcontrol-configs-migrate/references/sdk-ai-tracker-patterns.md +++ b/skills/ai-configs/configs-migrate/references/sdk-ai-tracker-patterns.md @@ -297,7 +297,7 @@ response = tracker.track_metrics_of( | Helper | Signature | Tier | Notes | |--------|-----------|------|-------| | `trackMetricsOf(extractor, func)` | `await tracker.trackMetricsOf((result) => extractor(result), async () => ...)` | **2 / 3** | **Canonical generic wrapper.** `extractor` maps provider response → `LDAIMetrics`. Use a provider package's bare `getAIMetricsFromResponse` for Tier 2 (`@launchdarkly/server-sdk-ai-openai`, `-langchain`, `-vercel`) or a small custom function for Tier 3. | -| `trackStreamMetricsOf(extractor, streamCreator)` | `tracker.trackStreamMetricsOf(async (chunks) => extractor(chunks), () => createStream())` | 2 / 3 | Stream variant. Does **not** capture TTFT automatically — if you need TTFT, use the manual pattern in [streaming-tracking.md](../../agentcontrol-configs-ai-metrics/references/streaming-tracking.md). | +| `trackStreamMetricsOf(extractor, streamCreator)` | `tracker.trackStreamMetricsOf(async (chunks) => extractor(chunks), () => createStream())` | 2 / 3 | Stream variant. Does **not** capture TTFT automatically — if you need TTFT, use the manual pattern in [streaming-tracking.md](../../configs-ai-metrics/references/streaming-tracking.md). | | `trackDurationOf(func)` | `await tracker.trackDurationOf(async () => ...)` | 4 | Wraps an async callable; captures duration only. Pair with explicit `trackTokens` + `trackSuccess`. | Example — OpenAI via `trackMetricsOf` + the provider package: ```typescript @@ -339,7 +339,7 @@ The managed runner handles message history, provider dispatch (via the installed ### Anthropic has no provider package today -Neither `@launchdarkly/server-sdk-ai-anthropic` nor `launchdarkly-server-sdk-ai-anthropic` exists as of this writing. For Anthropic direct calls, write a custom extractor and pass it to `track_metrics_of` / `trackMetricsOf` — see the Python example above or the full walk-through in [anthropic-tracking.md](../../agentcontrol-configs-ai-metrics/references/anthropic-tracking.md). If the app is open to LangChain, routing Anthropic through `ChatAnthropic` and the LangChain provider package recovers Tier 2 with zero extractor code. +Neither `@launchdarkly/server-sdk-ai-anthropic` nor `launchdarkly-server-sdk-ai-anthropic` exists as of this writing. For Anthropic direct calls, write a custom extractor and pass it to `track_metrics_of` / `trackMetricsOf` — see the Python example above or the full walk-through in [anthropic-tracking.md](../../configs-ai-metrics/references/anthropic-tracking.md). If the app is open to LangChain, routing Anthropic through `ChatAnthropic` and the LangChain provider package recovers Tier 2 with zero extractor code. ## Tier decision table @@ -352,7 +352,7 @@ Neither `@launchdarkly/server-sdk-ai-anthropic` nor `launchdarkly-server-sdk-ai- | Anthropic direct SDK | **3** | Custom extractor reading `response.usage.input_tokens` / `output_tokens` | | Bedrock Converse (no provider package) | **3** | Custom extractor reading `response.usage.inputTokens` / `outputTokens` (or route via LangChain for Tier 2) | | Gemini / Google GenAI, Cohere, custom HTTP | **3** | Custom extractor | -| Streaming response with TTFT required | **4** | Manual `trackTimeToFirstToken` + `trackDuration` + `trackTokens` + `trackSuccess` — see [streaming-tracking.md](../../agentcontrol-configs-ai-metrics/references/streaming-tracking.md) | +| Streaming response with TTFT required | **4** | Manual `trackTimeToFirstToken` + `trackDuration` + `trackTokens` + `trackSuccess` — see [streaming-tracking.md](../../configs-ai-metrics/references/streaming-tracking.md) | | Streaming response without TTFT (Node) | **2 / 3** | `trackStreamMetricsOf(extractor, streamFn)` | ## Streaming responses @@ -450,7 +450,7 @@ Run the checklist in order. Each step rules out one cause. ## Common gotchas -- **`model.parameters` vs `model.custom`.** `create_langchain_model` (Python) / `createLangChainModel` (Node) forwards every key in `model.parameters` to the provider SDK. App-scoped knobs (search result limits, retry budgets, feature toggles) **must** live in `model.custom` or the provider will crash at runtime with an unexpected-keyword-argument error. Read them with `ai_config.model.get_custom("key")`. Full walk-through with the MCP/REST-API caveat in [langchain-tracking.md § `model.parameters` vs `model.custom`](../../agentcontrol-configs-ai-metrics/references/langchain-tracking.md). +- **`model.parameters` vs `model.custom`.** `create_langchain_model` (Python) / `createLangChainModel` (Node) forwards every key in `model.parameters` to the provider SDK. App-scoped knobs (search result limits, retry budgets, feature toggles) **must** live in `model.custom` or the provider will crash at runtime with an unexpected-keyword-argument error. Read them with `ai_config.model.get_custom("key")`. Full walk-through with the MCP/REST-API caveat in [langchain-tracking.md § `model.parameters` vs `model.custom`](../../configs-ai-metrics/references/langchain-tracking.md). - **`track_tokens` token shape.** The Python `TokenUsage` dataclass requires `total` to be set — it is not derived. Compute `total = input + output` if the provider doesn't return one. - **`track_feedback` lifecycle.** The feedback call must be made on a tracker bound to the same `runId` that produced the response. If the thumbs-up comes in a later process, use the cross-process resumption pattern above — do **not** call `create_tracker()` again in the consumer, because that mints a *new* `runId`. - **OpenAI streaming tokens.** OpenAI only emits `usage` in the final chunk when `stream_options={"include_usage": True}` is passed. Without that flag, you have to tokenize manually — `tiktoken` for OpenAI models. diff --git a/skills/ai-configs/agentcontrol-configs-online-evals/README.md b/skills/ai-configs/configs-online-evals/README.md similarity index 67% rename from skills/ai-configs/agentcontrol-configs-online-evals/README.md rename to skills/ai-configs/configs-online-evals/README.md index 0f623d3..cb717bd 100644 --- a/skills/ai-configs/agentcontrol-configs-online-evals/README.md +++ b/skills/ai-configs/configs-online-evals/README.md @@ -12,12 +12,12 @@ This skill teaches agents how to: ## Installation (Local) -Copy `skills/ai-configs/agentcontrol-configs-online-evals/` into your agent client's skills path. +Copy `skills/ai-configs/configs-online-evals/` into your agent client's skills path. ## Prerequisites - LaunchDarkly API access token with `ai-configs:write` permission -- Existing AI Config with variations (use `agentcontrol-configs-create` skill) +- Existing AI Config with variations (use `configs-create` skill) - For custom judges: understanding of LLM-as-a-judge methodology ## Usage @@ -33,16 +33,16 @@ Create a custom judge that checks for scope creep in code changes ## Structure ``` -agentcontrol-configs-online-evals/ +configs-online-evals/ ├── SKILL.md └── README.md ``` ## Related -- [AI Config Create](../agentcontrol-configs-create/) - Create AI Configs first -- [AI Config Targeting](../agentcontrol-configs-targeting/) - Enable targeting on judges -- [AI Config Variations](../agentcontrol-configs-variations/) - Manage variations +- [AI Config Create](../configs-create/) - Create AI Configs first +- [AI Config Targeting](../configs-targeting/) - Enable targeting on judges +- [AI Config Variations](../configs-variations/) - Manage variations - [Online Evaluations Docs](https://docs.launchdarkly.com/home/ai-configs/online-evaluations) ## License diff --git a/skills/ai-configs/agentcontrol-configs-online-evals/SKILL.md b/skills/ai-configs/configs-online-evals/SKILL.md similarity index 98% rename from skills/ai-configs/agentcontrol-configs-online-evals/SKILL.md rename to skills/ai-configs/configs-online-evals/SKILL.md index 35fd1cf..bd99480 100644 --- a/skills/ai-configs/agentcontrol-configs-online-evals/SKILL.md +++ b/skills/ai-configs/configs-online-evals/SKILL.md @@ -1,5 +1,5 @@ --- -name: agentcontrol-configs-online-evals +name: configs-online-evals description: Attach judges to AI Config variations for automatic LLM-as-a-judge evaluation. Create custom judges, configure sampling rates, and monitor quality scores. compatibility: Requires LaunchDarkly API access token with ai-configs:write permission. SDK versions Python v0.20.0+ or Node.js v0.20.0+ for automatic metric recording and the consolidated `track_judge_result` / `trackJudgeResult` API. metadata: @@ -15,7 +15,7 @@ Attach judges to AI Config variations for automatic quality scoring using LLM-as - LaunchDarkly account with AI Configs enabled - API access token with write permissions -- Existing AI Config with variations (use `agentcontrol-configs-create` skill) +- Existing AI Config with variations (use `configs-create` skill) - For automatic metric recording and the consolidated judge-result API: Python AI SDK v0.20.0+ or Node.js AI SDK v0.20.0+ ## API Key Detection @@ -437,9 +437,9 @@ After attaching judges: ## Related Skills -- `agentcontrol-configs-create` - Create AI Configs and judges -- `agentcontrol-configs-targeting` - Configure targeting rules -- `agentcontrol-configs-variations` - Manage variations +- `configs-create` - Create AI Configs and judges +- `configs-targeting` - Configure targeting rules +- `configs-variations` - Manage variations ## References diff --git a/skills/ai-configs/agentcontrol-projects/README.md b/skills/ai-configs/configs-projects/README.md similarity index 93% rename from skills/ai-configs/agentcontrol-projects/README.md rename to skills/ai-configs/configs-projects/README.md index a6c1679..aeee2aa 100644 --- a/skills/ai-configs/agentcontrol-projects/README.md +++ b/skills/ai-configs/configs-projects/README.md @@ -17,7 +17,7 @@ For now, install by placing this skill directory where your agent client loads s Examples: -- **Generic**: copy `skills/ai-configs/agentcontrol-projects/` into your client's skills path +- **Generic**: copy `skills/ai-configs/configs-projects/` into your client's skills path ## Prerequisites @@ -44,7 +44,7 @@ Add LaunchDarkly project management to this codebase ## Structure ``` -agentcontrol-projects/ +configs-projects/ ├── SKILL.md ├── README.md └── references/ diff --git a/skills/ai-configs/agentcontrol-projects/SKILL.md b/skills/ai-configs/configs-projects/SKILL.md similarity index 94% rename from skills/ai-configs/agentcontrol-projects/SKILL.md rename to skills/ai-configs/configs-projects/SKILL.md index 21f0d75..ef53432 100644 --- a/skills/ai-configs/agentcontrol-projects/SKILL.md +++ b/skills/ai-configs/configs-projects/SKILL.md @@ -1,5 +1,5 @@ --- -name: agentcontrol-projects +name: configs-projects description: Guide for setting up LaunchDarkly projects in your codebase. Helps you assess your stack, choose the right approach, and integrate project management that makes sense for your architecture. compatibility: Requires LaunchDarkly API access token with projects:write permission or LaunchDarkly MCP server. metadata: @@ -221,16 +221,16 @@ ai-apac → Asia-Pacific Region After setting up projects: -1. **Create AI Configs** - Use the `agentcontrol-configs-create` skill -2. **Set up SDK Integration** - Use the `agentcontrol-sdk` skill -3. **Configure Targeting** - Use the `agentcontrol-configs-targeting` skill +1. **Create AI Configs** - Use the `configs-create` skill +2. **Set up SDK Integration** - Use the `configs-sdk` skill +3. **Configure Targeting** - Use the `configs-targeting` skill ## Related Skills -- `agentcontrol-configs-create` - Create AI Configs in projects -- `agentcontrol-sdk` - Integrate SDK in your application -- `agentcontrol-configs-targeting` - Configure AI Config targeting -- `agentcontrol-configs-variations` - Manage config variations +- `configs-create` - Create AI Configs in projects +- `configs-sdk` - Integrate SDK in your application +- `configs-targeting` - Configure AI Config targeting +- `configs-variations` - Manage config variations ## References diff --git a/skills/ai-configs/agentcontrol-projects/references/admin-tooling.md b/skills/ai-configs/configs-projects/references/admin-tooling.md similarity index 100% rename from skills/ai-configs/agentcontrol-projects/references/admin-tooling.md rename to skills/ai-configs/configs-projects/references/admin-tooling.md diff --git a/skills/ai-configs/agentcontrol-projects/references/env-config.md b/skills/ai-configs/configs-projects/references/env-config.md similarity index 99% rename from skills/ai-configs/agentcontrol-projects/references/env-config.md rename to skills/ai-configs/configs-projects/references/env-config.md index 21033e9..f383ce0 100644 --- a/skills/ai-configs/agentcontrol-projects/references/env-config.md +++ b/skills/ai-configs/configs-projects/references/env-config.md @@ -363,6 +363,6 @@ def verify_sdk_key(sdk_key: str): ## Next Steps -- [Integrate SDK in your application](../agentcontrol-sdk/SKILL.md) +- [Integrate SDK in your application](../configs-sdk/SKILL.md) - [Set up project cloning](project-cloning.md) - [Build automation scripts](iac-automation.md) diff --git a/skills/ai-configs/agentcontrol-projects/references/go-setup.md b/skills/ai-configs/configs-projects/references/go-setup.md similarity index 100% rename from skills/ai-configs/agentcontrol-projects/references/go-setup.md rename to skills/ai-configs/configs-projects/references/go-setup.md diff --git a/skills/ai-configs/agentcontrol-projects/references/iac-automation.md b/skills/ai-configs/configs-projects/references/iac-automation.md similarity index 100% rename from skills/ai-configs/agentcontrol-projects/references/iac-automation.md rename to skills/ai-configs/configs-projects/references/iac-automation.md diff --git a/skills/ai-configs/agentcontrol-projects/references/multi-language-setup.md b/skills/ai-configs/configs-projects/references/multi-language-setup.md similarity index 100% rename from skills/ai-configs/agentcontrol-projects/references/multi-language-setup.md rename to skills/ai-configs/configs-projects/references/multi-language-setup.md diff --git a/skills/ai-configs/agentcontrol-projects/references/nodejs-setup.md b/skills/ai-configs/configs-projects/references/nodejs-setup.md similarity index 100% rename from skills/ai-configs/agentcontrol-projects/references/nodejs-setup.md rename to skills/ai-configs/configs-projects/references/nodejs-setup.md diff --git a/skills/ai-configs/agentcontrol-projects/references/project-cloning.md b/skills/ai-configs/configs-projects/references/project-cloning.md similarity index 100% rename from skills/ai-configs/agentcontrol-projects/references/project-cloning.md rename to skills/ai-configs/configs-projects/references/project-cloning.md diff --git a/skills/ai-configs/agentcontrol-projects/references/python-setup.md b/skills/ai-configs/configs-projects/references/python-setup.md similarity index 100% rename from skills/ai-configs/agentcontrol-projects/references/python-setup.md rename to skills/ai-configs/configs-projects/references/python-setup.md diff --git a/skills/ai-configs/agentcontrol-projects/references/quick-start.md b/skills/ai-configs/configs-projects/references/quick-start.md similarity index 100% rename from skills/ai-configs/agentcontrol-projects/references/quick-start.md rename to skills/ai-configs/configs-projects/references/quick-start.md diff --git a/skills/ai-configs/agentcontrol-configs-snippets/SKILL.md b/skills/ai-configs/configs-snippets/SKILL.md similarity index 99% rename from skills/ai-configs/agentcontrol-configs-snippets/SKILL.md rename to skills/ai-configs/configs-snippets/SKILL.md index be0436a..c4e5393 100644 --- a/skills/ai-configs/agentcontrol-configs-snippets/SKILL.md +++ b/skills/ai-configs/configs-snippets/SKILL.md @@ -1,5 +1,5 @@ --- -name: agentcontrol-configs-snippets +name: configs-snippets description: "Create and manage prompt snippets — reusable text blocks referenced inside AI Config variation prompts. Keeps common instructions, personas, and guardrails consistent across multiple configs." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server diff --git a/skills/ai-configs/agentcontrol-configs-targeting/README.md b/skills/ai-configs/configs-targeting/README.md similarity index 63% rename from skills/ai-configs/agentcontrol-configs-targeting/README.md rename to skills/ai-configs/configs-targeting/README.md index e599dc6..9fe73dc 100644 --- a/skills/ai-configs/agentcontrol-configs-targeting/README.md +++ b/skills/ai-configs/configs-targeting/README.md @@ -13,13 +13,13 @@ This skill teaches agents how to: ## Installation (Local) -Copy `skills/ai-configs/agentcontrol-configs-targeting/` into your agent client's skills path. +Copy `skills/ai-configs/configs-targeting/` into your agent client's skills path. ## Prerequisites - LaunchDarkly API access token with `ai-configs:write` permission -- Existing AI Config with variations (use `agentcontrol-configs-create` skill) -- Understanding of contexts (see `agentcontrol-context-basic` skill) +- Existing AI Config with variations (use `configs-create` skill) +- Understanding of contexts (see `configs-context-basic` skill) ## Usage @@ -34,16 +34,16 @@ Add a percentage rollout: 60% to variation A, 40% to variation B for premium use ## Structure ``` -agentcontrol-configs-targeting/ +configs-targeting/ ├── SKILL.md └── README.md ``` ## Related -- [AI Config Create](../agentcontrol-configs-create/) - Create AI Configs first -- [AI Config Variations](../agentcontrol-configs-variations/) - Create variations to target -- [AI Config Online Evals](../agentcontrol-configs-online-evals/) - Attach judges +- [AI Config Create](../configs-create/) - Create AI Configs first +- [AI Config Variations](../configs-variations/) - Create variations to target +- [AI Config Online Evals](../configs-online-evals/) - Attach judges - [Targeting Docs](https://docs.launchdarkly.com/home/ai-configs/target) ## License diff --git a/skills/ai-configs/agentcontrol-configs-targeting/SKILL.md b/skills/ai-configs/configs-targeting/SKILL.md similarity index 97% rename from skills/ai-configs/agentcontrol-configs-targeting/SKILL.md rename to skills/ai-configs/configs-targeting/SKILL.md index 4ea5154..8ecf42b 100644 --- a/skills/ai-configs/agentcontrol-configs-targeting/SKILL.md +++ b/skills/ai-configs/configs-targeting/SKILL.md @@ -1,5 +1,5 @@ --- -name: agentcontrol-configs-targeting +name: configs-targeting description: Configure AI Config targeting rules to control which variations serve to different users. Enable percentage rollouts, attribute-based rules, segment targeting, and guarded rollouts. compatibility: Requires LaunchDarkly API access token with ai-configs:write permission. metadata: @@ -16,7 +16,7 @@ Configure targeting rules for AI Configs to control which variations serve to di - LaunchDarkly account with AI Configs enabled - API access token with write permissions - Project key and environment key -- Existing AI Config with variations (use `agentcontrol-configs-create` skill) +- Existing AI Config with variations (use `configs-create` skill) ## API Key Detection @@ -488,16 +488,16 @@ After configuring targeting: ``` https://app.launchdarkly.com/projects/{projectKey}/ai-configs/{configKey} ``` -2. **Monitor performance** with `agentcontrol-configs-ai-metrics` -3. **Attach judges** with `agentcontrol-configs-online-evals` +2. **Monitor performance** with `configs-ai-metrics` +3. **Attach judges** with `configs-online-evals` 4. **Set up guarded rollouts** for automatic regression detection ## Related Skills -- `agentcontrol-configs-create` - Create AI Configs with variations -- `agentcontrol-configs-variations` - Manage variations -- `agentcontrol-configs-online-evals` - Attach judges -- `agentcontrol-segments` - Create segments for targeting +- `configs-create` - Create AI Configs with variations +- `configs-variations` - Manage variations +- `configs-online-evals` - Attach judges +- `configs-segments` - Create segments for targeting ## References diff --git a/skills/ai-configs/agentcontrol-tools/README.md b/skills/ai-configs/configs-tools/README.md similarity index 76% rename from skills/ai-configs/agentcontrol-tools/README.md rename to skills/ai-configs/configs-tools/README.md index 8b1fe28..df42d50 100644 --- a/skills/ai-configs/agentcontrol-tools/README.md +++ b/skills/ai-configs/configs-tools/README.md @@ -12,7 +12,7 @@ This skill teaches agents how to: ## Installation (Local) -Copy `skills/ai-configs/agentcontrol-tools/` into your agent client's skills path. +Copy `skills/ai-configs/configs-tools/` into your agent client's skills path. ## Prerequisites @@ -31,15 +31,15 @@ Create tools for the content assistant to call our API ## Structure ``` -agentcontrol-tools/ +configs-tools/ ├── SKILL.md └── README.md ``` ## Related -- [AI Config Create](../agentcontrol-configs-create/): Create the config before adding tools -- [AI Config Variations](../agentcontrol-configs-variations/): Manage variations that tools attach to +- [AI Config Create](../configs-create/): Create the config before adding tools +- [AI Config Variations](../configs-variations/): Manage variations that tools attach to - [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) ## License diff --git a/skills/ai-configs/agentcontrol-tools/SKILL.md b/skills/ai-configs/configs-tools/SKILL.md similarity index 91% rename from skills/ai-configs/agentcontrol-tools/SKILL.md rename to skills/ai-configs/configs-tools/SKILL.md index 79d7660..7dccd17 100644 --- a/skills/ai-configs/agentcontrol-tools/SKILL.md +++ b/skills/ai-configs/configs-tools/SKILL.md @@ -1,5 +1,5 @@ --- -name: agentcontrol-tools +name: configs-tools description: "Give your AI agents capabilities through tools (function calling). Helps you identify what your AI needs to do, create tool definitions, and attach them to AI Config variations." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server @@ -211,13 +211,13 @@ for _ in range(MAX_STEPS): messages.append({"role": "user", "content": tool_results}) ``` -Per-provider tool-call payload shapes live in the `agentcontrol-configs-ai-metrics` references: +Per-provider tool-call payload shapes live in the `configs-ai-metrics` references: -- [openai-tracking.md](../agentcontrol-configs-ai-metrics/references/openai-tracking.md) — Chat Completions + Responses API -- [anthropic-tracking.md](../agentcontrol-configs-ai-metrics/references/anthropic-tracking.md) — `tool_use` blocks and `tool_result` payloads -- [bedrock-tracking.md](../agentcontrol-configs-ai-metrics/references/bedrock-tracking.md) — `toolUse` / `toolResult` Converse format -- [gemini-tracking.md](../agentcontrol-configs-ai-metrics/references/gemini-tracking.md) — `functionCalls` / `functionResponse` parts -- [langchain-tracking.md](../agentcontrol-configs-ai-metrics/references/langchain-tracking.md) — LangGraph tool loop inherits from `create_react_agent` +- [openai-tracking.md](../configs-ai-metrics/references/openai-tracking.md) — Chat Completions + Responses API +- [anthropic-tracking.md](../configs-ai-metrics/references/anthropic-tracking.md) — `tool_use` blocks and `tool_result` payloads +- [bedrock-tracking.md](../configs-ai-metrics/references/bedrock-tracking.md) — `toolUse` / `toolResult` Converse format +- [gemini-tracking.md](../configs-ai-metrics/references/gemini-tracking.md) — `functionCalls` / `functionResponse` parts +- [langchain-tracking.md](../configs-ai-metrics/references/langchain-tracking.md) — LangGraph tool loop inherits from `create_react_agent` ## Orchestrator Note @@ -240,5 +240,5 @@ LangGraph, CrewAI, and AutoGen often generate schemas from function definitions. ## Related Skills -- `agentcontrol-configs-create` -- Create config before attaching tools -- `agentcontrol-configs-variations` -- Manage variations with different tool sets +- `configs-create` -- Create config before attaching tools +- `configs-variations` -- Manage variations with different tool sets diff --git a/skills/ai-configs/agentcontrol-configs-update/README.md b/skills/ai-configs/configs-update/README.md similarity index 78% rename from skills/ai-configs/agentcontrol-configs-update/README.md rename to skills/ai-configs/configs-update/README.md index ca74615..1155f02 100644 --- a/skills/ai-configs/agentcontrol-configs-update/README.md +++ b/skills/ai-configs/configs-update/README.md @@ -13,7 +13,7 @@ This skill teaches agents how to: ## Installation (Local) -Copy `skills/ai-configs/agentcontrol-configs-update/` into your agent client's skills path. +Copy `skills/ai-configs/configs-update/` into your agent client's skills path. ## Prerequisites @@ -36,15 +36,15 @@ Archive the old chatbot config ## Structure ``` -agentcontrol-configs-update/ +configs-update/ ├── SKILL.md └── README.md ``` ## Related -- [AI Config Create](../agentcontrol-configs-create/): Create configs -- [AI Config Variations](../agentcontrol-configs-variations/): Add or test variations +- [AI Config Create](../configs-create/): Create configs +- [AI Config Variations](../configs-variations/): Add or test variations - [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) ## License diff --git a/skills/ai-configs/agentcontrol-configs-update/SKILL.md b/skills/ai-configs/configs-update/SKILL.md similarity index 95% rename from skills/ai-configs/agentcontrol-configs-update/SKILL.md rename to skills/ai-configs/configs-update/SKILL.md index 4cf3f40..183113c 100644 --- a/skills/ai-configs/agentcontrol-configs-update/SKILL.md +++ b/skills/ai-configs/configs-update/SKILL.md @@ -1,5 +1,5 @@ --- -name: agentcontrol-configs-update +name: configs-update description: "Update, archive, and delete LaunchDarkly AI Configs and their variations. Use when you need to modify config properties, change model parameters, update instructions or messages, archive unused configs, or permanently remove them." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server @@ -90,5 +90,5 @@ Use `get-ai-config` to confirm the response shows your updated values. ## Related Skills -- `agentcontrol-configs-variations` -- Create variations to test changes side-by-side -- `agentcontrol-tools` -- Update tool attachments +- `configs-variations` -- Create variations to test changes side-by-side +- `configs-tools` -- Update tool attachments diff --git a/skills/ai-configs/agentcontrol-configs-variations/README.md b/skills/ai-configs/configs-variations/README.md similarity index 71% rename from skills/ai-configs/agentcontrol-configs-variations/README.md rename to skills/ai-configs/configs-variations/README.md index 6a7f0b7..3d40d2c 100644 --- a/skills/ai-configs/agentcontrol-configs-variations/README.md +++ b/skills/ai-configs/configs-variations/README.md @@ -11,7 +11,7 @@ This skill teaches agents how to: ## Installation (Local) -Copy `skills/ai-configs/agentcontrol-configs-variations/` into your agent client's skills path. +Copy `skills/ai-configs/configs-variations/` into your agent client's skills path. ## Prerequisites @@ -30,16 +30,16 @@ Create variations to compare Claude vs GPT-4 for our agent ## Structure ``` -agentcontrol-configs-variations/ +configs-variations/ ├── SKILL.md └── README.md ``` ## Related -- [AI Config Create](../agentcontrol-configs-create/): Create the config first -- [AI Config Update](../agentcontrol-configs-update/): Modify existing variations -- [AI Config Tools](../agentcontrol-tools/): Attach tools to variations +- [AI Config Create](../configs-create/): Create the config first +- [AI Config Update](../configs-update/): Modify existing variations +- [AI Config Tools](../configs-tools/): Attach tools to variations - [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) ## License diff --git a/skills/ai-configs/agentcontrol-configs-variations/SKILL.md b/skills/ai-configs/configs-variations/SKILL.md similarity index 97% rename from skills/ai-configs/agentcontrol-configs-variations/SKILL.md rename to skills/ai-configs/configs-variations/SKILL.md index c028296..26c58e0 100644 --- a/skills/ai-configs/agentcontrol-configs-variations/SKILL.md +++ b/skills/ai-configs/configs-variations/SKILL.md @@ -1,5 +1,5 @@ --- -name: agentcontrol-configs-variations +name: configs-variations description: "Experiment with AI configurations by creating and managing variations. Helps you test different models, prompts, and parameters to find what works best through systematic experimentation." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server @@ -106,5 +106,5 @@ When the user wants to try a different model, prompt, or parameters, **always cr ## Related Skills -- `agentcontrol-configs-create` -- Create the initial config -- `agentcontrol-configs-update` -- Refine based on learnings +- `configs-create` -- Create the initial config +- `configs-update` -- Refine based on learnings From 4145cd6b38f86fe400e02e0268fbec062c8dc91a Mon Sep 17 00:00:00 2001 From: Scarlett Attensil Date: Tue, 19 May 2026 11:42:06 -0700 Subject: [PATCH 03/10] Move skills to skills/agentcontrol/ with bare names Per review (Paul's screenshot + Slack): parent folder ai-configs -> agentcontrol, and drop the configs- prefix so skills are bare names (create, migrate, projects, tools, agent-graphs, ...). Updates directories, SKILL.md frontmatter, skills.json, README catalog, .cursor-plugin path, and all cross-references. LaunchDarkly REST resource paths (/api/v2/.../ai-configs/), ai-configs:write scopes, docs URLs, and the list-ai-configs tool are intentionally left. --- .cursor-plugin/plugin.json | 2 +- README.md | 20 ++--- skills.json | 48 +++++------ .../agent-graphs}/SKILL.md | 2 +- .../ai-metrics}/SKILL.md | 14 ++-- .../references/anthropic-tracking.md | 0 .../references/bedrock-tracking.md | 0 .../ai-metrics}/references/gemini-tracking.md | 2 +- .../references/langchain-tracking.md | 2 +- .../ai-metrics}/references/metrics-api.md | 0 .../ai-metrics}/references/openai-tracking.md | 0 .../references/strands-tracking.md | 2 +- .../references/streaming-tracking.md | 0 .../create}/README.md | 10 +-- .../create}/SKILL.md | 18 ++--- .../custom-metrics}/SKILL.md | 14 ++-- .../migrate}/README.md | 20 ++--- .../migrate}/SKILL.md | 80 +++++++++---------- .../references/agent-graph-reference.md | 0 .../references/agent-mode-frameworks.md | 12 +-- .../references/before-after-examples.md | 0 .../references/fallback-defaults-pattern.md | 2 +- .../references/phase-1-analysis-checklist.md | 6 +- .../references/sdk-ai-tracker-patterns.md | 8 +- .../online-evals}/README.md | 12 +-- .../online-evals}/SKILL.md | 10 +-- .../projects}/README.md | 4 +- .../projects}/SKILL.md | 16 ++-- .../projects}/references/admin-tooling.md | 0 .../projects}/references/env-config.md | 2 +- .../projects}/references/go-setup.md | 0 .../projects}/references/iac-automation.md | 0 .../references/multi-language-setup.md | 0 .../projects}/references/nodejs-setup.md | 0 .../projects}/references/project-cloning.md | 0 .../projects}/references/python-setup.md | 0 .../projects}/references/quick-start.md | 0 .../snippets}/SKILL.md | 2 +- .../targeting}/README.md | 14 ++-- .../targeting}/SKILL.md | 16 ++-- .../tools}/README.md | 8 +- .../tools}/SKILL.md | 18 ++--- .../update}/README.md | 8 +- .../update}/SKILL.md | 6 +- .../variations}/README.md | 10 +-- .../variations}/SKILL.md | 6 +- 46 files changed, 197 insertions(+), 197 deletions(-) rename skills/{ai-configs/configs-agent-graphs => agentcontrol/agent-graphs}/SKILL.md (99%) rename skills/{ai-configs/configs-ai-metrics => agentcontrol/ai-metrics}/SKILL.md (93%) rename skills/{ai-configs/configs-ai-metrics => agentcontrol/ai-metrics}/references/anthropic-tracking.md (100%) rename skills/{ai-configs/configs-ai-metrics => agentcontrol/ai-metrics}/references/bedrock-tracking.md (100%) rename skills/{ai-configs/configs-ai-metrics => agentcontrol/ai-metrics}/references/gemini-tracking.md (99%) rename skills/{ai-configs/configs-ai-metrics => agentcontrol/ai-metrics}/references/langchain-tracking.md (97%) rename skills/{ai-configs/configs-ai-metrics => agentcontrol/ai-metrics}/references/metrics-api.md (100%) rename skills/{ai-configs/configs-ai-metrics => agentcontrol/ai-metrics}/references/openai-tracking.md (100%) rename skills/{ai-configs/configs-ai-metrics => agentcontrol/ai-metrics}/references/strands-tracking.md (93%) rename skills/{ai-configs/configs-ai-metrics => agentcontrol/ai-metrics}/references/streaming-tracking.md (100%) rename skills/{ai-configs/configs-create => agentcontrol/create}/README.md (76%) rename skills/{ai-configs/configs-create => agentcontrol/create}/SKILL.md (90%) rename skills/{ai-configs/configs-custom-metrics => agentcontrol/custom-metrics}/SKILL.md (97%) rename skills/{ai-configs/configs-migrate => agentcontrol/migrate}/README.md (78%) rename skills/{ai-configs/configs-migrate => agentcontrol/migrate}/SKILL.md (87%) rename skills/{ai-configs/configs-migrate => agentcontrol/migrate}/references/agent-graph-reference.md (100%) rename skills/{ai-configs/configs-migrate => agentcontrol/migrate}/references/agent-mode-frameworks.md (98%) rename skills/{ai-configs/configs-migrate => agentcontrol/migrate}/references/before-after-examples.md (100%) rename skills/{ai-configs/configs-migrate => agentcontrol/migrate}/references/fallback-defaults-pattern.md (99%) rename skills/{ai-configs/configs-migrate => agentcontrol/migrate}/references/phase-1-analysis-checklist.md (97%) rename skills/{ai-configs/configs-migrate => agentcontrol/migrate}/references/sdk-ai-tracker-patterns.md (98%) rename skills/{ai-configs/configs-online-evals => agentcontrol/online-evals}/README.md (71%) rename skills/{ai-configs/configs-online-evals => agentcontrol/online-evals}/SKILL.md (98%) rename skills/{ai-configs/configs-projects => agentcontrol/projects}/README.md (94%) rename skills/{ai-configs/configs-projects => agentcontrol/projects}/SKILL.md (95%) rename skills/{ai-configs/configs-projects => agentcontrol/projects}/references/admin-tooling.md (100%) rename skills/{ai-configs/configs-projects => agentcontrol/projects}/references/env-config.md (99%) rename skills/{ai-configs/configs-projects => agentcontrol/projects}/references/go-setup.md (100%) rename skills/{ai-configs/configs-projects => agentcontrol/projects}/references/iac-automation.md (100%) rename skills/{ai-configs/configs-projects => agentcontrol/projects}/references/multi-language-setup.md (100%) rename skills/{ai-configs/configs-projects => agentcontrol/projects}/references/nodejs-setup.md (100%) rename skills/{ai-configs/configs-projects => agentcontrol/projects}/references/project-cloning.md (100%) rename skills/{ai-configs/configs-projects => agentcontrol/projects}/references/python-setup.md (100%) rename skills/{ai-configs/configs-projects => agentcontrol/projects}/references/quick-start.md (100%) rename skills/{ai-configs/configs-snippets => agentcontrol/snippets}/SKILL.md (99%) rename skills/{ai-configs/configs-targeting => agentcontrol/targeting}/README.md (66%) rename skills/{ai-configs/configs-targeting => agentcontrol/targeting}/SKILL.md (97%) rename skills/{ai-configs/configs-tools => agentcontrol/tools}/README.md (78%) rename skills/{ai-configs/configs-tools => agentcontrol/tools}/SKILL.md (92%) rename skills/{ai-configs/configs-update => agentcontrol/update}/README.md (82%) rename skills/{ai-configs/configs-update => agentcontrol/update}/SKILL.md (96%) rename skills/{ai-configs/configs-variations => agentcontrol/variations}/README.md (75%) rename skills/{ai-configs/configs-variations => agentcontrol/variations}/SKILL.md (97%) diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json index cf14bfe..7d572df 100644 --- a/.cursor-plugin/plugin.json +++ b/.cursor-plugin/plugin.json @@ -21,7 +21,7 @@ "mcp" ], "skills": [ - "skills/ai-configs", + "skills/agentcontrol", "skills/feature-flags", "skills/metrics", "skills/onboarding" diff --git a/README.md b/README.md index 8f2b952..2fa1b0e 100644 --- a/README.md +++ b/README.md @@ -22,16 +22,16 @@ Agent Skills are modular, text-based playbooks that teach an agent how to perfor | Skill | Description | |-------|-------------| -| `ai-configs/configs-create` | Create AI Configs with variations for agent or completion mode | -| `ai-configs/configs-migrate` | Migrate an app with hardcoded LLM prompts to AI Configs in five stages (extract, wrap, tools, tracking, evals) | -| `ai-configs/configs-update` | Update and delete AI Configs, manage lifecycle | -| `ai-configs/configs-variations` | Manage AI Config variations for A/B testing | -| `ai-configs/configs-tools` | Create and attach tools for function calling | -| `ai-configs/configs-projects` | Create and manage projects to organize AI Configs | -| `ai-configs/configs-online-evals` | Attach LLM-as-a-judge evaluators to AI Configs | -| `ai-configs/configs-targeting` | Configure targeting rules for AI Config rollouts | -| `ai-configs/configs-snippets` | Create and manage reusable prompt snippets across AI Configs | -| `ai-configs/configs-agent-graphs` | Create and manage multi-agent graphs with routing and handoffs | +| `agentcontrol/create` | Create AI Configs with variations for agent or completion mode | +| `agentcontrol/migrate` | Migrate an app with hardcoded LLM prompts to AI Configs in five stages (extract, wrap, tools, tracking, evals) | +| `agentcontrol/update` | Update and delete AI Configs, manage lifecycle | +| `agentcontrol/variations` | Manage AI Config variations for A/B testing | +| `agentcontrol/tools` | Create and attach tools for function calling | +| `agentcontrol/projects` | Create and manage projects to organize AI Configs | +| `agentcontrol/online-evals` | Attach LLM-as-a-judge evaluators to AI Configs | +| `agentcontrol/targeting` | Configure targeting rules for AI Config rollouts | +| `agentcontrol/snippets` | Create and manage reusable prompt snippets across AI Configs | +| `agentcontrol/agent-graphs` | Create and manage multi-agent graphs with routing and handoffs | ### Experiments diff --git a/skills.json b/skills.json index 67b298c..7e5d921 100644 --- a/skills.json +++ b/skills.json @@ -1,94 +1,94 @@ { "skills": [ { - "name": "configs-agent-graphs", + "name": "agent-graphs", "description": "Create and manage agent graphs \u2014 directed graphs of AI Configs connected by edges with handoff logic. Use when building multi-agent workflows where configs route to each other.", - "path": "skills/ai-configs/configs-agent-graphs", + "path": "skills/agentcontrol/agent-graphs", "version": "0.1.0", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "configs-ai-metrics", + "name": "ai-metrics", "description": "Instrument an existing codebase with LaunchDarkly AI Config tracking. Walks the four-tier ladder (managed runner \u2192 provider package \u2192 custom extractor + trackMetricsOf \u2192 raw manual) and picks the lowest-ceremony option that still captures duration, tokens, and success/error.", - "path": "skills/ai-configs/configs-ai-metrics", + "path": "skills/agentcontrol/ai-metrics", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the LaunchDarkly server-side AI SDK (`launchdarkly-server-sdk-ai>=0.20.0` for Python or `@launchdarkly/server-sdk-ai>=0.20.0` for Node) and an existing AI Config." }, { - "name": "configs-create", + "name": "create", "description": "Create and configure AI Configs in LaunchDarkly. Helps you choose between agent vs completion mode, create the config, add variations with models and prompts, and verify the setup.", - "path": "skills/ai-configs/configs-create", + "path": "skills/agentcontrol/create", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "configs-custom-metrics", + "name": "custom-metrics", "description": "Create, track, retrieve, update, and delete custom business metrics for AI Configs. Covers full lifecycle: define metric kinds via API, emit events via SDK, and query results.", - "path": "skills/ai-configs/configs-custom-metrics", + "path": "skills/agentcontrol/custom-metrics", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the LaunchDarkly server SDK and a LaunchDarkly API token with the `writer` role for metric management." }, { - "name": "configs-migrate", + "name": "migrate", "description": "Migrate an application with hardcoded LLM prompts to a full LaunchDarkly AI Configs implementation in five stages: audit the code, wrap the call, move the tools, add tracking, attach evaluators. Use when the user wants to externalize model/prompt configuration, move from direct provider calls (OpenAI, Anthropic, Bedrock, Gemini, Strands) to a managed AI Config, or stage a full hardcoded-to-LaunchDarkly migration.", - "path": "skills/ai-configs/configs-migrate", + "path": "skills/agentcontrol/migrate", "version": "0.1.0", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "configs-online-evals", + "name": "online-evals", "description": "Attach judges to AI Config variations for automatic LLM-as-a-judge evaluation. Create custom judges, configure sampling rates, and monitor quality scores.", - "path": "skills/ai-configs/configs-online-evals", + "path": "skills/agentcontrol/online-evals", "version": "0.1.0", "compatibility": "Requires LaunchDarkly API access token with ai-configs:write permission. SDK versions Python v0.20.0+ or Node.js v0.20.0+ for automatic metric recording and the consolidated `track_judge_result` / `trackJudgeResult` API." }, { - "name": "configs-projects", + "name": "projects", "description": "Guide for setting up LaunchDarkly projects in your codebase. Helps you assess your stack, choose the right approach, and integrate project management that makes sense for your architecture.", - "path": "skills/ai-configs/configs-projects", + "path": "skills/agentcontrol/projects", "version": "0.4.0", "compatibility": "Requires LaunchDarkly API access token with projects:write permission or LaunchDarkly MCP server." }, { - "name": "configs-snippets", + "name": "snippets", "description": "Create and manage prompt snippets \u2014 reusable text blocks referenced inside AI Config variation prompts. Keeps common instructions, personas, and guardrails consistent across multiple configs.", - "path": "skills/ai-configs/configs-snippets", + "path": "skills/agentcontrol/snippets", "version": "0.1.0", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "configs-targeting", + "name": "targeting", "description": "Configure AI Config targeting rules to control which variations serve to different users. Enable percentage rollouts, attribute-based rules, segment targeting, and guarded rollouts.", - "path": "skills/ai-configs/configs-targeting", + "path": "skills/agentcontrol/targeting", "version": "0.1.0", "compatibility": "Requires LaunchDarkly API access token with ai-configs:write permission." }, { - "name": "configs-tools", + "name": "tools", "description": "Give your AI agents capabilities through tools (function calling). Helps you identify what your AI needs to do, create tool definitions, and attach them to AI Config variations.", - "path": "skills/ai-configs/configs-tools", + "path": "skills/agentcontrol/tools", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "configs-update", + "name": "update", "description": "Update, archive, and delete LaunchDarkly AI Configs and their variations. Use when you need to modify config properties, change model parameters, update instructions or messages, archive unused configs, or permanently remove them.", - "path": "skills/ai-configs/configs-update", + "path": "skills/agentcontrol/update", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "configs-variations", + "name": "variations", "description": "Experiment with AI configurations by creating and managing variations. Helps you test different models, prompts, and parameters to find what works best through systematic experimentation.", - "path": "skills/ai-configs/configs-variations", + "path": "skills/agentcontrol/variations", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" diff --git a/skills/ai-configs/configs-agent-graphs/SKILL.md b/skills/agentcontrol/agent-graphs/SKILL.md similarity index 99% rename from skills/ai-configs/configs-agent-graphs/SKILL.md rename to skills/agentcontrol/agent-graphs/SKILL.md index d5176fe..b067e1c 100644 --- a/skills/ai-configs/configs-agent-graphs/SKILL.md +++ b/skills/agentcontrol/agent-graphs/SKILL.md @@ -1,5 +1,5 @@ --- -name: configs-agent-graphs +name: agent-graphs description: "Create and manage agent graphs — directed graphs of configs connected by edges with handoff logic. Use when building multi-agent workflows where configs route to each other." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server diff --git a/skills/ai-configs/configs-ai-metrics/SKILL.md b/skills/agentcontrol/ai-metrics/SKILL.md similarity index 93% rename from skills/ai-configs/configs-ai-metrics/SKILL.md rename to skills/agentcontrol/ai-metrics/SKILL.md index 96170a3..7eedbb5 100644 --- a/skills/ai-configs/configs-ai-metrics/SKILL.md +++ b/skills/agentcontrol/ai-metrics/SKILL.md @@ -1,5 +1,5 @@ --- -name: configs-ai-metrics +name: ai-metrics description: "Instrument an existing codebase with LaunchDarkly AI Config tracking. Walks the four-tier ladder (managed runner → provider package → custom extractor + trackMetricsOf → raw manual) and picks the lowest-ceremony option that still captures duration, tokens, and success/error." license: Apache-2.0 compatibility: Requires the LaunchDarkly server-side AI SDK (`launchdarkly-server-sdk-ai>=0.20.0` for Python or `@launchdarkly/server-sdk-ai>=0.20.0` for Node) and an existing AI Config. @@ -38,12 +38,12 @@ Before picking a tier, find the provider call and answer these questions: - [ ] **Provider?** OpenAI, Anthropic, Bedrock, Gemini, Azure, custom HTTP? → cross-reference with the package availability matrix below. - [ ] **Streaming?** If yes, you'll need TTFT tracking, which means Tier 4 for the TTFT part even if the rest is Tier 2. - [ ] **Language?** Python or Node? Provider-package coverage differs between them. -- [ ] **Already using an AI Config?** If not, route to `configs-create` first — tracking requires a tracker, which is obtained by calling `create_tracker()` / `createTracker()` on the config object returned by `completion_config()` / `completionConfig()` / `createModel()`. +- [ ] **Already using an AI Config?** If not, route to `create` first — tracking requires a tracker, which is obtained by calling `create_tracker()` / `createTracker()` on the config object returned by `completion_config()` / `completionConfig()` / `createModel()`. - [ ] **On the current SDK API?** If the call site uses `aiclient.config(...)` / `aiClient.config(...)` or constructs an `AIConfig(...)` / `LDAIConfig` default, it's on the pre-0.20 surface. Migrate it as part of this work before adding tracking: - `aiclient.config(...)` → `aiclient.completion_config(...)` for one-shot/chat or `aiclient.agent_config(...)` for agent mode (mirror the call signature). Node is the same with camelCase. - `AIConfig(...)` default → `AICompletionConfigDefault(...)` or `AIAgentConfigDefault(...)` (Node: `LDAICompletionConfigDefault` / `LDAIAgentConfigDefault`). `AIConfig` is the base class the SDK returns; it isn't a valid default-value constructor — the typed `*Default` variants are. - If the result was being tuple-unpacked (`config, tracker = aiclient.config(...)`), drop the unpack — the new methods return a single config object. Obtain the tracker via `config.create_tracker()` / `aiConfig.createTracker()`. - - For deeper rewrites (call sites with hardcoded model/prompt as well), hand off to `configs-migrate` instead of doing the full migration here. + - For deeper rewrites (call sites with hardcoded model/prompt as well), hand off to `migrate` instead of doing the full migration here. ### 2. Look up your Tier-2 option @@ -103,7 +103,7 @@ Obtain a tracker via the factory on the config object: `tracker = config.create_ ## Related skills -- `configs-create` — prerequisite if the app doesn't have an AI Config yet -- `configs-custom-metrics` — business metrics (conversion, resolution, retention) layered on top of the AI metrics this skill captures -- `configs-online-evals` — automatic quality scoring (LLM-as-judge) on sampled live requests; complementary to the metrics here -- `configs-migrate` — Stage 4 of the hardcoded-to-AI-Configs migration delegates to this skill +- `create` — prerequisite if the app doesn't have an AI Config yet +- `custom-metrics` — business metrics (conversion, resolution, retention) layered on top of the AI metrics this skill captures +- `online-evals` — automatic quality scoring (LLM-as-judge) on sampled live requests; complementary to the metrics here +- `migrate` — Stage 4 of the hardcoded-to-AI-Configs migration delegates to this skill diff --git a/skills/ai-configs/configs-ai-metrics/references/anthropic-tracking.md b/skills/agentcontrol/ai-metrics/references/anthropic-tracking.md similarity index 100% rename from skills/ai-configs/configs-ai-metrics/references/anthropic-tracking.md rename to skills/agentcontrol/ai-metrics/references/anthropic-tracking.md diff --git a/skills/ai-configs/configs-ai-metrics/references/bedrock-tracking.md b/skills/agentcontrol/ai-metrics/references/bedrock-tracking.md similarity index 100% rename from skills/ai-configs/configs-ai-metrics/references/bedrock-tracking.md rename to skills/agentcontrol/ai-metrics/references/bedrock-tracking.md diff --git a/skills/ai-configs/configs-ai-metrics/references/gemini-tracking.md b/skills/agentcontrol/ai-metrics/references/gemini-tracking.md similarity index 99% rename from skills/ai-configs/configs-ai-metrics/references/gemini-tracking.md rename to skills/agentcontrol/ai-metrics/references/gemini-tracking.md index 9b14542..32c5bf9 100644 --- a/skills/ai-configs/configs-ai-metrics/references/gemini-tracking.md +++ b/skills/agentcontrol/ai-metrics/references/gemini-tracking.md @@ -191,7 +191,7 @@ gemini_tools = [ ] if ld_tools else [] ``` -Tool handlers stay in your application code — LaunchDarkly stores the schema, your application owns the behavior. For the full agent loop pattern (`MAX_STEPS`, `functionCalls` handling, `tracker.track_tool_call`), see the agent-mode section of `configs-tools`. +Tool handlers stay in your application code — LaunchDarkly stores the schema, your application owns the behavior. For the full agent loop pattern (`MAX_STEPS`, `functionCalls` handling, `tracker.track_tool_call`), see the agent-mode section of `tools`. ## Tier 2 option — route via LangChain diff --git a/skills/ai-configs/configs-ai-metrics/references/langchain-tracking.md b/skills/agentcontrol/ai-metrics/references/langchain-tracking.md similarity index 97% rename from skills/ai-configs/configs-ai-metrics/references/langchain-tracking.md rename to skills/agentcontrol/ai-metrics/references/langchain-tracking.md index c6e4e67..b3a084e 100644 --- a/skills/ai-configs/configs-ai-metrics/references/langchain-tracking.md +++ b/skills/agentcontrol/ai-metrics/references/langchain-tracking.md @@ -252,7 +252,7 @@ for (const msg of result.messages ?? []) { ## Binding AI-Config-attached tools with `build_structured_tools` -If the variation has tools attached (via `/configs-tools`), use `build_structured_tools` rather than hand-rolling a `TOOL_REGISTRY` / `resolve_tools` / `ALL_TOOLS` shape. The helper reads `ai_config.model.parameters.tools`, picks the matching entries from your `{name: callable}` registry, wraps them as LangChain `StructuredTool` instances, and preserves the LD tool key as the `StructuredTool.name` (so `ToolNode(...)` lookup works without a second mapping). +If the variation has tools attached (via `/tools`), use `build_structured_tools` rather than hand-rolling a `TOOL_REGISTRY` / `resolve_tools` / `ALL_TOOLS` shape. The helper reads `ai_config.model.parameters.tools`, picks the matching entries from your `{name: callable}` registry, wraps them as LangChain `StructuredTool` instances, and preserves the LD tool key as the `StructuredTool.name` (so `ToolNode(...)` lookup works without a second mapping). ```python # tools.py — implementations only; no manual schema, no resolve_tools() diff --git a/skills/ai-configs/configs-ai-metrics/references/metrics-api.md b/skills/agentcontrol/ai-metrics/references/metrics-api.md similarity index 100% rename from skills/ai-configs/configs-ai-metrics/references/metrics-api.md rename to skills/agentcontrol/ai-metrics/references/metrics-api.md diff --git a/skills/ai-configs/configs-ai-metrics/references/openai-tracking.md b/skills/agentcontrol/ai-metrics/references/openai-tracking.md similarity index 100% rename from skills/ai-configs/configs-ai-metrics/references/openai-tracking.md rename to skills/agentcontrol/ai-metrics/references/openai-tracking.md diff --git a/skills/ai-configs/configs-ai-metrics/references/strands-tracking.md b/skills/agentcontrol/ai-metrics/references/strands-tracking.md similarity index 93% rename from skills/ai-configs/configs-ai-metrics/references/strands-tracking.md rename to skills/agentcontrol/ai-metrics/references/strands-tracking.md index 1741254..062a006 100644 --- a/skills/ai-configs/configs-ai-metrics/references/strands-tracking.md +++ b/skills/agentcontrol/ai-metrics/references/strands-tracking.md @@ -79,7 +79,7 @@ Pick the style that matches the rest of the codebase — the two variants record ## Provider dispatch stays in your code -Strands model classes are provider-specific (`AnthropicModel`, `OpenAIModel`, `BedrockModel`). To serve more than one provider from a single AI Config key, dispatch on `agent_config.provider.name` before constructing the `Agent`. See [agent-mode-frameworks.md § Strands Agent](../../configs-migrate/references/agent-mode-frameworks.md) for the `create_strands_model` dispatcher, including the rule that `parameters.tools` must be dropped before being passed into the Strands model class (tools flow through the `Agent` constructor, not through model params). +Strands model classes are provider-specific (`AnthropicModel`, `OpenAIModel`, `BedrockModel`). To serve more than one provider from a single AI Config key, dispatch on `agent_config.provider.name` before constructing the `Agent`. See [agent-mode-frameworks.md § Strands Agent](../../migrate/references/agent-mode-frameworks.md) for the `create_strands_model` dispatcher, including the rule that `parameters.tools` must be dropped before being passed into the Strands model class (tools flow through the `Agent` constructor, not through model params). ## Always flush before exit diff --git a/skills/ai-configs/configs-ai-metrics/references/streaming-tracking.md b/skills/agentcontrol/ai-metrics/references/streaming-tracking.md similarity index 100% rename from skills/ai-configs/configs-ai-metrics/references/streaming-tracking.md rename to skills/agentcontrol/ai-metrics/references/streaming-tracking.md diff --git a/skills/ai-configs/configs-create/README.md b/skills/agentcontrol/create/README.md similarity index 76% rename from skills/ai-configs/configs-create/README.md rename to skills/agentcontrol/create/README.md index 264e703..5366713 100644 --- a/skills/ai-configs/configs-create/README.md +++ b/skills/agentcontrol/create/README.md @@ -12,7 +12,7 @@ This skill teaches agents how to: ## Installation (Local) -Copy `skills/ai-configs/configs-create/` into your agent client's skills path. +Copy `skills/agentcontrol/create/` into your agent client's skills path. ## Prerequisites @@ -31,16 +31,16 @@ Set up an AI config for content generation using Claude ## Structure ``` -configs-create/ +create/ ├── SKILL.md └── README.md ``` ## Related -- [AI Config Projects](../configs-projects/): Create projects first -- [AI Config Tools](../configs-tools/): Add tools after creating config -- [AI Config Variations](../configs-variations/): Add more variations for experimentation +- [AI Config Projects](../projects/): Create projects first +- [AI Config Tools](../tools/): Add tools after creating config +- [AI Config Variations](../variations/): Add more variations for experimentation - [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) ## License diff --git a/skills/ai-configs/configs-create/SKILL.md b/skills/agentcontrol/create/SKILL.md similarity index 90% rename from skills/ai-configs/configs-create/SKILL.md rename to skills/agentcontrol/create/SKILL.md index e2b1a4f..c5be3aa 100644 --- a/skills/ai-configs/configs-create/SKILL.md +++ b/skills/agentcontrol/create/SKILL.md @@ -1,5 +1,5 @@ --- -name: configs-create +name: create description: "Create and configure AI Configs in LaunchDarkly. Helps you choose between agent vs completion mode, create the config, add variations with models and prompts, and verify the setup." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server @@ -12,7 +12,7 @@ metadata: You're using a skill that will guide you through creating an AI Config in LaunchDarkly. Your job is to understand the use case, choose the right mode, create the config and its variations, and verify everything is set up correctly. -> **⚠️ This skill creates a config — it does not make it servable.** A freshly-created AI Config has its **fallthrough pointing at an auto-generated disabled variation**, not at the variation you just created. The SDK will return `ai_config.enabled=False` on every evaluation until you flip targeting on and point the fallthrough at your new variation. This is not a bug — it's the default state. **You must run `/configs-targeting` (or the equivalent REST / CLI call shown in Step 5) before verifying against the SDK**, or verification will look like the LD-served path is broken when it isn't. The single most common failure mode users hit with this skill is skipping the targeting step and spending time debugging `enabled=False` in their application code. +> **⚠️ This skill creates a config — it does not make it servable.** A freshly-created AI Config has its **fallthrough pointing at an auto-generated disabled variation**, not at the variation you just created. The SDK will return `ai_config.enabled=False` on every evaluation until you flip targeting on and point the fallthrough at your new variation. This is not a bug — it's the default state. **You must run `/targeting` (or the equivalent REST / CLI call shown in Step 5) before verifying against the SDK**, or verification will look like the LD-served path is broken when it isn't. The single most common failure mode users hit with this skill is skipping the targeting step and spending time debugging `enabled=False` in their application code. ## Prerequisites @@ -147,7 +147,7 @@ Print this checklist verbatim to the user after Step 4, then wait for confirmati > > 🔴 **The SDK will return `enabled=False` until you flip targeting on.** The fallthrough is currently pointing at an auto-generated disabled variation, not at the `{variationKey}` you just created. > -> **Next step — run `/configs-targeting`** with these inputs: +> **Next step — run `/targeting`** with these inputs: > - Project key: `{projectKey}` > - Config key: `{configKey}` > - Environment key: the env whose SDK key is in your `.env` (usually `test` or `production`) @@ -159,7 +159,7 @@ Print this checklist verbatim to the user after Step 4, then wait for confirmati #### Direct shortcut if the user wants to flip targeting without invoking the sibling skill -`configs-targeting` is the canonical path — it handles percentage rollouts, targeted rules, and variation-ID lookups. But for the simplest case ("promote the new variation to fallthrough in one environment"), you can run the underlying semantic PATCH yourself once you know the new variation's `_id`. +`targeting` is the canonical path — it handles percentage rollouts, targeted rules, and variation-ID lookups. But for the simplest case ("promote the new variation to fallthrough in one environment"), you can run the underlying semantic PATCH yourself once you know the new variation's `_id`. Get the variation ID (use `get-ai-config` MCP, or): ```bash @@ -203,7 +203,7 @@ The `create-ai-config-variation` tool validates this format and rejects invalid |-----------|--------| | Config already exists | Ask if user wants to update instead | | Variation shows "NO MODEL" | Use `update-ai-config-variation` to set modelConfigKey | -| Need to attach tools | Create tools first (`configs-tools` skill), then update the variation | +| Need to attach tools | Create tools first (`tools` skill), then update the variation | ## What NOT to Do @@ -213,10 +213,10 @@ The `create-ai-config-variation` tool validates this format and rejects invalid - Don't forget modelConfigKey (models won't show in the UI) - Don't omit `modelName` from the initial variation call. It is required at create time; setting it via a follow-up PATCH is a workaround for a bug, not the intended flow. The PATCH field is also `modelName`, not `name`. - Don't drop to raw `curl` + `jq` for verification. Use `get-ai-config` (MCP) — it returns a typed object and avoids brittle `jq` filters that break on response-shape variation. -- Don't consider the workflow complete until the user has been told to run `configs-targeting`. A created variation that isn't promoted to fallthrough returns `enabled=False` to every consumer. +- Don't consider the workflow complete until the user has been told to run `targeting`. A created variation that isn't promoted to fallthrough returns `enabled=False` to every consumer. ## Related Skills -- `configs-tools` -- Create tools before attaching -- `configs-variations` -- Add more variations for experimentation -- `configs-update` -- Modify configs based on learnings +- `tools` -- Create tools before attaching +- `variations` -- Add more variations for experimentation +- `update` -- Modify configs based on learnings diff --git a/skills/ai-configs/configs-custom-metrics/SKILL.md b/skills/agentcontrol/custom-metrics/SKILL.md similarity index 97% rename from skills/ai-configs/configs-custom-metrics/SKILL.md rename to skills/agentcontrol/custom-metrics/SKILL.md index b787dcd..880f133 100644 --- a/skills/ai-configs/configs-custom-metrics/SKILL.md +++ b/skills/agentcontrol/custom-metrics/SKILL.md @@ -1,5 +1,5 @@ --- -name: configs-custom-metrics +name: custom-metrics description: "Create, track, retrieve, update, and delete custom business metrics for AI Configs. Covers full lifecycle: define metric kinds via API, emit events via SDK, and query results." license: Apache-2.0 compatibility: Requires the LaunchDarkly server SDK and a LaunchDarkly API token with the `writer` role for metric management. @@ -14,9 +14,9 @@ Full lifecycle management of custom business metrics: create metric definitions ## Prerequisites -- LaunchDarkly SDK initialized (see `configs-sdk`) +- LaunchDarkly SDK initialized (see `sdk`) - LaunchDarkly API token with `writer` role for metric management -- Understanding of built-in AI metrics (see `configs-ai-metrics`) +- Understanding of built-in AI metrics (see `ai-metrics`) ## API Key Detection @@ -149,7 +149,7 @@ from ldclient import Context from ldclient.config import Config import ldclient -# Initialize (see configs-sdk for details) +# Initialize (see sdk for details) ldclient.set_config(Config("your-sdk-key")) ld_client = ldclient.get() @@ -492,9 +492,9 @@ Custom metrics appear in: ## Related Skills -- `configs-sdk` - SDK setup -- `configs-ai-metrics` - Built-in AI metrics (tokens, duration, cost) -- `configs-online-evals` - Quality metrics via judges +- `sdk` - SDK setup +- `ai-metrics` - Built-in AI metrics (tokens, duration, cost) +- `online-evals` - Quality metrics via judges ## References diff --git a/skills/ai-configs/configs-migrate/README.md b/skills/agentcontrol/migrate/README.md similarity index 78% rename from skills/ai-configs/configs-migrate/README.md rename to skills/agentcontrol/migrate/README.md index 34053c8..71f0df8 100644 --- a/skills/ai-configs/configs-migrate/README.md +++ b/skills/agentcontrol/migrate/README.md @@ -9,14 +9,14 @@ This skill orchestrates the full migration journey from hardcoded `openai.chat.c The five stages: 1. **Extract** hardcoded model names, prompts, and parameters (read-only) -2. **Wrap** the call site in `completion_config` / `completionConfig` with a safe fallback — delegates the config creation to `configs-create` -3. **Tools** — move function-calling schemas into LaunchDarkly — delegates to `configs-tools` +2. **Wrap** the call site in `completion_config` / `completionConfig` with a safe fallback — delegates the config creation to `create` +3. **Tools** — move function-calling schemas into LaunchDarkly — delegates to `tools` 4. **Tracking** — wire `track_duration`, `track_tokens`, `track_success`/`track_error`, optional `track_feedback` — inline, with a reference doc covering every SDK method in Python and Node side by side -5. **Evals** — attach judges for LLM-as-a-judge scoring — delegates to `configs-online-evals` +5. **Evals** — attach judges for LLM-as-a-judge scoring — delegates to `online-evals` ## Installation (Local) -Copy `skills/ai-configs/configs-migrate/` into your agent client's skills path. +Copy `skills/agentcontrol/migrate/` into your agent client's skills path. ## Prerequisites @@ -41,7 +41,7 @@ Wire up the AI tracker and attach accuracy + relevance judges to our existing co ## Structure ``` -configs-migrate/ +migrate/ ├── SKILL.md ├── README.md └── references/ @@ -55,11 +55,11 @@ configs-migrate/ ## Related -- [AI Config Create](../configs-create/): Delegated to by Stage 2 (wrap) -- [AI Config Tools](../configs-tools/): Delegated to by Stage 3 (tools) -- [AI Config Online Evals](../configs-online-evals/): Delegated to by Stage 5 (evals) -- [AI Config Variations](../configs-variations/): Next step after migration for A/B testing -- [AI Config Targeting](../configs-targeting/): Next step after migration for rollout control +- [AI Config Create](../create/): Delegated to by Stage 2 (wrap) +- [AI Config Tools](../tools/): Delegated to by Stage 3 (tools) +- [AI Config Online Evals](../online-evals/): Delegated to by Stage 5 (evals) +- [AI Config Variations](../variations/): Next step after migration for A/B testing +- [AI Config Targeting](../targeting/): Next step after migration for rollout control - [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) ## License diff --git a/skills/ai-configs/configs-migrate/SKILL.md b/skills/agentcontrol/migrate/SKILL.md similarity index 87% rename from skills/ai-configs/configs-migrate/SKILL.md rename to skills/agentcontrol/migrate/SKILL.md index 5a64b4d..5add0c3 100644 --- a/skills/ai-configs/configs-migrate/SKILL.md +++ b/skills/agentcontrol/migrate/SKILL.md @@ -1,5 +1,5 @@ --- -name: configs-migrate +name: migrate description: "Migrate an application with hardcoded LLM prompts to a full LaunchDarkly AI Configs implementation in five stages: audit the code, wrap the call, move the tools, add tracking, attach evaluators. Use when the user wants to externalize model/prompt configuration, move from direct provider calls (OpenAI, Anthropic, Bedrock, Gemini, Strands) to a managed AI Config, or stage a full hardcoded-to-LaunchDarkly migration." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server @@ -22,7 +22,7 @@ You're using a skill that will guide you through migrating an application from h > > 1. **Tracker in the wrong scope.** For an agent with a loop, mint `create_tracker()` once per user turn in a `setup_run` entry node — not inside `call_model`. Per-iteration factory calls produce N `runId`s and trip the at-most-once guards. See [agent-mode-frameworks.md § Custom `StateGraph`](references/agent-mode-frameworks.md). > 2. **`load_chat_model` wrapper reuse.** Templates like `langchain-ai/react-agent` ship a `load_chat_model(f"{provider}/{name}")` helper that wraps `init_chat_model(...)` and silently drops every variation parameter. **Delete it** (don't just avoid using it) and replace call sites with `create_langchain_model(ai_config)`. -> 3. **Fallthrough not flipped after `/configs-create`.** A freshly-created AI Config's fallthrough points at an auto-generated disabled variation, so the SDK returns `enabled=False` until `/configs-targeting` runs. Flip it before Stage 2 verification. +> 3. **Fallthrough not flipped after `/create`.** A freshly-created AI Config's fallthrough points at an auto-generated disabled variation, so the SDK returns `enabled=False` until `/targeting` runs. Flip it before Stage 2 verification. ## Coverage — which shapes are well-trodden vs require extrapolation @@ -30,16 +30,16 @@ The skill is optimized for Python and Node.js / TypeScript; other languages are | Shape | Python | Node.js | Reference | |-------|--------|---------|-----------| -| One-shot completion (direct OpenAI / Anthropic / Bedrock / Gemini call) | ✅ Worked example | ✅ Worked example | [before-after-examples.md](references/before-after-examples.md), per-provider docs in `configs-ai-metrics/references/` | -| Chat loop via managed runner (`ManagedModel`) | ✅ Tier 1 pattern | ✅ Tier 1 pattern | [configs-ai-metrics SKILL.md](../configs-ai-metrics/SKILL.md) | -| LangChain single-call | ✅ Worked example | ✅ Worked example | [langchain-tracking.md](../configs-ai-metrics/references/langchain-tracking.md) | +| One-shot completion (direct OpenAI / Anthropic / Bedrock / Gemini call) | ✅ Worked example | ✅ Worked example | [before-after-examples.md](references/before-after-examples.md), per-provider docs in `ai-metrics/references/` | +| Chat loop via managed runner (`ManagedModel`) | ✅ Tier 1 pattern | ✅ Tier 1 pattern | [ai-metrics SKILL.md](../ai-metrics/SKILL.md) | +| LangChain single-call | ✅ Worked example | ✅ Worked example | [langchain-tracking.md](../ai-metrics/references/langchain-tracking.md) | | LangGraph prebuilt agent (Python `langchain.agents.create_agent`, Node `createReactAgent`) | ✅ Worked example | ✅ Worked example | [agent-mode-frameworks.md § LangGraph](references/agent-mode-frameworks.md) | | LangGraph custom `StateGraph` with run-scoped tracker (setup_run + call_model + finalize) | ✅ Deep worked example | ⚠️ Mentioned — translate from Python | [agent-mode-frameworks.md § Custom `StateGraph`](references/agent-mode-frameworks.md) | | CrewAI `Agent` | ✅ Worked example | — (not a Node framework) | [agent-mode-frameworks.md § CrewAI](references/agent-mode-frameworks.md) | | Strands `Agent` | ✅ Worked example | ⚠️ BedrockModel + OpenAIModel only (no Anthropic) | [agent-mode-frameworks.md § Strands](references/agent-mode-frameworks.md) | | Custom ReAct loop (hand-rolled, any framework or none) | ✅ Worked example | ⚠️ Apply framework-agnostic invariants; translate from Python | [agent-mode-frameworks.md § Custom ReAct loop](references/agent-mode-frameworks.md) | -| Vercel AI SDK (`generateText` / `streamText`) | — (not a Python framework) | ⚠️ Provider package exists; no worked example in skill | `configs-ai-metrics` provider-package matrix | -| Streaming (SSE / WebSocket) | ⚠️ Delegated to `configs-ai-metrics` streaming doc | ⚠️ Same — use `trackStreamMetricsOf` + manual TTFT | [streaming-tracking.md](../configs-ai-metrics/references/streaming-tracking.md) | +| Vercel AI SDK (`generateText` / `streamText`) | — (not a Python framework) | ⚠️ Provider package exists; no worked example in skill | `ai-metrics` provider-package matrix | +| Streaming (SSE / WebSocket) | ⚠️ Delegated to `ai-metrics` streaming doc | ⚠️ Same — use `trackStreamMetricsOf` + manual TTFT | [streaming-tracking.md](../ai-metrics/references/streaming-tracking.md) | | Multi-agent graph (supervisor + workers) | ⚠️ Out of main scope; see reference | ⚠️ Out of main scope; see reference | [agent-graph-reference.md](references/agent-graph-reference.md) | | Non-LangGraph agent frameworks (Pydantic AI, DSPy, AutoGen, Haystack, LlamaIndex agents, Semantic Kernel) | ⚠️ Apply the three invariants; no framework-specific example | ⚠️ Same | [agent-mode-frameworks.md § Framework-agnostic invariants](references/agent-mode-frameworks.md) | | Go, Ruby, .NET | ℹ️ Install commands only | ℹ️ Install commands only | [phase-1-analysis-checklist.md § SDK routing table](references/phase-1-analysis-checklist.md) | @@ -67,11 +67,11 @@ If a CHANGELOG entry post-dates this skill and changes an API you're about to us **Hand-off model.** This skill does **not** auto-invoke other skills. At each stage that needs a LaunchDarkly write, this skill prepares the inputs (config key, mode, model, prompt, tool schemas, judge keys) and then **tells the user to run the next slash-command themselves**. After the user finishes that sibling skill, return to the next step here. Treat the "Delegate" lines below as next-step instructions, not auto-handoffs. **Sibling skills the user runs at each stage:** -- `configs-projects` — pre-Stage 2, only if no project exists yet -- `configs-create` — Stage 2 (creates the AI Config and first variation) -- `configs-tools` — Stage 3 (creates tool definitions and attaches them) -- `configs-targeting` — between Stage 2 and Stage 4 (promotes the new variation to fallthrough so the SDK actually serves it) -- `configs-online-evals` — Stage 5 (attaches judges, creates custom judges) +- `projects` — pre-Stage 2, only if no project exists yet +- `create` — Stage 2 (creates the AI Config and first variation) +- `tools` — Stage 3 (creates tool definitions and attaches them) +- `targeting` — between Stage 2 and Stage 4 (promotes the new variation to fallthrough so the SDK actually serves it) +- `online-evals` — Stage 5 (attaches judges, creates custom judges) ## Core Principles @@ -79,7 +79,7 @@ If a CHANGELOG entry post-dates this skill and changes an API you're about to us 2. **Replace config, not business logic.** The SDK call is a drop-in for the place where the model, parameters, and prompt are *defined* — not for the provider call itself. OpenAI/Anthropic/Bedrock calls stay where they are. 3. **Fallback mirrors current behavior.** The fallback passed to `completion_config` / `agent_config` must preserve the hardcoded values you removed, so the app is unchanged if LaunchDarkly is unreachable. 4. **Stages are ordered.** Wrap before you add tools. Add tools before you track. Track before you add evals. Skipping ahead produces configs without traffic, metrics without context, and judges with nothing to score. -5. **Hand off to focused skills, manually.** Each stage that needs a LaunchDarkly write tells the user to run a sibling slash-command (`/configs-create`, `/configs-tools`, `/configs-targeting`, `/configs-online-evals`) and waits for them to come back. This skill does **not** auto-invoke other skills. +5. **Hand off to focused skills, manually.** Each stage that needs a LaunchDarkly write tells the user to run a sibling slash-command (`/create`, `/tools`, `/targeting`, `/online-evals`) and waits for them to come back. This skill does **not** auto-invoke other skills. ## Workflow @@ -160,7 +160,7 @@ This is the first stage that writes code. It has nine sub-steps. - OpenAI: `launchdarkly-server-sdk-ai-openai>=0.4.0` (Python) / `@launchdarkly/server-sdk-ai-openai@^0.5.5` (Node) - LangChain / LangGraph: `launchdarkly-server-sdk-ai-langchain>=0.5.0` (Python) / `@launchdarkly/server-sdk-ai-langchain@^0.5.5` (Node) - Vercel AI SDK (Node only): `@launchdarkly/server-sdk-ai-vercel@^0.5.5` - - Anthropic, Gemini, Bedrock — no provider package published; use Tier-3 custom extractor (see `configs-ai-metrics`) + - Anthropic, Gemini, Bedrock — no provider package published; use Tier-3 custom extractor (see `ai-metrics`) 3. **Initialize `LDAIClient` once at startup.** Reuse any existing `LDClient` — do not create a second base client. Place the initialization in the same module that owns existing app config. @@ -207,11 +207,11 @@ This is the first stage that writes code. It has nine sub-steps. const aiClient = initAi(ldClient); ``` -4. **Hand off to `configs-create`.** Print the extracted model, prompt/instructions, parameters, and mode from the Stage 1 manifest, then tell the user: *"Run `/configs-create` with these inputs, then come back here."* Supply the config key you want the code to call (e.g. `chat-assistant`). Do not attempt to auto-invoke the sibling skill — wait for the user to finish it before continuing. +4. **Hand off to `create`.** Print the extracted model, prompt/instructions, parameters, and mode from the Stage 1 manifest, then tell the user: *"Run `/create` with these inputs, then come back here."* Supply the config key you want the code to call (e.g. `chat-assistant`). Do not attempt to auto-invoke the sibling skill — wait for the user to finish it before continuing. - **After `configs-create` finishes, the user must also run `/configs-targeting` to promote the new variation to fallthrough.** A freshly created variation returns `enabled=False` to every consumer until targeting is updated. Skip this and Stage 2 verification (sub-step 9 below) will silently take the fallback path on every request. + **After `create` finishes, the user must also run `/targeting` to promote the new variation to fallthrough.** A freshly created variation returns `enabled=False` to every consumer until targeting is updated. Skip this and Stage 2 verification (sub-step 9 below) will silently take the fallback path on every request. -5. **Rewrite template placeholders to Mustache syntax.** If the hardcoded prompt interpolates runtime values with Python `.format()`, f-strings, JS template literals, or any other non-Mustache syntax (e.g. `{system_time}`, `${userName}`, `%(topic)s`), rewrite every placeholder to `{{ variable }}` Mustache form. Do this in **both** the file you're about to send to `/configs-create` *and* the fallback string you'll write in sub-step 6. The AI SDK interpolates variables through a Mustache renderer on the LD-served path *and* the fallback path using the fourth-argument `variables` dict to `completion_config(...)` / `completionConfig(...)`. Leaving a Python-style `{system_time}` literal in the fallback ships a silent regression when LaunchDarkly is unreachable — the renderer won't match the single-brace form and the literal `{system_time}` goes to the provider as part of the prompt. +5. **Rewrite template placeholders to Mustache syntax.** If the hardcoded prompt interpolates runtime values with Python `.format()`, f-strings, JS template literals, or any other non-Mustache syntax (e.g. `{system_time}`, `${userName}`, `%(topic)s`), rewrite every placeholder to `{{ variable }}` Mustache form. Do this in **both** the file you're about to send to `/create` *and* the fallback string you'll write in sub-step 6. The AI SDK interpolates variables through a Mustache renderer on the LD-served path *and* the fallback path using the fourth-argument `variables` dict to `completion_config(...)` / `completionConfig(...)`. Leaving a Python-style `{system_time}` literal in the fallback ships a silent regression when LaunchDarkly is unreachable — the renderer won't match the single-brace form and the literal `{system_time}` goes to the provider as part of the prompt. **Before:** ```python @@ -318,7 +318,7 @@ This is the first stage that writes code. It has nine sub-steps. 9. **Verify.** Run the app with a valid `LD_SDK_KEY`; confirm the call succeeds and the response matches pre-migration output. Then temporarily set `LD_SDK_KEY=sdk-invalid` (or unset it) and confirm the fallback path runs without error. Both paths must work before moving to Stage 3. -Delegate: **`configs-create`** (sub-step 4). +Delegate: **`create`** (sub-step 4). ### Step 3: Move tools into the config (Stage 3) @@ -338,7 +338,7 @@ Skip this step if the audited app has no function calling / tools. Otherwise: For LangChain/LangGraph tools defined with `@tool`, extract the schema via `tool.args_schema.model_json_schema()` (or the equivalent Pydantic `model_json_schema()` call). For plain async callables used as tools (common in custom StateGraph shapes), LangChain infers the schema from the function signature at bind time — extract it via `StructuredTool.from_function(fn).args_schema.model_json_schema()`. Do not hand-write the schema. -2. **Hand off to `configs-tools`.** Print the extracted tool names, descriptions, and schemas, then tell the user: *"Run `/configs-tools` with these tools and the variation key, then come back here."* The sibling skill creates tool definitions (`create-ai-tool`) and attaches them to the variation (`update-ai-config-variation`). Wait for the user to finish before proceeding to sub-step 3. Do not auto-invoke. +2. **Hand off to `tools`.** Print the extracted tool names, descriptions, and schemas, then tell the user: *"Run `/tools` with these tools and the variation key, then come back here."* The sibling skill creates tool definitions (`create-ai-tool`) and attaches them to the variation (`update-ai-config-variation`). Wait for the user to finish before proceeding to sub-step 3. Do not auto-invoke. 3. **Replace the hardcoded tools array at the call site** with a read from `config.tools` (or the SDK equivalent for your language). Load the actual implementation functions dynamically from the tool names — see [agent-mode-frameworks.md](references/agent-mode-frameworks.md) for the dynamic-tool-factory pattern from the devrel agents tutorial. @@ -346,13 +346,13 @@ Skip this step if the audited app has no function calling / tools. Otherwise: 4. **Verify.** Run the app; confirm the tool flows still execute correctly. `get-ai-config` (via the delegate) confirms the tools are attached server-side. -Delegate: **`configs-tools`** (sub-step 2). +Delegate: **`tools`** (sub-step 2). ### Step 4: Instrument the tracker (Stage 4) -Delegate: **`configs-ai-metrics`** wires the per-request `tracker.track_*` calls (duration, tokens, success/error, feedback) around the provider call. Use **`configs-custom-metrics`** alongside it if the app needs business metrics beyond the built-in AI ones. Note: do not confuse this with `launchdarkly-metric-instrument`, which is for `ldClient.track()` feature metrics — a different API. See [sdk-ai-tracker-patterns.md](references/sdk-ai-tracker-patterns.md) for the full per-method Python + Node matrix that the delegate skill draws on. +Delegate: **`ai-metrics`** wires the per-request `tracker.track_*` calls (duration, tokens, success/error, feedback) around the provider call. Use **`custom-metrics`** alongside it if the app needs business metrics beyond the built-in AI ones. Note: do not confuse this with `launchdarkly-metric-instrument`, which is for `ldClient.track()` feature metrics — a different API. See [sdk-ai-tracker-patterns.md](references/sdk-ai-tracker-patterns.md) for the full per-method Python + Node matrix that the delegate skill draws on. -Hand off: print the AI Config key, variation key, provider, and whether the call is streaming, then tell the user: *"Run `/configs-ai-metrics` with these inputs, then come back here."* Do not auto-invoke. Return here for sub-step 5 (verify) once they're done. +Hand off: print the AI Config key, variation key, provider, and whether the call is streaming, then tell the user: *"Run `/ai-metrics` with these inputs, then come back here."* Do not auto-invoke. Return here for sub-step 5 (verify) once they're done. 1. **Create the tracker.** Obtain a per-execution tracker via the factory on the config returned in Stage 2: `tracker = config.create_tracker()` (Python) or `const tracker = aiConfig.createTracker();` (Node). Call the factory **once per user turn** and reuse the returned `tracker` for every tracking call in that turn — each call mints a fresh `runId` that tags every event emitted from the turn so they can be correlated via exported events or downstream queries. (The Monitoring tab aggregates today; run-level grouping is a downstream concern — but the `runId` is also what the SDK's at-most-once guards are keyed on, so minting a new one mid-turn breaks the guard semantics regardless of where the events end up.) @@ -404,7 +404,7 @@ Hand off: print the AI Config key, variation key, provider, and whether the call ); ``` - For Anthropic direct, Bedrock (no provider package), Gemini, and custom HTTP, write a small extractor returning `LDAIMetrics` — see the delegate skill's [anthropic-tracking.md](../configs-ai-metrics/references/anthropic-tracking.md), [bedrock-tracking.md](../configs-ai-metrics/references/bedrock-tracking.md), and [gemini-tracking.md](../configs-ai-metrics/references/gemini-tracking.md). LangChain single-node and LangGraph go through the `launchdarkly-server-sdk-ai-langchain` / `@launchdarkly/server-sdk-ai-langchain` provider package. Build the model with `create_langchain_model(config)` (Python) / `createLangChainModel(config)` (Node) — both forward all variation parameters — and track with `get_ai_metrics_from_response` / `getAIMetricsFromResponse`. See [langchain-tracking.md](../configs-ai-metrics/references/langchain-tracking.md). + For Anthropic direct, Bedrock (no provider package), Gemini, and custom HTTP, write a small extractor returning `LDAIMetrics` — see the delegate skill's [anthropic-tracking.md](../ai-metrics/references/anthropic-tracking.md), [bedrock-tracking.md](../ai-metrics/references/bedrock-tracking.md), and [gemini-tracking.md](../ai-metrics/references/gemini-tracking.md). LangChain single-node and LangGraph go through the `launchdarkly-server-sdk-ai-langchain` / `@launchdarkly/server-sdk-ai-langchain` provider package. Build the model with `create_langchain_model(config)` (Python) / `createLangChainModel(config)` (Node) — both forward all variation parameters — and track with `get_ai_metrics_from_response` / `getAIMetricsFromResponse`. See [langchain-tracking.md](../ai-metrics/references/langchain-tracking.md). 4. **Wire feedback tracking if the app has thumbs-up/down UI.** Both SDKs expose `trackFeedback` with a `{kind}` argument. @@ -442,11 +442,11 @@ Hand off: print the AI Config key, variation key, provider, and whether the call - Run the `default` variation (or whichever variation mirrors the pre-migration hardcoded behavior) against the dataset first — this is the baseline. - Clone it into a second variation pointing at a **different model family** (e.g., if the baseline is `anthropic/claude-sonnet-4-5`, clone to `openai/gpt-4o` or `openai/gpt-4o-mini`). The comparison is most informative across families, not across siblings. - Attach the built-in **Accuracy** judge with a pass threshold of **0.85**, and run both variations against the same dataset. - - Promote the winner to fallthrough via `/configs-targeting` only if it beats the baseline on Accuracy and does not regress on Relevance or Toxicity. + - Promote the winner to fallthrough via `/targeting` only if it beats the baseline on Accuracy and does not regress on Relevance or Toxicity. Write this shape into the project's `datasets/README.md` (or equivalent) so the comparison pattern is reproducible after the migration ships. -3. **Hand off to `configs-online-evals`** — only for UI-attached judges (completion mode) or to create custom judge AI Configs that will be referenced by the programmatic path. Tell the user: *"Run `/configs-online-evals` with these inputs, then come back here."* Do not auto-invoke. Pass: +3. **Hand off to `online-evals`** — only for UI-attached judges (completion mode) or to create custom judge AI Configs that will be referenced by the programmatic path. Tell the user: *"Run `/online-evals` with these inputs, then come back here."* Do not auto-invoke. Pass: - The parent AI Config key and variation key - A list of built-in judges (Accuracy, Relevance, Toxicity) or custom judge keys to create/attach - Target environment @@ -480,14 +480,14 @@ Hand off: print the AI Config key, variation key, provider, and whether the call - **`sampling_rate` is a parameter on `evaluate()`**, not on `create_judge`. It defaults to `1.0` (evaluate every call). For live paths, pass something lower (0.1–0.25) to control cost. - **`evaluate()` returns a `JudgeResult`** (never `None`). Check `result.sampled` to know whether the evaluation actually ran, and call `track_judge_result(result)`. Node uses `trackJudgeResult(result)` and `LDJudgeResult` with the same `sampled` field. - **Ask the user which judge AI Config key to use.** LaunchDarkly ships three built-in judges — Accuracy, Relevance, Toxicity — but the actual AI Config **keys** for the built-ins are not canonical SDK constants and aren't documented. Have the user open **AI Configs > Library** in the LD UI and copy the key of the judge they want to reference, or create a custom judge AI Config via `configs-create` first. + **Ask the user which judge AI Config key to use.** LaunchDarkly ships three built-in judges — Accuracy, Relevance, Toxicity — but the actual AI Config **keys** for the built-ins are not canonical SDK constants and aren't documented. Have the user open **AI Configs > Library** in the LD UI and copy the key of the judge they want to reference, or create a custom judge AI Config via `create` first. 5. **Verify.** - **UI-attached auto judges:** trigger a request in staging, open the Monitoring tab → "Evaluator metrics" dropdown. Scores appear within 1–2 minutes at the configured sampling rate. - **Programmatic direct-judge:** hit the wrapped endpoint and confirm `track_judge_result` lands on the parent config's Monitoring tab. - **Offline eval:** run the dataset through the LD Playground, compare baseline vs new-variation scores side by side. No runtime wiring required. -Delegate: **`configs-online-evals`** (sub-step 3, optional — only for UI-attached judges or custom-judge creation; offline eval doesn't delegate). +Delegate: **`online-evals`** (sub-step 3, optional — only for UI-attached judges or custom-judge creation; offline eval doesn't delegate). ## Edge Cases @@ -498,9 +498,9 @@ Delegate: **`configs-online-evals`** (sub-step 3, optional — only for UI-attac | Retry wrapper around the provider call | The tracker is minted once at the top of the user turn; the retry loop is inside that scope. Every retry attempt shares the same `runId`. Tracker calls (`track_duration` / `track_tokens` / `track_success` / `track_error`) live *outside* the retry body — one call at the end of the turn, on the success path or the final-failure path | | App has no tools — Stage 3 skipped | Move directly from Stage 2 verification to Stage 4 (tracking) | | Mode mismatch: user said agent, audit shows one-shot chat | Choose completion mode unless the app uses a LangGraph prebuilt agent (`langchain.agents.create_agent` in Python or `createReactAgent` in Node), CrewAI `Agent`, Strands `Agent`, or a similar goal-driven framework | -| App uses Strands Agents (Python) | Agent mode. Build a `create_strands_model` dispatcher keyed on `agent_config.provider.name` that returns `AnthropicModel(model_id=..., max_tokens=...)` or `OpenAIModel(model_id=..., params=...)`. Drop `parameters.tools` before passing params to the model class — Strands receives tools via `Agent(tools=[...])`. Tracking is Tier 3: wrap `invoke_async` with `tracker.track_duration_of(...)` and record tokens from `result.metrics.accumulated_usage`. See [agent-mode-frameworks.md § Strands Agent](references/agent-mode-frameworks.md) and [strands-tracking.md](../configs-ai-metrics/references/strands-tracking.md) | +| App uses Strands Agents (Python) | Agent mode. Build a `create_strands_model` dispatcher keyed on `agent_config.provider.name` that returns `AnthropicModel(model_id=..., max_tokens=...)` or `OpenAIModel(model_id=..., params=...)`. Drop `parameters.tools` before passing params to the model class — Strands receives tools via `Agent(tools=[...])`. Tracking is Tier 3: wrap `invoke_async` with `tracker.track_duration_of(...)` and record tokens from `result.metrics.accumulated_usage`. See [agent-mode-frameworks.md § Strands Agent](references/agent-mode-frameworks.md) and [strands-tracking.md](../ai-metrics/references/strands-tracking.md) | | Strands app on TypeScript | TS SDK ships `BedrockModel` and `OpenAIModel` only — cannot serve Anthropic-backed variations. Use the Python SDK if multi-provider variations are required | -| TypeScript app using Anthropic SDK | No `trackAnthropicMetrics` helper exists. Use Tier 3: `trackMetricsOf` with a small custom extractor that reads `response.usage.input_tokens` / `response.usage.output_tokens` and returns `LDAIMetrics`. See [anthropic-tracking.md](../configs-ai-metrics/references/anthropic-tracking.md) in the `configs-ai-metrics` skill for the exact extractor | +| TypeScript app using Anthropic SDK | No `trackAnthropicMetrics` helper exists. Use Tier 3: `trackMetricsOf` with a small custom extractor that reads `response.usage.input_tokens` / `response.usage.output_tokens` and returns `LDAIMetrics`. See [anthropic-tracking.md](../ai-metrics/references/anthropic-tracking.md) in the `ai-metrics` skill for the exact extractor | | Fallback would silently crash because `LD_SDK_KEY` is missing | Log a startup warning; proceed with the fallback. Never raise at import time | | Multi-agent graph (supervisor + workers) | Stop after migrating a single agent. Agent Graph Definitions are available in **both** SDKs — Python via `launchdarkly-server-sdk-ai.agent_graph` and Node via the graph API in `@launchdarkly/server-sdk-ai`. Read [agent-graph-reference.md](references/agent-graph-reference.md) for the graph-level migration path — it is deliberately out of this skill's main scope | | Single-agent (ReAct, tool loop) + agent mode | Default to offline eval via the LD Playground + Datasets for Stage 5. UI-attached judges are completion-only today, and programmatic direct-judge adds per-call cost that is usually not worth it until after the migration is live and stable. Point at the [Offline Evals guide](https://docs.launchdarkly.com/guides/ai-configs/offline-evaluations) | @@ -525,17 +525,17 @@ These are ordered by how likely they are to show up as a first-run failure. The - **If the repo already contains a `load_chat_model(f"{provider}/{name}")` helper, delete it — don't just avoid using it.** This exact shape ships with `langchain-ai/react-agent` and is copied into dozens of derivative repos; look for `utils.load_chat_model`, `utils.build_model`, or any one-arg `init_chat_model` wrapper that splits a `"provider/model"` string. Re-using it is the first-run failure mode: every variation parameter (temperature, max_tokens, top_p, stop sequences) silently drops on the floor because `init_chat_model` only receives the name and provider. `create_langchain_model(ai_config)` is a one-for-one replacement that forwards the whole `model.parameters` dict. Replace every call site, then delete the wrapper file-side so the next reader can't reach for it. - **Same rule applies to hand-rolled `resolve_tools` / `TOOL_REGISTRY` / `ALL_TOOLS` helpers.** If the template already has a `resolve_tools(tool_keys)` or an `ALL_TOOLS` module-level list, import `build_structured_tools` from `ldai_langchain.langchain_helper` and delete the hand-rolled version. `build_structured_tools(ai_config, TOOL_REGISTRY_DICT)` reads `ai_config.model.parameters.tools` and wraps the matching callables as LangChain `StructuredTool`s with the LD tool key as the `StructuredTool.name` — so `ToolNode` lookup works without a second mapping. Don't leave both in the repo. -- Don't put app-scoped knobs directly in `model.parameters`. `create_langchain_model` forwards every key in `parameters` to the provider SDK via `init_chat_model`, so a `max_search_results` / `retry_budget` / `feature_toggle` entry will crash the provider with an unexpected-keyword-argument error. The correct home is `model.custom`, which the provider helpers ignore and the app reads via `ai_config.model.get_custom("key")`. The MCP `update-ai-config-variation` tool does not currently expose top-level `custom`, so pick one of two paths: (a) PATCH the variation via the REST API to set `model.custom` directly, or (b) set it via MCP inside `parameters.custom` (as a nested dict) and use a defensive accessor that reads both locations. Full walk-through with code samples in [langchain-tracking.md § MCP caveat](../configs-ai-metrics/references/langchain-tracking.md). +- Don't put app-scoped knobs directly in `model.parameters`. `create_langchain_model` forwards every key in `parameters` to the provider SDK via `init_chat_model`, so a `max_search_results` / `retry_budget` / `feature_toggle` entry will crash the provider with an unexpected-keyword-argument error. The correct home is `model.custom`, which the provider helpers ignore and the app reads via `ai_config.model.get_custom("key")`. The MCP `update-ai-config-variation` tool does not currently expose top-level `custom`, so pick one of two paths: (a) PATCH the variation via the REST API to set `model.custom` directly, or (b) set it via MCP inside `parameters.custom` (as a nested dict) and use a defensive accessor that reads both locations. Full walk-through with code samples in [langchain-tracking.md § MCP caveat](../ai-metrics/references/langchain-tracking.md). - Don't re-encode tool schemas inside the fallback. When LaunchDarkly is unreachable the fallback should run without tools (or with whatever minimal provider-bound parameters the app needs to keep operating). Building a `_FALLBACK_TOOLS` array that duplicates the AI Config's tool schema re-introduces the hardcoded config the migration was supposed to move out of code. - Don't import `LaunchDarklyCallbackHandler` from `ldai.langchain` — neither the class nor the dotted module path exists. The Python LangChain helper package is `ldai_langchain` (top-level module, underscore). Use `create_langchain_model(config)` + `track_metrics_of_async(get_ai_metrics_from_response, lambda: llm.ainvoke(messages))` as the canonical pattern. ### Stage / handoff discipline - Don't skip Step 1 even when the user says "just wrap it." Without the audit, the fallback will drift from the hardcoded behavior. -- Don't delegate to `configs-create` before extracting the prompt and model — the delegate needs them as inputs. -- Don't try to attach tools during initial `setup-ai-config`. Tool attachment is a separate step owned by `configs-tools`. -- Don't claim you "delegated to `configs-create`" or any other sibling skill. This skill does not auto-invoke. At each handoff, print the inputs and tell the user to run the sibling slash-command, then wait. Anything else misleads the user about what just happened. -- Don't skip the `/configs-targeting` step between Stage 2 and Stage 4. A freshly created variation returns `enabled=False` until targeting promotes it to fallthrough — Stage 2 verification will silently take the fallback path on every request. +- Don't delegate to `create` before extracting the prompt and model — the delegate needs them as inputs. +- Don't try to attach tools during initial `setup-ai-config`. Tool attachment is a separate step owned by `tools`. +- Don't claim you "delegated to `create`" or any other sibling skill. This skill does not auto-invoke. At each handoff, print the inputs and tell the user to run the sibling slash-command, then wait. Anything else misleads the user about what just happened. +- Don't skip the `/targeting` step between Stage 2 and Stage 4. A freshly created variation returns `enabled=False` until targeting promotes it to fallthrough — Stage 2 verification will silently take the fallback path on every request. - Don't attempt a multi-agent graph migration in one pass. Migrate a single agent first; use [agent-graph-reference.md](references/agent-graph-reference.md) as the next-step read. ### Stage 5 evaluations @@ -554,12 +554,12 @@ These are ordered by how likely they are to show up as a first-run failure. The ## Related Skills -- `configs-create` — called by Stage 2 to create the config -- `configs-tools` — called by Stage 3 to create and attach tool definitions -- `configs-online-evals` — called by Stage 5 to attach judges -- `configs-variations` — add variations for A/B testing after migration is complete -- `configs-targeting` — roll out new variations to users after migration is complete -- `configs-update` — modify config properties as your app evolves +- `create` — called by Stage 2 to create the config +- `tools` — called by Stage 3 to create and attach tool definitions +- `online-evals` — called by Stage 5 to attach judges +- `variations` — add variations for A/B testing after migration is complete +- `targeting` — roll out new variations to users after migration is complete +- `update` — modify config properties as your app evolves - `launchdarkly-metric-instrument` — for `ldClient.track()` feature metrics (NOT for AI tracker calls) ## References diff --git a/skills/ai-configs/configs-migrate/references/agent-graph-reference.md b/skills/agentcontrol/migrate/references/agent-graph-reference.md similarity index 100% rename from skills/ai-configs/configs-migrate/references/agent-graph-reference.md rename to skills/agentcontrol/migrate/references/agent-graph-reference.md diff --git a/skills/ai-configs/configs-migrate/references/agent-mode-frameworks.md b/skills/agentcontrol/migrate/references/agent-mode-frameworks.md similarity index 98% rename from skills/ai-configs/configs-migrate/references/agent-mode-frameworks.md rename to skills/agentcontrol/migrate/references/agent-mode-frameworks.md index 859e191..86cf549 100644 --- a/skills/ai-configs/configs-migrate/references/agent-mode-frameworks.md +++ b/skills/agentcontrol/migrate/references/agent-mode-frameworks.md @@ -16,9 +16,9 @@ Completion mode is the default and covers direct provider calls (OpenAI, Anthrop Agent mode returns an `instructions` string. Completion mode returns a `messages` array. Both modes support tools, parameters, and the same tracker — the only difference is the input shape the SDK returns to you. -**Caveat:** judges cannot be attached to agent-mode variations via the LaunchDarkly UI. Agent mode evaluations must go through the programmatic judge API (`create_judge(...).evaluate(input, output)`). See `configs-online-evals` for the programmatic path. +**Caveat:** judges cannot be attached to agent-mode variations via the LaunchDarkly UI. Agent mode evaluations must go through the programmatic judge API (`create_judge(...).evaluate(input, output)`). See `online-evals` for the programmatic path. -**Model construction for LangChain / LangGraph.** When the framework runs on top of LangChain (which includes LangGraph's prebuilt agent and most custom graphs), build the chat model with `create_langchain_model(ai_config)` (Python) or `createLangChainModel(aiConfig)` (Node). These helpers forward every variation parameter (`temperature`, `max_tokens`, `top_p`, …) and handle LaunchDarkly→LangChain provider-name mapping internally. Do not hand-roll `init_chat_model(model=..., model_provider=...)` — it silently drops every variation parameter. See [langchain-tracking.md](../../configs-ai-metrics/references/langchain-tracking.md) for the canonical single-model and LangGraph patterns, including the SDK helpers `sum_token_usage_from_messages` / `get_tool_calls_from_response` (Python, `ldai_langchain`) used inside the `track_metrics_of_async` / `trackMetricsOf` extractor. +**Model construction for LangChain / LangGraph.** When the framework runs on top of LangChain (which includes LangGraph's prebuilt agent and most custom graphs), build the chat model with `create_langchain_model(ai_config)` (Python) or `createLangChainModel(aiConfig)` (Node). These helpers forward every variation parameter (`temperature`, `max_tokens`, `top_p`, …) and handle LaunchDarkly→LangChain provider-name mapping internally. Do not hand-roll `init_chat_model(model=..., model_provider=...)` — it silently drops every variation parameter. See [langchain-tracking.md](../../ai-metrics/references/langchain-tracking.md) for the canonical single-model and LangGraph patterns, including the SDK helpers `sum_token_usage_from_messages` / `get_tool_calls_from_response` (Python, `ldai_langchain`) used inside the `track_metrics_of_async` / `trackMetricsOf` extractor. ## Framework-agnostic invariants for the run-scoped pattern @@ -291,7 +291,7 @@ async def run_turn(ai_client, user_id: str, user_input: str): **Key points:** - `system_prompt=agent_config.instructions` — the instructions string replaces the hardcoded system prompt. - `create_strands_model(agent_config)` is the provider-dispatch seam. Add a branch per provider the variation can serve. -- The tracker is Tier 3: `tracker.track_duration_of(...)` + an explicit `track_tokens` call fed by `track_strands_metrics`. See [strands-tracking.md](../../configs-ai-metrics/references/strands-tracking.md) for the single-call `track_metrics_of_async` variant and the per-field breakdown of `accumulated_usage`. +- The tracker is Tier 3: `tracker.track_duration_of(...)` + an explicit `track_tokens` call fed by `track_strands_metrics`. See [strands-tracking.md](../../ai-metrics/references/strands-tracking.md) for the single-call `track_metrics_of_async` variant and the per-field breakdown of `accumulated_usage`. - Always `ldclient.get().flush()` before process exit in short-lived scripts — trailing events can otherwise be lost. **TypeScript caveat.** The Strands TypeScript SDK ships `BedrockModel` and `OpenAIModel` only — it cannot run Anthropic-backed variations. If the app needs to serve both OpenAI and Anthropic from a single AI Config, use the Python SDK. @@ -799,12 +799,12 @@ agent = create_agent( **What this gives you:** - Toggle a tool on/off by editing the AI Config in LaunchDarkly — no redeploy needed to remove a tool from production -- Roll out a new tool to 5% of users by editing targeting rules (combined with `configs-targeting`) +- Roll out a new tool to 5% of users by editing targeting rules (combined with `targeting`) - Keep the actual tool implementation code in the repo; only metadata lives in LaunchDarkly ### Extracting schemas from existing hardcoded tools -If your tools are defined with LangChain `@tool` or Pydantic `BaseModel` schemas, extract the JSON schema programmatically to pass to `configs-tools` during Stage 3: +If your tools are defined with LangChain `@tool` or Pydantic `BaseModel` schemas, extract the JSON schema programmatically to pass to `tools` during Stage 3: ```python from my_tools import search_kb # a @tool-decorated function @@ -813,7 +813,7 @@ schema = search_kb.args_schema.model_json_schema() # schema is a dict ready to pass as the tool's parameters field ``` -Do not hand-write the schema — LangChain already generated it from the function signature, and Pydantic will keep it in sync. The `configs-tools` delegate accepts raw JSON Schema for the parameters field. +Do not hand-write the schema — LangChain already generated it from the function signature, and Pydantic will keep it in sync. The `tools` delegate accepts raw JSON Schema for the parameters field. ### Dynamic schemas from LaunchDarkly diff --git a/skills/ai-configs/configs-migrate/references/before-after-examples.md b/skills/agentcontrol/migrate/references/before-after-examples.md similarity index 100% rename from skills/ai-configs/configs-migrate/references/before-after-examples.md rename to skills/agentcontrol/migrate/references/before-after-examples.md diff --git a/skills/ai-configs/configs-migrate/references/fallback-defaults-pattern.md b/skills/agentcontrol/migrate/references/fallback-defaults-pattern.md similarity index 99% rename from skills/ai-configs/configs-migrate/references/fallback-defaults-pattern.md rename to skills/agentcontrol/migrate/references/fallback-defaults-pattern.md index 4a02575..a2bba78 100644 --- a/skills/ai-configs/configs-migrate/references/fallback-defaults-pattern.md +++ b/skills/agentcontrol/migrate/references/fallback-defaults-pattern.md @@ -305,7 +305,7 @@ config = ai_client.agent_config( The SDK runs both paths through the same Mustache renderer. Leaving a Python-style `{var}` literal in the fallback ships a silent regression: LaunchDarkly serves correctly-interpolated output; the fallback ships the unrendered literal, or (worse) a value frozen at import time. -Same rule applies to JS template literals and any other non-Mustache scheme — rewrite to `{{ variable }}` before handing off to `/configs-create`. +Same rule applies to JS template literals and any other non-Mustache scheme — rewrite to `{{ variable }}` before handing off to `/create`. ## Critical rules (apply to all three patterns) diff --git a/skills/ai-configs/configs-migrate/references/phase-1-analysis-checklist.md b/skills/agentcontrol/migrate/references/phase-1-analysis-checklist.md similarity index 97% rename from skills/ai-configs/configs-migrate/references/phase-1-analysis-checklist.md rename to skills/agentcontrol/migrate/references/phase-1-analysis-checklist.md index 8f86d9d..be0a6d4 100644 --- a/skills/ai-configs/configs-migrate/references/phase-1-analysis-checklist.md +++ b/skills/agentcontrol/migrate/references/phase-1-analysis-checklist.md @@ -151,7 +151,7 @@ Feeds into Stage 2 (install + wrap). Quoted from the `ai-configs-relaunch-guides | LangChain / LangGraph | `@launchdarkly/server-sdk-ai-langchain` | `createLangChainModel(config)` (forwards all variation parameters and handles provider-name mapping) + `getAIMetricsFromResponse` with `trackMetricsOf` | | Vercel AI SDK | `@launchdarkly/server-sdk-ai-vercel` | `getAIMetricsFromResponse` + `trackMetricsOf`, or `VercelRunnerFactory.createVercelModel(aiConfig)` for a managed runner | -Python currently ships helper packages for OpenAI (`ldai_openai`) and LangChain (`ldai_langchain`). The LangChain Python package exposes `create_langchain_model(config)` (builds a LangChain chat model from the AI Config, forwarding every variation parameter and mapping LD provider names to LangChain equivalents), `convert_messages_to_langchain`, and `get_ai_metrics_from_response` — the same package covers LangGraph. Use `create_langchain_model(config)` + `track_metrics_of_async(get_ai_metrics_from_response, lambda: llm.ainvoke(messages))` as the canonical single-call pattern. See [langchain-tracking.md](../../configs-ai-metrics/references/langchain-tracking.md) for both LangChain and LangGraph patterns and [sdk-ai-tracker-patterns.md](sdk-ai-tracker-patterns.md) for the full tracker-method matrix. +Python currently ships helper packages for OpenAI (`ldai_openai`) and LangChain (`ldai_langchain`). The LangChain Python package exposes `create_langchain_model(config)` (builds a LangChain chat model from the AI Config, forwarding every variation parameter and mapping LD provider names to LangChain equivalents), `convert_messages_to_langchain`, and `get_ai_metrics_from_response` — the same package covers LangGraph. Use `create_langchain_model(config)` + `track_metrics_of_async(get_ai_metrics_from_response, lambda: llm.ainvoke(messages))` as the canonical single-call pattern. See [langchain-tracking.md](../../ai-metrics/references/langchain-tracking.md) for both LangChain and LangGraph patterns and [sdk-ai-tracker-patterns.md](sdk-ai-tracker-patterns.md) for the full tracker-method matrix. ## Phase 1 output format @@ -183,9 +183,9 @@ Coverage totals: N hardcoded code targets · M externalized prompt f Proposed plan: Stage 1 (Audit): Read-only manifest of hardcoded targets; flag placeholders for Mustache rewrite and knobs for model.custom Stage 2 (Wrap): Install SDK, create AI Config 'chat-assistant', inline fallback mirrors current values (Mustache syntax), rewrite the call site - Stage 3 (Tools): Skipped (no function calling) / Attach 2 tools via configs-tools + Stage 3 (Tools): Skipped (no function calling) / Attach 2 tools via tools Stage 4 (Tracking): Inline tracker wiring (track_duration + track_tokens + track_success/error) — run-scoped tracker for agent loops - Stage 5 (Evals): Attach built-in 'accuracy' judge at 0.25 sampling via configs-online-evals + Stage 5 (Evals): Attach built-in 'accuracy' judge at 0.25 sampling via online-evals ``` ## STOP diff --git a/skills/ai-configs/configs-migrate/references/sdk-ai-tracker-patterns.md b/skills/agentcontrol/migrate/references/sdk-ai-tracker-patterns.md similarity index 98% rename from skills/ai-configs/configs-migrate/references/sdk-ai-tracker-patterns.md rename to skills/agentcontrol/migrate/references/sdk-ai-tracker-patterns.md index c755d4a..2316556 100644 --- a/skills/ai-configs/configs-migrate/references/sdk-ai-tracker-patterns.md +++ b/skills/agentcontrol/migrate/references/sdk-ai-tracker-patterns.md @@ -297,7 +297,7 @@ response = tracker.track_metrics_of( | Helper | Signature | Tier | Notes | |--------|-----------|------|-------| | `trackMetricsOf(extractor, func)` | `await tracker.trackMetricsOf((result) => extractor(result), async () => ...)` | **2 / 3** | **Canonical generic wrapper.** `extractor` maps provider response → `LDAIMetrics`. Use a provider package's bare `getAIMetricsFromResponse` for Tier 2 (`@launchdarkly/server-sdk-ai-openai`, `-langchain`, `-vercel`) or a small custom function for Tier 3. | -| `trackStreamMetricsOf(extractor, streamCreator)` | `tracker.trackStreamMetricsOf(async (chunks) => extractor(chunks), () => createStream())` | 2 / 3 | Stream variant. Does **not** capture TTFT automatically — if you need TTFT, use the manual pattern in [streaming-tracking.md](../../configs-ai-metrics/references/streaming-tracking.md). | +| `trackStreamMetricsOf(extractor, streamCreator)` | `tracker.trackStreamMetricsOf(async (chunks) => extractor(chunks), () => createStream())` | 2 / 3 | Stream variant. Does **not** capture TTFT automatically — if you need TTFT, use the manual pattern in [streaming-tracking.md](../../ai-metrics/references/streaming-tracking.md). | | `trackDurationOf(func)` | `await tracker.trackDurationOf(async () => ...)` | 4 | Wraps an async callable; captures duration only. Pair with explicit `trackTokens` + `trackSuccess`. | Example — OpenAI via `trackMetricsOf` + the provider package: ```typescript @@ -339,7 +339,7 @@ The managed runner handles message history, provider dispatch (via the installed ### Anthropic has no provider package today -Neither `@launchdarkly/server-sdk-ai-anthropic` nor `launchdarkly-server-sdk-ai-anthropic` exists as of this writing. For Anthropic direct calls, write a custom extractor and pass it to `track_metrics_of` / `trackMetricsOf` — see the Python example above or the full walk-through in [anthropic-tracking.md](../../configs-ai-metrics/references/anthropic-tracking.md). If the app is open to LangChain, routing Anthropic through `ChatAnthropic` and the LangChain provider package recovers Tier 2 with zero extractor code. +Neither `@launchdarkly/server-sdk-ai-anthropic` nor `launchdarkly-server-sdk-ai-anthropic` exists as of this writing. For Anthropic direct calls, write a custom extractor and pass it to `track_metrics_of` / `trackMetricsOf` — see the Python example above or the full walk-through in [anthropic-tracking.md](../../ai-metrics/references/anthropic-tracking.md). If the app is open to LangChain, routing Anthropic through `ChatAnthropic` and the LangChain provider package recovers Tier 2 with zero extractor code. ## Tier decision table @@ -352,7 +352,7 @@ Neither `@launchdarkly/server-sdk-ai-anthropic` nor `launchdarkly-server-sdk-ai- | Anthropic direct SDK | **3** | Custom extractor reading `response.usage.input_tokens` / `output_tokens` | | Bedrock Converse (no provider package) | **3** | Custom extractor reading `response.usage.inputTokens` / `outputTokens` (or route via LangChain for Tier 2) | | Gemini / Google GenAI, Cohere, custom HTTP | **3** | Custom extractor | -| Streaming response with TTFT required | **4** | Manual `trackTimeToFirstToken` + `trackDuration` + `trackTokens` + `trackSuccess` — see [streaming-tracking.md](../../configs-ai-metrics/references/streaming-tracking.md) | +| Streaming response with TTFT required | **4** | Manual `trackTimeToFirstToken` + `trackDuration` + `trackTokens` + `trackSuccess` — see [streaming-tracking.md](../../ai-metrics/references/streaming-tracking.md) | | Streaming response without TTFT (Node) | **2 / 3** | `trackStreamMetricsOf(extractor, streamFn)` | ## Streaming responses @@ -450,7 +450,7 @@ Run the checklist in order. Each step rules out one cause. ## Common gotchas -- **`model.parameters` vs `model.custom`.** `create_langchain_model` (Python) / `createLangChainModel` (Node) forwards every key in `model.parameters` to the provider SDK. App-scoped knobs (search result limits, retry budgets, feature toggles) **must** live in `model.custom` or the provider will crash at runtime with an unexpected-keyword-argument error. Read them with `ai_config.model.get_custom("key")`. Full walk-through with the MCP/REST-API caveat in [langchain-tracking.md § `model.parameters` vs `model.custom`](../../configs-ai-metrics/references/langchain-tracking.md). +- **`model.parameters` vs `model.custom`.** `create_langchain_model` (Python) / `createLangChainModel` (Node) forwards every key in `model.parameters` to the provider SDK. App-scoped knobs (search result limits, retry budgets, feature toggles) **must** live in `model.custom` or the provider will crash at runtime with an unexpected-keyword-argument error. Read them with `ai_config.model.get_custom("key")`. Full walk-through with the MCP/REST-API caveat in [langchain-tracking.md § `model.parameters` vs `model.custom`](../../ai-metrics/references/langchain-tracking.md). - **`track_tokens` token shape.** The Python `TokenUsage` dataclass requires `total` to be set — it is not derived. Compute `total = input + output` if the provider doesn't return one. - **`track_feedback` lifecycle.** The feedback call must be made on a tracker bound to the same `runId` that produced the response. If the thumbs-up comes in a later process, use the cross-process resumption pattern above — do **not** call `create_tracker()` again in the consumer, because that mints a *new* `runId`. - **OpenAI streaming tokens.** OpenAI only emits `usage` in the final chunk when `stream_options={"include_usage": True}` is passed. Without that flag, you have to tokenize manually — `tiktoken` for OpenAI models. diff --git a/skills/ai-configs/configs-online-evals/README.md b/skills/agentcontrol/online-evals/README.md similarity index 71% rename from skills/ai-configs/configs-online-evals/README.md rename to skills/agentcontrol/online-evals/README.md index cb717bd..3dc0b8e 100644 --- a/skills/ai-configs/configs-online-evals/README.md +++ b/skills/agentcontrol/online-evals/README.md @@ -12,12 +12,12 @@ This skill teaches agents how to: ## Installation (Local) -Copy `skills/ai-configs/configs-online-evals/` into your agent client's skills path. +Copy `skills/agentcontrol/online-evals/` into your agent client's skills path. ## Prerequisites - LaunchDarkly API access token with `ai-configs:write` permission -- Existing AI Config with variations (use `configs-create` skill) +- Existing AI Config with variations (use `create` skill) - For custom judges: understanding of LLM-as-a-judge methodology ## Usage @@ -33,16 +33,16 @@ Create a custom judge that checks for scope creep in code changes ## Structure ``` -configs-online-evals/ +online-evals/ ├── SKILL.md └── README.md ``` ## Related -- [AI Config Create](../configs-create/) - Create AI Configs first -- [AI Config Targeting](../configs-targeting/) - Enable targeting on judges -- [AI Config Variations](../configs-variations/) - Manage variations +- [AI Config Create](../create/) - Create AI Configs first +- [AI Config Targeting](../targeting/) - Enable targeting on judges +- [AI Config Variations](../variations/) - Manage variations - [Online Evaluations Docs](https://docs.launchdarkly.com/home/ai-configs/online-evaluations) ## License diff --git a/skills/ai-configs/configs-online-evals/SKILL.md b/skills/agentcontrol/online-evals/SKILL.md similarity index 98% rename from skills/ai-configs/configs-online-evals/SKILL.md rename to skills/agentcontrol/online-evals/SKILL.md index bd99480..a024285 100644 --- a/skills/ai-configs/configs-online-evals/SKILL.md +++ b/skills/agentcontrol/online-evals/SKILL.md @@ -1,5 +1,5 @@ --- -name: configs-online-evals +name: online-evals description: Attach judges to AI Config variations for automatic LLM-as-a-judge evaluation. Create custom judges, configure sampling rates, and monitor quality scores. compatibility: Requires LaunchDarkly API access token with ai-configs:write permission. SDK versions Python v0.20.0+ or Node.js v0.20.0+ for automatic metric recording and the consolidated `track_judge_result` / `trackJudgeResult` API. metadata: @@ -15,7 +15,7 @@ Attach judges to AI Config variations for automatic quality scoring using LLM-as - LaunchDarkly account with AI Configs enabled - API access token with write permissions -- Existing AI Config with variations (use `configs-create` skill) +- Existing AI Config with variations (use `create` skill) - For automatic metric recording and the consolidated judge-result API: Python AI SDK v0.20.0+ or Node.js AI SDK v0.20.0+ ## API Key Detection @@ -437,9 +437,9 @@ After attaching judges: ## Related Skills -- `configs-create` - Create AI Configs and judges -- `configs-targeting` - Configure targeting rules -- `configs-variations` - Manage variations +- `create` - Create AI Configs and judges +- `targeting` - Configure targeting rules +- `variations` - Manage variations ## References diff --git a/skills/ai-configs/configs-projects/README.md b/skills/agentcontrol/projects/README.md similarity index 94% rename from skills/ai-configs/configs-projects/README.md rename to skills/agentcontrol/projects/README.md index aeee2aa..0decb32 100644 --- a/skills/ai-configs/configs-projects/README.md +++ b/skills/agentcontrol/projects/README.md @@ -17,7 +17,7 @@ For now, install by placing this skill directory where your agent client loads s Examples: -- **Generic**: copy `skills/ai-configs/configs-projects/` into your client's skills path +- **Generic**: copy `skills/agentcontrol/projects/` into your client's skills path ## Prerequisites @@ -44,7 +44,7 @@ Add LaunchDarkly project management to this codebase ## Structure ``` -configs-projects/ +projects/ ├── SKILL.md ├── README.md └── references/ diff --git a/skills/ai-configs/configs-projects/SKILL.md b/skills/agentcontrol/projects/SKILL.md similarity index 95% rename from skills/ai-configs/configs-projects/SKILL.md rename to skills/agentcontrol/projects/SKILL.md index ef53432..0871fe6 100644 --- a/skills/ai-configs/configs-projects/SKILL.md +++ b/skills/agentcontrol/projects/SKILL.md @@ -1,5 +1,5 @@ --- -name: configs-projects +name: projects description: Guide for setting up LaunchDarkly projects in your codebase. Helps you assess your stack, choose the right approach, and integrate project management that makes sense for your architecture. compatibility: Requires LaunchDarkly API access token with projects:write permission or LaunchDarkly MCP server. metadata: @@ -221,16 +221,16 @@ ai-apac → Asia-Pacific Region After setting up projects: -1. **Create AI Configs** - Use the `configs-create` skill -2. **Set up SDK Integration** - Use the `configs-sdk` skill -3. **Configure Targeting** - Use the `configs-targeting` skill +1. **Create AI Configs** - Use the `create` skill +2. **Set up SDK Integration** - Use the `sdk` skill +3. **Configure Targeting** - Use the `targeting` skill ## Related Skills -- `configs-create` - Create AI Configs in projects -- `configs-sdk` - Integrate SDK in your application -- `configs-targeting` - Configure AI Config targeting -- `configs-variations` - Manage config variations +- `create` - Create AI Configs in projects +- `sdk` - Integrate SDK in your application +- `targeting` - Configure AI Config targeting +- `variations` - Manage config variations ## References diff --git a/skills/ai-configs/configs-projects/references/admin-tooling.md b/skills/agentcontrol/projects/references/admin-tooling.md similarity index 100% rename from skills/ai-configs/configs-projects/references/admin-tooling.md rename to skills/agentcontrol/projects/references/admin-tooling.md diff --git a/skills/ai-configs/configs-projects/references/env-config.md b/skills/agentcontrol/projects/references/env-config.md similarity index 99% rename from skills/ai-configs/configs-projects/references/env-config.md rename to skills/agentcontrol/projects/references/env-config.md index f383ce0..0353901 100644 --- a/skills/ai-configs/configs-projects/references/env-config.md +++ b/skills/agentcontrol/projects/references/env-config.md @@ -363,6 +363,6 @@ def verify_sdk_key(sdk_key: str): ## Next Steps -- [Integrate SDK in your application](../configs-sdk/SKILL.md) +- [Integrate SDK in your application](../sdk/SKILL.md) - [Set up project cloning](project-cloning.md) - [Build automation scripts](iac-automation.md) diff --git a/skills/ai-configs/configs-projects/references/go-setup.md b/skills/agentcontrol/projects/references/go-setup.md similarity index 100% rename from skills/ai-configs/configs-projects/references/go-setup.md rename to skills/agentcontrol/projects/references/go-setup.md diff --git a/skills/ai-configs/configs-projects/references/iac-automation.md b/skills/agentcontrol/projects/references/iac-automation.md similarity index 100% rename from skills/ai-configs/configs-projects/references/iac-automation.md rename to skills/agentcontrol/projects/references/iac-automation.md diff --git a/skills/ai-configs/configs-projects/references/multi-language-setup.md b/skills/agentcontrol/projects/references/multi-language-setup.md similarity index 100% rename from skills/ai-configs/configs-projects/references/multi-language-setup.md rename to skills/agentcontrol/projects/references/multi-language-setup.md diff --git a/skills/ai-configs/configs-projects/references/nodejs-setup.md b/skills/agentcontrol/projects/references/nodejs-setup.md similarity index 100% rename from skills/ai-configs/configs-projects/references/nodejs-setup.md rename to skills/agentcontrol/projects/references/nodejs-setup.md diff --git a/skills/ai-configs/configs-projects/references/project-cloning.md b/skills/agentcontrol/projects/references/project-cloning.md similarity index 100% rename from skills/ai-configs/configs-projects/references/project-cloning.md rename to skills/agentcontrol/projects/references/project-cloning.md diff --git a/skills/ai-configs/configs-projects/references/python-setup.md b/skills/agentcontrol/projects/references/python-setup.md similarity index 100% rename from skills/ai-configs/configs-projects/references/python-setup.md rename to skills/agentcontrol/projects/references/python-setup.md diff --git a/skills/ai-configs/configs-projects/references/quick-start.md b/skills/agentcontrol/projects/references/quick-start.md similarity index 100% rename from skills/ai-configs/configs-projects/references/quick-start.md rename to skills/agentcontrol/projects/references/quick-start.md diff --git a/skills/ai-configs/configs-snippets/SKILL.md b/skills/agentcontrol/snippets/SKILL.md similarity index 99% rename from skills/ai-configs/configs-snippets/SKILL.md rename to skills/agentcontrol/snippets/SKILL.md index c4e5393..ecea329 100644 --- a/skills/ai-configs/configs-snippets/SKILL.md +++ b/skills/agentcontrol/snippets/SKILL.md @@ -1,5 +1,5 @@ --- -name: configs-snippets +name: snippets description: "Create and manage prompt snippets — reusable text blocks referenced inside AI Config variation prompts. Keeps common instructions, personas, and guardrails consistent across multiple configs." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server diff --git a/skills/ai-configs/configs-targeting/README.md b/skills/agentcontrol/targeting/README.md similarity index 66% rename from skills/ai-configs/configs-targeting/README.md rename to skills/agentcontrol/targeting/README.md index 9fe73dc..4566f64 100644 --- a/skills/ai-configs/configs-targeting/README.md +++ b/skills/agentcontrol/targeting/README.md @@ -13,13 +13,13 @@ This skill teaches agents how to: ## Installation (Local) -Copy `skills/ai-configs/configs-targeting/` into your agent client's skills path. +Copy `skills/agentcontrol/targeting/` into your agent client's skills path. ## Prerequisites - LaunchDarkly API access token with `ai-configs:write` permission -- Existing AI Config with variations (use `configs-create` skill) -- Understanding of contexts (see `configs-context-basic` skill) +- Existing AI Config with variations (use `create` skill) +- Understanding of contexts (see `context-basic` skill) ## Usage @@ -34,16 +34,16 @@ Add a percentage rollout: 60% to variation A, 40% to variation B for premium use ## Structure ``` -configs-targeting/ +targeting/ ├── SKILL.md └── README.md ``` ## Related -- [AI Config Create](../configs-create/) - Create AI Configs first -- [AI Config Variations](../configs-variations/) - Create variations to target -- [AI Config Online Evals](../configs-online-evals/) - Attach judges +- [AI Config Create](../create/) - Create AI Configs first +- [AI Config Variations](../variations/) - Create variations to target +- [AI Config Online Evals](../online-evals/) - Attach judges - [Targeting Docs](https://docs.launchdarkly.com/home/ai-configs/target) ## License diff --git a/skills/ai-configs/configs-targeting/SKILL.md b/skills/agentcontrol/targeting/SKILL.md similarity index 97% rename from skills/ai-configs/configs-targeting/SKILL.md rename to skills/agentcontrol/targeting/SKILL.md index 8ecf42b..2d06b2b 100644 --- a/skills/ai-configs/configs-targeting/SKILL.md +++ b/skills/agentcontrol/targeting/SKILL.md @@ -1,5 +1,5 @@ --- -name: configs-targeting +name: targeting description: Configure AI Config targeting rules to control which variations serve to different users. Enable percentage rollouts, attribute-based rules, segment targeting, and guarded rollouts. compatibility: Requires LaunchDarkly API access token with ai-configs:write permission. metadata: @@ -16,7 +16,7 @@ Configure targeting rules for AI Configs to control which variations serve to di - LaunchDarkly account with AI Configs enabled - API access token with write permissions - Project key and environment key -- Existing AI Config with variations (use `configs-create` skill) +- Existing AI Config with variations (use `create` skill) ## API Key Detection @@ -488,16 +488,16 @@ After configuring targeting: ``` https://app.launchdarkly.com/projects/{projectKey}/ai-configs/{configKey} ``` -2. **Monitor performance** with `configs-ai-metrics` -3. **Attach judges** with `configs-online-evals` +2. **Monitor performance** with `ai-metrics` +3. **Attach judges** with `online-evals` 4. **Set up guarded rollouts** for automatic regression detection ## Related Skills -- `configs-create` - Create AI Configs with variations -- `configs-variations` - Manage variations -- `configs-online-evals` - Attach judges -- `configs-segments` - Create segments for targeting +- `create` - Create AI Configs with variations +- `variations` - Manage variations +- `online-evals` - Attach judges +- `segments` - Create segments for targeting ## References diff --git a/skills/ai-configs/configs-tools/README.md b/skills/agentcontrol/tools/README.md similarity index 78% rename from skills/ai-configs/configs-tools/README.md rename to skills/agentcontrol/tools/README.md index df42d50..3bf63a0 100644 --- a/skills/ai-configs/configs-tools/README.md +++ b/skills/agentcontrol/tools/README.md @@ -12,7 +12,7 @@ This skill teaches agents how to: ## Installation (Local) -Copy `skills/ai-configs/configs-tools/` into your agent client's skills path. +Copy `skills/agentcontrol/tools/` into your agent client's skills path. ## Prerequisites @@ -31,15 +31,15 @@ Create tools for the content assistant to call our API ## Structure ``` -configs-tools/ +tools/ ├── SKILL.md └── README.md ``` ## Related -- [AI Config Create](../configs-create/): Create the config before adding tools -- [AI Config Variations](../configs-variations/): Manage variations that tools attach to +- [AI Config Create](../create/): Create the config before adding tools +- [AI Config Variations](../variations/): Manage variations that tools attach to - [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) ## License diff --git a/skills/ai-configs/configs-tools/SKILL.md b/skills/agentcontrol/tools/SKILL.md similarity index 92% rename from skills/ai-configs/configs-tools/SKILL.md rename to skills/agentcontrol/tools/SKILL.md index 7dccd17..cb17288 100644 --- a/skills/ai-configs/configs-tools/SKILL.md +++ b/skills/agentcontrol/tools/SKILL.md @@ -1,5 +1,5 @@ --- -name: configs-tools +name: tools description: "Give your AI agents capabilities through tools (function calling). Helps you identify what your AI needs to do, create tool definitions, and attach them to AI Config variations." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server @@ -211,13 +211,13 @@ for _ in range(MAX_STEPS): messages.append({"role": "user", "content": tool_results}) ``` -Per-provider tool-call payload shapes live in the `configs-ai-metrics` references: +Per-provider tool-call payload shapes live in the `ai-metrics` references: -- [openai-tracking.md](../configs-ai-metrics/references/openai-tracking.md) — Chat Completions + Responses API -- [anthropic-tracking.md](../configs-ai-metrics/references/anthropic-tracking.md) — `tool_use` blocks and `tool_result` payloads -- [bedrock-tracking.md](../configs-ai-metrics/references/bedrock-tracking.md) — `toolUse` / `toolResult` Converse format -- [gemini-tracking.md](../configs-ai-metrics/references/gemini-tracking.md) — `functionCalls` / `functionResponse` parts -- [langchain-tracking.md](../configs-ai-metrics/references/langchain-tracking.md) — LangGraph tool loop inherits from `create_react_agent` +- [openai-tracking.md](../ai-metrics/references/openai-tracking.md) — Chat Completions + Responses API +- [anthropic-tracking.md](../ai-metrics/references/anthropic-tracking.md) — `tool_use` blocks and `tool_result` payloads +- [bedrock-tracking.md](../ai-metrics/references/bedrock-tracking.md) — `toolUse` / `toolResult` Converse format +- [gemini-tracking.md](../ai-metrics/references/gemini-tracking.md) — `functionCalls` / `functionResponse` parts +- [langchain-tracking.md](../ai-metrics/references/langchain-tracking.md) — LangGraph tool loop inherits from `create_react_agent` ## Orchestrator Note @@ -240,5 +240,5 @@ LangGraph, CrewAI, and AutoGen often generate schemas from function definitions. ## Related Skills -- `configs-create` -- Create config before attaching tools -- `configs-variations` -- Manage variations with different tool sets +- `create` -- Create config before attaching tools +- `variations` -- Manage variations with different tool sets diff --git a/skills/ai-configs/configs-update/README.md b/skills/agentcontrol/update/README.md similarity index 82% rename from skills/ai-configs/configs-update/README.md rename to skills/agentcontrol/update/README.md index 1155f02..d90648a 100644 --- a/skills/ai-configs/configs-update/README.md +++ b/skills/agentcontrol/update/README.md @@ -13,7 +13,7 @@ This skill teaches agents how to: ## Installation (Local) -Copy `skills/ai-configs/configs-update/` into your agent client's skills path. +Copy `skills/agentcontrol/update/` into your agent client's skills path. ## Prerequisites @@ -36,15 +36,15 @@ Archive the old chatbot config ## Structure ``` -configs-update/ +update/ ├── SKILL.md └── README.md ``` ## Related -- [AI Config Create](../configs-create/): Create configs -- [AI Config Variations](../configs-variations/): Add or test variations +- [AI Config Create](../create/): Create configs +- [AI Config Variations](../variations/): Add or test variations - [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) ## License diff --git a/skills/ai-configs/configs-update/SKILL.md b/skills/agentcontrol/update/SKILL.md similarity index 96% rename from skills/ai-configs/configs-update/SKILL.md rename to skills/agentcontrol/update/SKILL.md index 183113c..67250ae 100644 --- a/skills/ai-configs/configs-update/SKILL.md +++ b/skills/agentcontrol/update/SKILL.md @@ -1,5 +1,5 @@ --- -name: configs-update +name: update description: "Update, archive, and delete LaunchDarkly AI Configs and their variations. Use when you need to modify config properties, change model parameters, update instructions or messages, archive unused configs, or permanently remove them." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server @@ -90,5 +90,5 @@ Use `get-ai-config` to confirm the response shows your updated values. ## Related Skills -- `configs-variations` -- Create variations to test changes side-by-side -- `configs-tools` -- Update tool attachments +- `variations` -- Create variations to test changes side-by-side +- `tools` -- Update tool attachments diff --git a/skills/ai-configs/configs-variations/README.md b/skills/agentcontrol/variations/README.md similarity index 75% rename from skills/ai-configs/configs-variations/README.md rename to skills/agentcontrol/variations/README.md index 3d40d2c..7d89218 100644 --- a/skills/ai-configs/configs-variations/README.md +++ b/skills/agentcontrol/variations/README.md @@ -11,7 +11,7 @@ This skill teaches agents how to: ## Installation (Local) -Copy `skills/ai-configs/configs-variations/` into your agent client's skills path. +Copy `skills/agentcontrol/variations/` into your agent client's skills path. ## Prerequisites @@ -30,16 +30,16 @@ Create variations to compare Claude vs GPT-4 for our agent ## Structure ``` -configs-variations/ +variations/ ├── SKILL.md └── README.md ``` ## Related -- [AI Config Create](../configs-create/): Create the config first -- [AI Config Update](../configs-update/): Modify existing variations -- [AI Config Tools](../configs-tools/): Attach tools to variations +- [AI Config Create](../create/): Create the config first +- [AI Config Update](../update/): Modify existing variations +- [AI Config Tools](../tools/): Attach tools to variations - [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) ## License diff --git a/skills/ai-configs/configs-variations/SKILL.md b/skills/agentcontrol/variations/SKILL.md similarity index 97% rename from skills/ai-configs/configs-variations/SKILL.md rename to skills/agentcontrol/variations/SKILL.md index 26c58e0..0586178 100644 --- a/skills/ai-configs/configs-variations/SKILL.md +++ b/skills/agentcontrol/variations/SKILL.md @@ -1,5 +1,5 @@ --- -name: configs-variations +name: variations description: "Experiment with AI configurations by creating and managing variations. Helps you test different models, prompts, and parameters to find what works best through systematic experimentation." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server @@ -106,5 +106,5 @@ When the user wants to try a different model, prompt, or parameters, **always cr ## Related Skills -- `configs-create` -- Create the initial config -- `configs-update` -- Refine based on learnings +- `create` -- Create the initial config +- `update` -- Refine based on learnings From 1141dd1bc995c62cf909e3a385b2ddf52272c422 Mon Sep 17 00:00:00 2001 From: Scarlett Attensil Date: Tue, 19 May 2026 11:48:43 -0700 Subject: [PATCH 04/10] Finalize naming: configs-* for config skills, bare for projects/tools - Parent folder: skills/agentcontrol/ - Config-related skills keep configs- prefix (configs-create, configs-migrate, configs-update, configs-variations, configs-targeting, configs-online-evals, configs-snippets, configs-agent-graphs, configs-ai-metrics, configs-custom-metrics) - Non-config skills are bare: projects, tools - Fixed the 7 stale top-level symlinks (were aiconfig-* -> deleted ai-configs/aiconfig-*); now renamed to new skill names and repointed into skills/agentcontrol/ - Updated skills.json, README catalog, .cursor-plugin path, and all cross-references. LD REST paths, ai-configs:write scopes, docs URLs, example tags, and the list-ai-configs tool intentionally untouched. --- README.md | 16 ++--- skills.json | 40 +++++------ .../SKILL.md | 2 +- .../SKILL.md | 14 ++-- .../references/anthropic-tracking.md | 0 .../references/bedrock-tracking.md | 0 .../references/gemini-tracking.md | 0 .../references/langchain-tracking.md | 0 .../references/metrics-api.md | 0 .../references/openai-tracking.md | 0 .../references/strands-tracking.md | 2 +- .../references/streaming-tracking.md | 0 .../{create => configs-create}/README.md | 6 +- .../{create => configs-create}/SKILL.md | 14 ++-- .../SKILL.md | 8 +-- .../{migrate => configs-migrate}/README.md | 16 ++--- .../{migrate => configs-migrate}/SKILL.md | 68 +++++++++---------- .../references/agent-graph-reference.md | 0 .../references/agent-mode-frameworks.md | 8 +-- .../references/before-after-examples.md | 0 .../references/fallback-defaults-pattern.md | 2 +- .../references/phase-1-analysis-checklist.md | 4 +- .../references/sdk-ai-tracker-patterns.md | 8 +-- .../README.md | 12 ++-- .../SKILL.md | 10 +-- .../{snippets => configs-snippets}/SKILL.md | 2 +- .../README.md | 12 ++-- .../{targeting => configs-targeting}/SKILL.md | 14 ++-- .../{update => configs-update}/README.md | 8 +-- .../{update => configs-update}/SKILL.md | 4 +- .../README.md | 8 +-- .../SKILL.md | 6 +- skills/agentcontrol/projects/SKILL.md | 10 +-- skills/agentcontrol/tools/README.md | 4 +- skills/agentcontrol/tools/SKILL.md | 16 ++--- skills/aiconfig-create | 1 - skills/aiconfig-online-evals | 1 - skills/aiconfig-projects | 1 - skills/aiconfig-targeting | 1 - skills/aiconfig-tools | 1 - skills/aiconfig-update | 1 - skills/aiconfig-variations | 1 - skills/configs-create | 1 + skills/configs-online-evals | 1 + skills/configs-targeting | 1 + skills/configs-update | 1 + skills/configs-variations | 1 + skills/projects | 1 + skills/tools | 1 + 49 files changed, 164 insertions(+), 164 deletions(-) rename skills/agentcontrol/{agent-graphs => configs-agent-graphs}/SKILL.md (99%) rename skills/agentcontrol/{ai-metrics => configs-ai-metrics}/SKILL.md (93%) rename skills/agentcontrol/{ai-metrics => configs-ai-metrics}/references/anthropic-tracking.md (100%) rename skills/agentcontrol/{ai-metrics => configs-ai-metrics}/references/bedrock-tracking.md (100%) rename skills/agentcontrol/{ai-metrics => configs-ai-metrics}/references/gemini-tracking.md (100%) rename skills/agentcontrol/{ai-metrics => configs-ai-metrics}/references/langchain-tracking.md (100%) rename skills/agentcontrol/{ai-metrics => configs-ai-metrics}/references/metrics-api.md (100%) rename skills/agentcontrol/{ai-metrics => configs-ai-metrics}/references/openai-tracking.md (100%) rename skills/agentcontrol/{ai-metrics => configs-ai-metrics}/references/strands-tracking.md (93%) rename skills/agentcontrol/{ai-metrics => configs-ai-metrics}/references/streaming-tracking.md (100%) rename skills/agentcontrol/{create => configs-create}/README.md (86%) rename skills/agentcontrol/{create => configs-create}/SKILL.md (91%) rename skills/agentcontrol/{custom-metrics => configs-custom-metrics}/SKILL.md (98%) rename skills/agentcontrol/{migrate => configs-migrate}/README.md (82%) rename skills/agentcontrol/{migrate => configs-migrate}/SKILL.md (89%) rename skills/agentcontrol/{migrate => configs-migrate}/references/agent-graph-reference.md (100%) rename skills/agentcontrol/{migrate => configs-migrate}/references/agent-mode-frameworks.md (98%) rename skills/agentcontrol/{migrate => configs-migrate}/references/before-after-examples.md (100%) rename skills/agentcontrol/{migrate => configs-migrate}/references/fallback-defaults-pattern.md (99%) rename skills/agentcontrol/{migrate => configs-migrate}/references/phase-1-analysis-checklist.md (98%) rename skills/agentcontrol/{migrate => configs-migrate}/references/sdk-ai-tracker-patterns.md (98%) rename skills/agentcontrol/{online-evals => configs-online-evals}/README.md (70%) rename skills/agentcontrol/{online-evals => configs-online-evals}/SKILL.md (98%) rename skills/agentcontrol/{snippets => configs-snippets}/SKILL.md (99%) rename skills/agentcontrol/{targeting => configs-targeting}/README.md (71%) rename skills/agentcontrol/{targeting => configs-targeting}/SKILL.md (97%) rename skills/agentcontrol/{update => configs-update}/README.md (82%) rename skills/agentcontrol/{update => configs-update}/SKILL.md (97%) rename skills/agentcontrol/{variations => configs-variations}/README.md (80%) rename skills/agentcontrol/{variations => configs-variations}/SKILL.md (97%) delete mode 120000 skills/aiconfig-create delete mode 120000 skills/aiconfig-online-evals delete mode 120000 skills/aiconfig-projects delete mode 120000 skills/aiconfig-targeting delete mode 120000 skills/aiconfig-tools delete mode 120000 skills/aiconfig-update delete mode 120000 skills/aiconfig-variations create mode 120000 skills/configs-create create mode 120000 skills/configs-online-evals create mode 120000 skills/configs-targeting create mode 120000 skills/configs-update create mode 120000 skills/configs-variations create mode 120000 skills/projects create mode 120000 skills/tools diff --git a/README.md b/README.md index 2fa1b0e..60b7306 100644 --- a/README.md +++ b/README.md @@ -22,16 +22,16 @@ Agent Skills are modular, text-based playbooks that teach an agent how to perfor | Skill | Description | |-------|-------------| -| `agentcontrol/create` | Create AI Configs with variations for agent or completion mode | -| `agentcontrol/migrate` | Migrate an app with hardcoded LLM prompts to AI Configs in five stages (extract, wrap, tools, tracking, evals) | -| `agentcontrol/update` | Update and delete AI Configs, manage lifecycle | -| `agentcontrol/variations` | Manage AI Config variations for A/B testing | +| `agentcontrol/configs-create` | Create AI Configs with variations for agent or completion mode | +| `agentcontrol/configs-migrate` | Migrate an app with hardcoded LLM prompts to AI Configs in five stages (extract, wrap, tools, tracking, evals) | +| `agentcontrol/configs-update` | Update and delete AI Configs, manage lifecycle | +| `agentcontrol/configs-variations` | Manage AI Config variations for A/B testing | | `agentcontrol/tools` | Create and attach tools for function calling | | `agentcontrol/projects` | Create and manage projects to organize AI Configs | -| `agentcontrol/online-evals` | Attach LLM-as-a-judge evaluators to AI Configs | -| `agentcontrol/targeting` | Configure targeting rules for AI Config rollouts | -| `agentcontrol/snippets` | Create and manage reusable prompt snippets across AI Configs | -| `agentcontrol/agent-graphs` | Create and manage multi-agent graphs with routing and handoffs | +| `agentcontrol/configs-online-evals` | Attach LLM-as-a-judge evaluators to AI Configs | +| `agentcontrol/configs-targeting` | Configure targeting rules for AI Config rollouts | +| `agentcontrol/configs-snippets` | Create and manage reusable prompt snippets across AI Configs | +| `agentcontrol/configs-agent-graphs` | Create and manage multi-agent graphs with routing and handoffs | ### Experiments diff --git a/skills.json b/skills.json index 7e5d921..ce4c026 100644 --- a/skills.json +++ b/skills.json @@ -1,49 +1,49 @@ { "skills": [ { - "name": "agent-graphs", + "name": "configs-agent-graphs", "description": "Create and manage agent graphs \u2014 directed graphs of AI Configs connected by edges with handoff logic. Use when building multi-agent workflows where configs route to each other.", - "path": "skills/agentcontrol/agent-graphs", + "path": "skills/agentcontrol/configs-agent-graphs", "version": "0.1.0", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "ai-metrics", + "name": "configs-ai-metrics", "description": "Instrument an existing codebase with LaunchDarkly AI Config tracking. Walks the four-tier ladder (managed runner \u2192 provider package \u2192 custom extractor + trackMetricsOf \u2192 raw manual) and picks the lowest-ceremony option that still captures duration, tokens, and success/error.", - "path": "skills/agentcontrol/ai-metrics", + "path": "skills/agentcontrol/configs-ai-metrics", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the LaunchDarkly server-side AI SDK (`launchdarkly-server-sdk-ai>=0.20.0` for Python or `@launchdarkly/server-sdk-ai>=0.20.0` for Node) and an existing AI Config." }, { - "name": "create", + "name": "configs-create", "description": "Create and configure AI Configs in LaunchDarkly. Helps you choose between agent vs completion mode, create the config, add variations with models and prompts, and verify the setup.", - "path": "skills/agentcontrol/create", + "path": "skills/agentcontrol/configs-create", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "custom-metrics", + "name": "configs-custom-metrics", "description": "Create, track, retrieve, update, and delete custom business metrics for AI Configs. Covers full lifecycle: define metric kinds via API, emit events via SDK, and query results.", - "path": "skills/agentcontrol/custom-metrics", + "path": "skills/agentcontrol/configs-custom-metrics", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the LaunchDarkly server SDK and a LaunchDarkly API token with the `writer` role for metric management." }, { - "name": "migrate", + "name": "configs-migrate", "description": "Migrate an application with hardcoded LLM prompts to a full LaunchDarkly AI Configs implementation in five stages: audit the code, wrap the call, move the tools, add tracking, attach evaluators. Use when the user wants to externalize model/prompt configuration, move from direct provider calls (OpenAI, Anthropic, Bedrock, Gemini, Strands) to a managed AI Config, or stage a full hardcoded-to-LaunchDarkly migration.", - "path": "skills/agentcontrol/migrate", + "path": "skills/agentcontrol/configs-migrate", "version": "0.1.0", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "online-evals", + "name": "configs-online-evals", "description": "Attach judges to AI Config variations for automatic LLM-as-a-judge evaluation. Create custom judges, configure sampling rates, and monitor quality scores.", - "path": "skills/agentcontrol/online-evals", + "path": "skills/agentcontrol/configs-online-evals", "version": "0.1.0", "compatibility": "Requires LaunchDarkly API access token with ai-configs:write permission. SDK versions Python v0.20.0+ or Node.js v0.20.0+ for automatic metric recording and the consolidated `track_judge_result` / `trackJudgeResult` API." }, @@ -55,17 +55,17 @@ "compatibility": "Requires LaunchDarkly API access token with projects:write permission or LaunchDarkly MCP server." }, { - "name": "snippets", + "name": "configs-snippets", "description": "Create and manage prompt snippets \u2014 reusable text blocks referenced inside AI Config variation prompts. Keeps common instructions, personas, and guardrails consistent across multiple configs.", - "path": "skills/agentcontrol/snippets", + "path": "skills/agentcontrol/configs-snippets", "version": "0.1.0", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "targeting", + "name": "configs-targeting", "description": "Configure AI Config targeting rules to control which variations serve to different users. Enable percentage rollouts, attribute-based rules, segment targeting, and guarded rollouts.", - "path": "skills/agentcontrol/targeting", + "path": "skills/agentcontrol/configs-targeting", "version": "0.1.0", "compatibility": "Requires LaunchDarkly API access token with ai-configs:write permission." }, @@ -78,17 +78,17 @@ "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "update", + "name": "configs-update", "description": "Update, archive, and delete LaunchDarkly AI Configs and their variations. Use when you need to modify config properties, change model parameters, update instructions or messages, archive unused configs, or permanently remove them.", - "path": "skills/agentcontrol/update", + "path": "skills/agentcontrol/configs-update", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "variations", + "name": "configs-variations", "description": "Experiment with AI configurations by creating and managing variations. Helps you test different models, prompts, and parameters to find what works best through systematic experimentation.", - "path": "skills/agentcontrol/variations", + "path": "skills/agentcontrol/configs-variations", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" diff --git a/skills/agentcontrol/agent-graphs/SKILL.md b/skills/agentcontrol/configs-agent-graphs/SKILL.md similarity index 99% rename from skills/agentcontrol/agent-graphs/SKILL.md rename to skills/agentcontrol/configs-agent-graphs/SKILL.md index b067e1c..d5176fe 100644 --- a/skills/agentcontrol/agent-graphs/SKILL.md +++ b/skills/agentcontrol/configs-agent-graphs/SKILL.md @@ -1,5 +1,5 @@ --- -name: agent-graphs +name: configs-agent-graphs description: "Create and manage agent graphs — directed graphs of configs connected by edges with handoff logic. Use when building multi-agent workflows where configs route to each other." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server diff --git a/skills/agentcontrol/ai-metrics/SKILL.md b/skills/agentcontrol/configs-ai-metrics/SKILL.md similarity index 93% rename from skills/agentcontrol/ai-metrics/SKILL.md rename to skills/agentcontrol/configs-ai-metrics/SKILL.md index 7eedbb5..96170a3 100644 --- a/skills/agentcontrol/ai-metrics/SKILL.md +++ b/skills/agentcontrol/configs-ai-metrics/SKILL.md @@ -1,5 +1,5 @@ --- -name: ai-metrics +name: configs-ai-metrics description: "Instrument an existing codebase with LaunchDarkly AI Config tracking. Walks the four-tier ladder (managed runner → provider package → custom extractor + trackMetricsOf → raw manual) and picks the lowest-ceremony option that still captures duration, tokens, and success/error." license: Apache-2.0 compatibility: Requires the LaunchDarkly server-side AI SDK (`launchdarkly-server-sdk-ai>=0.20.0` for Python or `@launchdarkly/server-sdk-ai>=0.20.0` for Node) and an existing AI Config. @@ -38,12 +38,12 @@ Before picking a tier, find the provider call and answer these questions: - [ ] **Provider?** OpenAI, Anthropic, Bedrock, Gemini, Azure, custom HTTP? → cross-reference with the package availability matrix below. - [ ] **Streaming?** If yes, you'll need TTFT tracking, which means Tier 4 for the TTFT part even if the rest is Tier 2. - [ ] **Language?** Python or Node? Provider-package coverage differs between them. -- [ ] **Already using an AI Config?** If not, route to `create` first — tracking requires a tracker, which is obtained by calling `create_tracker()` / `createTracker()` on the config object returned by `completion_config()` / `completionConfig()` / `createModel()`. +- [ ] **Already using an AI Config?** If not, route to `configs-create` first — tracking requires a tracker, which is obtained by calling `create_tracker()` / `createTracker()` on the config object returned by `completion_config()` / `completionConfig()` / `createModel()`. - [ ] **On the current SDK API?** If the call site uses `aiclient.config(...)` / `aiClient.config(...)` or constructs an `AIConfig(...)` / `LDAIConfig` default, it's on the pre-0.20 surface. Migrate it as part of this work before adding tracking: - `aiclient.config(...)` → `aiclient.completion_config(...)` for one-shot/chat or `aiclient.agent_config(...)` for agent mode (mirror the call signature). Node is the same with camelCase. - `AIConfig(...)` default → `AICompletionConfigDefault(...)` or `AIAgentConfigDefault(...)` (Node: `LDAICompletionConfigDefault` / `LDAIAgentConfigDefault`). `AIConfig` is the base class the SDK returns; it isn't a valid default-value constructor — the typed `*Default` variants are. - If the result was being tuple-unpacked (`config, tracker = aiclient.config(...)`), drop the unpack — the new methods return a single config object. Obtain the tracker via `config.create_tracker()` / `aiConfig.createTracker()`. - - For deeper rewrites (call sites with hardcoded model/prompt as well), hand off to `migrate` instead of doing the full migration here. + - For deeper rewrites (call sites with hardcoded model/prompt as well), hand off to `configs-migrate` instead of doing the full migration here. ### 2. Look up your Tier-2 option @@ -103,7 +103,7 @@ Obtain a tracker via the factory on the config object: `tracker = config.create_ ## Related skills -- `create` — prerequisite if the app doesn't have an AI Config yet -- `custom-metrics` — business metrics (conversion, resolution, retention) layered on top of the AI metrics this skill captures -- `online-evals` — automatic quality scoring (LLM-as-judge) on sampled live requests; complementary to the metrics here -- `migrate` — Stage 4 of the hardcoded-to-AI-Configs migration delegates to this skill +- `configs-create` — prerequisite if the app doesn't have an AI Config yet +- `configs-custom-metrics` — business metrics (conversion, resolution, retention) layered on top of the AI metrics this skill captures +- `configs-online-evals` — automatic quality scoring (LLM-as-judge) on sampled live requests; complementary to the metrics here +- `configs-migrate` — Stage 4 of the hardcoded-to-AI-Configs migration delegates to this skill diff --git a/skills/agentcontrol/ai-metrics/references/anthropic-tracking.md b/skills/agentcontrol/configs-ai-metrics/references/anthropic-tracking.md similarity index 100% rename from skills/agentcontrol/ai-metrics/references/anthropic-tracking.md rename to skills/agentcontrol/configs-ai-metrics/references/anthropic-tracking.md diff --git a/skills/agentcontrol/ai-metrics/references/bedrock-tracking.md b/skills/agentcontrol/configs-ai-metrics/references/bedrock-tracking.md similarity index 100% rename from skills/agentcontrol/ai-metrics/references/bedrock-tracking.md rename to skills/agentcontrol/configs-ai-metrics/references/bedrock-tracking.md diff --git a/skills/agentcontrol/ai-metrics/references/gemini-tracking.md b/skills/agentcontrol/configs-ai-metrics/references/gemini-tracking.md similarity index 100% rename from skills/agentcontrol/ai-metrics/references/gemini-tracking.md rename to skills/agentcontrol/configs-ai-metrics/references/gemini-tracking.md diff --git a/skills/agentcontrol/ai-metrics/references/langchain-tracking.md b/skills/agentcontrol/configs-ai-metrics/references/langchain-tracking.md similarity index 100% rename from skills/agentcontrol/ai-metrics/references/langchain-tracking.md rename to skills/agentcontrol/configs-ai-metrics/references/langchain-tracking.md diff --git a/skills/agentcontrol/ai-metrics/references/metrics-api.md b/skills/agentcontrol/configs-ai-metrics/references/metrics-api.md similarity index 100% rename from skills/agentcontrol/ai-metrics/references/metrics-api.md rename to skills/agentcontrol/configs-ai-metrics/references/metrics-api.md diff --git a/skills/agentcontrol/ai-metrics/references/openai-tracking.md b/skills/agentcontrol/configs-ai-metrics/references/openai-tracking.md similarity index 100% rename from skills/agentcontrol/ai-metrics/references/openai-tracking.md rename to skills/agentcontrol/configs-ai-metrics/references/openai-tracking.md diff --git a/skills/agentcontrol/ai-metrics/references/strands-tracking.md b/skills/agentcontrol/configs-ai-metrics/references/strands-tracking.md similarity index 93% rename from skills/agentcontrol/ai-metrics/references/strands-tracking.md rename to skills/agentcontrol/configs-ai-metrics/references/strands-tracking.md index 062a006..1741254 100644 --- a/skills/agentcontrol/ai-metrics/references/strands-tracking.md +++ b/skills/agentcontrol/configs-ai-metrics/references/strands-tracking.md @@ -79,7 +79,7 @@ Pick the style that matches the rest of the codebase — the two variants record ## Provider dispatch stays in your code -Strands model classes are provider-specific (`AnthropicModel`, `OpenAIModel`, `BedrockModel`). To serve more than one provider from a single AI Config key, dispatch on `agent_config.provider.name` before constructing the `Agent`. See [agent-mode-frameworks.md § Strands Agent](../../migrate/references/agent-mode-frameworks.md) for the `create_strands_model` dispatcher, including the rule that `parameters.tools` must be dropped before being passed into the Strands model class (tools flow through the `Agent` constructor, not through model params). +Strands model classes are provider-specific (`AnthropicModel`, `OpenAIModel`, `BedrockModel`). To serve more than one provider from a single AI Config key, dispatch on `agent_config.provider.name` before constructing the `Agent`. See [agent-mode-frameworks.md § Strands Agent](../../configs-migrate/references/agent-mode-frameworks.md) for the `create_strands_model` dispatcher, including the rule that `parameters.tools` must be dropped before being passed into the Strands model class (tools flow through the `Agent` constructor, not through model params). ## Always flush before exit diff --git a/skills/agentcontrol/ai-metrics/references/streaming-tracking.md b/skills/agentcontrol/configs-ai-metrics/references/streaming-tracking.md similarity index 100% rename from skills/agentcontrol/ai-metrics/references/streaming-tracking.md rename to skills/agentcontrol/configs-ai-metrics/references/streaming-tracking.md diff --git a/skills/agentcontrol/create/README.md b/skills/agentcontrol/configs-create/README.md similarity index 86% rename from skills/agentcontrol/create/README.md rename to skills/agentcontrol/configs-create/README.md index 5366713..cf21ee7 100644 --- a/skills/agentcontrol/create/README.md +++ b/skills/agentcontrol/configs-create/README.md @@ -12,7 +12,7 @@ This skill teaches agents how to: ## Installation (Local) -Copy `skills/agentcontrol/create/` into your agent client's skills path. +Copy `skills/agentcontrol/configs-create/` into your agent client's skills path. ## Prerequisites @@ -31,7 +31,7 @@ Set up an AI config for content generation using Claude ## Structure ``` -create/ +configs-create/ ├── SKILL.md └── README.md ``` @@ -40,7 +40,7 @@ create/ - [AI Config Projects](../projects/): Create projects first - [AI Config Tools](../tools/): Add tools after creating config -- [AI Config Variations](../variations/): Add more variations for experimentation +- [AI Config Variations](../configs-variations/): Add more variations for experimentation - [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) ## License diff --git a/skills/agentcontrol/create/SKILL.md b/skills/agentcontrol/configs-create/SKILL.md similarity index 91% rename from skills/agentcontrol/create/SKILL.md rename to skills/agentcontrol/configs-create/SKILL.md index c5be3aa..5a50668 100644 --- a/skills/agentcontrol/create/SKILL.md +++ b/skills/agentcontrol/configs-create/SKILL.md @@ -1,5 +1,5 @@ --- -name: create +name: configs-create description: "Create and configure AI Configs in LaunchDarkly. Helps you choose between agent vs completion mode, create the config, add variations with models and prompts, and verify the setup." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server @@ -12,7 +12,7 @@ metadata: You're using a skill that will guide you through creating an AI Config in LaunchDarkly. Your job is to understand the use case, choose the right mode, create the config and its variations, and verify everything is set up correctly. -> **⚠️ This skill creates a config — it does not make it servable.** A freshly-created AI Config has its **fallthrough pointing at an auto-generated disabled variation**, not at the variation you just created. The SDK will return `ai_config.enabled=False` on every evaluation until you flip targeting on and point the fallthrough at your new variation. This is not a bug — it's the default state. **You must run `/targeting` (or the equivalent REST / CLI call shown in Step 5) before verifying against the SDK**, or verification will look like the LD-served path is broken when it isn't. The single most common failure mode users hit with this skill is skipping the targeting step and spending time debugging `enabled=False` in their application code. +> **⚠️ This skill creates a config — it does not make it servable.** A freshly-created AI Config has its **fallthrough pointing at an auto-generated disabled variation**, not at the variation you just created. The SDK will return `ai_config.enabled=False` on every evaluation until you flip targeting on and point the fallthrough at your new variation. This is not a bug — it's the default state. **You must run `/configs-targeting` (or the equivalent REST / CLI call shown in Step 5) before verifying against the SDK**, or verification will look like the LD-served path is broken when it isn't. The single most common failure mode users hit with this skill is skipping the targeting step and spending time debugging `enabled=False` in their application code. ## Prerequisites @@ -147,7 +147,7 @@ Print this checklist verbatim to the user after Step 4, then wait for confirmati > > 🔴 **The SDK will return `enabled=False` until you flip targeting on.** The fallthrough is currently pointing at an auto-generated disabled variation, not at the `{variationKey}` you just created. > -> **Next step — run `/targeting`** with these inputs: +> **Next step — run `/configs-targeting`** with these inputs: > - Project key: `{projectKey}` > - Config key: `{configKey}` > - Environment key: the env whose SDK key is in your `.env` (usually `test` or `production`) @@ -159,7 +159,7 @@ Print this checklist verbatim to the user after Step 4, then wait for confirmati #### Direct shortcut if the user wants to flip targeting without invoking the sibling skill -`targeting` is the canonical path — it handles percentage rollouts, targeted rules, and variation-ID lookups. But for the simplest case ("promote the new variation to fallthrough in one environment"), you can run the underlying semantic PATCH yourself once you know the new variation's `_id`. +`configs-targeting` is the canonical path — it handles percentage rollouts, targeted rules, and variation-ID lookups. But for the simplest case ("promote the new variation to fallthrough in one environment"), you can run the underlying semantic PATCH yourself once you know the new variation's `_id`. Get the variation ID (use `get-ai-config` MCP, or): ```bash @@ -213,10 +213,10 @@ The `create-ai-config-variation` tool validates this format and rejects invalid - Don't forget modelConfigKey (models won't show in the UI) - Don't omit `modelName` from the initial variation call. It is required at create time; setting it via a follow-up PATCH is a workaround for a bug, not the intended flow. The PATCH field is also `modelName`, not `name`. - Don't drop to raw `curl` + `jq` for verification. Use `get-ai-config` (MCP) — it returns a typed object and avoids brittle `jq` filters that break on response-shape variation. -- Don't consider the workflow complete until the user has been told to run `targeting`. A created variation that isn't promoted to fallthrough returns `enabled=False` to every consumer. +- Don't consider the workflow complete until the user has been told to run `configs-targeting`. A created variation that isn't promoted to fallthrough returns `enabled=False` to every consumer. ## Related Skills - `tools` -- Create tools before attaching -- `variations` -- Add more variations for experimentation -- `update` -- Modify configs based on learnings +- `configs-variations` -- Add more variations for experimentation +- `configs-update` -- Modify configs based on learnings diff --git a/skills/agentcontrol/custom-metrics/SKILL.md b/skills/agentcontrol/configs-custom-metrics/SKILL.md similarity index 98% rename from skills/agentcontrol/custom-metrics/SKILL.md rename to skills/agentcontrol/configs-custom-metrics/SKILL.md index 880f133..794603e 100644 --- a/skills/agentcontrol/custom-metrics/SKILL.md +++ b/skills/agentcontrol/configs-custom-metrics/SKILL.md @@ -1,5 +1,5 @@ --- -name: custom-metrics +name: configs-custom-metrics description: "Create, track, retrieve, update, and delete custom business metrics for AI Configs. Covers full lifecycle: define metric kinds via API, emit events via SDK, and query results." license: Apache-2.0 compatibility: Requires the LaunchDarkly server SDK and a LaunchDarkly API token with the `writer` role for metric management. @@ -16,7 +16,7 @@ Full lifecycle management of custom business metrics: create metric definitions - LaunchDarkly SDK initialized (see `sdk`) - LaunchDarkly API token with `writer` role for metric management -- Understanding of built-in AI metrics (see `ai-metrics`) +- Understanding of built-in AI metrics (see `configs-ai-metrics`) ## API Key Detection @@ -493,8 +493,8 @@ Custom metrics appear in: ## Related Skills - `sdk` - SDK setup -- `ai-metrics` - Built-in AI metrics (tokens, duration, cost) -- `online-evals` - Quality metrics via judges +- `configs-ai-metrics` - Built-in AI metrics (tokens, duration, cost) +- `configs-online-evals` - Quality metrics via judges ## References diff --git a/skills/agentcontrol/migrate/README.md b/skills/agentcontrol/configs-migrate/README.md similarity index 82% rename from skills/agentcontrol/migrate/README.md rename to skills/agentcontrol/configs-migrate/README.md index 71f0df8..c2473c4 100644 --- a/skills/agentcontrol/migrate/README.md +++ b/skills/agentcontrol/configs-migrate/README.md @@ -9,14 +9,14 @@ This skill orchestrates the full migration journey from hardcoded `openai.chat.c The five stages: 1. **Extract** hardcoded model names, prompts, and parameters (read-only) -2. **Wrap** the call site in `completion_config` / `completionConfig` with a safe fallback — delegates the config creation to `create` +2. **Wrap** the call site in `completion_config` / `completionConfig` with a safe fallback — delegates the config creation to `configs-create` 3. **Tools** — move function-calling schemas into LaunchDarkly — delegates to `tools` 4. **Tracking** — wire `track_duration`, `track_tokens`, `track_success`/`track_error`, optional `track_feedback` — inline, with a reference doc covering every SDK method in Python and Node side by side -5. **Evals** — attach judges for LLM-as-a-judge scoring — delegates to `online-evals` +5. **Evals** — attach judges for LLM-as-a-judge scoring — delegates to `configs-online-evals` ## Installation (Local) -Copy `skills/agentcontrol/migrate/` into your agent client's skills path. +Copy `skills/agentcontrol/configs-migrate/` into your agent client's skills path. ## Prerequisites @@ -41,7 +41,7 @@ Wire up the AI tracker and attach accuracy + relevance judges to our existing co ## Structure ``` -migrate/ +configs-migrate/ ├── SKILL.md ├── README.md └── references/ @@ -55,11 +55,11 @@ migrate/ ## Related -- [AI Config Create](../create/): Delegated to by Stage 2 (wrap) +- [AI Config Create](../configs-create/): Delegated to by Stage 2 (wrap) - [AI Config Tools](../tools/): Delegated to by Stage 3 (tools) -- [AI Config Online Evals](../online-evals/): Delegated to by Stage 5 (evals) -- [AI Config Variations](../variations/): Next step after migration for A/B testing -- [AI Config Targeting](../targeting/): Next step after migration for rollout control +- [AI Config Online Evals](../configs-online-evals/): Delegated to by Stage 5 (evals) +- [AI Config Variations](../configs-variations/): Next step after migration for A/B testing +- [AI Config Targeting](../configs-targeting/): Next step after migration for rollout control - [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) ## License diff --git a/skills/agentcontrol/migrate/SKILL.md b/skills/agentcontrol/configs-migrate/SKILL.md similarity index 89% rename from skills/agentcontrol/migrate/SKILL.md rename to skills/agentcontrol/configs-migrate/SKILL.md index 5add0c3..c90ca2e 100644 --- a/skills/agentcontrol/migrate/SKILL.md +++ b/skills/agentcontrol/configs-migrate/SKILL.md @@ -1,5 +1,5 @@ --- -name: migrate +name: configs-migrate description: "Migrate an application with hardcoded LLM prompts to a full LaunchDarkly AI Configs implementation in five stages: audit the code, wrap the call, move the tools, add tracking, attach evaluators. Use when the user wants to externalize model/prompt configuration, move from direct provider calls (OpenAI, Anthropic, Bedrock, Gemini, Strands) to a managed AI Config, or stage a full hardcoded-to-LaunchDarkly migration." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server @@ -22,7 +22,7 @@ You're using a skill that will guide you through migrating an application from h > > 1. **Tracker in the wrong scope.** For an agent with a loop, mint `create_tracker()` once per user turn in a `setup_run` entry node — not inside `call_model`. Per-iteration factory calls produce N `runId`s and trip the at-most-once guards. See [agent-mode-frameworks.md § Custom `StateGraph`](references/agent-mode-frameworks.md). > 2. **`load_chat_model` wrapper reuse.** Templates like `langchain-ai/react-agent` ship a `load_chat_model(f"{provider}/{name}")` helper that wraps `init_chat_model(...)` and silently drops every variation parameter. **Delete it** (don't just avoid using it) and replace call sites with `create_langchain_model(ai_config)`. -> 3. **Fallthrough not flipped after `/create`.** A freshly-created AI Config's fallthrough points at an auto-generated disabled variation, so the SDK returns `enabled=False` until `/targeting` runs. Flip it before Stage 2 verification. +> 3. **Fallthrough not flipped after `/configs-create`.** A freshly-created AI Config's fallthrough points at an auto-generated disabled variation, so the SDK returns `enabled=False` until `/configs-targeting` runs. Flip it before Stage 2 verification. ## Coverage — which shapes are well-trodden vs require extrapolation @@ -30,16 +30,16 @@ The skill is optimized for Python and Node.js / TypeScript; other languages are | Shape | Python | Node.js | Reference | |-------|--------|---------|-----------| -| One-shot completion (direct OpenAI / Anthropic / Bedrock / Gemini call) | ✅ Worked example | ✅ Worked example | [before-after-examples.md](references/before-after-examples.md), per-provider docs in `ai-metrics/references/` | -| Chat loop via managed runner (`ManagedModel`) | ✅ Tier 1 pattern | ✅ Tier 1 pattern | [ai-metrics SKILL.md](../ai-metrics/SKILL.md) | -| LangChain single-call | ✅ Worked example | ✅ Worked example | [langchain-tracking.md](../ai-metrics/references/langchain-tracking.md) | +| One-shot completion (direct OpenAI / Anthropic / Bedrock / Gemini call) | ✅ Worked example | ✅ Worked example | [before-after-examples.md](references/before-after-examples.md), per-provider docs in `configs-ai-metrics/references/` | +| Chat loop via managed runner (`ManagedModel`) | ✅ Tier 1 pattern | ✅ Tier 1 pattern | [configs-ai-metrics SKILL.md](../configs-ai-metrics/SKILL.md) | +| LangChain single-call | ✅ Worked example | ✅ Worked example | [langchain-tracking.md](../configs-ai-metrics/references/langchain-tracking.md) | | LangGraph prebuilt agent (Python `langchain.agents.create_agent`, Node `createReactAgent`) | ✅ Worked example | ✅ Worked example | [agent-mode-frameworks.md § LangGraph](references/agent-mode-frameworks.md) | | LangGraph custom `StateGraph` with run-scoped tracker (setup_run + call_model + finalize) | ✅ Deep worked example | ⚠️ Mentioned — translate from Python | [agent-mode-frameworks.md § Custom `StateGraph`](references/agent-mode-frameworks.md) | | CrewAI `Agent` | ✅ Worked example | — (not a Node framework) | [agent-mode-frameworks.md § CrewAI](references/agent-mode-frameworks.md) | | Strands `Agent` | ✅ Worked example | ⚠️ BedrockModel + OpenAIModel only (no Anthropic) | [agent-mode-frameworks.md § Strands](references/agent-mode-frameworks.md) | | Custom ReAct loop (hand-rolled, any framework or none) | ✅ Worked example | ⚠️ Apply framework-agnostic invariants; translate from Python | [agent-mode-frameworks.md § Custom ReAct loop](references/agent-mode-frameworks.md) | -| Vercel AI SDK (`generateText` / `streamText`) | — (not a Python framework) | ⚠️ Provider package exists; no worked example in skill | `ai-metrics` provider-package matrix | -| Streaming (SSE / WebSocket) | ⚠️ Delegated to `ai-metrics` streaming doc | ⚠️ Same — use `trackStreamMetricsOf` + manual TTFT | [streaming-tracking.md](../ai-metrics/references/streaming-tracking.md) | +| Vercel AI SDK (`generateText` / `streamText`) | — (not a Python framework) | ⚠️ Provider package exists; no worked example in skill | `configs-ai-metrics` provider-package matrix | +| Streaming (SSE / WebSocket) | ⚠️ Delegated to `configs-ai-metrics` streaming doc | ⚠️ Same — use `trackStreamMetricsOf` + manual TTFT | [streaming-tracking.md](../configs-ai-metrics/references/streaming-tracking.md) | | Multi-agent graph (supervisor + workers) | ⚠️ Out of main scope; see reference | ⚠️ Out of main scope; see reference | [agent-graph-reference.md](references/agent-graph-reference.md) | | Non-LangGraph agent frameworks (Pydantic AI, DSPy, AutoGen, Haystack, LlamaIndex agents, Semantic Kernel) | ⚠️ Apply the three invariants; no framework-specific example | ⚠️ Same | [agent-mode-frameworks.md § Framework-agnostic invariants](references/agent-mode-frameworks.md) | | Go, Ruby, .NET | ℹ️ Install commands only | ℹ️ Install commands only | [phase-1-analysis-checklist.md § SDK routing table](references/phase-1-analysis-checklist.md) | @@ -68,10 +68,10 @@ If a CHANGELOG entry post-dates this skill and changes an API you're about to us **Sibling skills the user runs at each stage:** - `projects` — pre-Stage 2, only if no project exists yet -- `create` — Stage 2 (creates the AI Config and first variation) +- `configs-create` — Stage 2 (creates the AI Config and first variation) - `tools` — Stage 3 (creates tool definitions and attaches them) -- `targeting` — between Stage 2 and Stage 4 (promotes the new variation to fallthrough so the SDK actually serves it) -- `online-evals` — Stage 5 (attaches judges, creates custom judges) +- `configs-targeting` — between Stage 2 and Stage 4 (promotes the new variation to fallthrough so the SDK actually serves it) +- `configs-online-evals` — Stage 5 (attaches judges, creates custom judges) ## Core Principles @@ -79,7 +79,7 @@ If a CHANGELOG entry post-dates this skill and changes an API you're about to us 2. **Replace config, not business logic.** The SDK call is a drop-in for the place where the model, parameters, and prompt are *defined* — not for the provider call itself. OpenAI/Anthropic/Bedrock calls stay where they are. 3. **Fallback mirrors current behavior.** The fallback passed to `completion_config` / `agent_config` must preserve the hardcoded values you removed, so the app is unchanged if LaunchDarkly is unreachable. 4. **Stages are ordered.** Wrap before you add tools. Add tools before you track. Track before you add evals. Skipping ahead produces configs without traffic, metrics without context, and judges with nothing to score. -5. **Hand off to focused skills, manually.** Each stage that needs a LaunchDarkly write tells the user to run a sibling slash-command (`/create`, `/tools`, `/targeting`, `/online-evals`) and waits for them to come back. This skill does **not** auto-invoke other skills. +5. **Hand off to focused skills, manually.** Each stage that needs a LaunchDarkly write tells the user to run a sibling slash-command (`/configs-create`, `/tools`, `/configs-targeting`, `/configs-online-evals`) and waits for them to come back. This skill does **not** auto-invoke other skills. ## Workflow @@ -160,7 +160,7 @@ This is the first stage that writes code. It has nine sub-steps. - OpenAI: `launchdarkly-server-sdk-ai-openai>=0.4.0` (Python) / `@launchdarkly/server-sdk-ai-openai@^0.5.5` (Node) - LangChain / LangGraph: `launchdarkly-server-sdk-ai-langchain>=0.5.0` (Python) / `@launchdarkly/server-sdk-ai-langchain@^0.5.5` (Node) - Vercel AI SDK (Node only): `@launchdarkly/server-sdk-ai-vercel@^0.5.5` - - Anthropic, Gemini, Bedrock — no provider package published; use Tier-3 custom extractor (see `ai-metrics`) + - Anthropic, Gemini, Bedrock — no provider package published; use Tier-3 custom extractor (see `configs-ai-metrics`) 3. **Initialize `LDAIClient` once at startup.** Reuse any existing `LDClient` — do not create a second base client. Place the initialization in the same module that owns existing app config. @@ -207,11 +207,11 @@ This is the first stage that writes code. It has nine sub-steps. const aiClient = initAi(ldClient); ``` -4. **Hand off to `create`.** Print the extracted model, prompt/instructions, parameters, and mode from the Stage 1 manifest, then tell the user: *"Run `/create` with these inputs, then come back here."* Supply the config key you want the code to call (e.g. `chat-assistant`). Do not attempt to auto-invoke the sibling skill — wait for the user to finish it before continuing. +4. **Hand off to `configs-create`.** Print the extracted model, prompt/instructions, parameters, and mode from the Stage 1 manifest, then tell the user: *"Run `/configs-create` with these inputs, then come back here."* Supply the config key you want the code to call (e.g. `chat-assistant`). Do not attempt to auto-invoke the sibling skill — wait for the user to finish it before continuing. - **After `create` finishes, the user must also run `/targeting` to promote the new variation to fallthrough.** A freshly created variation returns `enabled=False` to every consumer until targeting is updated. Skip this and Stage 2 verification (sub-step 9 below) will silently take the fallback path on every request. + **After `configs-create` finishes, the user must also run `/configs-targeting` to promote the new variation to fallthrough.** A freshly created variation returns `enabled=False` to every consumer until targeting is updated. Skip this and Stage 2 verification (sub-step 9 below) will silently take the fallback path on every request. -5. **Rewrite template placeholders to Mustache syntax.** If the hardcoded prompt interpolates runtime values with Python `.format()`, f-strings, JS template literals, or any other non-Mustache syntax (e.g. `{system_time}`, `${userName}`, `%(topic)s`), rewrite every placeholder to `{{ variable }}` Mustache form. Do this in **both** the file you're about to send to `/create` *and* the fallback string you'll write in sub-step 6. The AI SDK interpolates variables through a Mustache renderer on the LD-served path *and* the fallback path using the fourth-argument `variables` dict to `completion_config(...)` / `completionConfig(...)`. Leaving a Python-style `{system_time}` literal in the fallback ships a silent regression when LaunchDarkly is unreachable — the renderer won't match the single-brace form and the literal `{system_time}` goes to the provider as part of the prompt. +5. **Rewrite template placeholders to Mustache syntax.** If the hardcoded prompt interpolates runtime values with Python `.format()`, f-strings, JS template literals, or any other non-Mustache syntax (e.g. `{system_time}`, `${userName}`, `%(topic)s`), rewrite every placeholder to `{{ variable }}` Mustache form. Do this in **both** the file you're about to send to `/configs-create` *and* the fallback string you'll write in sub-step 6. The AI SDK interpolates variables through a Mustache renderer on the LD-served path *and* the fallback path using the fourth-argument `variables` dict to `completion_config(...)` / `completionConfig(...)`. Leaving a Python-style `{system_time}` literal in the fallback ships a silent regression when LaunchDarkly is unreachable — the renderer won't match the single-brace form and the literal `{system_time}` goes to the provider as part of the prompt. **Before:** ```python @@ -318,7 +318,7 @@ This is the first stage that writes code. It has nine sub-steps. 9. **Verify.** Run the app with a valid `LD_SDK_KEY`; confirm the call succeeds and the response matches pre-migration output. Then temporarily set `LD_SDK_KEY=sdk-invalid` (or unset it) and confirm the fallback path runs without error. Both paths must work before moving to Stage 3. -Delegate: **`create`** (sub-step 4). +Delegate: **`configs-create`** (sub-step 4). ### Step 3: Move tools into the config (Stage 3) @@ -350,9 +350,9 @@ Delegate: **`tools`** (sub-step 2). ### Step 4: Instrument the tracker (Stage 4) -Delegate: **`ai-metrics`** wires the per-request `tracker.track_*` calls (duration, tokens, success/error, feedback) around the provider call. Use **`custom-metrics`** alongside it if the app needs business metrics beyond the built-in AI ones. Note: do not confuse this with `launchdarkly-metric-instrument`, which is for `ldClient.track()` feature metrics — a different API. See [sdk-ai-tracker-patterns.md](references/sdk-ai-tracker-patterns.md) for the full per-method Python + Node matrix that the delegate skill draws on. +Delegate: **`configs-ai-metrics`** wires the per-request `tracker.track_*` calls (duration, tokens, success/error, feedback) around the provider call. Use **`configs-custom-metrics`** alongside it if the app needs business metrics beyond the built-in AI ones. Note: do not confuse this with `launchdarkly-metric-instrument`, which is for `ldClient.track()` feature metrics — a different API. See [sdk-ai-tracker-patterns.md](references/sdk-ai-tracker-patterns.md) for the full per-method Python + Node matrix that the delegate skill draws on. -Hand off: print the AI Config key, variation key, provider, and whether the call is streaming, then tell the user: *"Run `/ai-metrics` with these inputs, then come back here."* Do not auto-invoke. Return here for sub-step 5 (verify) once they're done. +Hand off: print the AI Config key, variation key, provider, and whether the call is streaming, then tell the user: *"Run `/configs-ai-metrics` with these inputs, then come back here."* Do not auto-invoke. Return here for sub-step 5 (verify) once they're done. 1. **Create the tracker.** Obtain a per-execution tracker via the factory on the config returned in Stage 2: `tracker = config.create_tracker()` (Python) or `const tracker = aiConfig.createTracker();` (Node). Call the factory **once per user turn** and reuse the returned `tracker` for every tracking call in that turn — each call mints a fresh `runId` that tags every event emitted from the turn so they can be correlated via exported events or downstream queries. (The Monitoring tab aggregates today; run-level grouping is a downstream concern — but the `runId` is also what the SDK's at-most-once guards are keyed on, so minting a new one mid-turn breaks the guard semantics regardless of where the events end up.) @@ -404,7 +404,7 @@ Hand off: print the AI Config key, variation key, provider, and whether the call ); ``` - For Anthropic direct, Bedrock (no provider package), Gemini, and custom HTTP, write a small extractor returning `LDAIMetrics` — see the delegate skill's [anthropic-tracking.md](../ai-metrics/references/anthropic-tracking.md), [bedrock-tracking.md](../ai-metrics/references/bedrock-tracking.md), and [gemini-tracking.md](../ai-metrics/references/gemini-tracking.md). LangChain single-node and LangGraph go through the `launchdarkly-server-sdk-ai-langchain` / `@launchdarkly/server-sdk-ai-langchain` provider package. Build the model with `create_langchain_model(config)` (Python) / `createLangChainModel(config)` (Node) — both forward all variation parameters — and track with `get_ai_metrics_from_response` / `getAIMetricsFromResponse`. See [langchain-tracking.md](../ai-metrics/references/langchain-tracking.md). + For Anthropic direct, Bedrock (no provider package), Gemini, and custom HTTP, write a small extractor returning `LDAIMetrics` — see the delegate skill's [anthropic-tracking.md](../configs-ai-metrics/references/anthropic-tracking.md), [bedrock-tracking.md](../configs-ai-metrics/references/bedrock-tracking.md), and [gemini-tracking.md](../configs-ai-metrics/references/gemini-tracking.md). LangChain single-node and LangGraph go through the `launchdarkly-server-sdk-ai-langchain` / `@launchdarkly/server-sdk-ai-langchain` provider package. Build the model with `create_langchain_model(config)` (Python) / `createLangChainModel(config)` (Node) — both forward all variation parameters — and track with `get_ai_metrics_from_response` / `getAIMetricsFromResponse`. See [langchain-tracking.md](../configs-ai-metrics/references/langchain-tracking.md). 4. **Wire feedback tracking if the app has thumbs-up/down UI.** Both SDKs expose `trackFeedback` with a `{kind}` argument. @@ -442,11 +442,11 @@ Hand off: print the AI Config key, variation key, provider, and whether the call - Run the `default` variation (or whichever variation mirrors the pre-migration hardcoded behavior) against the dataset first — this is the baseline. - Clone it into a second variation pointing at a **different model family** (e.g., if the baseline is `anthropic/claude-sonnet-4-5`, clone to `openai/gpt-4o` or `openai/gpt-4o-mini`). The comparison is most informative across families, not across siblings. - Attach the built-in **Accuracy** judge with a pass threshold of **0.85**, and run both variations against the same dataset. - - Promote the winner to fallthrough via `/targeting` only if it beats the baseline on Accuracy and does not regress on Relevance or Toxicity. + - Promote the winner to fallthrough via `/configs-targeting` only if it beats the baseline on Accuracy and does not regress on Relevance or Toxicity. Write this shape into the project's `datasets/README.md` (or equivalent) so the comparison pattern is reproducible after the migration ships. -3. **Hand off to `online-evals`** — only for UI-attached judges (completion mode) or to create custom judge AI Configs that will be referenced by the programmatic path. Tell the user: *"Run `/online-evals` with these inputs, then come back here."* Do not auto-invoke. Pass: +3. **Hand off to `configs-online-evals`** — only for UI-attached judges (completion mode) or to create custom judge AI Configs that will be referenced by the programmatic path. Tell the user: *"Run `/configs-online-evals` with these inputs, then come back here."* Do not auto-invoke. Pass: - The parent AI Config key and variation key - A list of built-in judges (Accuracy, Relevance, Toxicity) or custom judge keys to create/attach - Target environment @@ -480,14 +480,14 @@ Hand off: print the AI Config key, variation key, provider, and whether the call - **`sampling_rate` is a parameter on `evaluate()`**, not on `create_judge`. It defaults to `1.0` (evaluate every call). For live paths, pass something lower (0.1–0.25) to control cost. - **`evaluate()` returns a `JudgeResult`** (never `None`). Check `result.sampled` to know whether the evaluation actually ran, and call `track_judge_result(result)`. Node uses `trackJudgeResult(result)` and `LDJudgeResult` with the same `sampled` field. - **Ask the user which judge AI Config key to use.** LaunchDarkly ships three built-in judges — Accuracy, Relevance, Toxicity — but the actual AI Config **keys** for the built-ins are not canonical SDK constants and aren't documented. Have the user open **AI Configs > Library** in the LD UI and copy the key of the judge they want to reference, or create a custom judge AI Config via `create` first. + **Ask the user which judge AI Config key to use.** LaunchDarkly ships three built-in judges — Accuracy, Relevance, Toxicity — but the actual AI Config **keys** for the built-ins are not canonical SDK constants and aren't documented. Have the user open **AI Configs > Library** in the LD UI and copy the key of the judge they want to reference, or create a custom judge AI Config via `configs-create` first. 5. **Verify.** - **UI-attached auto judges:** trigger a request in staging, open the Monitoring tab → "Evaluator metrics" dropdown. Scores appear within 1–2 minutes at the configured sampling rate. - **Programmatic direct-judge:** hit the wrapped endpoint and confirm `track_judge_result` lands on the parent config's Monitoring tab. - **Offline eval:** run the dataset through the LD Playground, compare baseline vs new-variation scores side by side. No runtime wiring required. -Delegate: **`online-evals`** (sub-step 3, optional — only for UI-attached judges or custom-judge creation; offline eval doesn't delegate). +Delegate: **`configs-online-evals`** (sub-step 3, optional — only for UI-attached judges or custom-judge creation; offline eval doesn't delegate). ## Edge Cases @@ -498,9 +498,9 @@ Delegate: **`online-evals`** (sub-step 3, optional — only for UI-attached judg | Retry wrapper around the provider call | The tracker is minted once at the top of the user turn; the retry loop is inside that scope. Every retry attempt shares the same `runId`. Tracker calls (`track_duration` / `track_tokens` / `track_success` / `track_error`) live *outside* the retry body — one call at the end of the turn, on the success path or the final-failure path | | App has no tools — Stage 3 skipped | Move directly from Stage 2 verification to Stage 4 (tracking) | | Mode mismatch: user said agent, audit shows one-shot chat | Choose completion mode unless the app uses a LangGraph prebuilt agent (`langchain.agents.create_agent` in Python or `createReactAgent` in Node), CrewAI `Agent`, Strands `Agent`, or a similar goal-driven framework | -| App uses Strands Agents (Python) | Agent mode. Build a `create_strands_model` dispatcher keyed on `agent_config.provider.name` that returns `AnthropicModel(model_id=..., max_tokens=...)` or `OpenAIModel(model_id=..., params=...)`. Drop `parameters.tools` before passing params to the model class — Strands receives tools via `Agent(tools=[...])`. Tracking is Tier 3: wrap `invoke_async` with `tracker.track_duration_of(...)` and record tokens from `result.metrics.accumulated_usage`. See [agent-mode-frameworks.md § Strands Agent](references/agent-mode-frameworks.md) and [strands-tracking.md](../ai-metrics/references/strands-tracking.md) | +| App uses Strands Agents (Python) | Agent mode. Build a `create_strands_model` dispatcher keyed on `agent_config.provider.name` that returns `AnthropicModel(model_id=..., max_tokens=...)` or `OpenAIModel(model_id=..., params=...)`. Drop `parameters.tools` before passing params to the model class — Strands receives tools via `Agent(tools=[...])`. Tracking is Tier 3: wrap `invoke_async` with `tracker.track_duration_of(...)` and record tokens from `result.metrics.accumulated_usage`. See [agent-mode-frameworks.md § Strands Agent](references/agent-mode-frameworks.md) and [strands-tracking.md](../configs-ai-metrics/references/strands-tracking.md) | | Strands app on TypeScript | TS SDK ships `BedrockModel` and `OpenAIModel` only — cannot serve Anthropic-backed variations. Use the Python SDK if multi-provider variations are required | -| TypeScript app using Anthropic SDK | No `trackAnthropicMetrics` helper exists. Use Tier 3: `trackMetricsOf` with a small custom extractor that reads `response.usage.input_tokens` / `response.usage.output_tokens` and returns `LDAIMetrics`. See [anthropic-tracking.md](../ai-metrics/references/anthropic-tracking.md) in the `ai-metrics` skill for the exact extractor | +| TypeScript app using Anthropic SDK | No `trackAnthropicMetrics` helper exists. Use Tier 3: `trackMetricsOf` with a small custom extractor that reads `response.usage.input_tokens` / `response.usage.output_tokens` and returns `LDAIMetrics`. See [anthropic-tracking.md](../configs-ai-metrics/references/anthropic-tracking.md) in the `configs-ai-metrics` skill for the exact extractor | | Fallback would silently crash because `LD_SDK_KEY` is missing | Log a startup warning; proceed with the fallback. Never raise at import time | | Multi-agent graph (supervisor + workers) | Stop after migrating a single agent. Agent Graph Definitions are available in **both** SDKs — Python via `launchdarkly-server-sdk-ai.agent_graph` and Node via the graph API in `@launchdarkly/server-sdk-ai`. Read [agent-graph-reference.md](references/agent-graph-reference.md) for the graph-level migration path — it is deliberately out of this skill's main scope | | Single-agent (ReAct, tool loop) + agent mode | Default to offline eval via the LD Playground + Datasets for Stage 5. UI-attached judges are completion-only today, and programmatic direct-judge adds per-call cost that is usually not worth it until after the migration is live and stable. Point at the [Offline Evals guide](https://docs.launchdarkly.com/guides/ai-configs/offline-evaluations) | @@ -525,17 +525,17 @@ These are ordered by how likely they are to show up as a first-run failure. The - **If the repo already contains a `load_chat_model(f"{provider}/{name}")` helper, delete it — don't just avoid using it.** This exact shape ships with `langchain-ai/react-agent` and is copied into dozens of derivative repos; look for `utils.load_chat_model`, `utils.build_model`, or any one-arg `init_chat_model` wrapper that splits a `"provider/model"` string. Re-using it is the first-run failure mode: every variation parameter (temperature, max_tokens, top_p, stop sequences) silently drops on the floor because `init_chat_model` only receives the name and provider. `create_langchain_model(ai_config)` is a one-for-one replacement that forwards the whole `model.parameters` dict. Replace every call site, then delete the wrapper file-side so the next reader can't reach for it. - **Same rule applies to hand-rolled `resolve_tools` / `TOOL_REGISTRY` / `ALL_TOOLS` helpers.** If the template already has a `resolve_tools(tool_keys)` or an `ALL_TOOLS` module-level list, import `build_structured_tools` from `ldai_langchain.langchain_helper` and delete the hand-rolled version. `build_structured_tools(ai_config, TOOL_REGISTRY_DICT)` reads `ai_config.model.parameters.tools` and wraps the matching callables as LangChain `StructuredTool`s with the LD tool key as the `StructuredTool.name` — so `ToolNode` lookup works without a second mapping. Don't leave both in the repo. -- Don't put app-scoped knobs directly in `model.parameters`. `create_langchain_model` forwards every key in `parameters` to the provider SDK via `init_chat_model`, so a `max_search_results` / `retry_budget` / `feature_toggle` entry will crash the provider with an unexpected-keyword-argument error. The correct home is `model.custom`, which the provider helpers ignore and the app reads via `ai_config.model.get_custom("key")`. The MCP `update-ai-config-variation` tool does not currently expose top-level `custom`, so pick one of two paths: (a) PATCH the variation via the REST API to set `model.custom` directly, or (b) set it via MCP inside `parameters.custom` (as a nested dict) and use a defensive accessor that reads both locations. Full walk-through with code samples in [langchain-tracking.md § MCP caveat](../ai-metrics/references/langchain-tracking.md). +- Don't put app-scoped knobs directly in `model.parameters`. `create_langchain_model` forwards every key in `parameters` to the provider SDK via `init_chat_model`, so a `max_search_results` / `retry_budget` / `feature_toggle` entry will crash the provider with an unexpected-keyword-argument error. The correct home is `model.custom`, which the provider helpers ignore and the app reads via `ai_config.model.get_custom("key")`. The MCP `update-ai-config-variation` tool does not currently expose top-level `custom`, so pick one of two paths: (a) PATCH the variation via the REST API to set `model.custom` directly, or (b) set it via MCP inside `parameters.custom` (as a nested dict) and use a defensive accessor that reads both locations. Full walk-through with code samples in [langchain-tracking.md § MCP caveat](../configs-ai-metrics/references/langchain-tracking.md). - Don't re-encode tool schemas inside the fallback. When LaunchDarkly is unreachable the fallback should run without tools (or with whatever minimal provider-bound parameters the app needs to keep operating). Building a `_FALLBACK_TOOLS` array that duplicates the AI Config's tool schema re-introduces the hardcoded config the migration was supposed to move out of code. - Don't import `LaunchDarklyCallbackHandler` from `ldai.langchain` — neither the class nor the dotted module path exists. The Python LangChain helper package is `ldai_langchain` (top-level module, underscore). Use `create_langchain_model(config)` + `track_metrics_of_async(get_ai_metrics_from_response, lambda: llm.ainvoke(messages))` as the canonical pattern. ### Stage / handoff discipline - Don't skip Step 1 even when the user says "just wrap it." Without the audit, the fallback will drift from the hardcoded behavior. -- Don't delegate to `create` before extracting the prompt and model — the delegate needs them as inputs. +- Don't delegate to `configs-create` before extracting the prompt and model — the delegate needs them as inputs. - Don't try to attach tools during initial `setup-ai-config`. Tool attachment is a separate step owned by `tools`. -- Don't claim you "delegated to `create`" or any other sibling skill. This skill does not auto-invoke. At each handoff, print the inputs and tell the user to run the sibling slash-command, then wait. Anything else misleads the user about what just happened. -- Don't skip the `/targeting` step between Stage 2 and Stage 4. A freshly created variation returns `enabled=False` until targeting promotes it to fallthrough — Stage 2 verification will silently take the fallback path on every request. +- Don't claim you "delegated to `configs-create`" or any other sibling skill. This skill does not auto-invoke. At each handoff, print the inputs and tell the user to run the sibling slash-command, then wait. Anything else misleads the user about what just happened. +- Don't skip the `/configs-targeting` step between Stage 2 and Stage 4. A freshly created variation returns `enabled=False` until targeting promotes it to fallthrough — Stage 2 verification will silently take the fallback path on every request. - Don't attempt a multi-agent graph migration in one pass. Migrate a single agent first; use [agent-graph-reference.md](references/agent-graph-reference.md) as the next-step read. ### Stage 5 evaluations @@ -554,12 +554,12 @@ These are ordered by how likely they are to show up as a first-run failure. The ## Related Skills -- `create` — called by Stage 2 to create the config +- `configs-create` — called by Stage 2 to create the config - `tools` — called by Stage 3 to create and attach tool definitions -- `online-evals` — called by Stage 5 to attach judges -- `variations` — add variations for A/B testing after migration is complete -- `targeting` — roll out new variations to users after migration is complete -- `update` — modify config properties as your app evolves +- `configs-online-evals` — called by Stage 5 to attach judges +- `configs-variations` — add variations for A/B testing after migration is complete +- `configs-targeting` — roll out new variations to users after migration is complete +- `configs-update` — modify config properties as your app evolves - `launchdarkly-metric-instrument` — for `ldClient.track()` feature metrics (NOT for AI tracker calls) ## References diff --git a/skills/agentcontrol/migrate/references/agent-graph-reference.md b/skills/agentcontrol/configs-migrate/references/agent-graph-reference.md similarity index 100% rename from skills/agentcontrol/migrate/references/agent-graph-reference.md rename to skills/agentcontrol/configs-migrate/references/agent-graph-reference.md diff --git a/skills/agentcontrol/migrate/references/agent-mode-frameworks.md b/skills/agentcontrol/configs-migrate/references/agent-mode-frameworks.md similarity index 98% rename from skills/agentcontrol/migrate/references/agent-mode-frameworks.md rename to skills/agentcontrol/configs-migrate/references/agent-mode-frameworks.md index 86cf549..57f8a74 100644 --- a/skills/agentcontrol/migrate/references/agent-mode-frameworks.md +++ b/skills/agentcontrol/configs-migrate/references/agent-mode-frameworks.md @@ -16,9 +16,9 @@ Completion mode is the default and covers direct provider calls (OpenAI, Anthrop Agent mode returns an `instructions` string. Completion mode returns a `messages` array. Both modes support tools, parameters, and the same tracker — the only difference is the input shape the SDK returns to you. -**Caveat:** judges cannot be attached to agent-mode variations via the LaunchDarkly UI. Agent mode evaluations must go through the programmatic judge API (`create_judge(...).evaluate(input, output)`). See `online-evals` for the programmatic path. +**Caveat:** judges cannot be attached to agent-mode variations via the LaunchDarkly UI. Agent mode evaluations must go through the programmatic judge API (`create_judge(...).evaluate(input, output)`). See `configs-online-evals` for the programmatic path. -**Model construction for LangChain / LangGraph.** When the framework runs on top of LangChain (which includes LangGraph's prebuilt agent and most custom graphs), build the chat model with `create_langchain_model(ai_config)` (Python) or `createLangChainModel(aiConfig)` (Node). These helpers forward every variation parameter (`temperature`, `max_tokens`, `top_p`, …) and handle LaunchDarkly→LangChain provider-name mapping internally. Do not hand-roll `init_chat_model(model=..., model_provider=...)` — it silently drops every variation parameter. See [langchain-tracking.md](../../ai-metrics/references/langchain-tracking.md) for the canonical single-model and LangGraph patterns, including the SDK helpers `sum_token_usage_from_messages` / `get_tool_calls_from_response` (Python, `ldai_langchain`) used inside the `track_metrics_of_async` / `trackMetricsOf` extractor. +**Model construction for LangChain / LangGraph.** When the framework runs on top of LangChain (which includes LangGraph's prebuilt agent and most custom graphs), build the chat model with `create_langchain_model(ai_config)` (Python) or `createLangChainModel(aiConfig)` (Node). These helpers forward every variation parameter (`temperature`, `max_tokens`, `top_p`, …) and handle LaunchDarkly→LangChain provider-name mapping internally. Do not hand-roll `init_chat_model(model=..., model_provider=...)` — it silently drops every variation parameter. See [langchain-tracking.md](../../configs-ai-metrics/references/langchain-tracking.md) for the canonical single-model and LangGraph patterns, including the SDK helpers `sum_token_usage_from_messages` / `get_tool_calls_from_response` (Python, `ldai_langchain`) used inside the `track_metrics_of_async` / `trackMetricsOf` extractor. ## Framework-agnostic invariants for the run-scoped pattern @@ -291,7 +291,7 @@ async def run_turn(ai_client, user_id: str, user_input: str): **Key points:** - `system_prompt=agent_config.instructions` — the instructions string replaces the hardcoded system prompt. - `create_strands_model(agent_config)` is the provider-dispatch seam. Add a branch per provider the variation can serve. -- The tracker is Tier 3: `tracker.track_duration_of(...)` + an explicit `track_tokens` call fed by `track_strands_metrics`. See [strands-tracking.md](../../ai-metrics/references/strands-tracking.md) for the single-call `track_metrics_of_async` variant and the per-field breakdown of `accumulated_usage`. +- The tracker is Tier 3: `tracker.track_duration_of(...)` + an explicit `track_tokens` call fed by `track_strands_metrics`. See [strands-tracking.md](../../configs-ai-metrics/references/strands-tracking.md) for the single-call `track_metrics_of_async` variant and the per-field breakdown of `accumulated_usage`. - Always `ldclient.get().flush()` before process exit in short-lived scripts — trailing events can otherwise be lost. **TypeScript caveat.** The Strands TypeScript SDK ships `BedrockModel` and `OpenAIModel` only — it cannot run Anthropic-backed variations. If the app needs to serve both OpenAI and Anthropic from a single AI Config, use the Python SDK. @@ -799,7 +799,7 @@ agent = create_agent( **What this gives you:** - Toggle a tool on/off by editing the AI Config in LaunchDarkly — no redeploy needed to remove a tool from production -- Roll out a new tool to 5% of users by editing targeting rules (combined with `targeting`) +- Roll out a new tool to 5% of users by editing targeting rules (combined with `configs-targeting`) - Keep the actual tool implementation code in the repo; only metadata lives in LaunchDarkly ### Extracting schemas from existing hardcoded tools diff --git a/skills/agentcontrol/migrate/references/before-after-examples.md b/skills/agentcontrol/configs-migrate/references/before-after-examples.md similarity index 100% rename from skills/agentcontrol/migrate/references/before-after-examples.md rename to skills/agentcontrol/configs-migrate/references/before-after-examples.md diff --git a/skills/agentcontrol/migrate/references/fallback-defaults-pattern.md b/skills/agentcontrol/configs-migrate/references/fallback-defaults-pattern.md similarity index 99% rename from skills/agentcontrol/migrate/references/fallback-defaults-pattern.md rename to skills/agentcontrol/configs-migrate/references/fallback-defaults-pattern.md index a2bba78..4a02575 100644 --- a/skills/agentcontrol/migrate/references/fallback-defaults-pattern.md +++ b/skills/agentcontrol/configs-migrate/references/fallback-defaults-pattern.md @@ -305,7 +305,7 @@ config = ai_client.agent_config( The SDK runs both paths through the same Mustache renderer. Leaving a Python-style `{var}` literal in the fallback ships a silent regression: LaunchDarkly serves correctly-interpolated output; the fallback ships the unrendered literal, or (worse) a value frozen at import time. -Same rule applies to JS template literals and any other non-Mustache scheme — rewrite to `{{ variable }}` before handing off to `/create`. +Same rule applies to JS template literals and any other non-Mustache scheme — rewrite to `{{ variable }}` before handing off to `/configs-create`. ## Critical rules (apply to all three patterns) diff --git a/skills/agentcontrol/migrate/references/phase-1-analysis-checklist.md b/skills/agentcontrol/configs-migrate/references/phase-1-analysis-checklist.md similarity index 98% rename from skills/agentcontrol/migrate/references/phase-1-analysis-checklist.md rename to skills/agentcontrol/configs-migrate/references/phase-1-analysis-checklist.md index be0a6d4..a95e260 100644 --- a/skills/agentcontrol/migrate/references/phase-1-analysis-checklist.md +++ b/skills/agentcontrol/configs-migrate/references/phase-1-analysis-checklist.md @@ -151,7 +151,7 @@ Feeds into Stage 2 (install + wrap). Quoted from the `ai-configs-relaunch-guides | LangChain / LangGraph | `@launchdarkly/server-sdk-ai-langchain` | `createLangChainModel(config)` (forwards all variation parameters and handles provider-name mapping) + `getAIMetricsFromResponse` with `trackMetricsOf` | | Vercel AI SDK | `@launchdarkly/server-sdk-ai-vercel` | `getAIMetricsFromResponse` + `trackMetricsOf`, or `VercelRunnerFactory.createVercelModel(aiConfig)` for a managed runner | -Python currently ships helper packages for OpenAI (`ldai_openai`) and LangChain (`ldai_langchain`). The LangChain Python package exposes `create_langchain_model(config)` (builds a LangChain chat model from the AI Config, forwarding every variation parameter and mapping LD provider names to LangChain equivalents), `convert_messages_to_langchain`, and `get_ai_metrics_from_response` — the same package covers LangGraph. Use `create_langchain_model(config)` + `track_metrics_of_async(get_ai_metrics_from_response, lambda: llm.ainvoke(messages))` as the canonical single-call pattern. See [langchain-tracking.md](../../ai-metrics/references/langchain-tracking.md) for both LangChain and LangGraph patterns and [sdk-ai-tracker-patterns.md](sdk-ai-tracker-patterns.md) for the full tracker-method matrix. +Python currently ships helper packages for OpenAI (`ldai_openai`) and LangChain (`ldai_langchain`). The LangChain Python package exposes `create_langchain_model(config)` (builds a LangChain chat model from the AI Config, forwarding every variation parameter and mapping LD provider names to LangChain equivalents), `convert_messages_to_langchain`, and `get_ai_metrics_from_response` — the same package covers LangGraph. Use `create_langchain_model(config)` + `track_metrics_of_async(get_ai_metrics_from_response, lambda: llm.ainvoke(messages))` as the canonical single-call pattern. See [langchain-tracking.md](../../configs-ai-metrics/references/langchain-tracking.md) for both LangChain and LangGraph patterns and [sdk-ai-tracker-patterns.md](sdk-ai-tracker-patterns.md) for the full tracker-method matrix. ## Phase 1 output format @@ -185,7 +185,7 @@ Proposed plan: Stage 2 (Wrap): Install SDK, create AI Config 'chat-assistant', inline fallback mirrors current values (Mustache syntax), rewrite the call site Stage 3 (Tools): Skipped (no function calling) / Attach 2 tools via tools Stage 4 (Tracking): Inline tracker wiring (track_duration + track_tokens + track_success/error) — run-scoped tracker for agent loops - Stage 5 (Evals): Attach built-in 'accuracy' judge at 0.25 sampling via online-evals + Stage 5 (Evals): Attach built-in 'accuracy' judge at 0.25 sampling via configs-online-evals ``` ## STOP diff --git a/skills/agentcontrol/migrate/references/sdk-ai-tracker-patterns.md b/skills/agentcontrol/configs-migrate/references/sdk-ai-tracker-patterns.md similarity index 98% rename from skills/agentcontrol/migrate/references/sdk-ai-tracker-patterns.md rename to skills/agentcontrol/configs-migrate/references/sdk-ai-tracker-patterns.md index 2316556..c755d4a 100644 --- a/skills/agentcontrol/migrate/references/sdk-ai-tracker-patterns.md +++ b/skills/agentcontrol/configs-migrate/references/sdk-ai-tracker-patterns.md @@ -297,7 +297,7 @@ response = tracker.track_metrics_of( | Helper | Signature | Tier | Notes | |--------|-----------|------|-------| | `trackMetricsOf(extractor, func)` | `await tracker.trackMetricsOf((result) => extractor(result), async () => ...)` | **2 / 3** | **Canonical generic wrapper.** `extractor` maps provider response → `LDAIMetrics`. Use a provider package's bare `getAIMetricsFromResponse` for Tier 2 (`@launchdarkly/server-sdk-ai-openai`, `-langchain`, `-vercel`) or a small custom function for Tier 3. | -| `trackStreamMetricsOf(extractor, streamCreator)` | `tracker.trackStreamMetricsOf(async (chunks) => extractor(chunks), () => createStream())` | 2 / 3 | Stream variant. Does **not** capture TTFT automatically — if you need TTFT, use the manual pattern in [streaming-tracking.md](../../ai-metrics/references/streaming-tracking.md). | +| `trackStreamMetricsOf(extractor, streamCreator)` | `tracker.trackStreamMetricsOf(async (chunks) => extractor(chunks), () => createStream())` | 2 / 3 | Stream variant. Does **not** capture TTFT automatically — if you need TTFT, use the manual pattern in [streaming-tracking.md](../../configs-ai-metrics/references/streaming-tracking.md). | | `trackDurationOf(func)` | `await tracker.trackDurationOf(async () => ...)` | 4 | Wraps an async callable; captures duration only. Pair with explicit `trackTokens` + `trackSuccess`. | Example — OpenAI via `trackMetricsOf` + the provider package: ```typescript @@ -339,7 +339,7 @@ The managed runner handles message history, provider dispatch (via the installed ### Anthropic has no provider package today -Neither `@launchdarkly/server-sdk-ai-anthropic` nor `launchdarkly-server-sdk-ai-anthropic` exists as of this writing. For Anthropic direct calls, write a custom extractor and pass it to `track_metrics_of` / `trackMetricsOf` — see the Python example above or the full walk-through in [anthropic-tracking.md](../../ai-metrics/references/anthropic-tracking.md). If the app is open to LangChain, routing Anthropic through `ChatAnthropic` and the LangChain provider package recovers Tier 2 with zero extractor code. +Neither `@launchdarkly/server-sdk-ai-anthropic` nor `launchdarkly-server-sdk-ai-anthropic` exists as of this writing. For Anthropic direct calls, write a custom extractor and pass it to `track_metrics_of` / `trackMetricsOf` — see the Python example above or the full walk-through in [anthropic-tracking.md](../../configs-ai-metrics/references/anthropic-tracking.md). If the app is open to LangChain, routing Anthropic through `ChatAnthropic` and the LangChain provider package recovers Tier 2 with zero extractor code. ## Tier decision table @@ -352,7 +352,7 @@ Neither `@launchdarkly/server-sdk-ai-anthropic` nor `launchdarkly-server-sdk-ai- | Anthropic direct SDK | **3** | Custom extractor reading `response.usage.input_tokens` / `output_tokens` | | Bedrock Converse (no provider package) | **3** | Custom extractor reading `response.usage.inputTokens` / `outputTokens` (or route via LangChain for Tier 2) | | Gemini / Google GenAI, Cohere, custom HTTP | **3** | Custom extractor | -| Streaming response with TTFT required | **4** | Manual `trackTimeToFirstToken` + `trackDuration` + `trackTokens` + `trackSuccess` — see [streaming-tracking.md](../../ai-metrics/references/streaming-tracking.md) | +| Streaming response with TTFT required | **4** | Manual `trackTimeToFirstToken` + `trackDuration` + `trackTokens` + `trackSuccess` — see [streaming-tracking.md](../../configs-ai-metrics/references/streaming-tracking.md) | | Streaming response without TTFT (Node) | **2 / 3** | `trackStreamMetricsOf(extractor, streamFn)` | ## Streaming responses @@ -450,7 +450,7 @@ Run the checklist in order. Each step rules out one cause. ## Common gotchas -- **`model.parameters` vs `model.custom`.** `create_langchain_model` (Python) / `createLangChainModel` (Node) forwards every key in `model.parameters` to the provider SDK. App-scoped knobs (search result limits, retry budgets, feature toggles) **must** live in `model.custom` or the provider will crash at runtime with an unexpected-keyword-argument error. Read them with `ai_config.model.get_custom("key")`. Full walk-through with the MCP/REST-API caveat in [langchain-tracking.md § `model.parameters` vs `model.custom`](../../ai-metrics/references/langchain-tracking.md). +- **`model.parameters` vs `model.custom`.** `create_langchain_model` (Python) / `createLangChainModel` (Node) forwards every key in `model.parameters` to the provider SDK. App-scoped knobs (search result limits, retry budgets, feature toggles) **must** live in `model.custom` or the provider will crash at runtime with an unexpected-keyword-argument error. Read them with `ai_config.model.get_custom("key")`. Full walk-through with the MCP/REST-API caveat in [langchain-tracking.md § `model.parameters` vs `model.custom`](../../configs-ai-metrics/references/langchain-tracking.md). - **`track_tokens` token shape.** The Python `TokenUsage` dataclass requires `total` to be set — it is not derived. Compute `total = input + output` if the provider doesn't return one. - **`track_feedback` lifecycle.** The feedback call must be made on a tracker bound to the same `runId` that produced the response. If the thumbs-up comes in a later process, use the cross-process resumption pattern above — do **not** call `create_tracker()` again in the consumer, because that mints a *new* `runId`. - **OpenAI streaming tokens.** OpenAI only emits `usage` in the final chunk when `stream_options={"include_usage": True}` is passed. Without that flag, you have to tokenize manually — `tiktoken` for OpenAI models. diff --git a/skills/agentcontrol/online-evals/README.md b/skills/agentcontrol/configs-online-evals/README.md similarity index 70% rename from skills/agentcontrol/online-evals/README.md rename to skills/agentcontrol/configs-online-evals/README.md index 3dc0b8e..056da3b 100644 --- a/skills/agentcontrol/online-evals/README.md +++ b/skills/agentcontrol/configs-online-evals/README.md @@ -12,12 +12,12 @@ This skill teaches agents how to: ## Installation (Local) -Copy `skills/agentcontrol/online-evals/` into your agent client's skills path. +Copy `skills/agentcontrol/configs-online-evals/` into your agent client's skills path. ## Prerequisites - LaunchDarkly API access token with `ai-configs:write` permission -- Existing AI Config with variations (use `create` skill) +- Existing AI Config with variations (use `configs-create` skill) - For custom judges: understanding of LLM-as-a-judge methodology ## Usage @@ -33,16 +33,16 @@ Create a custom judge that checks for scope creep in code changes ## Structure ``` -online-evals/ +configs-online-evals/ ├── SKILL.md └── README.md ``` ## Related -- [AI Config Create](../create/) - Create AI Configs first -- [AI Config Targeting](../targeting/) - Enable targeting on judges -- [AI Config Variations](../variations/) - Manage variations +- [AI Config Create](../configs-create/) - Create AI Configs first +- [AI Config Targeting](../configs-targeting/) - Enable targeting on judges +- [AI Config Variations](../configs-variations/) - Manage variations - [Online Evaluations Docs](https://docs.launchdarkly.com/home/ai-configs/online-evaluations) ## License diff --git a/skills/agentcontrol/online-evals/SKILL.md b/skills/agentcontrol/configs-online-evals/SKILL.md similarity index 98% rename from skills/agentcontrol/online-evals/SKILL.md rename to skills/agentcontrol/configs-online-evals/SKILL.md index a024285..bd99480 100644 --- a/skills/agentcontrol/online-evals/SKILL.md +++ b/skills/agentcontrol/configs-online-evals/SKILL.md @@ -1,5 +1,5 @@ --- -name: online-evals +name: configs-online-evals description: Attach judges to AI Config variations for automatic LLM-as-a-judge evaluation. Create custom judges, configure sampling rates, and monitor quality scores. compatibility: Requires LaunchDarkly API access token with ai-configs:write permission. SDK versions Python v0.20.0+ or Node.js v0.20.0+ for automatic metric recording and the consolidated `track_judge_result` / `trackJudgeResult` API. metadata: @@ -15,7 +15,7 @@ Attach judges to AI Config variations for automatic quality scoring using LLM-as - LaunchDarkly account with AI Configs enabled - API access token with write permissions -- Existing AI Config with variations (use `create` skill) +- Existing AI Config with variations (use `configs-create` skill) - For automatic metric recording and the consolidated judge-result API: Python AI SDK v0.20.0+ or Node.js AI SDK v0.20.0+ ## API Key Detection @@ -437,9 +437,9 @@ After attaching judges: ## Related Skills -- `create` - Create AI Configs and judges -- `targeting` - Configure targeting rules -- `variations` - Manage variations +- `configs-create` - Create AI Configs and judges +- `configs-targeting` - Configure targeting rules +- `configs-variations` - Manage variations ## References diff --git a/skills/agentcontrol/snippets/SKILL.md b/skills/agentcontrol/configs-snippets/SKILL.md similarity index 99% rename from skills/agentcontrol/snippets/SKILL.md rename to skills/agentcontrol/configs-snippets/SKILL.md index ecea329..c4e5393 100644 --- a/skills/agentcontrol/snippets/SKILL.md +++ b/skills/agentcontrol/configs-snippets/SKILL.md @@ -1,5 +1,5 @@ --- -name: snippets +name: configs-snippets description: "Create and manage prompt snippets — reusable text blocks referenced inside AI Config variation prompts. Keeps common instructions, personas, and guardrails consistent across multiple configs." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server diff --git a/skills/agentcontrol/targeting/README.md b/skills/agentcontrol/configs-targeting/README.md similarity index 71% rename from skills/agentcontrol/targeting/README.md rename to skills/agentcontrol/configs-targeting/README.md index 4566f64..1240e0e 100644 --- a/skills/agentcontrol/targeting/README.md +++ b/skills/agentcontrol/configs-targeting/README.md @@ -13,12 +13,12 @@ This skill teaches agents how to: ## Installation (Local) -Copy `skills/agentcontrol/targeting/` into your agent client's skills path. +Copy `skills/agentcontrol/configs-targeting/` into your agent client's skills path. ## Prerequisites - LaunchDarkly API access token with `ai-configs:write` permission -- Existing AI Config with variations (use `create` skill) +- Existing AI Config with variations (use `configs-create` skill) - Understanding of contexts (see `context-basic` skill) ## Usage @@ -34,16 +34,16 @@ Add a percentage rollout: 60% to variation A, 40% to variation B for premium use ## Structure ``` -targeting/ +configs-targeting/ ├── SKILL.md └── README.md ``` ## Related -- [AI Config Create](../create/) - Create AI Configs first -- [AI Config Variations](../variations/) - Create variations to target -- [AI Config Online Evals](../online-evals/) - Attach judges +- [AI Config Create](../configs-create/) - Create AI Configs first +- [AI Config Variations](../configs-variations/) - Create variations to target +- [AI Config Online Evals](../configs-online-evals/) - Attach judges - [Targeting Docs](https://docs.launchdarkly.com/home/ai-configs/target) ## License diff --git a/skills/agentcontrol/targeting/SKILL.md b/skills/agentcontrol/configs-targeting/SKILL.md similarity index 97% rename from skills/agentcontrol/targeting/SKILL.md rename to skills/agentcontrol/configs-targeting/SKILL.md index 2d06b2b..47bebb6 100644 --- a/skills/agentcontrol/targeting/SKILL.md +++ b/skills/agentcontrol/configs-targeting/SKILL.md @@ -1,5 +1,5 @@ --- -name: targeting +name: configs-targeting description: Configure AI Config targeting rules to control which variations serve to different users. Enable percentage rollouts, attribute-based rules, segment targeting, and guarded rollouts. compatibility: Requires LaunchDarkly API access token with ai-configs:write permission. metadata: @@ -16,7 +16,7 @@ Configure targeting rules for AI Configs to control which variations serve to di - LaunchDarkly account with AI Configs enabled - API access token with write permissions - Project key and environment key -- Existing AI Config with variations (use `create` skill) +- Existing AI Config with variations (use `configs-create` skill) ## API Key Detection @@ -488,15 +488,15 @@ After configuring targeting: ``` https://app.launchdarkly.com/projects/{projectKey}/ai-configs/{configKey} ``` -2. **Monitor performance** with `ai-metrics` -3. **Attach judges** with `online-evals` +2. **Monitor performance** with `configs-ai-metrics` +3. **Attach judges** with `configs-online-evals` 4. **Set up guarded rollouts** for automatic regression detection ## Related Skills -- `create` - Create AI Configs with variations -- `variations` - Manage variations -- `online-evals` - Attach judges +- `configs-create` - Create AI Configs with variations +- `configs-variations` - Manage variations +- `configs-online-evals` - Attach judges - `segments` - Create segments for targeting ## References diff --git a/skills/agentcontrol/update/README.md b/skills/agentcontrol/configs-update/README.md similarity index 82% rename from skills/agentcontrol/update/README.md rename to skills/agentcontrol/configs-update/README.md index d90648a..4582294 100644 --- a/skills/agentcontrol/update/README.md +++ b/skills/agentcontrol/configs-update/README.md @@ -13,7 +13,7 @@ This skill teaches agents how to: ## Installation (Local) -Copy `skills/agentcontrol/update/` into your agent client's skills path. +Copy `skills/agentcontrol/configs-update/` into your agent client's skills path. ## Prerequisites @@ -36,15 +36,15 @@ Archive the old chatbot config ## Structure ``` -update/ +configs-update/ ├── SKILL.md └── README.md ``` ## Related -- [AI Config Create](../create/): Create configs -- [AI Config Variations](../variations/): Add or test variations +- [AI Config Create](../configs-create/): Create configs +- [AI Config Variations](../configs-variations/): Add or test variations - [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) ## License diff --git a/skills/agentcontrol/update/SKILL.md b/skills/agentcontrol/configs-update/SKILL.md similarity index 97% rename from skills/agentcontrol/update/SKILL.md rename to skills/agentcontrol/configs-update/SKILL.md index 67250ae..e05d2c1 100644 --- a/skills/agentcontrol/update/SKILL.md +++ b/skills/agentcontrol/configs-update/SKILL.md @@ -1,5 +1,5 @@ --- -name: update +name: configs-update description: "Update, archive, and delete LaunchDarkly AI Configs and their variations. Use when you need to modify config properties, change model parameters, update instructions or messages, archive unused configs, or permanently remove them." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server @@ -90,5 +90,5 @@ Use `get-ai-config` to confirm the response shows your updated values. ## Related Skills -- `variations` -- Create variations to test changes side-by-side +- `configs-variations` -- Create variations to test changes side-by-side - `tools` -- Update tool attachments diff --git a/skills/agentcontrol/variations/README.md b/skills/agentcontrol/configs-variations/README.md similarity index 80% rename from skills/agentcontrol/variations/README.md rename to skills/agentcontrol/configs-variations/README.md index 7d89218..307ca2e 100644 --- a/skills/agentcontrol/variations/README.md +++ b/skills/agentcontrol/configs-variations/README.md @@ -11,7 +11,7 @@ This skill teaches agents how to: ## Installation (Local) -Copy `skills/agentcontrol/variations/` into your agent client's skills path. +Copy `skills/agentcontrol/configs-variations/` into your agent client's skills path. ## Prerequisites @@ -30,15 +30,15 @@ Create variations to compare Claude vs GPT-4 for our agent ## Structure ``` -variations/ +configs-variations/ ├── SKILL.md └── README.md ``` ## Related -- [AI Config Create](../create/): Create the config first -- [AI Config Update](../update/): Modify existing variations +- [AI Config Create](../configs-create/): Create the config first +- [AI Config Update](../configs-update/): Modify existing variations - [AI Config Tools](../tools/): Attach tools to variations - [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) diff --git a/skills/agentcontrol/variations/SKILL.md b/skills/agentcontrol/configs-variations/SKILL.md similarity index 97% rename from skills/agentcontrol/variations/SKILL.md rename to skills/agentcontrol/configs-variations/SKILL.md index 0586178..26c58e0 100644 --- a/skills/agentcontrol/variations/SKILL.md +++ b/skills/agentcontrol/configs-variations/SKILL.md @@ -1,5 +1,5 @@ --- -name: variations +name: configs-variations description: "Experiment with AI configurations by creating and managing variations. Helps you test different models, prompts, and parameters to find what works best through systematic experimentation." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server @@ -106,5 +106,5 @@ When the user wants to try a different model, prompt, or parameters, **always cr ## Related Skills -- `create` -- Create the initial config -- `update` -- Refine based on learnings +- `configs-create` -- Create the initial config +- `configs-update` -- Refine based on learnings diff --git a/skills/agentcontrol/projects/SKILL.md b/skills/agentcontrol/projects/SKILL.md index 0871fe6..ddb94af 100644 --- a/skills/agentcontrol/projects/SKILL.md +++ b/skills/agentcontrol/projects/SKILL.md @@ -221,16 +221,16 @@ ai-apac → Asia-Pacific Region After setting up projects: -1. **Create AI Configs** - Use the `create` skill +1. **Create AI Configs** - Use the `configs-create` skill 2. **Set up SDK Integration** - Use the `sdk` skill -3. **Configure Targeting** - Use the `targeting` skill +3. **Configure Targeting** - Use the `configs-targeting` skill ## Related Skills -- `create` - Create AI Configs in projects +- `configs-create` - Create AI Configs in projects - `sdk` - Integrate SDK in your application -- `targeting` - Configure AI Config targeting -- `variations` - Manage config variations +- `configs-targeting` - Configure AI Config targeting +- `configs-variations` - Manage config variations ## References diff --git a/skills/agentcontrol/tools/README.md b/skills/agentcontrol/tools/README.md index 3bf63a0..19f427a 100644 --- a/skills/agentcontrol/tools/README.md +++ b/skills/agentcontrol/tools/README.md @@ -38,8 +38,8 @@ tools/ ## Related -- [AI Config Create](../create/): Create the config before adding tools -- [AI Config Variations](../variations/): Manage variations that tools attach to +- [AI Config Create](../configs-create/): Create the config before adding tools +- [AI Config Variations](../configs-variations/): Manage variations that tools attach to - [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) ## License diff --git a/skills/agentcontrol/tools/SKILL.md b/skills/agentcontrol/tools/SKILL.md index cb17288..e4c8957 100644 --- a/skills/agentcontrol/tools/SKILL.md +++ b/skills/agentcontrol/tools/SKILL.md @@ -211,13 +211,13 @@ for _ in range(MAX_STEPS): messages.append({"role": "user", "content": tool_results}) ``` -Per-provider tool-call payload shapes live in the `ai-metrics` references: +Per-provider tool-call payload shapes live in the `configs-ai-metrics` references: -- [openai-tracking.md](../ai-metrics/references/openai-tracking.md) — Chat Completions + Responses API -- [anthropic-tracking.md](../ai-metrics/references/anthropic-tracking.md) — `tool_use` blocks and `tool_result` payloads -- [bedrock-tracking.md](../ai-metrics/references/bedrock-tracking.md) — `toolUse` / `toolResult` Converse format -- [gemini-tracking.md](../ai-metrics/references/gemini-tracking.md) — `functionCalls` / `functionResponse` parts -- [langchain-tracking.md](../ai-metrics/references/langchain-tracking.md) — LangGraph tool loop inherits from `create_react_agent` +- [openai-tracking.md](../configs-ai-metrics/references/openai-tracking.md) — Chat Completions + Responses API +- [anthropic-tracking.md](../configs-ai-metrics/references/anthropic-tracking.md) — `tool_use` blocks and `tool_result` payloads +- [bedrock-tracking.md](../configs-ai-metrics/references/bedrock-tracking.md) — `toolUse` / `toolResult` Converse format +- [gemini-tracking.md](../configs-ai-metrics/references/gemini-tracking.md) — `functionCalls` / `functionResponse` parts +- [langchain-tracking.md](../configs-ai-metrics/references/langchain-tracking.md) — LangGraph tool loop inherits from `create_react_agent` ## Orchestrator Note @@ -240,5 +240,5 @@ LangGraph, CrewAI, and AutoGen often generate schemas from function definitions. ## Related Skills -- `create` -- Create config before attaching tools -- `variations` -- Manage variations with different tool sets +- `configs-create` -- Create config before attaching tools +- `configs-variations` -- Manage variations with different tool sets diff --git a/skills/aiconfig-create b/skills/aiconfig-create deleted file mode 120000 index 60cf28a..0000000 --- a/skills/aiconfig-create +++ /dev/null @@ -1 +0,0 @@ -ai-configs/aiconfig-create \ No newline at end of file diff --git a/skills/aiconfig-online-evals b/skills/aiconfig-online-evals deleted file mode 120000 index 689a81a..0000000 --- a/skills/aiconfig-online-evals +++ /dev/null @@ -1 +0,0 @@ -ai-configs/aiconfig-online-evals \ No newline at end of file diff --git a/skills/aiconfig-projects b/skills/aiconfig-projects deleted file mode 120000 index 991a943..0000000 --- a/skills/aiconfig-projects +++ /dev/null @@ -1 +0,0 @@ -ai-configs/aiconfig-projects \ No newline at end of file diff --git a/skills/aiconfig-targeting b/skills/aiconfig-targeting deleted file mode 120000 index 756a566..0000000 --- a/skills/aiconfig-targeting +++ /dev/null @@ -1 +0,0 @@ -ai-configs/aiconfig-targeting \ No newline at end of file diff --git a/skills/aiconfig-tools b/skills/aiconfig-tools deleted file mode 120000 index faa4c34..0000000 --- a/skills/aiconfig-tools +++ /dev/null @@ -1 +0,0 @@ -ai-configs/aiconfig-tools \ No newline at end of file diff --git a/skills/aiconfig-update b/skills/aiconfig-update deleted file mode 120000 index e1e9d31..0000000 --- a/skills/aiconfig-update +++ /dev/null @@ -1 +0,0 @@ -ai-configs/aiconfig-update \ No newline at end of file diff --git a/skills/aiconfig-variations b/skills/aiconfig-variations deleted file mode 120000 index f1c714d..0000000 --- a/skills/aiconfig-variations +++ /dev/null @@ -1 +0,0 @@ -ai-configs/aiconfig-variations \ No newline at end of file diff --git a/skills/configs-create b/skills/configs-create new file mode 120000 index 0000000..c55a91b --- /dev/null +++ b/skills/configs-create @@ -0,0 +1 @@ +agentcontrol/configs-create \ No newline at end of file diff --git a/skills/configs-online-evals b/skills/configs-online-evals new file mode 120000 index 0000000..c80f182 --- /dev/null +++ b/skills/configs-online-evals @@ -0,0 +1 @@ +agentcontrol/configs-online-evals \ No newline at end of file diff --git a/skills/configs-targeting b/skills/configs-targeting new file mode 120000 index 0000000..4cf3e85 --- /dev/null +++ b/skills/configs-targeting @@ -0,0 +1 @@ +agentcontrol/configs-targeting \ No newline at end of file diff --git a/skills/configs-update b/skills/configs-update new file mode 120000 index 0000000..6dd07da --- /dev/null +++ b/skills/configs-update @@ -0,0 +1 @@ +agentcontrol/configs-update \ No newline at end of file diff --git a/skills/configs-variations b/skills/configs-variations new file mode 120000 index 0000000..79453ca --- /dev/null +++ b/skills/configs-variations @@ -0,0 +1 @@ +agentcontrol/configs-variations \ No newline at end of file diff --git a/skills/projects b/skills/projects new file mode 120000 index 0000000..79b64fa --- /dev/null +++ b/skills/projects @@ -0,0 +1 @@ +agentcontrol/projects \ No newline at end of file diff --git a/skills/tools b/skills/tools new file mode 120000 index 0000000..8c65647 --- /dev/null +++ b/skills/tools @@ -0,0 +1 @@ +agentcontrol/tools \ No newline at end of file From edf49b5f911639108eb19fa8b97d3b0b8446afef Mon Sep 17 00:00:00 2001 From: Scarlett Attensil Date: Tue, 19 May 2026 11:51:41 -0700 Subject: [PATCH 05/10] Regenerate skills.json via generate_catalog (fixes Validate CI) skills.json must be sorted/emitted by scripts/generate_catalog.py; the manual edits left it out of order, failing the Validate Skills check. Regenerated; also picks up the configs-agent-graphs description tweak (AI Configs -> configs). --- skills.json | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/skills.json b/skills.json index ce4c026..ca45797 100644 --- a/skills.json +++ b/skills.json @@ -1,8 +1,16 @@ { "skills": [ + { + "name": "apply", + "description": "Apply LaunchDarkly SDK onboarding: install dependency (or dual-SDK pair), configure env and secrets with consent, add init at entrypoint(s), verify compile. Nested under sdk-install; next is run.", + "path": "skills/onboarding/sdk-install/apply", + "version": "0.2.0", + "license": "Apache-2.0", + "compatibility": "Requires integration plan and LaunchDarkly credentials (see parent onboarding)" + }, { "name": "configs-agent-graphs", - "description": "Create and manage agent graphs \u2014 directed graphs of AI Configs connected by edges with handoff logic. Use when building multi-agent workflows where configs route to each other.", + "description": "Create and manage agent graphs \u2014 directed graphs of configs connected by edges with handoff logic. Use when building multi-agent workflows where configs route to each other.", "path": "skills/agentcontrol/configs-agent-graphs", "version": "0.1.0", "license": "Apache-2.0", @@ -47,13 +55,6 @@ "version": "0.1.0", "compatibility": "Requires LaunchDarkly API access token with ai-configs:write permission. SDK versions Python v0.20.0+ or Node.js v0.20.0+ for automatic metric recording and the consolidated `track_judge_result` / `trackJudgeResult` API." }, - { - "name": "projects", - "description": "Guide for setting up LaunchDarkly projects in your codebase. Helps you assess your stack, choose the right approach, and integrate project management that makes sense for your architecture.", - "path": "skills/agentcontrol/projects", - "version": "0.4.0", - "compatibility": "Requires LaunchDarkly API access token with projects:write permission or LaunchDarkly MCP server." - }, { "name": "configs-snippets", "description": "Create and manage prompt snippets \u2014 reusable text blocks referenced inside AI Config variation prompts. Keeps common instructions, personas, and guardrails consistent across multiple configs.", @@ -69,14 +70,6 @@ "version": "0.1.0", "compatibility": "Requires LaunchDarkly API access token with ai-configs:write permission." }, - { - "name": "tools", - "description": "Give your AI agents capabilities through tools (function calling). Helps you identify what your AI needs to do, create tool definitions, and attach them to AI Config variations.", - "path": "skills/agentcontrol/tools", - "version": "1.0.0-experimental", - "license": "Apache-2.0", - "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" - }, { "name": "configs-update", "description": "Update, archive, and delete LaunchDarkly AI Configs and their variations. Use when you need to modify config properties, change model parameters, update instructions or messages, archive unused configs, or permanently remove them.", @@ -93,14 +86,6 @@ "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, - { - "name": "apply", - "description": "Apply LaunchDarkly SDK onboarding: install dependency (or dual-SDK pair), configure env and secrets with consent, add init at entrypoint(s), verify compile. Nested under sdk-install; next is run.", - "path": "skills/onboarding/sdk-install/apply", - "version": "0.2.0", - "license": "Apache-2.0", - "compatibility": "Requires integration plan and LaunchDarkly credentials (see parent onboarding)" - }, { "name": "detect", "description": "Detect repository stack for LaunchDarkly SDK onboarding: languages, frameworks, package managers, monorepo targets, entrypoints, existing LD usage. Nested under sdk-install; next is plan.", @@ -266,6 +251,13 @@ "license": "Apache-2.0", "compatibility": "Requires completed or equivalent detect context (see sibling detect skill)" }, + { + "name": "projects", + "description": "Guide for setting up LaunchDarkly projects in your codebase. Helps you assess your stack, choose the right approach, and integrate project management that makes sense for your architecture.", + "path": "skills/agentcontrol/projects", + "version": "0.4.0", + "compatibility": "Requires LaunchDarkly API access token with projects:write permission or LaunchDarkly MCP server." + }, { "name": "sdk-install", "description": "Install and initialize the correct LaunchDarkly SDK during onboarding by running nested skills in order: detect, plan, apply. Parent onboarding Step 6 is first flag.", @@ -273,6 +265,14 @@ "version": "0.2.0", "license": "Apache-2.0", "compatibility": "Requires a supported language/framework in the project. SDK credentials are required by [Apply](apply/SKILL.md), not for [Detect](detect/SKILL.md) / [Plan](plan/SKILL.md) alone (see parent onboarding **Prerequisites**)." + }, + { + "name": "tools", + "description": "Give your AI agents capabilities through tools (function calling). Helps you identify what your AI needs to do, create tool definitions, and attach them to AI Config variations.", + "path": "skills/agentcontrol/tools", + "version": "1.0.0-experimental", + "license": "Apache-2.0", + "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" } ] } From 5cf9e21a6f385fa88c1326f75db8dba4b94f9249 Mon Sep 17 00:00:00 2001 From: Scarlett Attensil Date: Tue, 19 May 2026 11:58:51 -0700 Subject: [PATCH 06/10] Rename configs-ai-metrics -> configs-built-in-metrics Per request: the built-in/AI metrics skill is now configs-built-in-metrics (pairs with configs-custom-metrics). Updates directory, SKILL.md frontmatter, all cross-references, and regenerates skills.json. --- skills.json | 4 ++-- .../SKILL.md | 2 +- .../references/anthropic-tracking.md | 0 .../references/bedrock-tracking.md | 0 .../references/gemini-tracking.md | 0 .../references/langchain-tracking.md | 0 .../references/metrics-api.md | 0 .../references/openai-tracking.md | 0 .../references/strands-tracking.md | 0 .../references/streaming-tracking.md | 0 .../configs-custom-metrics/SKILL.md | 4 ++-- skills/agentcontrol/configs-migrate/SKILL.md | 24 +++++++++---------- .../references/agent-mode-frameworks.md | 4 ++-- .../references/phase-1-analysis-checklist.md | 2 +- .../references/sdk-ai-tracker-patterns.md | 8 +++---- .../agentcontrol/configs-targeting/SKILL.md | 2 +- skills/agentcontrol/tools/SKILL.md | 12 +++++----- 17 files changed, 31 insertions(+), 31 deletions(-) rename skills/agentcontrol/{configs-ai-metrics => configs-built-in-metrics}/SKILL.md (99%) rename skills/agentcontrol/{configs-ai-metrics => configs-built-in-metrics}/references/anthropic-tracking.md (100%) rename skills/agentcontrol/{configs-ai-metrics => configs-built-in-metrics}/references/bedrock-tracking.md (100%) rename skills/agentcontrol/{configs-ai-metrics => configs-built-in-metrics}/references/gemini-tracking.md (100%) rename skills/agentcontrol/{configs-ai-metrics => configs-built-in-metrics}/references/langchain-tracking.md (100%) rename skills/agentcontrol/{configs-ai-metrics => configs-built-in-metrics}/references/metrics-api.md (100%) rename skills/agentcontrol/{configs-ai-metrics => configs-built-in-metrics}/references/openai-tracking.md (100%) rename skills/agentcontrol/{configs-ai-metrics => configs-built-in-metrics}/references/strands-tracking.md (100%) rename skills/agentcontrol/{configs-ai-metrics => configs-built-in-metrics}/references/streaming-tracking.md (100%) diff --git a/skills.json b/skills.json index ca45797..cb18deb 100644 --- a/skills.json +++ b/skills.json @@ -17,9 +17,9 @@ "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, { - "name": "configs-ai-metrics", + "name": "configs-built-in-metrics", "description": "Instrument an existing codebase with LaunchDarkly AI Config tracking. Walks the four-tier ladder (managed runner \u2192 provider package \u2192 custom extractor + trackMetricsOf \u2192 raw manual) and picks the lowest-ceremony option that still captures duration, tokens, and success/error.", - "path": "skills/agentcontrol/configs-ai-metrics", + "path": "skills/agentcontrol/configs-built-in-metrics", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the LaunchDarkly server-side AI SDK (`launchdarkly-server-sdk-ai>=0.20.0` for Python or `@launchdarkly/server-sdk-ai>=0.20.0` for Node) and an existing AI Config." diff --git a/skills/agentcontrol/configs-ai-metrics/SKILL.md b/skills/agentcontrol/configs-built-in-metrics/SKILL.md similarity index 99% rename from skills/agentcontrol/configs-ai-metrics/SKILL.md rename to skills/agentcontrol/configs-built-in-metrics/SKILL.md index 96170a3..78eff9a 100644 --- a/skills/agentcontrol/configs-ai-metrics/SKILL.md +++ b/skills/agentcontrol/configs-built-in-metrics/SKILL.md @@ -1,5 +1,5 @@ --- -name: configs-ai-metrics +name: configs-built-in-metrics description: "Instrument an existing codebase with LaunchDarkly AI Config tracking. Walks the four-tier ladder (managed runner → provider package → custom extractor + trackMetricsOf → raw manual) and picks the lowest-ceremony option that still captures duration, tokens, and success/error." license: Apache-2.0 compatibility: Requires the LaunchDarkly server-side AI SDK (`launchdarkly-server-sdk-ai>=0.20.0` for Python or `@launchdarkly/server-sdk-ai>=0.20.0` for Node) and an existing AI Config. diff --git a/skills/agentcontrol/configs-ai-metrics/references/anthropic-tracking.md b/skills/agentcontrol/configs-built-in-metrics/references/anthropic-tracking.md similarity index 100% rename from skills/agentcontrol/configs-ai-metrics/references/anthropic-tracking.md rename to skills/agentcontrol/configs-built-in-metrics/references/anthropic-tracking.md diff --git a/skills/agentcontrol/configs-ai-metrics/references/bedrock-tracking.md b/skills/agentcontrol/configs-built-in-metrics/references/bedrock-tracking.md similarity index 100% rename from skills/agentcontrol/configs-ai-metrics/references/bedrock-tracking.md rename to skills/agentcontrol/configs-built-in-metrics/references/bedrock-tracking.md diff --git a/skills/agentcontrol/configs-ai-metrics/references/gemini-tracking.md b/skills/agentcontrol/configs-built-in-metrics/references/gemini-tracking.md similarity index 100% rename from skills/agentcontrol/configs-ai-metrics/references/gemini-tracking.md rename to skills/agentcontrol/configs-built-in-metrics/references/gemini-tracking.md diff --git a/skills/agentcontrol/configs-ai-metrics/references/langchain-tracking.md b/skills/agentcontrol/configs-built-in-metrics/references/langchain-tracking.md similarity index 100% rename from skills/agentcontrol/configs-ai-metrics/references/langchain-tracking.md rename to skills/agentcontrol/configs-built-in-metrics/references/langchain-tracking.md diff --git a/skills/agentcontrol/configs-ai-metrics/references/metrics-api.md b/skills/agentcontrol/configs-built-in-metrics/references/metrics-api.md similarity index 100% rename from skills/agentcontrol/configs-ai-metrics/references/metrics-api.md rename to skills/agentcontrol/configs-built-in-metrics/references/metrics-api.md diff --git a/skills/agentcontrol/configs-ai-metrics/references/openai-tracking.md b/skills/agentcontrol/configs-built-in-metrics/references/openai-tracking.md similarity index 100% rename from skills/agentcontrol/configs-ai-metrics/references/openai-tracking.md rename to skills/agentcontrol/configs-built-in-metrics/references/openai-tracking.md diff --git a/skills/agentcontrol/configs-ai-metrics/references/strands-tracking.md b/skills/agentcontrol/configs-built-in-metrics/references/strands-tracking.md similarity index 100% rename from skills/agentcontrol/configs-ai-metrics/references/strands-tracking.md rename to skills/agentcontrol/configs-built-in-metrics/references/strands-tracking.md diff --git a/skills/agentcontrol/configs-ai-metrics/references/streaming-tracking.md b/skills/agentcontrol/configs-built-in-metrics/references/streaming-tracking.md similarity index 100% rename from skills/agentcontrol/configs-ai-metrics/references/streaming-tracking.md rename to skills/agentcontrol/configs-built-in-metrics/references/streaming-tracking.md diff --git a/skills/agentcontrol/configs-custom-metrics/SKILL.md b/skills/agentcontrol/configs-custom-metrics/SKILL.md index 794603e..be25a8f 100644 --- a/skills/agentcontrol/configs-custom-metrics/SKILL.md +++ b/skills/agentcontrol/configs-custom-metrics/SKILL.md @@ -16,7 +16,7 @@ Full lifecycle management of custom business metrics: create metric definitions - LaunchDarkly SDK initialized (see `sdk`) - LaunchDarkly API token with `writer` role for metric management -- Understanding of built-in AI metrics (see `configs-ai-metrics`) +- Understanding of built-in AI metrics (see `configs-built-in-metrics`) ## API Key Detection @@ -493,7 +493,7 @@ Custom metrics appear in: ## Related Skills - `sdk` - SDK setup -- `configs-ai-metrics` - Built-in AI metrics (tokens, duration, cost) +- `configs-built-in-metrics` - Built-in AI metrics (tokens, duration, cost) - `configs-online-evals` - Quality metrics via judges ## References diff --git a/skills/agentcontrol/configs-migrate/SKILL.md b/skills/agentcontrol/configs-migrate/SKILL.md index c90ca2e..b950344 100644 --- a/skills/agentcontrol/configs-migrate/SKILL.md +++ b/skills/agentcontrol/configs-migrate/SKILL.md @@ -30,16 +30,16 @@ The skill is optimized for Python and Node.js / TypeScript; other languages are | Shape | Python | Node.js | Reference | |-------|--------|---------|-----------| -| One-shot completion (direct OpenAI / Anthropic / Bedrock / Gemini call) | ✅ Worked example | ✅ Worked example | [before-after-examples.md](references/before-after-examples.md), per-provider docs in `configs-ai-metrics/references/` | -| Chat loop via managed runner (`ManagedModel`) | ✅ Tier 1 pattern | ✅ Tier 1 pattern | [configs-ai-metrics SKILL.md](../configs-ai-metrics/SKILL.md) | -| LangChain single-call | ✅ Worked example | ✅ Worked example | [langchain-tracking.md](../configs-ai-metrics/references/langchain-tracking.md) | +| One-shot completion (direct OpenAI / Anthropic / Bedrock / Gemini call) | ✅ Worked example | ✅ Worked example | [before-after-examples.md](references/before-after-examples.md), per-provider docs in `configs-built-in-metrics/references/` | +| Chat loop via managed runner (`ManagedModel`) | ✅ Tier 1 pattern | ✅ Tier 1 pattern | [configs-built-in-metrics SKILL.md](../configs-built-in-metrics/SKILL.md) | +| LangChain single-call | ✅ Worked example | ✅ Worked example | [langchain-tracking.md](../configs-built-in-metrics/references/langchain-tracking.md) | | LangGraph prebuilt agent (Python `langchain.agents.create_agent`, Node `createReactAgent`) | ✅ Worked example | ✅ Worked example | [agent-mode-frameworks.md § LangGraph](references/agent-mode-frameworks.md) | | LangGraph custom `StateGraph` with run-scoped tracker (setup_run + call_model + finalize) | ✅ Deep worked example | ⚠️ Mentioned — translate from Python | [agent-mode-frameworks.md § Custom `StateGraph`](references/agent-mode-frameworks.md) | | CrewAI `Agent` | ✅ Worked example | — (not a Node framework) | [agent-mode-frameworks.md § CrewAI](references/agent-mode-frameworks.md) | | Strands `Agent` | ✅ Worked example | ⚠️ BedrockModel + OpenAIModel only (no Anthropic) | [agent-mode-frameworks.md § Strands](references/agent-mode-frameworks.md) | | Custom ReAct loop (hand-rolled, any framework or none) | ✅ Worked example | ⚠️ Apply framework-agnostic invariants; translate from Python | [agent-mode-frameworks.md § Custom ReAct loop](references/agent-mode-frameworks.md) | -| Vercel AI SDK (`generateText` / `streamText`) | — (not a Python framework) | ⚠️ Provider package exists; no worked example in skill | `configs-ai-metrics` provider-package matrix | -| Streaming (SSE / WebSocket) | ⚠️ Delegated to `configs-ai-metrics` streaming doc | ⚠️ Same — use `trackStreamMetricsOf` + manual TTFT | [streaming-tracking.md](../configs-ai-metrics/references/streaming-tracking.md) | +| Vercel AI SDK (`generateText` / `streamText`) | — (not a Python framework) | ⚠️ Provider package exists; no worked example in skill | `configs-built-in-metrics` provider-package matrix | +| Streaming (SSE / WebSocket) | ⚠️ Delegated to `configs-built-in-metrics` streaming doc | ⚠️ Same — use `trackStreamMetricsOf` + manual TTFT | [streaming-tracking.md](../configs-built-in-metrics/references/streaming-tracking.md) | | Multi-agent graph (supervisor + workers) | ⚠️ Out of main scope; see reference | ⚠️ Out of main scope; see reference | [agent-graph-reference.md](references/agent-graph-reference.md) | | Non-LangGraph agent frameworks (Pydantic AI, DSPy, AutoGen, Haystack, LlamaIndex agents, Semantic Kernel) | ⚠️ Apply the three invariants; no framework-specific example | ⚠️ Same | [agent-mode-frameworks.md § Framework-agnostic invariants](references/agent-mode-frameworks.md) | | Go, Ruby, .NET | ℹ️ Install commands only | ℹ️ Install commands only | [phase-1-analysis-checklist.md § SDK routing table](references/phase-1-analysis-checklist.md) | @@ -160,7 +160,7 @@ This is the first stage that writes code. It has nine sub-steps. - OpenAI: `launchdarkly-server-sdk-ai-openai>=0.4.0` (Python) / `@launchdarkly/server-sdk-ai-openai@^0.5.5` (Node) - LangChain / LangGraph: `launchdarkly-server-sdk-ai-langchain>=0.5.0` (Python) / `@launchdarkly/server-sdk-ai-langchain@^0.5.5` (Node) - Vercel AI SDK (Node only): `@launchdarkly/server-sdk-ai-vercel@^0.5.5` - - Anthropic, Gemini, Bedrock — no provider package published; use Tier-3 custom extractor (see `configs-ai-metrics`) + - Anthropic, Gemini, Bedrock — no provider package published; use Tier-3 custom extractor (see `configs-built-in-metrics`) 3. **Initialize `LDAIClient` once at startup.** Reuse any existing `LDClient` — do not create a second base client. Place the initialization in the same module that owns existing app config. @@ -350,9 +350,9 @@ Delegate: **`tools`** (sub-step 2). ### Step 4: Instrument the tracker (Stage 4) -Delegate: **`configs-ai-metrics`** wires the per-request `tracker.track_*` calls (duration, tokens, success/error, feedback) around the provider call. Use **`configs-custom-metrics`** alongside it if the app needs business metrics beyond the built-in AI ones. Note: do not confuse this with `launchdarkly-metric-instrument`, which is for `ldClient.track()` feature metrics — a different API. See [sdk-ai-tracker-patterns.md](references/sdk-ai-tracker-patterns.md) for the full per-method Python + Node matrix that the delegate skill draws on. +Delegate: **`configs-built-in-metrics`** wires the per-request `tracker.track_*` calls (duration, tokens, success/error, feedback) around the provider call. Use **`configs-custom-metrics`** alongside it if the app needs business metrics beyond the built-in AI ones. Note: do not confuse this with `launchdarkly-metric-instrument`, which is for `ldClient.track()` feature metrics — a different API. See [sdk-ai-tracker-patterns.md](references/sdk-ai-tracker-patterns.md) for the full per-method Python + Node matrix that the delegate skill draws on. -Hand off: print the AI Config key, variation key, provider, and whether the call is streaming, then tell the user: *"Run `/configs-ai-metrics` with these inputs, then come back here."* Do not auto-invoke. Return here for sub-step 5 (verify) once they're done. +Hand off: print the AI Config key, variation key, provider, and whether the call is streaming, then tell the user: *"Run `/configs-built-in-metrics` with these inputs, then come back here."* Do not auto-invoke. Return here for sub-step 5 (verify) once they're done. 1. **Create the tracker.** Obtain a per-execution tracker via the factory on the config returned in Stage 2: `tracker = config.create_tracker()` (Python) or `const tracker = aiConfig.createTracker();` (Node). Call the factory **once per user turn** and reuse the returned `tracker` for every tracking call in that turn — each call mints a fresh `runId` that tags every event emitted from the turn so they can be correlated via exported events or downstream queries. (The Monitoring tab aggregates today; run-level grouping is a downstream concern — but the `runId` is also what the SDK's at-most-once guards are keyed on, so minting a new one mid-turn breaks the guard semantics regardless of where the events end up.) @@ -404,7 +404,7 @@ Hand off: print the AI Config key, variation key, provider, and whether the call ); ``` - For Anthropic direct, Bedrock (no provider package), Gemini, and custom HTTP, write a small extractor returning `LDAIMetrics` — see the delegate skill's [anthropic-tracking.md](../configs-ai-metrics/references/anthropic-tracking.md), [bedrock-tracking.md](../configs-ai-metrics/references/bedrock-tracking.md), and [gemini-tracking.md](../configs-ai-metrics/references/gemini-tracking.md). LangChain single-node and LangGraph go through the `launchdarkly-server-sdk-ai-langchain` / `@launchdarkly/server-sdk-ai-langchain` provider package. Build the model with `create_langchain_model(config)` (Python) / `createLangChainModel(config)` (Node) — both forward all variation parameters — and track with `get_ai_metrics_from_response` / `getAIMetricsFromResponse`. See [langchain-tracking.md](../configs-ai-metrics/references/langchain-tracking.md). + For Anthropic direct, Bedrock (no provider package), Gemini, and custom HTTP, write a small extractor returning `LDAIMetrics` — see the delegate skill's [anthropic-tracking.md](../configs-built-in-metrics/references/anthropic-tracking.md), [bedrock-tracking.md](../configs-built-in-metrics/references/bedrock-tracking.md), and [gemini-tracking.md](../configs-built-in-metrics/references/gemini-tracking.md). LangChain single-node and LangGraph go through the `launchdarkly-server-sdk-ai-langchain` / `@launchdarkly/server-sdk-ai-langchain` provider package. Build the model with `create_langchain_model(config)` (Python) / `createLangChainModel(config)` (Node) — both forward all variation parameters — and track with `get_ai_metrics_from_response` / `getAIMetricsFromResponse`. See [langchain-tracking.md](../configs-built-in-metrics/references/langchain-tracking.md). 4. **Wire feedback tracking if the app has thumbs-up/down UI.** Both SDKs expose `trackFeedback` with a `{kind}` argument. @@ -498,9 +498,9 @@ Delegate: **`configs-online-evals`** (sub-step 3, optional — only for UI-attac | Retry wrapper around the provider call | The tracker is minted once at the top of the user turn; the retry loop is inside that scope. Every retry attempt shares the same `runId`. Tracker calls (`track_duration` / `track_tokens` / `track_success` / `track_error`) live *outside* the retry body — one call at the end of the turn, on the success path or the final-failure path | | App has no tools — Stage 3 skipped | Move directly from Stage 2 verification to Stage 4 (tracking) | | Mode mismatch: user said agent, audit shows one-shot chat | Choose completion mode unless the app uses a LangGraph prebuilt agent (`langchain.agents.create_agent` in Python or `createReactAgent` in Node), CrewAI `Agent`, Strands `Agent`, or a similar goal-driven framework | -| App uses Strands Agents (Python) | Agent mode. Build a `create_strands_model` dispatcher keyed on `agent_config.provider.name` that returns `AnthropicModel(model_id=..., max_tokens=...)` or `OpenAIModel(model_id=..., params=...)`. Drop `parameters.tools` before passing params to the model class — Strands receives tools via `Agent(tools=[...])`. Tracking is Tier 3: wrap `invoke_async` with `tracker.track_duration_of(...)` and record tokens from `result.metrics.accumulated_usage`. See [agent-mode-frameworks.md § Strands Agent](references/agent-mode-frameworks.md) and [strands-tracking.md](../configs-ai-metrics/references/strands-tracking.md) | +| App uses Strands Agents (Python) | Agent mode. Build a `create_strands_model` dispatcher keyed on `agent_config.provider.name` that returns `AnthropicModel(model_id=..., max_tokens=...)` or `OpenAIModel(model_id=..., params=...)`. Drop `parameters.tools` before passing params to the model class — Strands receives tools via `Agent(tools=[...])`. Tracking is Tier 3: wrap `invoke_async` with `tracker.track_duration_of(...)` and record tokens from `result.metrics.accumulated_usage`. See [agent-mode-frameworks.md § Strands Agent](references/agent-mode-frameworks.md) and [strands-tracking.md](../configs-built-in-metrics/references/strands-tracking.md) | | Strands app on TypeScript | TS SDK ships `BedrockModel` and `OpenAIModel` only — cannot serve Anthropic-backed variations. Use the Python SDK if multi-provider variations are required | -| TypeScript app using Anthropic SDK | No `trackAnthropicMetrics` helper exists. Use Tier 3: `trackMetricsOf` with a small custom extractor that reads `response.usage.input_tokens` / `response.usage.output_tokens` and returns `LDAIMetrics`. See [anthropic-tracking.md](../configs-ai-metrics/references/anthropic-tracking.md) in the `configs-ai-metrics` skill for the exact extractor | +| TypeScript app using Anthropic SDK | No `trackAnthropicMetrics` helper exists. Use Tier 3: `trackMetricsOf` with a small custom extractor that reads `response.usage.input_tokens` / `response.usage.output_tokens` and returns `LDAIMetrics`. See [anthropic-tracking.md](../configs-built-in-metrics/references/anthropic-tracking.md) in the `configs-built-in-metrics` skill for the exact extractor | | Fallback would silently crash because `LD_SDK_KEY` is missing | Log a startup warning; proceed with the fallback. Never raise at import time | | Multi-agent graph (supervisor + workers) | Stop after migrating a single agent. Agent Graph Definitions are available in **both** SDKs — Python via `launchdarkly-server-sdk-ai.agent_graph` and Node via the graph API in `@launchdarkly/server-sdk-ai`. Read [agent-graph-reference.md](references/agent-graph-reference.md) for the graph-level migration path — it is deliberately out of this skill's main scope | | Single-agent (ReAct, tool loop) + agent mode | Default to offline eval via the LD Playground + Datasets for Stage 5. UI-attached judges are completion-only today, and programmatic direct-judge adds per-call cost that is usually not worth it until after the migration is live and stable. Point at the [Offline Evals guide](https://docs.launchdarkly.com/guides/ai-configs/offline-evaluations) | @@ -525,7 +525,7 @@ These are ordered by how likely they are to show up as a first-run failure. The - **If the repo already contains a `load_chat_model(f"{provider}/{name}")` helper, delete it — don't just avoid using it.** This exact shape ships with `langchain-ai/react-agent` and is copied into dozens of derivative repos; look for `utils.load_chat_model`, `utils.build_model`, or any one-arg `init_chat_model` wrapper that splits a `"provider/model"` string. Re-using it is the first-run failure mode: every variation parameter (temperature, max_tokens, top_p, stop sequences) silently drops on the floor because `init_chat_model` only receives the name and provider. `create_langchain_model(ai_config)` is a one-for-one replacement that forwards the whole `model.parameters` dict. Replace every call site, then delete the wrapper file-side so the next reader can't reach for it. - **Same rule applies to hand-rolled `resolve_tools` / `TOOL_REGISTRY` / `ALL_TOOLS` helpers.** If the template already has a `resolve_tools(tool_keys)` or an `ALL_TOOLS` module-level list, import `build_structured_tools` from `ldai_langchain.langchain_helper` and delete the hand-rolled version. `build_structured_tools(ai_config, TOOL_REGISTRY_DICT)` reads `ai_config.model.parameters.tools` and wraps the matching callables as LangChain `StructuredTool`s with the LD tool key as the `StructuredTool.name` — so `ToolNode` lookup works without a second mapping. Don't leave both in the repo. -- Don't put app-scoped knobs directly in `model.parameters`. `create_langchain_model` forwards every key in `parameters` to the provider SDK via `init_chat_model`, so a `max_search_results` / `retry_budget` / `feature_toggle` entry will crash the provider with an unexpected-keyword-argument error. The correct home is `model.custom`, which the provider helpers ignore and the app reads via `ai_config.model.get_custom("key")`. The MCP `update-ai-config-variation` tool does not currently expose top-level `custom`, so pick one of two paths: (a) PATCH the variation via the REST API to set `model.custom` directly, or (b) set it via MCP inside `parameters.custom` (as a nested dict) and use a defensive accessor that reads both locations. Full walk-through with code samples in [langchain-tracking.md § MCP caveat](../configs-ai-metrics/references/langchain-tracking.md). +- Don't put app-scoped knobs directly in `model.parameters`. `create_langchain_model` forwards every key in `parameters` to the provider SDK via `init_chat_model`, so a `max_search_results` / `retry_budget` / `feature_toggle` entry will crash the provider with an unexpected-keyword-argument error. The correct home is `model.custom`, which the provider helpers ignore and the app reads via `ai_config.model.get_custom("key")`. The MCP `update-ai-config-variation` tool does not currently expose top-level `custom`, so pick one of two paths: (a) PATCH the variation via the REST API to set `model.custom` directly, or (b) set it via MCP inside `parameters.custom` (as a nested dict) and use a defensive accessor that reads both locations. Full walk-through with code samples in [langchain-tracking.md § MCP caveat](../configs-built-in-metrics/references/langchain-tracking.md). - Don't re-encode tool schemas inside the fallback. When LaunchDarkly is unreachable the fallback should run without tools (or with whatever minimal provider-bound parameters the app needs to keep operating). Building a `_FALLBACK_TOOLS` array that duplicates the AI Config's tool schema re-introduces the hardcoded config the migration was supposed to move out of code. - Don't import `LaunchDarklyCallbackHandler` from `ldai.langchain` — neither the class nor the dotted module path exists. The Python LangChain helper package is `ldai_langchain` (top-level module, underscore). Use `create_langchain_model(config)` + `track_metrics_of_async(get_ai_metrics_from_response, lambda: llm.ainvoke(messages))` as the canonical pattern. diff --git a/skills/agentcontrol/configs-migrate/references/agent-mode-frameworks.md b/skills/agentcontrol/configs-migrate/references/agent-mode-frameworks.md index 57f8a74..b883e23 100644 --- a/skills/agentcontrol/configs-migrate/references/agent-mode-frameworks.md +++ b/skills/agentcontrol/configs-migrate/references/agent-mode-frameworks.md @@ -18,7 +18,7 @@ Agent mode returns an `instructions` string. Completion mode returns a `messages **Caveat:** judges cannot be attached to agent-mode variations via the LaunchDarkly UI. Agent mode evaluations must go through the programmatic judge API (`create_judge(...).evaluate(input, output)`). See `configs-online-evals` for the programmatic path. -**Model construction for LangChain / LangGraph.** When the framework runs on top of LangChain (which includes LangGraph's prebuilt agent and most custom graphs), build the chat model with `create_langchain_model(ai_config)` (Python) or `createLangChainModel(aiConfig)` (Node). These helpers forward every variation parameter (`temperature`, `max_tokens`, `top_p`, …) and handle LaunchDarkly→LangChain provider-name mapping internally. Do not hand-roll `init_chat_model(model=..., model_provider=...)` — it silently drops every variation parameter. See [langchain-tracking.md](../../configs-ai-metrics/references/langchain-tracking.md) for the canonical single-model and LangGraph patterns, including the SDK helpers `sum_token_usage_from_messages` / `get_tool_calls_from_response` (Python, `ldai_langchain`) used inside the `track_metrics_of_async` / `trackMetricsOf` extractor. +**Model construction for LangChain / LangGraph.** When the framework runs on top of LangChain (which includes LangGraph's prebuilt agent and most custom graphs), build the chat model with `create_langchain_model(ai_config)` (Python) or `createLangChainModel(aiConfig)` (Node). These helpers forward every variation parameter (`temperature`, `max_tokens`, `top_p`, …) and handle LaunchDarkly→LangChain provider-name mapping internally. Do not hand-roll `init_chat_model(model=..., model_provider=...)` — it silently drops every variation parameter. See [langchain-tracking.md](../../configs-built-in-metrics/references/langchain-tracking.md) for the canonical single-model and LangGraph patterns, including the SDK helpers `sum_token_usage_from_messages` / `get_tool_calls_from_response` (Python, `ldai_langchain`) used inside the `track_metrics_of_async` / `trackMetricsOf` extractor. ## Framework-agnostic invariants for the run-scoped pattern @@ -291,7 +291,7 @@ async def run_turn(ai_client, user_id: str, user_input: str): **Key points:** - `system_prompt=agent_config.instructions` — the instructions string replaces the hardcoded system prompt. - `create_strands_model(agent_config)` is the provider-dispatch seam. Add a branch per provider the variation can serve. -- The tracker is Tier 3: `tracker.track_duration_of(...)` + an explicit `track_tokens` call fed by `track_strands_metrics`. See [strands-tracking.md](../../configs-ai-metrics/references/strands-tracking.md) for the single-call `track_metrics_of_async` variant and the per-field breakdown of `accumulated_usage`. +- The tracker is Tier 3: `tracker.track_duration_of(...)` + an explicit `track_tokens` call fed by `track_strands_metrics`. See [strands-tracking.md](../../configs-built-in-metrics/references/strands-tracking.md) for the single-call `track_metrics_of_async` variant and the per-field breakdown of `accumulated_usage`. - Always `ldclient.get().flush()` before process exit in short-lived scripts — trailing events can otherwise be lost. **TypeScript caveat.** The Strands TypeScript SDK ships `BedrockModel` and `OpenAIModel` only — it cannot run Anthropic-backed variations. If the app needs to serve both OpenAI and Anthropic from a single AI Config, use the Python SDK. diff --git a/skills/agentcontrol/configs-migrate/references/phase-1-analysis-checklist.md b/skills/agentcontrol/configs-migrate/references/phase-1-analysis-checklist.md index a95e260..d0f2686 100644 --- a/skills/agentcontrol/configs-migrate/references/phase-1-analysis-checklist.md +++ b/skills/agentcontrol/configs-migrate/references/phase-1-analysis-checklist.md @@ -151,7 +151,7 @@ Feeds into Stage 2 (install + wrap). Quoted from the `ai-configs-relaunch-guides | LangChain / LangGraph | `@launchdarkly/server-sdk-ai-langchain` | `createLangChainModel(config)` (forwards all variation parameters and handles provider-name mapping) + `getAIMetricsFromResponse` with `trackMetricsOf` | | Vercel AI SDK | `@launchdarkly/server-sdk-ai-vercel` | `getAIMetricsFromResponse` + `trackMetricsOf`, or `VercelRunnerFactory.createVercelModel(aiConfig)` for a managed runner | -Python currently ships helper packages for OpenAI (`ldai_openai`) and LangChain (`ldai_langchain`). The LangChain Python package exposes `create_langchain_model(config)` (builds a LangChain chat model from the AI Config, forwarding every variation parameter and mapping LD provider names to LangChain equivalents), `convert_messages_to_langchain`, and `get_ai_metrics_from_response` — the same package covers LangGraph. Use `create_langchain_model(config)` + `track_metrics_of_async(get_ai_metrics_from_response, lambda: llm.ainvoke(messages))` as the canonical single-call pattern. See [langchain-tracking.md](../../configs-ai-metrics/references/langchain-tracking.md) for both LangChain and LangGraph patterns and [sdk-ai-tracker-patterns.md](sdk-ai-tracker-patterns.md) for the full tracker-method matrix. +Python currently ships helper packages for OpenAI (`ldai_openai`) and LangChain (`ldai_langchain`). The LangChain Python package exposes `create_langchain_model(config)` (builds a LangChain chat model from the AI Config, forwarding every variation parameter and mapping LD provider names to LangChain equivalents), `convert_messages_to_langchain`, and `get_ai_metrics_from_response` — the same package covers LangGraph. Use `create_langchain_model(config)` + `track_metrics_of_async(get_ai_metrics_from_response, lambda: llm.ainvoke(messages))` as the canonical single-call pattern. See [langchain-tracking.md](../../configs-built-in-metrics/references/langchain-tracking.md) for both LangChain and LangGraph patterns and [sdk-ai-tracker-patterns.md](sdk-ai-tracker-patterns.md) for the full tracker-method matrix. ## Phase 1 output format diff --git a/skills/agentcontrol/configs-migrate/references/sdk-ai-tracker-patterns.md b/skills/agentcontrol/configs-migrate/references/sdk-ai-tracker-patterns.md index c755d4a..3b881be 100644 --- a/skills/agentcontrol/configs-migrate/references/sdk-ai-tracker-patterns.md +++ b/skills/agentcontrol/configs-migrate/references/sdk-ai-tracker-patterns.md @@ -297,7 +297,7 @@ response = tracker.track_metrics_of( | Helper | Signature | Tier | Notes | |--------|-----------|------|-------| | `trackMetricsOf(extractor, func)` | `await tracker.trackMetricsOf((result) => extractor(result), async () => ...)` | **2 / 3** | **Canonical generic wrapper.** `extractor` maps provider response → `LDAIMetrics`. Use a provider package's bare `getAIMetricsFromResponse` for Tier 2 (`@launchdarkly/server-sdk-ai-openai`, `-langchain`, `-vercel`) or a small custom function for Tier 3. | -| `trackStreamMetricsOf(extractor, streamCreator)` | `tracker.trackStreamMetricsOf(async (chunks) => extractor(chunks), () => createStream())` | 2 / 3 | Stream variant. Does **not** capture TTFT automatically — if you need TTFT, use the manual pattern in [streaming-tracking.md](../../configs-ai-metrics/references/streaming-tracking.md). | +| `trackStreamMetricsOf(extractor, streamCreator)` | `tracker.trackStreamMetricsOf(async (chunks) => extractor(chunks), () => createStream())` | 2 / 3 | Stream variant. Does **not** capture TTFT automatically — if you need TTFT, use the manual pattern in [streaming-tracking.md](../../configs-built-in-metrics/references/streaming-tracking.md). | | `trackDurationOf(func)` | `await tracker.trackDurationOf(async () => ...)` | 4 | Wraps an async callable; captures duration only. Pair with explicit `trackTokens` + `trackSuccess`. | Example — OpenAI via `trackMetricsOf` + the provider package: ```typescript @@ -339,7 +339,7 @@ The managed runner handles message history, provider dispatch (via the installed ### Anthropic has no provider package today -Neither `@launchdarkly/server-sdk-ai-anthropic` nor `launchdarkly-server-sdk-ai-anthropic` exists as of this writing. For Anthropic direct calls, write a custom extractor and pass it to `track_metrics_of` / `trackMetricsOf` — see the Python example above or the full walk-through in [anthropic-tracking.md](../../configs-ai-metrics/references/anthropic-tracking.md). If the app is open to LangChain, routing Anthropic through `ChatAnthropic` and the LangChain provider package recovers Tier 2 with zero extractor code. +Neither `@launchdarkly/server-sdk-ai-anthropic` nor `launchdarkly-server-sdk-ai-anthropic` exists as of this writing. For Anthropic direct calls, write a custom extractor and pass it to `track_metrics_of` / `trackMetricsOf` — see the Python example above or the full walk-through in [anthropic-tracking.md](../../configs-built-in-metrics/references/anthropic-tracking.md). If the app is open to LangChain, routing Anthropic through `ChatAnthropic` and the LangChain provider package recovers Tier 2 with zero extractor code. ## Tier decision table @@ -352,7 +352,7 @@ Neither `@launchdarkly/server-sdk-ai-anthropic` nor `launchdarkly-server-sdk-ai- | Anthropic direct SDK | **3** | Custom extractor reading `response.usage.input_tokens` / `output_tokens` | | Bedrock Converse (no provider package) | **3** | Custom extractor reading `response.usage.inputTokens` / `outputTokens` (or route via LangChain for Tier 2) | | Gemini / Google GenAI, Cohere, custom HTTP | **3** | Custom extractor | -| Streaming response with TTFT required | **4** | Manual `trackTimeToFirstToken` + `trackDuration` + `trackTokens` + `trackSuccess` — see [streaming-tracking.md](../../configs-ai-metrics/references/streaming-tracking.md) | +| Streaming response with TTFT required | **4** | Manual `trackTimeToFirstToken` + `trackDuration` + `trackTokens` + `trackSuccess` — see [streaming-tracking.md](../../configs-built-in-metrics/references/streaming-tracking.md) | | Streaming response without TTFT (Node) | **2 / 3** | `trackStreamMetricsOf(extractor, streamFn)` | ## Streaming responses @@ -450,7 +450,7 @@ Run the checklist in order. Each step rules out one cause. ## Common gotchas -- **`model.parameters` vs `model.custom`.** `create_langchain_model` (Python) / `createLangChainModel` (Node) forwards every key in `model.parameters` to the provider SDK. App-scoped knobs (search result limits, retry budgets, feature toggles) **must** live in `model.custom` or the provider will crash at runtime with an unexpected-keyword-argument error. Read them with `ai_config.model.get_custom("key")`. Full walk-through with the MCP/REST-API caveat in [langchain-tracking.md § `model.parameters` vs `model.custom`](../../configs-ai-metrics/references/langchain-tracking.md). +- **`model.parameters` vs `model.custom`.** `create_langchain_model` (Python) / `createLangChainModel` (Node) forwards every key in `model.parameters` to the provider SDK. App-scoped knobs (search result limits, retry budgets, feature toggles) **must** live in `model.custom` or the provider will crash at runtime with an unexpected-keyword-argument error. Read them with `ai_config.model.get_custom("key")`. Full walk-through with the MCP/REST-API caveat in [langchain-tracking.md § `model.parameters` vs `model.custom`](../../configs-built-in-metrics/references/langchain-tracking.md). - **`track_tokens` token shape.** The Python `TokenUsage` dataclass requires `total` to be set — it is not derived. Compute `total = input + output` if the provider doesn't return one. - **`track_feedback` lifecycle.** The feedback call must be made on a tracker bound to the same `runId` that produced the response. If the thumbs-up comes in a later process, use the cross-process resumption pattern above — do **not** call `create_tracker()` again in the consumer, because that mints a *new* `runId`. - **OpenAI streaming tokens.** OpenAI only emits `usage` in the final chunk when `stream_options={"include_usage": True}` is passed. Without that flag, you have to tokenize manually — `tiktoken` for OpenAI models. diff --git a/skills/agentcontrol/configs-targeting/SKILL.md b/skills/agentcontrol/configs-targeting/SKILL.md index 47bebb6..aa5b904 100644 --- a/skills/agentcontrol/configs-targeting/SKILL.md +++ b/skills/agentcontrol/configs-targeting/SKILL.md @@ -488,7 +488,7 @@ After configuring targeting: ``` https://app.launchdarkly.com/projects/{projectKey}/ai-configs/{configKey} ``` -2. **Monitor performance** with `configs-ai-metrics` +2. **Monitor performance** with `configs-built-in-metrics` 3. **Attach judges** with `configs-online-evals` 4. **Set up guarded rollouts** for automatic regression detection diff --git a/skills/agentcontrol/tools/SKILL.md b/skills/agentcontrol/tools/SKILL.md index e4c8957..8160160 100644 --- a/skills/agentcontrol/tools/SKILL.md +++ b/skills/agentcontrol/tools/SKILL.md @@ -211,13 +211,13 @@ for _ in range(MAX_STEPS): messages.append({"role": "user", "content": tool_results}) ``` -Per-provider tool-call payload shapes live in the `configs-ai-metrics` references: +Per-provider tool-call payload shapes live in the `configs-built-in-metrics` references: -- [openai-tracking.md](../configs-ai-metrics/references/openai-tracking.md) — Chat Completions + Responses API -- [anthropic-tracking.md](../configs-ai-metrics/references/anthropic-tracking.md) — `tool_use` blocks and `tool_result` payloads -- [bedrock-tracking.md](../configs-ai-metrics/references/bedrock-tracking.md) — `toolUse` / `toolResult` Converse format -- [gemini-tracking.md](../configs-ai-metrics/references/gemini-tracking.md) — `functionCalls` / `functionResponse` parts -- [langchain-tracking.md](../configs-ai-metrics/references/langchain-tracking.md) — LangGraph tool loop inherits from `create_react_agent` +- [openai-tracking.md](../configs-built-in-metrics/references/openai-tracking.md) — Chat Completions + Responses API +- [anthropic-tracking.md](../configs-built-in-metrics/references/anthropic-tracking.md) — `tool_use` blocks and `tool_result` payloads +- [bedrock-tracking.md](../configs-built-in-metrics/references/bedrock-tracking.md) — `toolUse` / `toolResult` Converse format +- [gemini-tracking.md](../configs-built-in-metrics/references/gemini-tracking.md) — `functionCalls` / `functionResponse` parts +- [langchain-tracking.md](../configs-built-in-metrics/references/langchain-tracking.md) — LangGraph tool loop inherits from `create_react_agent` ## Orchestrator Note From 76e001115b348b9d85354a5c58d3eee473057214 Mon Sep 17 00:00:00 2001 From: Scarlett Attensil Date: Tue, 19 May 2026 12:02:28 -0700 Subject: [PATCH 07/10] Make agent-graphs, online-evals, migrate, snippets bare (review) Per Paul's review: these four drop the configs- prefix, joining projects/tools as bare names. configs- now only on create, targeting, update, variations, built-in-metrics, custom-metrics. Repointed the online-evals symlink; regenerated skills.json. --- README.md | 8 +-- skills.json | 62 +++++++++---------- .../SKILL.md | 2 +- .../configs-built-in-metrics/SKILL.md | 6 +- .../references/strands-tracking.md | 2 +- .../configs-custom-metrics/SKILL.md | 2 +- .../agentcontrol/configs-targeting/README.md | 2 +- .../agentcontrol/configs-targeting/SKILL.md | 4 +- .../{configs-migrate => migrate}/README.md | 8 +-- .../{configs-migrate => migrate}/SKILL.md | 12 ++-- .../references/agent-graph-reference.md | 0 .../references/agent-mode-frameworks.md | 2 +- .../references/before-after-examples.md | 0 .../references/fallback-defaults-pattern.md | 0 .../references/phase-1-analysis-checklist.md | 2 +- .../references/sdk-ai-tracker-patterns.md | 0 .../README.md | 4 +- .../SKILL.md | 2 +- .../{configs-snippets => snippets}/SKILL.md | 2 +- skills/configs-online-evals | 1 - skills/online-evals | 1 + 21 files changed, 61 insertions(+), 61 deletions(-) rename skills/agentcontrol/{configs-agent-graphs => agent-graphs}/SKILL.md (99%) rename skills/agentcontrol/{configs-migrate => migrate}/README.md (92%) rename skills/agentcontrol/{configs-migrate => migrate}/SKILL.md (98%) rename skills/agentcontrol/{configs-migrate => migrate}/references/agent-graph-reference.md (100%) rename skills/agentcontrol/{configs-migrate => migrate}/references/agent-mode-frameworks.md (99%) rename skills/agentcontrol/{configs-migrate => migrate}/references/before-after-examples.md (100%) rename skills/agentcontrol/{configs-migrate => migrate}/references/fallback-defaults-pattern.md (100%) rename skills/agentcontrol/{configs-migrate => migrate}/references/phase-1-analysis-checklist.md (99%) rename skills/agentcontrol/{configs-migrate => migrate}/references/sdk-ai-tracker-patterns.md (100%) rename skills/agentcontrol/{configs-online-evals => online-evals}/README.md (91%) rename skills/agentcontrol/{configs-online-evals => online-evals}/SKILL.md (99%) rename skills/agentcontrol/{configs-snippets => snippets}/SKILL.md (99%) delete mode 120000 skills/configs-online-evals create mode 120000 skills/online-evals diff --git a/README.md b/README.md index 60b7306..46229fe 100644 --- a/README.md +++ b/README.md @@ -23,15 +23,15 @@ Agent Skills are modular, text-based playbooks that teach an agent how to perfor | Skill | Description | |-------|-------------| | `agentcontrol/configs-create` | Create AI Configs with variations for agent or completion mode | -| `agentcontrol/configs-migrate` | Migrate an app with hardcoded LLM prompts to AI Configs in five stages (extract, wrap, tools, tracking, evals) | +| `agentcontrol/migrate` | Migrate an app with hardcoded LLM prompts to AI Configs in five stages (extract, wrap, tools, tracking, evals) | | `agentcontrol/configs-update` | Update and delete AI Configs, manage lifecycle | | `agentcontrol/configs-variations` | Manage AI Config variations for A/B testing | | `agentcontrol/tools` | Create and attach tools for function calling | | `agentcontrol/projects` | Create and manage projects to organize AI Configs | -| `agentcontrol/configs-online-evals` | Attach LLM-as-a-judge evaluators to AI Configs | +| `agentcontrol/online-evals` | Attach LLM-as-a-judge evaluators to AI Configs | | `agentcontrol/configs-targeting` | Configure targeting rules for AI Config rollouts | -| `agentcontrol/configs-snippets` | Create and manage reusable prompt snippets across AI Configs | -| `agentcontrol/configs-agent-graphs` | Create and manage multi-agent graphs with routing and handoffs | +| `agentcontrol/snippets` | Create and manage reusable prompt snippets across AI Configs | +| `agentcontrol/agent-graphs` | Create and manage multi-agent graphs with routing and handoffs | ### Experiments diff --git a/skills.json b/skills.json index cb18deb..318bc15 100644 --- a/skills.json +++ b/skills.json @@ -1,5 +1,13 @@ { "skills": [ + { + "name": "agent-graphs", + "description": "Create and manage agent graphs \u2014 directed graphs of configs connected by edges with handoff logic. Use when building multi-agent workflows where configs route to each other.", + "path": "skills/agentcontrol/agent-graphs", + "version": "0.1.0", + "license": "Apache-2.0", + "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" + }, { "name": "apply", "description": "Apply LaunchDarkly SDK onboarding: install dependency (or dual-SDK pair), configure env and secrets with consent, add init at entrypoint(s), verify compile. Nested under sdk-install; next is run.", @@ -8,14 +16,6 @@ "license": "Apache-2.0", "compatibility": "Requires integration plan and LaunchDarkly credentials (see parent onboarding)" }, - { - "name": "configs-agent-graphs", - "description": "Create and manage agent graphs \u2014 directed graphs of configs connected by edges with handoff logic. Use when building multi-agent workflows where configs route to each other.", - "path": "skills/agentcontrol/configs-agent-graphs", - "version": "0.1.0", - "license": "Apache-2.0", - "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" - }, { "name": "configs-built-in-metrics", "description": "Instrument an existing codebase with LaunchDarkly AI Config tracking. Walks the four-tier ladder (managed runner \u2192 provider package \u2192 custom extractor + trackMetricsOf \u2192 raw manual) and picks the lowest-ceremony option that still captures duration, tokens, and success/error.", @@ -40,29 +40,6 @@ "license": "Apache-2.0", "compatibility": "Requires the LaunchDarkly server SDK and a LaunchDarkly API token with the `writer` role for metric management." }, - { - "name": "configs-migrate", - "description": "Migrate an application with hardcoded LLM prompts to a full LaunchDarkly AI Configs implementation in five stages: audit the code, wrap the call, move the tools, add tracking, attach evaluators. Use when the user wants to externalize model/prompt configuration, move from direct provider calls (OpenAI, Anthropic, Bedrock, Gemini, Strands) to a managed AI Config, or stage a full hardcoded-to-LaunchDarkly migration.", - "path": "skills/agentcontrol/configs-migrate", - "version": "0.1.0", - "license": "Apache-2.0", - "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" - }, - { - "name": "configs-online-evals", - "description": "Attach judges to AI Config variations for automatic LLM-as-a-judge evaluation. Create custom judges, configure sampling rates, and monitor quality scores.", - "path": "skills/agentcontrol/configs-online-evals", - "version": "0.1.0", - "compatibility": "Requires LaunchDarkly API access token with ai-configs:write permission. SDK versions Python v0.20.0+ or Node.js v0.20.0+ for automatic metric recording and the consolidated `track_judge_result` / `trackJudgeResult` API." - }, - { - "name": "configs-snippets", - "description": "Create and manage prompt snippets \u2014 reusable text blocks referenced inside AI Config variation prompts. Keeps common instructions, personas, and guardrails consistent across multiple configs.", - "path": "skills/agentcontrol/configs-snippets", - "version": "0.1.0", - "license": "Apache-2.0", - "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" - }, { "name": "configs-targeting", "description": "Configure AI Config targeting rules to control which variations serve to different users. Enable percentage rollouts, attribute-based rules, segment targeting, and guarded rollouts.", @@ -225,6 +202,14 @@ "license": "Apache-2.0", "compatibility": "Requires an MCP-compatible coding agent and a LaunchDarkly account" }, + { + "name": "migrate", + "description": "Migrate an application with hardcoded LLM prompts to a full LaunchDarkly AI Configs implementation in five stages: audit the code, wrap the call, move the tools, add tracking, attach evaluators. Use when the user wants to externalize model/prompt configuration, move from direct provider calls (OpenAI, Anthropic, Bedrock, Gemini, Strands) to a managed AI Config, or stage a full hardcoded-to-LaunchDarkly migration.", + "path": "skills/agentcontrol/migrate", + "version": "0.1.0", + "license": "Apache-2.0", + "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" + }, { "name": "onboarding", "description": "Onboard a project to LaunchDarkly: kickoff roadmap, resumable log, explore repo, MCP, companion flag skills, nested SDK install (detect/plan/apply), first flag. Use when adding LaunchDarkly, setting up or integrating feature flags in a project, SDK integration, or 'onboard me'.", @@ -243,6 +228,13 @@ "setup" ] }, + { + "name": "online-evals", + "description": "Attach judges to AI Config variations for automatic LLM-as-a-judge evaluation. Create custom judges, configure sampling rates, and monitor quality scores.", + "path": "skills/agentcontrol/online-evals", + "version": "0.1.0", + "compatibility": "Requires LaunchDarkly API access token with ai-configs:write permission. SDK versions Python v0.20.0+ or Node.js v0.20.0+ for automatic metric recording and the consolidated `track_judge_result` / `trackJudgeResult` API." + }, { "name": "plan", "description": "Generate a minimal LaunchDarkly SDK integration plan from detected stack: choose SDK type(s), dual-SDK server+client when required, files to change, env conventions. Nested under sdk-install; follows detect, precedes apply.", @@ -266,6 +258,14 @@ "license": "Apache-2.0", "compatibility": "Requires a supported language/framework in the project. SDK credentials are required by [Apply](apply/SKILL.md), not for [Detect](detect/SKILL.md) / [Plan](plan/SKILL.md) alone (see parent onboarding **Prerequisites**)." }, + { + "name": "snippets", + "description": "Create and manage prompt snippets \u2014 reusable text blocks referenced inside AI Config variation prompts. Keeps common instructions, personas, and guardrails consistent across multiple configs.", + "path": "skills/agentcontrol/snippets", + "version": "0.1.0", + "license": "Apache-2.0", + "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" + }, { "name": "tools", "description": "Give your AI agents capabilities through tools (function calling). Helps you identify what your AI needs to do, create tool definitions, and attach them to AI Config variations.", diff --git a/skills/agentcontrol/configs-agent-graphs/SKILL.md b/skills/agentcontrol/agent-graphs/SKILL.md similarity index 99% rename from skills/agentcontrol/configs-agent-graphs/SKILL.md rename to skills/agentcontrol/agent-graphs/SKILL.md index d5176fe..b067e1c 100644 --- a/skills/agentcontrol/configs-agent-graphs/SKILL.md +++ b/skills/agentcontrol/agent-graphs/SKILL.md @@ -1,5 +1,5 @@ --- -name: configs-agent-graphs +name: agent-graphs description: "Create and manage agent graphs — directed graphs of configs connected by edges with handoff logic. Use when building multi-agent workflows where configs route to each other." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server diff --git a/skills/agentcontrol/configs-built-in-metrics/SKILL.md b/skills/agentcontrol/configs-built-in-metrics/SKILL.md index 78eff9a..65b5be5 100644 --- a/skills/agentcontrol/configs-built-in-metrics/SKILL.md +++ b/skills/agentcontrol/configs-built-in-metrics/SKILL.md @@ -43,7 +43,7 @@ Before picking a tier, find the provider call and answer these questions: - `aiclient.config(...)` → `aiclient.completion_config(...)` for one-shot/chat or `aiclient.agent_config(...)` for agent mode (mirror the call signature). Node is the same with camelCase. - `AIConfig(...)` default → `AICompletionConfigDefault(...)` or `AIAgentConfigDefault(...)` (Node: `LDAICompletionConfigDefault` / `LDAIAgentConfigDefault`). `AIConfig` is the base class the SDK returns; it isn't a valid default-value constructor — the typed `*Default` variants are. - If the result was being tuple-unpacked (`config, tracker = aiclient.config(...)`), drop the unpack — the new methods return a single config object. Obtain the tracker via `config.create_tracker()` / `aiConfig.createTracker()`. - - For deeper rewrites (call sites with hardcoded model/prompt as well), hand off to `configs-migrate` instead of doing the full migration here. + - For deeper rewrites (call sites with hardcoded model/prompt as well), hand off to `migrate` instead of doing the full migration here. ### 2. Look up your Tier-2 option @@ -105,5 +105,5 @@ Obtain a tracker via the factory on the config object: `tracker = config.create_ - `configs-create` — prerequisite if the app doesn't have an AI Config yet - `configs-custom-metrics` — business metrics (conversion, resolution, retention) layered on top of the AI metrics this skill captures -- `configs-online-evals` — automatic quality scoring (LLM-as-judge) on sampled live requests; complementary to the metrics here -- `configs-migrate` — Stage 4 of the hardcoded-to-AI-Configs migration delegates to this skill +- `online-evals` — automatic quality scoring (LLM-as-judge) on sampled live requests; complementary to the metrics here +- `migrate` — Stage 4 of the hardcoded-to-AI-Configs migration delegates to this skill diff --git a/skills/agentcontrol/configs-built-in-metrics/references/strands-tracking.md b/skills/agentcontrol/configs-built-in-metrics/references/strands-tracking.md index 1741254..062a006 100644 --- a/skills/agentcontrol/configs-built-in-metrics/references/strands-tracking.md +++ b/skills/agentcontrol/configs-built-in-metrics/references/strands-tracking.md @@ -79,7 +79,7 @@ Pick the style that matches the rest of the codebase — the two variants record ## Provider dispatch stays in your code -Strands model classes are provider-specific (`AnthropicModel`, `OpenAIModel`, `BedrockModel`). To serve more than one provider from a single AI Config key, dispatch on `agent_config.provider.name` before constructing the `Agent`. See [agent-mode-frameworks.md § Strands Agent](../../configs-migrate/references/agent-mode-frameworks.md) for the `create_strands_model` dispatcher, including the rule that `parameters.tools` must be dropped before being passed into the Strands model class (tools flow through the `Agent` constructor, not through model params). +Strands model classes are provider-specific (`AnthropicModel`, `OpenAIModel`, `BedrockModel`). To serve more than one provider from a single AI Config key, dispatch on `agent_config.provider.name` before constructing the `Agent`. See [agent-mode-frameworks.md § Strands Agent](../../migrate/references/agent-mode-frameworks.md) for the `create_strands_model` dispatcher, including the rule that `parameters.tools` must be dropped before being passed into the Strands model class (tools flow through the `Agent` constructor, not through model params). ## Always flush before exit diff --git a/skills/agentcontrol/configs-custom-metrics/SKILL.md b/skills/agentcontrol/configs-custom-metrics/SKILL.md index be25a8f..1b66981 100644 --- a/skills/agentcontrol/configs-custom-metrics/SKILL.md +++ b/skills/agentcontrol/configs-custom-metrics/SKILL.md @@ -494,7 +494,7 @@ Custom metrics appear in: - `sdk` - SDK setup - `configs-built-in-metrics` - Built-in AI metrics (tokens, duration, cost) -- `configs-online-evals` - Quality metrics via judges +- `online-evals` - Quality metrics via judges ## References diff --git a/skills/agentcontrol/configs-targeting/README.md b/skills/agentcontrol/configs-targeting/README.md index 1240e0e..073b85c 100644 --- a/skills/agentcontrol/configs-targeting/README.md +++ b/skills/agentcontrol/configs-targeting/README.md @@ -43,7 +43,7 @@ configs-targeting/ - [AI Config Create](../configs-create/) - Create AI Configs first - [AI Config Variations](../configs-variations/) - Create variations to target -- [AI Config Online Evals](../configs-online-evals/) - Attach judges +- [AI Config Online Evals](../online-evals/) - Attach judges - [Targeting Docs](https://docs.launchdarkly.com/home/ai-configs/target) ## License diff --git a/skills/agentcontrol/configs-targeting/SKILL.md b/skills/agentcontrol/configs-targeting/SKILL.md index aa5b904..7a98990 100644 --- a/skills/agentcontrol/configs-targeting/SKILL.md +++ b/skills/agentcontrol/configs-targeting/SKILL.md @@ -489,14 +489,14 @@ After configuring targeting: https://app.launchdarkly.com/projects/{projectKey}/ai-configs/{configKey} ``` 2. **Monitor performance** with `configs-built-in-metrics` -3. **Attach judges** with `configs-online-evals` +3. **Attach judges** with `online-evals` 4. **Set up guarded rollouts** for automatic regression detection ## Related Skills - `configs-create` - Create AI Configs with variations - `configs-variations` - Manage variations -- `configs-online-evals` - Attach judges +- `online-evals` - Attach judges - `segments` - Create segments for targeting ## References diff --git a/skills/agentcontrol/configs-migrate/README.md b/skills/agentcontrol/migrate/README.md similarity index 92% rename from skills/agentcontrol/configs-migrate/README.md rename to skills/agentcontrol/migrate/README.md index c2473c4..268997b 100644 --- a/skills/agentcontrol/configs-migrate/README.md +++ b/skills/agentcontrol/migrate/README.md @@ -12,11 +12,11 @@ The five stages: 2. **Wrap** the call site in `completion_config` / `completionConfig` with a safe fallback — delegates the config creation to `configs-create` 3. **Tools** — move function-calling schemas into LaunchDarkly — delegates to `tools` 4. **Tracking** — wire `track_duration`, `track_tokens`, `track_success`/`track_error`, optional `track_feedback` — inline, with a reference doc covering every SDK method in Python and Node side by side -5. **Evals** — attach judges for LLM-as-a-judge scoring — delegates to `configs-online-evals` +5. **Evals** — attach judges for LLM-as-a-judge scoring — delegates to `online-evals` ## Installation (Local) -Copy `skills/agentcontrol/configs-migrate/` into your agent client's skills path. +Copy `skills/agentcontrol/migrate/` into your agent client's skills path. ## Prerequisites @@ -41,7 +41,7 @@ Wire up the AI tracker and attach accuracy + relevance judges to our existing co ## Structure ``` -configs-migrate/ +migrate/ ├── SKILL.md ├── README.md └── references/ @@ -57,7 +57,7 @@ configs-migrate/ - [AI Config Create](../configs-create/): Delegated to by Stage 2 (wrap) - [AI Config Tools](../tools/): Delegated to by Stage 3 (tools) -- [AI Config Online Evals](../configs-online-evals/): Delegated to by Stage 5 (evals) +- [AI Config Online Evals](../online-evals/): Delegated to by Stage 5 (evals) - [AI Config Variations](../configs-variations/): Next step after migration for A/B testing - [AI Config Targeting](../configs-targeting/): Next step after migration for rollout control - [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) diff --git a/skills/agentcontrol/configs-migrate/SKILL.md b/skills/agentcontrol/migrate/SKILL.md similarity index 98% rename from skills/agentcontrol/configs-migrate/SKILL.md rename to skills/agentcontrol/migrate/SKILL.md index b950344..ab500e4 100644 --- a/skills/agentcontrol/configs-migrate/SKILL.md +++ b/skills/agentcontrol/migrate/SKILL.md @@ -1,5 +1,5 @@ --- -name: configs-migrate +name: migrate description: "Migrate an application with hardcoded LLM prompts to a full LaunchDarkly AI Configs implementation in five stages: audit the code, wrap the call, move the tools, add tracking, attach evaluators. Use when the user wants to externalize model/prompt configuration, move from direct provider calls (OpenAI, Anthropic, Bedrock, Gemini, Strands) to a managed AI Config, or stage a full hardcoded-to-LaunchDarkly migration." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server @@ -71,7 +71,7 @@ If a CHANGELOG entry post-dates this skill and changes an API you're about to us - `configs-create` — Stage 2 (creates the AI Config and first variation) - `tools` — Stage 3 (creates tool definitions and attaches them) - `configs-targeting` — between Stage 2 and Stage 4 (promotes the new variation to fallthrough so the SDK actually serves it) -- `configs-online-evals` — Stage 5 (attaches judges, creates custom judges) +- `online-evals` — Stage 5 (attaches judges, creates custom judges) ## Core Principles @@ -79,7 +79,7 @@ If a CHANGELOG entry post-dates this skill and changes an API you're about to us 2. **Replace config, not business logic.** The SDK call is a drop-in for the place where the model, parameters, and prompt are *defined* — not for the provider call itself. OpenAI/Anthropic/Bedrock calls stay where they are. 3. **Fallback mirrors current behavior.** The fallback passed to `completion_config` / `agent_config` must preserve the hardcoded values you removed, so the app is unchanged if LaunchDarkly is unreachable. 4. **Stages are ordered.** Wrap before you add tools. Add tools before you track. Track before you add evals. Skipping ahead produces configs without traffic, metrics without context, and judges with nothing to score. -5. **Hand off to focused skills, manually.** Each stage that needs a LaunchDarkly write tells the user to run a sibling slash-command (`/configs-create`, `/tools`, `/configs-targeting`, `/configs-online-evals`) and waits for them to come back. This skill does **not** auto-invoke other skills. +5. **Hand off to focused skills, manually.** Each stage that needs a LaunchDarkly write tells the user to run a sibling slash-command (`/configs-create`, `/tools`, `/configs-targeting`, `/online-evals`) and waits for them to come back. This skill does **not** auto-invoke other skills. ## Workflow @@ -446,7 +446,7 @@ Hand off: print the AI Config key, variation key, provider, and whether the call Write this shape into the project's `datasets/README.md` (or equivalent) so the comparison pattern is reproducible after the migration ships. -3. **Hand off to `configs-online-evals`** — only for UI-attached judges (completion mode) or to create custom judge AI Configs that will be referenced by the programmatic path. Tell the user: *"Run `/configs-online-evals` with these inputs, then come back here."* Do not auto-invoke. Pass: +3. **Hand off to `online-evals`** — only for UI-attached judges (completion mode) or to create custom judge AI Configs that will be referenced by the programmatic path. Tell the user: *"Run `/online-evals` with these inputs, then come back here."* Do not auto-invoke. Pass: - The parent AI Config key and variation key - A list of built-in judges (Accuracy, Relevance, Toxicity) or custom judge keys to create/attach - Target environment @@ -487,7 +487,7 @@ Hand off: print the AI Config key, variation key, provider, and whether the call - **Programmatic direct-judge:** hit the wrapped endpoint and confirm `track_judge_result` lands on the parent config's Monitoring tab. - **Offline eval:** run the dataset through the LD Playground, compare baseline vs new-variation scores side by side. No runtime wiring required. -Delegate: **`configs-online-evals`** (sub-step 3, optional — only for UI-attached judges or custom-judge creation; offline eval doesn't delegate). +Delegate: **`online-evals`** (sub-step 3, optional — only for UI-attached judges or custom-judge creation; offline eval doesn't delegate). ## Edge Cases @@ -556,7 +556,7 @@ These are ordered by how likely they are to show up as a first-run failure. The - `configs-create` — called by Stage 2 to create the config - `tools` — called by Stage 3 to create and attach tool definitions -- `configs-online-evals` — called by Stage 5 to attach judges +- `online-evals` — called by Stage 5 to attach judges - `configs-variations` — add variations for A/B testing after migration is complete - `configs-targeting` — roll out new variations to users after migration is complete - `configs-update` — modify config properties as your app evolves diff --git a/skills/agentcontrol/configs-migrate/references/agent-graph-reference.md b/skills/agentcontrol/migrate/references/agent-graph-reference.md similarity index 100% rename from skills/agentcontrol/configs-migrate/references/agent-graph-reference.md rename to skills/agentcontrol/migrate/references/agent-graph-reference.md diff --git a/skills/agentcontrol/configs-migrate/references/agent-mode-frameworks.md b/skills/agentcontrol/migrate/references/agent-mode-frameworks.md similarity index 99% rename from skills/agentcontrol/configs-migrate/references/agent-mode-frameworks.md rename to skills/agentcontrol/migrate/references/agent-mode-frameworks.md index b883e23..2d2f7ac 100644 --- a/skills/agentcontrol/configs-migrate/references/agent-mode-frameworks.md +++ b/skills/agentcontrol/migrate/references/agent-mode-frameworks.md @@ -16,7 +16,7 @@ Completion mode is the default and covers direct provider calls (OpenAI, Anthrop Agent mode returns an `instructions` string. Completion mode returns a `messages` array. Both modes support tools, parameters, and the same tracker — the only difference is the input shape the SDK returns to you. -**Caveat:** judges cannot be attached to agent-mode variations via the LaunchDarkly UI. Agent mode evaluations must go through the programmatic judge API (`create_judge(...).evaluate(input, output)`). See `configs-online-evals` for the programmatic path. +**Caveat:** judges cannot be attached to agent-mode variations via the LaunchDarkly UI. Agent mode evaluations must go through the programmatic judge API (`create_judge(...).evaluate(input, output)`). See `online-evals` for the programmatic path. **Model construction for LangChain / LangGraph.** When the framework runs on top of LangChain (which includes LangGraph's prebuilt agent and most custom graphs), build the chat model with `create_langchain_model(ai_config)` (Python) or `createLangChainModel(aiConfig)` (Node). These helpers forward every variation parameter (`temperature`, `max_tokens`, `top_p`, …) and handle LaunchDarkly→LangChain provider-name mapping internally. Do not hand-roll `init_chat_model(model=..., model_provider=...)` — it silently drops every variation parameter. See [langchain-tracking.md](../../configs-built-in-metrics/references/langchain-tracking.md) for the canonical single-model and LangGraph patterns, including the SDK helpers `sum_token_usage_from_messages` / `get_tool_calls_from_response` (Python, `ldai_langchain`) used inside the `track_metrics_of_async` / `trackMetricsOf` extractor. diff --git a/skills/agentcontrol/configs-migrate/references/before-after-examples.md b/skills/agentcontrol/migrate/references/before-after-examples.md similarity index 100% rename from skills/agentcontrol/configs-migrate/references/before-after-examples.md rename to skills/agentcontrol/migrate/references/before-after-examples.md diff --git a/skills/agentcontrol/configs-migrate/references/fallback-defaults-pattern.md b/skills/agentcontrol/migrate/references/fallback-defaults-pattern.md similarity index 100% rename from skills/agentcontrol/configs-migrate/references/fallback-defaults-pattern.md rename to skills/agentcontrol/migrate/references/fallback-defaults-pattern.md diff --git a/skills/agentcontrol/configs-migrate/references/phase-1-analysis-checklist.md b/skills/agentcontrol/migrate/references/phase-1-analysis-checklist.md similarity index 99% rename from skills/agentcontrol/configs-migrate/references/phase-1-analysis-checklist.md rename to skills/agentcontrol/migrate/references/phase-1-analysis-checklist.md index d0f2686..5e5fadb 100644 --- a/skills/agentcontrol/configs-migrate/references/phase-1-analysis-checklist.md +++ b/skills/agentcontrol/migrate/references/phase-1-analysis-checklist.md @@ -185,7 +185,7 @@ Proposed plan: Stage 2 (Wrap): Install SDK, create AI Config 'chat-assistant', inline fallback mirrors current values (Mustache syntax), rewrite the call site Stage 3 (Tools): Skipped (no function calling) / Attach 2 tools via tools Stage 4 (Tracking): Inline tracker wiring (track_duration + track_tokens + track_success/error) — run-scoped tracker for agent loops - Stage 5 (Evals): Attach built-in 'accuracy' judge at 0.25 sampling via configs-online-evals + Stage 5 (Evals): Attach built-in 'accuracy' judge at 0.25 sampling via online-evals ``` ## STOP diff --git a/skills/agentcontrol/configs-migrate/references/sdk-ai-tracker-patterns.md b/skills/agentcontrol/migrate/references/sdk-ai-tracker-patterns.md similarity index 100% rename from skills/agentcontrol/configs-migrate/references/sdk-ai-tracker-patterns.md rename to skills/agentcontrol/migrate/references/sdk-ai-tracker-patterns.md diff --git a/skills/agentcontrol/configs-online-evals/README.md b/skills/agentcontrol/online-evals/README.md similarity index 91% rename from skills/agentcontrol/configs-online-evals/README.md rename to skills/agentcontrol/online-evals/README.md index 056da3b..156955e 100644 --- a/skills/agentcontrol/configs-online-evals/README.md +++ b/skills/agentcontrol/online-evals/README.md @@ -12,7 +12,7 @@ This skill teaches agents how to: ## Installation (Local) -Copy `skills/agentcontrol/configs-online-evals/` into your agent client's skills path. +Copy `skills/agentcontrol/online-evals/` into your agent client's skills path. ## Prerequisites @@ -33,7 +33,7 @@ Create a custom judge that checks for scope creep in code changes ## Structure ``` -configs-online-evals/ +online-evals/ ├── SKILL.md └── README.md ``` diff --git a/skills/agentcontrol/configs-online-evals/SKILL.md b/skills/agentcontrol/online-evals/SKILL.md similarity index 99% rename from skills/agentcontrol/configs-online-evals/SKILL.md rename to skills/agentcontrol/online-evals/SKILL.md index bd99480..1c939a0 100644 --- a/skills/agentcontrol/configs-online-evals/SKILL.md +++ b/skills/agentcontrol/online-evals/SKILL.md @@ -1,5 +1,5 @@ --- -name: configs-online-evals +name: online-evals description: Attach judges to AI Config variations for automatic LLM-as-a-judge evaluation. Create custom judges, configure sampling rates, and monitor quality scores. compatibility: Requires LaunchDarkly API access token with ai-configs:write permission. SDK versions Python v0.20.0+ or Node.js v0.20.0+ for automatic metric recording and the consolidated `track_judge_result` / `trackJudgeResult` API. metadata: diff --git a/skills/agentcontrol/configs-snippets/SKILL.md b/skills/agentcontrol/snippets/SKILL.md similarity index 99% rename from skills/agentcontrol/configs-snippets/SKILL.md rename to skills/agentcontrol/snippets/SKILL.md index c4e5393..ecea329 100644 --- a/skills/agentcontrol/configs-snippets/SKILL.md +++ b/skills/agentcontrol/snippets/SKILL.md @@ -1,5 +1,5 @@ --- -name: configs-snippets +name: snippets description: "Create and manage prompt snippets — reusable text blocks referenced inside AI Config variation prompts. Keeps common instructions, personas, and guardrails consistent across multiple configs." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server diff --git a/skills/configs-online-evals b/skills/configs-online-evals deleted file mode 120000 index c80f182..0000000 --- a/skills/configs-online-evals +++ /dev/null @@ -1 +0,0 @@ -agentcontrol/configs-online-evals \ No newline at end of file diff --git a/skills/online-evals b/skills/online-evals new file mode 120000 index 0000000..5774aaa --- /dev/null +++ b/skills/online-evals @@ -0,0 +1 @@ +agentcontrol/online-evals \ No newline at end of file From b715a908a9010e93a65ae53d511200e726412b2c Mon Sep 17 00:00:00 2001 From: Scarlett Attensil Date: Tue, 19 May 2026 12:06:59 -0700 Subject: [PATCH 08/10] Make custom-metrics and built-in-metrics bare Drop configs- prefix from both metrics skills. configs- now only on create, targeting, update, variations. Regenerated skills.json; no symlinks affected (neither was symlinked). --- skills.json | 20 ++++++++-------- .../SKILL.md | 4 ++-- .../references/anthropic-tracking.md | 0 .../references/bedrock-tracking.md | 0 .../references/gemini-tracking.md | 0 .../references/langchain-tracking.md | 0 .../references/metrics-api.md | 0 .../references/openai-tracking.md | 0 .../references/strands-tracking.md | 0 .../references/streaming-tracking.md | 0 .../agentcontrol/configs-targeting/SKILL.md | 2 +- .../SKILL.md | 6 ++--- skills/agentcontrol/migrate/SKILL.md | 24 +++++++++---------- .../references/agent-mode-frameworks.md | 4 ++-- .../references/phase-1-analysis-checklist.md | 2 +- .../references/sdk-ai-tracker-patterns.md | 8 +++---- skills/agentcontrol/tools/SKILL.md | 12 +++++----- 17 files changed, 41 insertions(+), 41 deletions(-) rename skills/agentcontrol/{configs-built-in-metrics => built-in-metrics}/SKILL.md (98%) rename skills/agentcontrol/{configs-built-in-metrics => built-in-metrics}/references/anthropic-tracking.md (100%) rename skills/agentcontrol/{configs-built-in-metrics => built-in-metrics}/references/bedrock-tracking.md (100%) rename skills/agentcontrol/{configs-built-in-metrics => built-in-metrics}/references/gemini-tracking.md (100%) rename skills/agentcontrol/{configs-built-in-metrics => built-in-metrics}/references/langchain-tracking.md (100%) rename skills/agentcontrol/{configs-built-in-metrics => built-in-metrics}/references/metrics-api.md (100%) rename skills/agentcontrol/{configs-built-in-metrics => built-in-metrics}/references/openai-tracking.md (100%) rename skills/agentcontrol/{configs-built-in-metrics => built-in-metrics}/references/strands-tracking.md (100%) rename skills/agentcontrol/{configs-built-in-metrics => built-in-metrics}/references/streaming-tracking.md (100%) rename skills/agentcontrol/{configs-custom-metrics => custom-metrics}/SKILL.md (98%) diff --git a/skills.json b/skills.json index 318bc15..4f2caa2 100644 --- a/skills.json +++ b/skills.json @@ -17,9 +17,9 @@ "compatibility": "Requires integration plan and LaunchDarkly credentials (see parent onboarding)" }, { - "name": "configs-built-in-metrics", + "name": "built-in-metrics", "description": "Instrument an existing codebase with LaunchDarkly AI Config tracking. Walks the four-tier ladder (managed runner \u2192 provider package \u2192 custom extractor + trackMetricsOf \u2192 raw manual) and picks the lowest-ceremony option that still captures duration, tokens, and success/error.", - "path": "skills/agentcontrol/configs-built-in-metrics", + "path": "skills/agentcontrol/built-in-metrics", "version": "1.0.0-experimental", "license": "Apache-2.0", "compatibility": "Requires the LaunchDarkly server-side AI SDK (`launchdarkly-server-sdk-ai>=0.20.0` for Python or `@launchdarkly/server-sdk-ai>=0.20.0` for Node) and an existing AI Config." @@ -32,14 +32,6 @@ "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, - { - "name": "configs-custom-metrics", - "description": "Create, track, retrieve, update, and delete custom business metrics for AI Configs. Covers full lifecycle: define metric kinds via API, emit events via SDK, and query results.", - "path": "skills/agentcontrol/configs-custom-metrics", - "version": "1.0.0-experimental", - "license": "Apache-2.0", - "compatibility": "Requires the LaunchDarkly server SDK and a LaunchDarkly API token with the `writer` role for metric management." - }, { "name": "configs-targeting", "description": "Configure AI Config targeting rules to control which variations serve to different users. Enable percentage rollouts, attribute-based rules, segment targeting, and guarded rollouts.", @@ -63,6 +55,14 @@ "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, + { + "name": "custom-metrics", + "description": "Create, track, retrieve, update, and delete custom business metrics for AI Configs. Covers full lifecycle: define metric kinds via API, emit events via SDK, and query results.", + "path": "skills/agentcontrol/custom-metrics", + "version": "1.0.0-experimental", + "license": "Apache-2.0", + "compatibility": "Requires the LaunchDarkly server SDK and a LaunchDarkly API token with the `writer` role for metric management." + }, { "name": "detect", "description": "Detect repository stack for LaunchDarkly SDK onboarding: languages, frameworks, package managers, monorepo targets, entrypoints, existing LD usage. Nested under sdk-install; next is plan.", diff --git a/skills/agentcontrol/configs-built-in-metrics/SKILL.md b/skills/agentcontrol/built-in-metrics/SKILL.md similarity index 98% rename from skills/agentcontrol/configs-built-in-metrics/SKILL.md rename to skills/agentcontrol/built-in-metrics/SKILL.md index 65b5be5..4b93473 100644 --- a/skills/agentcontrol/configs-built-in-metrics/SKILL.md +++ b/skills/agentcontrol/built-in-metrics/SKILL.md @@ -1,5 +1,5 @@ --- -name: configs-built-in-metrics +name: built-in-metrics description: "Instrument an existing codebase with LaunchDarkly AI Config tracking. Walks the four-tier ladder (managed runner → provider package → custom extractor + trackMetricsOf → raw manual) and picks the lowest-ceremony option that still captures duration, tokens, and success/error." license: Apache-2.0 compatibility: Requires the LaunchDarkly server-side AI SDK (`launchdarkly-server-sdk-ai>=0.20.0` for Python or `@launchdarkly/server-sdk-ai>=0.20.0` for Node) and an existing AI Config. @@ -104,6 +104,6 @@ Obtain a tracker via the factory on the config object: `tracker = config.create_ ## Related skills - `configs-create` — prerequisite if the app doesn't have an AI Config yet -- `configs-custom-metrics` — business metrics (conversion, resolution, retention) layered on top of the AI metrics this skill captures +- `custom-metrics` — business metrics (conversion, resolution, retention) layered on top of the AI metrics this skill captures - `online-evals` — automatic quality scoring (LLM-as-judge) on sampled live requests; complementary to the metrics here - `migrate` — Stage 4 of the hardcoded-to-AI-Configs migration delegates to this skill diff --git a/skills/agentcontrol/configs-built-in-metrics/references/anthropic-tracking.md b/skills/agentcontrol/built-in-metrics/references/anthropic-tracking.md similarity index 100% rename from skills/agentcontrol/configs-built-in-metrics/references/anthropic-tracking.md rename to skills/agentcontrol/built-in-metrics/references/anthropic-tracking.md diff --git a/skills/agentcontrol/configs-built-in-metrics/references/bedrock-tracking.md b/skills/agentcontrol/built-in-metrics/references/bedrock-tracking.md similarity index 100% rename from skills/agentcontrol/configs-built-in-metrics/references/bedrock-tracking.md rename to skills/agentcontrol/built-in-metrics/references/bedrock-tracking.md diff --git a/skills/agentcontrol/configs-built-in-metrics/references/gemini-tracking.md b/skills/agentcontrol/built-in-metrics/references/gemini-tracking.md similarity index 100% rename from skills/agentcontrol/configs-built-in-metrics/references/gemini-tracking.md rename to skills/agentcontrol/built-in-metrics/references/gemini-tracking.md diff --git a/skills/agentcontrol/configs-built-in-metrics/references/langchain-tracking.md b/skills/agentcontrol/built-in-metrics/references/langchain-tracking.md similarity index 100% rename from skills/agentcontrol/configs-built-in-metrics/references/langchain-tracking.md rename to skills/agentcontrol/built-in-metrics/references/langchain-tracking.md diff --git a/skills/agentcontrol/configs-built-in-metrics/references/metrics-api.md b/skills/agentcontrol/built-in-metrics/references/metrics-api.md similarity index 100% rename from skills/agentcontrol/configs-built-in-metrics/references/metrics-api.md rename to skills/agentcontrol/built-in-metrics/references/metrics-api.md diff --git a/skills/agentcontrol/configs-built-in-metrics/references/openai-tracking.md b/skills/agentcontrol/built-in-metrics/references/openai-tracking.md similarity index 100% rename from skills/agentcontrol/configs-built-in-metrics/references/openai-tracking.md rename to skills/agentcontrol/built-in-metrics/references/openai-tracking.md diff --git a/skills/agentcontrol/configs-built-in-metrics/references/strands-tracking.md b/skills/agentcontrol/built-in-metrics/references/strands-tracking.md similarity index 100% rename from skills/agentcontrol/configs-built-in-metrics/references/strands-tracking.md rename to skills/agentcontrol/built-in-metrics/references/strands-tracking.md diff --git a/skills/agentcontrol/configs-built-in-metrics/references/streaming-tracking.md b/skills/agentcontrol/built-in-metrics/references/streaming-tracking.md similarity index 100% rename from skills/agentcontrol/configs-built-in-metrics/references/streaming-tracking.md rename to skills/agentcontrol/built-in-metrics/references/streaming-tracking.md diff --git a/skills/agentcontrol/configs-targeting/SKILL.md b/skills/agentcontrol/configs-targeting/SKILL.md index 7a98990..7903507 100644 --- a/skills/agentcontrol/configs-targeting/SKILL.md +++ b/skills/agentcontrol/configs-targeting/SKILL.md @@ -488,7 +488,7 @@ After configuring targeting: ``` https://app.launchdarkly.com/projects/{projectKey}/ai-configs/{configKey} ``` -2. **Monitor performance** with `configs-built-in-metrics` +2. **Monitor performance** with `built-in-metrics` 3. **Attach judges** with `online-evals` 4. **Set up guarded rollouts** for automatic regression detection diff --git a/skills/agentcontrol/configs-custom-metrics/SKILL.md b/skills/agentcontrol/custom-metrics/SKILL.md similarity index 98% rename from skills/agentcontrol/configs-custom-metrics/SKILL.md rename to skills/agentcontrol/custom-metrics/SKILL.md index 1b66981..cca2111 100644 --- a/skills/agentcontrol/configs-custom-metrics/SKILL.md +++ b/skills/agentcontrol/custom-metrics/SKILL.md @@ -1,5 +1,5 @@ --- -name: configs-custom-metrics +name: custom-metrics description: "Create, track, retrieve, update, and delete custom business metrics for AI Configs. Covers full lifecycle: define metric kinds via API, emit events via SDK, and query results." license: Apache-2.0 compatibility: Requires the LaunchDarkly server SDK and a LaunchDarkly API token with the `writer` role for metric management. @@ -16,7 +16,7 @@ Full lifecycle management of custom business metrics: create metric definitions - LaunchDarkly SDK initialized (see `sdk`) - LaunchDarkly API token with `writer` role for metric management -- Understanding of built-in AI metrics (see `configs-built-in-metrics`) +- Understanding of built-in AI metrics (see `built-in-metrics`) ## API Key Detection @@ -493,7 +493,7 @@ Custom metrics appear in: ## Related Skills - `sdk` - SDK setup -- `configs-built-in-metrics` - Built-in AI metrics (tokens, duration, cost) +- `built-in-metrics` - Built-in AI metrics (tokens, duration, cost) - `online-evals` - Quality metrics via judges ## References diff --git a/skills/agentcontrol/migrate/SKILL.md b/skills/agentcontrol/migrate/SKILL.md index ab500e4..fd0667e 100644 --- a/skills/agentcontrol/migrate/SKILL.md +++ b/skills/agentcontrol/migrate/SKILL.md @@ -30,16 +30,16 @@ The skill is optimized for Python and Node.js / TypeScript; other languages are | Shape | Python | Node.js | Reference | |-------|--------|---------|-----------| -| One-shot completion (direct OpenAI / Anthropic / Bedrock / Gemini call) | ✅ Worked example | ✅ Worked example | [before-after-examples.md](references/before-after-examples.md), per-provider docs in `configs-built-in-metrics/references/` | -| Chat loop via managed runner (`ManagedModel`) | ✅ Tier 1 pattern | ✅ Tier 1 pattern | [configs-built-in-metrics SKILL.md](../configs-built-in-metrics/SKILL.md) | -| LangChain single-call | ✅ Worked example | ✅ Worked example | [langchain-tracking.md](../configs-built-in-metrics/references/langchain-tracking.md) | +| One-shot completion (direct OpenAI / Anthropic / Bedrock / Gemini call) | ✅ Worked example | ✅ Worked example | [before-after-examples.md](references/before-after-examples.md), per-provider docs in `built-in-metrics/references/` | +| Chat loop via managed runner (`ManagedModel`) | ✅ Tier 1 pattern | ✅ Tier 1 pattern | [built-in-metrics SKILL.md](../built-in-metrics/SKILL.md) | +| LangChain single-call | ✅ Worked example | ✅ Worked example | [langchain-tracking.md](../built-in-metrics/references/langchain-tracking.md) | | LangGraph prebuilt agent (Python `langchain.agents.create_agent`, Node `createReactAgent`) | ✅ Worked example | ✅ Worked example | [agent-mode-frameworks.md § LangGraph](references/agent-mode-frameworks.md) | | LangGraph custom `StateGraph` with run-scoped tracker (setup_run + call_model + finalize) | ✅ Deep worked example | ⚠️ Mentioned — translate from Python | [agent-mode-frameworks.md § Custom `StateGraph`](references/agent-mode-frameworks.md) | | CrewAI `Agent` | ✅ Worked example | — (not a Node framework) | [agent-mode-frameworks.md § CrewAI](references/agent-mode-frameworks.md) | | Strands `Agent` | ✅ Worked example | ⚠️ BedrockModel + OpenAIModel only (no Anthropic) | [agent-mode-frameworks.md § Strands](references/agent-mode-frameworks.md) | | Custom ReAct loop (hand-rolled, any framework or none) | ✅ Worked example | ⚠️ Apply framework-agnostic invariants; translate from Python | [agent-mode-frameworks.md § Custom ReAct loop](references/agent-mode-frameworks.md) | -| Vercel AI SDK (`generateText` / `streamText`) | — (not a Python framework) | ⚠️ Provider package exists; no worked example in skill | `configs-built-in-metrics` provider-package matrix | -| Streaming (SSE / WebSocket) | ⚠️ Delegated to `configs-built-in-metrics` streaming doc | ⚠️ Same — use `trackStreamMetricsOf` + manual TTFT | [streaming-tracking.md](../configs-built-in-metrics/references/streaming-tracking.md) | +| Vercel AI SDK (`generateText` / `streamText`) | — (not a Python framework) | ⚠️ Provider package exists; no worked example in skill | `built-in-metrics` provider-package matrix | +| Streaming (SSE / WebSocket) | ⚠️ Delegated to `built-in-metrics` streaming doc | ⚠️ Same — use `trackStreamMetricsOf` + manual TTFT | [streaming-tracking.md](../built-in-metrics/references/streaming-tracking.md) | | Multi-agent graph (supervisor + workers) | ⚠️ Out of main scope; see reference | ⚠️ Out of main scope; see reference | [agent-graph-reference.md](references/agent-graph-reference.md) | | Non-LangGraph agent frameworks (Pydantic AI, DSPy, AutoGen, Haystack, LlamaIndex agents, Semantic Kernel) | ⚠️ Apply the three invariants; no framework-specific example | ⚠️ Same | [agent-mode-frameworks.md § Framework-agnostic invariants](references/agent-mode-frameworks.md) | | Go, Ruby, .NET | ℹ️ Install commands only | ℹ️ Install commands only | [phase-1-analysis-checklist.md § SDK routing table](references/phase-1-analysis-checklist.md) | @@ -160,7 +160,7 @@ This is the first stage that writes code. It has nine sub-steps. - OpenAI: `launchdarkly-server-sdk-ai-openai>=0.4.0` (Python) / `@launchdarkly/server-sdk-ai-openai@^0.5.5` (Node) - LangChain / LangGraph: `launchdarkly-server-sdk-ai-langchain>=0.5.0` (Python) / `@launchdarkly/server-sdk-ai-langchain@^0.5.5` (Node) - Vercel AI SDK (Node only): `@launchdarkly/server-sdk-ai-vercel@^0.5.5` - - Anthropic, Gemini, Bedrock — no provider package published; use Tier-3 custom extractor (see `configs-built-in-metrics`) + - Anthropic, Gemini, Bedrock — no provider package published; use Tier-3 custom extractor (see `built-in-metrics`) 3. **Initialize `LDAIClient` once at startup.** Reuse any existing `LDClient` — do not create a second base client. Place the initialization in the same module that owns existing app config. @@ -350,9 +350,9 @@ Delegate: **`tools`** (sub-step 2). ### Step 4: Instrument the tracker (Stage 4) -Delegate: **`configs-built-in-metrics`** wires the per-request `tracker.track_*` calls (duration, tokens, success/error, feedback) around the provider call. Use **`configs-custom-metrics`** alongside it if the app needs business metrics beyond the built-in AI ones. Note: do not confuse this with `launchdarkly-metric-instrument`, which is for `ldClient.track()` feature metrics — a different API. See [sdk-ai-tracker-patterns.md](references/sdk-ai-tracker-patterns.md) for the full per-method Python + Node matrix that the delegate skill draws on. +Delegate: **`built-in-metrics`** wires the per-request `tracker.track_*` calls (duration, tokens, success/error, feedback) around the provider call. Use **`custom-metrics`** alongside it if the app needs business metrics beyond the built-in AI ones. Note: do not confuse this with `launchdarkly-metric-instrument`, which is for `ldClient.track()` feature metrics — a different API. See [sdk-ai-tracker-patterns.md](references/sdk-ai-tracker-patterns.md) for the full per-method Python + Node matrix that the delegate skill draws on. -Hand off: print the AI Config key, variation key, provider, and whether the call is streaming, then tell the user: *"Run `/configs-built-in-metrics` with these inputs, then come back here."* Do not auto-invoke. Return here for sub-step 5 (verify) once they're done. +Hand off: print the AI Config key, variation key, provider, and whether the call is streaming, then tell the user: *"Run `/built-in-metrics` with these inputs, then come back here."* Do not auto-invoke. Return here for sub-step 5 (verify) once they're done. 1. **Create the tracker.** Obtain a per-execution tracker via the factory on the config returned in Stage 2: `tracker = config.create_tracker()` (Python) or `const tracker = aiConfig.createTracker();` (Node). Call the factory **once per user turn** and reuse the returned `tracker` for every tracking call in that turn — each call mints a fresh `runId` that tags every event emitted from the turn so they can be correlated via exported events or downstream queries. (The Monitoring tab aggregates today; run-level grouping is a downstream concern — but the `runId` is also what the SDK's at-most-once guards are keyed on, so minting a new one mid-turn breaks the guard semantics regardless of where the events end up.) @@ -404,7 +404,7 @@ Hand off: print the AI Config key, variation key, provider, and whether the call ); ``` - For Anthropic direct, Bedrock (no provider package), Gemini, and custom HTTP, write a small extractor returning `LDAIMetrics` — see the delegate skill's [anthropic-tracking.md](../configs-built-in-metrics/references/anthropic-tracking.md), [bedrock-tracking.md](../configs-built-in-metrics/references/bedrock-tracking.md), and [gemini-tracking.md](../configs-built-in-metrics/references/gemini-tracking.md). LangChain single-node and LangGraph go through the `launchdarkly-server-sdk-ai-langchain` / `@launchdarkly/server-sdk-ai-langchain` provider package. Build the model with `create_langchain_model(config)` (Python) / `createLangChainModel(config)` (Node) — both forward all variation parameters — and track with `get_ai_metrics_from_response` / `getAIMetricsFromResponse`. See [langchain-tracking.md](../configs-built-in-metrics/references/langchain-tracking.md). + For Anthropic direct, Bedrock (no provider package), Gemini, and custom HTTP, write a small extractor returning `LDAIMetrics` — see the delegate skill's [anthropic-tracking.md](../built-in-metrics/references/anthropic-tracking.md), [bedrock-tracking.md](../built-in-metrics/references/bedrock-tracking.md), and [gemini-tracking.md](../built-in-metrics/references/gemini-tracking.md). LangChain single-node and LangGraph go through the `launchdarkly-server-sdk-ai-langchain` / `@launchdarkly/server-sdk-ai-langchain` provider package. Build the model with `create_langchain_model(config)` (Python) / `createLangChainModel(config)` (Node) — both forward all variation parameters — and track with `get_ai_metrics_from_response` / `getAIMetricsFromResponse`. See [langchain-tracking.md](../built-in-metrics/references/langchain-tracking.md). 4. **Wire feedback tracking if the app has thumbs-up/down UI.** Both SDKs expose `trackFeedback` with a `{kind}` argument. @@ -498,9 +498,9 @@ Delegate: **`online-evals`** (sub-step 3, optional — only for UI-attached judg | Retry wrapper around the provider call | The tracker is minted once at the top of the user turn; the retry loop is inside that scope. Every retry attempt shares the same `runId`. Tracker calls (`track_duration` / `track_tokens` / `track_success` / `track_error`) live *outside* the retry body — one call at the end of the turn, on the success path or the final-failure path | | App has no tools — Stage 3 skipped | Move directly from Stage 2 verification to Stage 4 (tracking) | | Mode mismatch: user said agent, audit shows one-shot chat | Choose completion mode unless the app uses a LangGraph prebuilt agent (`langchain.agents.create_agent` in Python or `createReactAgent` in Node), CrewAI `Agent`, Strands `Agent`, or a similar goal-driven framework | -| App uses Strands Agents (Python) | Agent mode. Build a `create_strands_model` dispatcher keyed on `agent_config.provider.name` that returns `AnthropicModel(model_id=..., max_tokens=...)` or `OpenAIModel(model_id=..., params=...)`. Drop `parameters.tools` before passing params to the model class — Strands receives tools via `Agent(tools=[...])`. Tracking is Tier 3: wrap `invoke_async` with `tracker.track_duration_of(...)` and record tokens from `result.metrics.accumulated_usage`. See [agent-mode-frameworks.md § Strands Agent](references/agent-mode-frameworks.md) and [strands-tracking.md](../configs-built-in-metrics/references/strands-tracking.md) | +| App uses Strands Agents (Python) | Agent mode. Build a `create_strands_model` dispatcher keyed on `agent_config.provider.name` that returns `AnthropicModel(model_id=..., max_tokens=...)` or `OpenAIModel(model_id=..., params=...)`. Drop `parameters.tools` before passing params to the model class — Strands receives tools via `Agent(tools=[...])`. Tracking is Tier 3: wrap `invoke_async` with `tracker.track_duration_of(...)` and record tokens from `result.metrics.accumulated_usage`. See [agent-mode-frameworks.md § Strands Agent](references/agent-mode-frameworks.md) and [strands-tracking.md](../built-in-metrics/references/strands-tracking.md) | | Strands app on TypeScript | TS SDK ships `BedrockModel` and `OpenAIModel` only — cannot serve Anthropic-backed variations. Use the Python SDK if multi-provider variations are required | -| TypeScript app using Anthropic SDK | No `trackAnthropicMetrics` helper exists. Use Tier 3: `trackMetricsOf` with a small custom extractor that reads `response.usage.input_tokens` / `response.usage.output_tokens` and returns `LDAIMetrics`. See [anthropic-tracking.md](../configs-built-in-metrics/references/anthropic-tracking.md) in the `configs-built-in-metrics` skill for the exact extractor | +| TypeScript app using Anthropic SDK | No `trackAnthropicMetrics` helper exists. Use Tier 3: `trackMetricsOf` with a small custom extractor that reads `response.usage.input_tokens` / `response.usage.output_tokens` and returns `LDAIMetrics`. See [anthropic-tracking.md](../built-in-metrics/references/anthropic-tracking.md) in the `built-in-metrics` skill for the exact extractor | | Fallback would silently crash because `LD_SDK_KEY` is missing | Log a startup warning; proceed with the fallback. Never raise at import time | | Multi-agent graph (supervisor + workers) | Stop after migrating a single agent. Agent Graph Definitions are available in **both** SDKs — Python via `launchdarkly-server-sdk-ai.agent_graph` and Node via the graph API in `@launchdarkly/server-sdk-ai`. Read [agent-graph-reference.md](references/agent-graph-reference.md) for the graph-level migration path — it is deliberately out of this skill's main scope | | Single-agent (ReAct, tool loop) + agent mode | Default to offline eval via the LD Playground + Datasets for Stage 5. UI-attached judges are completion-only today, and programmatic direct-judge adds per-call cost that is usually not worth it until after the migration is live and stable. Point at the [Offline Evals guide](https://docs.launchdarkly.com/guides/ai-configs/offline-evaluations) | @@ -525,7 +525,7 @@ These are ordered by how likely they are to show up as a first-run failure. The - **If the repo already contains a `load_chat_model(f"{provider}/{name}")` helper, delete it — don't just avoid using it.** This exact shape ships with `langchain-ai/react-agent` and is copied into dozens of derivative repos; look for `utils.load_chat_model`, `utils.build_model`, or any one-arg `init_chat_model` wrapper that splits a `"provider/model"` string. Re-using it is the first-run failure mode: every variation parameter (temperature, max_tokens, top_p, stop sequences) silently drops on the floor because `init_chat_model` only receives the name and provider. `create_langchain_model(ai_config)` is a one-for-one replacement that forwards the whole `model.parameters` dict. Replace every call site, then delete the wrapper file-side so the next reader can't reach for it. - **Same rule applies to hand-rolled `resolve_tools` / `TOOL_REGISTRY` / `ALL_TOOLS` helpers.** If the template already has a `resolve_tools(tool_keys)` or an `ALL_TOOLS` module-level list, import `build_structured_tools` from `ldai_langchain.langchain_helper` and delete the hand-rolled version. `build_structured_tools(ai_config, TOOL_REGISTRY_DICT)` reads `ai_config.model.parameters.tools` and wraps the matching callables as LangChain `StructuredTool`s with the LD tool key as the `StructuredTool.name` — so `ToolNode` lookup works without a second mapping. Don't leave both in the repo. -- Don't put app-scoped knobs directly in `model.parameters`. `create_langchain_model` forwards every key in `parameters` to the provider SDK via `init_chat_model`, so a `max_search_results` / `retry_budget` / `feature_toggle` entry will crash the provider with an unexpected-keyword-argument error. The correct home is `model.custom`, which the provider helpers ignore and the app reads via `ai_config.model.get_custom("key")`. The MCP `update-ai-config-variation` tool does not currently expose top-level `custom`, so pick one of two paths: (a) PATCH the variation via the REST API to set `model.custom` directly, or (b) set it via MCP inside `parameters.custom` (as a nested dict) and use a defensive accessor that reads both locations. Full walk-through with code samples in [langchain-tracking.md § MCP caveat](../configs-built-in-metrics/references/langchain-tracking.md). +- Don't put app-scoped knobs directly in `model.parameters`. `create_langchain_model` forwards every key in `parameters` to the provider SDK via `init_chat_model`, so a `max_search_results` / `retry_budget` / `feature_toggle` entry will crash the provider with an unexpected-keyword-argument error. The correct home is `model.custom`, which the provider helpers ignore and the app reads via `ai_config.model.get_custom("key")`. The MCP `update-ai-config-variation` tool does not currently expose top-level `custom`, so pick one of two paths: (a) PATCH the variation via the REST API to set `model.custom` directly, or (b) set it via MCP inside `parameters.custom` (as a nested dict) and use a defensive accessor that reads both locations. Full walk-through with code samples in [langchain-tracking.md § MCP caveat](../built-in-metrics/references/langchain-tracking.md). - Don't re-encode tool schemas inside the fallback. When LaunchDarkly is unreachable the fallback should run without tools (or with whatever minimal provider-bound parameters the app needs to keep operating). Building a `_FALLBACK_TOOLS` array that duplicates the AI Config's tool schema re-introduces the hardcoded config the migration was supposed to move out of code. - Don't import `LaunchDarklyCallbackHandler` from `ldai.langchain` — neither the class nor the dotted module path exists. The Python LangChain helper package is `ldai_langchain` (top-level module, underscore). Use `create_langchain_model(config)` + `track_metrics_of_async(get_ai_metrics_from_response, lambda: llm.ainvoke(messages))` as the canonical pattern. diff --git a/skills/agentcontrol/migrate/references/agent-mode-frameworks.md b/skills/agentcontrol/migrate/references/agent-mode-frameworks.md index 2d2f7ac..cf892d0 100644 --- a/skills/agentcontrol/migrate/references/agent-mode-frameworks.md +++ b/skills/agentcontrol/migrate/references/agent-mode-frameworks.md @@ -18,7 +18,7 @@ Agent mode returns an `instructions` string. Completion mode returns a `messages **Caveat:** judges cannot be attached to agent-mode variations via the LaunchDarkly UI. Agent mode evaluations must go through the programmatic judge API (`create_judge(...).evaluate(input, output)`). See `online-evals` for the programmatic path. -**Model construction for LangChain / LangGraph.** When the framework runs on top of LangChain (which includes LangGraph's prebuilt agent and most custom graphs), build the chat model with `create_langchain_model(ai_config)` (Python) or `createLangChainModel(aiConfig)` (Node). These helpers forward every variation parameter (`temperature`, `max_tokens`, `top_p`, …) and handle LaunchDarkly→LangChain provider-name mapping internally. Do not hand-roll `init_chat_model(model=..., model_provider=...)` — it silently drops every variation parameter. See [langchain-tracking.md](../../configs-built-in-metrics/references/langchain-tracking.md) for the canonical single-model and LangGraph patterns, including the SDK helpers `sum_token_usage_from_messages` / `get_tool_calls_from_response` (Python, `ldai_langchain`) used inside the `track_metrics_of_async` / `trackMetricsOf` extractor. +**Model construction for LangChain / LangGraph.** When the framework runs on top of LangChain (which includes LangGraph's prebuilt agent and most custom graphs), build the chat model with `create_langchain_model(ai_config)` (Python) or `createLangChainModel(aiConfig)` (Node). These helpers forward every variation parameter (`temperature`, `max_tokens`, `top_p`, …) and handle LaunchDarkly→LangChain provider-name mapping internally. Do not hand-roll `init_chat_model(model=..., model_provider=...)` — it silently drops every variation parameter. See [langchain-tracking.md](../../built-in-metrics/references/langchain-tracking.md) for the canonical single-model and LangGraph patterns, including the SDK helpers `sum_token_usage_from_messages` / `get_tool_calls_from_response` (Python, `ldai_langchain`) used inside the `track_metrics_of_async` / `trackMetricsOf` extractor. ## Framework-agnostic invariants for the run-scoped pattern @@ -291,7 +291,7 @@ async def run_turn(ai_client, user_id: str, user_input: str): **Key points:** - `system_prompt=agent_config.instructions` — the instructions string replaces the hardcoded system prompt. - `create_strands_model(agent_config)` is the provider-dispatch seam. Add a branch per provider the variation can serve. -- The tracker is Tier 3: `tracker.track_duration_of(...)` + an explicit `track_tokens` call fed by `track_strands_metrics`. See [strands-tracking.md](../../configs-built-in-metrics/references/strands-tracking.md) for the single-call `track_metrics_of_async` variant and the per-field breakdown of `accumulated_usage`. +- The tracker is Tier 3: `tracker.track_duration_of(...)` + an explicit `track_tokens` call fed by `track_strands_metrics`. See [strands-tracking.md](../../built-in-metrics/references/strands-tracking.md) for the single-call `track_metrics_of_async` variant and the per-field breakdown of `accumulated_usage`. - Always `ldclient.get().flush()` before process exit in short-lived scripts — trailing events can otherwise be lost. **TypeScript caveat.** The Strands TypeScript SDK ships `BedrockModel` and `OpenAIModel` only — it cannot run Anthropic-backed variations. If the app needs to serve both OpenAI and Anthropic from a single AI Config, use the Python SDK. diff --git a/skills/agentcontrol/migrate/references/phase-1-analysis-checklist.md b/skills/agentcontrol/migrate/references/phase-1-analysis-checklist.md index 5e5fadb..da16f00 100644 --- a/skills/agentcontrol/migrate/references/phase-1-analysis-checklist.md +++ b/skills/agentcontrol/migrate/references/phase-1-analysis-checklist.md @@ -151,7 +151,7 @@ Feeds into Stage 2 (install + wrap). Quoted from the `ai-configs-relaunch-guides | LangChain / LangGraph | `@launchdarkly/server-sdk-ai-langchain` | `createLangChainModel(config)` (forwards all variation parameters and handles provider-name mapping) + `getAIMetricsFromResponse` with `trackMetricsOf` | | Vercel AI SDK | `@launchdarkly/server-sdk-ai-vercel` | `getAIMetricsFromResponse` + `trackMetricsOf`, or `VercelRunnerFactory.createVercelModel(aiConfig)` for a managed runner | -Python currently ships helper packages for OpenAI (`ldai_openai`) and LangChain (`ldai_langchain`). The LangChain Python package exposes `create_langchain_model(config)` (builds a LangChain chat model from the AI Config, forwarding every variation parameter and mapping LD provider names to LangChain equivalents), `convert_messages_to_langchain`, and `get_ai_metrics_from_response` — the same package covers LangGraph. Use `create_langchain_model(config)` + `track_metrics_of_async(get_ai_metrics_from_response, lambda: llm.ainvoke(messages))` as the canonical single-call pattern. See [langchain-tracking.md](../../configs-built-in-metrics/references/langchain-tracking.md) for both LangChain and LangGraph patterns and [sdk-ai-tracker-patterns.md](sdk-ai-tracker-patterns.md) for the full tracker-method matrix. +Python currently ships helper packages for OpenAI (`ldai_openai`) and LangChain (`ldai_langchain`). The LangChain Python package exposes `create_langchain_model(config)` (builds a LangChain chat model from the AI Config, forwarding every variation parameter and mapping LD provider names to LangChain equivalents), `convert_messages_to_langchain`, and `get_ai_metrics_from_response` — the same package covers LangGraph. Use `create_langchain_model(config)` + `track_metrics_of_async(get_ai_metrics_from_response, lambda: llm.ainvoke(messages))` as the canonical single-call pattern. See [langchain-tracking.md](../../built-in-metrics/references/langchain-tracking.md) for both LangChain and LangGraph patterns and [sdk-ai-tracker-patterns.md](sdk-ai-tracker-patterns.md) for the full tracker-method matrix. ## Phase 1 output format diff --git a/skills/agentcontrol/migrate/references/sdk-ai-tracker-patterns.md b/skills/agentcontrol/migrate/references/sdk-ai-tracker-patterns.md index 3b881be..c5446f3 100644 --- a/skills/agentcontrol/migrate/references/sdk-ai-tracker-patterns.md +++ b/skills/agentcontrol/migrate/references/sdk-ai-tracker-patterns.md @@ -297,7 +297,7 @@ response = tracker.track_metrics_of( | Helper | Signature | Tier | Notes | |--------|-----------|------|-------| | `trackMetricsOf(extractor, func)` | `await tracker.trackMetricsOf((result) => extractor(result), async () => ...)` | **2 / 3** | **Canonical generic wrapper.** `extractor` maps provider response → `LDAIMetrics`. Use a provider package's bare `getAIMetricsFromResponse` for Tier 2 (`@launchdarkly/server-sdk-ai-openai`, `-langchain`, `-vercel`) or a small custom function for Tier 3. | -| `trackStreamMetricsOf(extractor, streamCreator)` | `tracker.trackStreamMetricsOf(async (chunks) => extractor(chunks), () => createStream())` | 2 / 3 | Stream variant. Does **not** capture TTFT automatically — if you need TTFT, use the manual pattern in [streaming-tracking.md](../../configs-built-in-metrics/references/streaming-tracking.md). | +| `trackStreamMetricsOf(extractor, streamCreator)` | `tracker.trackStreamMetricsOf(async (chunks) => extractor(chunks), () => createStream())` | 2 / 3 | Stream variant. Does **not** capture TTFT automatically — if you need TTFT, use the manual pattern in [streaming-tracking.md](../../built-in-metrics/references/streaming-tracking.md). | | `trackDurationOf(func)` | `await tracker.trackDurationOf(async () => ...)` | 4 | Wraps an async callable; captures duration only. Pair with explicit `trackTokens` + `trackSuccess`. | Example — OpenAI via `trackMetricsOf` + the provider package: ```typescript @@ -339,7 +339,7 @@ The managed runner handles message history, provider dispatch (via the installed ### Anthropic has no provider package today -Neither `@launchdarkly/server-sdk-ai-anthropic` nor `launchdarkly-server-sdk-ai-anthropic` exists as of this writing. For Anthropic direct calls, write a custom extractor and pass it to `track_metrics_of` / `trackMetricsOf` — see the Python example above or the full walk-through in [anthropic-tracking.md](../../configs-built-in-metrics/references/anthropic-tracking.md). If the app is open to LangChain, routing Anthropic through `ChatAnthropic` and the LangChain provider package recovers Tier 2 with zero extractor code. +Neither `@launchdarkly/server-sdk-ai-anthropic` nor `launchdarkly-server-sdk-ai-anthropic` exists as of this writing. For Anthropic direct calls, write a custom extractor and pass it to `track_metrics_of` / `trackMetricsOf` — see the Python example above or the full walk-through in [anthropic-tracking.md](../../built-in-metrics/references/anthropic-tracking.md). If the app is open to LangChain, routing Anthropic through `ChatAnthropic` and the LangChain provider package recovers Tier 2 with zero extractor code. ## Tier decision table @@ -352,7 +352,7 @@ Neither `@launchdarkly/server-sdk-ai-anthropic` nor `launchdarkly-server-sdk-ai- | Anthropic direct SDK | **3** | Custom extractor reading `response.usage.input_tokens` / `output_tokens` | | Bedrock Converse (no provider package) | **3** | Custom extractor reading `response.usage.inputTokens` / `outputTokens` (or route via LangChain for Tier 2) | | Gemini / Google GenAI, Cohere, custom HTTP | **3** | Custom extractor | -| Streaming response with TTFT required | **4** | Manual `trackTimeToFirstToken` + `trackDuration` + `trackTokens` + `trackSuccess` — see [streaming-tracking.md](../../configs-built-in-metrics/references/streaming-tracking.md) | +| Streaming response with TTFT required | **4** | Manual `trackTimeToFirstToken` + `trackDuration` + `trackTokens` + `trackSuccess` — see [streaming-tracking.md](../../built-in-metrics/references/streaming-tracking.md) | | Streaming response without TTFT (Node) | **2 / 3** | `trackStreamMetricsOf(extractor, streamFn)` | ## Streaming responses @@ -450,7 +450,7 @@ Run the checklist in order. Each step rules out one cause. ## Common gotchas -- **`model.parameters` vs `model.custom`.** `create_langchain_model` (Python) / `createLangChainModel` (Node) forwards every key in `model.parameters` to the provider SDK. App-scoped knobs (search result limits, retry budgets, feature toggles) **must** live in `model.custom` or the provider will crash at runtime with an unexpected-keyword-argument error. Read them with `ai_config.model.get_custom("key")`. Full walk-through with the MCP/REST-API caveat in [langchain-tracking.md § `model.parameters` vs `model.custom`](../../configs-built-in-metrics/references/langchain-tracking.md). +- **`model.parameters` vs `model.custom`.** `create_langchain_model` (Python) / `createLangChainModel` (Node) forwards every key in `model.parameters` to the provider SDK. App-scoped knobs (search result limits, retry budgets, feature toggles) **must** live in `model.custom` or the provider will crash at runtime with an unexpected-keyword-argument error. Read them with `ai_config.model.get_custom("key")`. Full walk-through with the MCP/REST-API caveat in [langchain-tracking.md § `model.parameters` vs `model.custom`](../../built-in-metrics/references/langchain-tracking.md). - **`track_tokens` token shape.** The Python `TokenUsage` dataclass requires `total` to be set — it is not derived. Compute `total = input + output` if the provider doesn't return one. - **`track_feedback` lifecycle.** The feedback call must be made on a tracker bound to the same `runId` that produced the response. If the thumbs-up comes in a later process, use the cross-process resumption pattern above — do **not** call `create_tracker()` again in the consumer, because that mints a *new* `runId`. - **OpenAI streaming tokens.** OpenAI only emits `usage` in the final chunk when `stream_options={"include_usage": True}` is passed. Without that flag, you have to tokenize manually — `tiktoken` for OpenAI models. diff --git a/skills/agentcontrol/tools/SKILL.md b/skills/agentcontrol/tools/SKILL.md index 8160160..ba8d16b 100644 --- a/skills/agentcontrol/tools/SKILL.md +++ b/skills/agentcontrol/tools/SKILL.md @@ -211,13 +211,13 @@ for _ in range(MAX_STEPS): messages.append({"role": "user", "content": tool_results}) ``` -Per-provider tool-call payload shapes live in the `configs-built-in-metrics` references: +Per-provider tool-call payload shapes live in the `built-in-metrics` references: -- [openai-tracking.md](../configs-built-in-metrics/references/openai-tracking.md) — Chat Completions + Responses API -- [anthropic-tracking.md](../configs-built-in-metrics/references/anthropic-tracking.md) — `tool_use` blocks and `tool_result` payloads -- [bedrock-tracking.md](../configs-built-in-metrics/references/bedrock-tracking.md) — `toolUse` / `toolResult` Converse format -- [gemini-tracking.md](../configs-built-in-metrics/references/gemini-tracking.md) — `functionCalls` / `functionResponse` parts -- [langchain-tracking.md](../configs-built-in-metrics/references/langchain-tracking.md) — LangGraph tool loop inherits from `create_react_agent` +- [openai-tracking.md](../built-in-metrics/references/openai-tracking.md) — Chat Completions + Responses API +- [anthropic-tracking.md](../built-in-metrics/references/anthropic-tracking.md) — `tool_use` blocks and `tool_result` payloads +- [bedrock-tracking.md](../built-in-metrics/references/bedrock-tracking.md) — `toolUse` / `toolResult` Converse format +- [gemini-tracking.md](../built-in-metrics/references/gemini-tracking.md) — `functionCalls` / `functionResponse` parts +- [langchain-tracking.md](../built-in-metrics/references/langchain-tracking.md) — LangGraph tool loop inherits from `create_react_agent` ## Orchestrator Note From 129ca111d08ad74115d286238d67815851fc13ed Mon Sep 17 00:00:00 2001 From: Scarlett Attensil Date: Tue, 19 May 2026 13:43:33 -0700 Subject: [PATCH 09/10] Add deprecation-redirect stubs for old aiconfig-* names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit External docs/automation reference the old aiconfig-* skill names. The host invokes skills by their 'name:' field and the repo enforces name == dir, so a config-alias or symlink can't forward old names. The only constraint-compatible mechanism is a stub skill: 12 dirs named aiconfig-* whose body redirects to the new skill (configs-create, migrate, projects, etc.). Regenerated skills.json (40 skills total). These are temporary — remove once external references are migrated. --- skills.json | 84 +++++++++++++++++++ .../aiconfig-agent-graphs/SKILL.md | 15 ++++ .../agentcontrol/aiconfig-ai-metrics/SKILL.md | 15 ++++ skills/agentcontrol/aiconfig-create/SKILL.md | 15 ++++ .../aiconfig-custom-metrics/SKILL.md | 15 ++++ skills/agentcontrol/aiconfig-migrate/SKILL.md | 15 ++++ .../aiconfig-online-evals/SKILL.md | 15 ++++ .../agentcontrol/aiconfig-projects/SKILL.md | 15 ++++ .../agentcontrol/aiconfig-snippets/SKILL.md | 15 ++++ .../agentcontrol/aiconfig-targeting/SKILL.md | 15 ++++ skills/agentcontrol/aiconfig-tools/SKILL.md | 15 ++++ skills/agentcontrol/aiconfig-update/SKILL.md | 15 ++++ .../agentcontrol/aiconfig-variations/SKILL.md | 15 ++++ 13 files changed, 264 insertions(+) create mode 100644 skills/agentcontrol/aiconfig-agent-graphs/SKILL.md create mode 100644 skills/agentcontrol/aiconfig-ai-metrics/SKILL.md create mode 100644 skills/agentcontrol/aiconfig-create/SKILL.md create mode 100644 skills/agentcontrol/aiconfig-custom-metrics/SKILL.md create mode 100644 skills/agentcontrol/aiconfig-migrate/SKILL.md create mode 100644 skills/agentcontrol/aiconfig-online-evals/SKILL.md create mode 100644 skills/agentcontrol/aiconfig-projects/SKILL.md create mode 100644 skills/agentcontrol/aiconfig-snippets/SKILL.md create mode 100644 skills/agentcontrol/aiconfig-targeting/SKILL.md create mode 100644 skills/agentcontrol/aiconfig-tools/SKILL.md create mode 100644 skills/agentcontrol/aiconfig-update/SKILL.md create mode 100644 skills/agentcontrol/aiconfig-variations/SKILL.md diff --git a/skills.json b/skills.json index 4f2caa2..62bdcd6 100644 --- a/skills.json +++ b/skills.json @@ -8,6 +8,90 @@ "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server" }, + { + "name": "aiconfig-agent-graphs", + "description": "DEPRECATED redirect \u2014 this skill was renamed to agent-graphs. Do not use this skill; invoke agent-graphs instead. Kept only so old references to aiconfig-agent-graphs still point users to the new name.", + "path": "skills/agentcontrol/aiconfig-agent-graphs", + "version": "0.1.0", + "license": "Apache-2.0" + }, + { + "name": "aiconfig-ai-metrics", + "description": "DEPRECATED redirect \u2014 this skill was renamed to built-in-metrics. Do not use this skill; invoke built-in-metrics instead. Kept only so old references to aiconfig-ai-metrics still point users to the new name.", + "path": "skills/agentcontrol/aiconfig-ai-metrics", + "version": "0.1.0", + "license": "Apache-2.0" + }, + { + "name": "aiconfig-create", + "description": "DEPRECATED redirect \u2014 this skill was renamed to configs-create. Do not use this skill; invoke configs-create instead. Kept only so old references to aiconfig-create still point users to the new name.", + "path": "skills/agentcontrol/aiconfig-create", + "version": "0.1.0", + "license": "Apache-2.0" + }, + { + "name": "aiconfig-custom-metrics", + "description": "DEPRECATED redirect \u2014 this skill was renamed to custom-metrics. Do not use this skill; invoke custom-metrics instead. Kept only so old references to aiconfig-custom-metrics still point users to the new name.", + "path": "skills/agentcontrol/aiconfig-custom-metrics", + "version": "0.1.0", + "license": "Apache-2.0" + }, + { + "name": "aiconfig-migrate", + "description": "DEPRECATED redirect \u2014 this skill was renamed to migrate. Do not use this skill; invoke migrate instead. Kept only so old references to aiconfig-migrate still point users to the new name.", + "path": "skills/agentcontrol/aiconfig-migrate", + "version": "0.1.0", + "license": "Apache-2.0" + }, + { + "name": "aiconfig-online-evals", + "description": "DEPRECATED redirect \u2014 this skill was renamed to online-evals. Do not use this skill; invoke online-evals instead. Kept only so old references to aiconfig-online-evals still point users to the new name.", + "path": "skills/agentcontrol/aiconfig-online-evals", + "version": "0.1.0", + "license": "Apache-2.0" + }, + { + "name": "aiconfig-projects", + "description": "DEPRECATED redirect \u2014 this skill was renamed to projects. Do not use this skill; invoke projects instead. Kept only so old references to aiconfig-projects still point users to the new name.", + "path": "skills/agentcontrol/aiconfig-projects", + "version": "0.1.0", + "license": "Apache-2.0" + }, + { + "name": "aiconfig-snippets", + "description": "DEPRECATED redirect \u2014 this skill was renamed to snippets. Do not use this skill; invoke snippets instead. Kept only so old references to aiconfig-snippets still point users to the new name.", + "path": "skills/agentcontrol/aiconfig-snippets", + "version": "0.1.0", + "license": "Apache-2.0" + }, + { + "name": "aiconfig-targeting", + "description": "DEPRECATED redirect \u2014 this skill was renamed to configs-targeting. Do not use this skill; invoke configs-targeting instead. Kept only so old references to aiconfig-targeting still point users to the new name.", + "path": "skills/agentcontrol/aiconfig-targeting", + "version": "0.1.0", + "license": "Apache-2.0" + }, + { + "name": "aiconfig-tools", + "description": "DEPRECATED redirect \u2014 this skill was renamed to tools. Do not use this skill; invoke tools instead. Kept only so old references to aiconfig-tools still point users to the new name.", + "path": "skills/agentcontrol/aiconfig-tools", + "version": "0.1.0", + "license": "Apache-2.0" + }, + { + "name": "aiconfig-update", + "description": "DEPRECATED redirect \u2014 this skill was renamed to configs-update. Do not use this skill; invoke configs-update instead. Kept only so old references to aiconfig-update still point users to the new name.", + "path": "skills/agentcontrol/aiconfig-update", + "version": "0.1.0", + "license": "Apache-2.0" + }, + { + "name": "aiconfig-variations", + "description": "DEPRECATED redirect \u2014 this skill was renamed to configs-variations. Do not use this skill; invoke configs-variations instead. Kept only so old references to aiconfig-variations still point users to the new name.", + "path": "skills/agentcontrol/aiconfig-variations", + "version": "0.1.0", + "license": "Apache-2.0" + }, { "name": "apply", "description": "Apply LaunchDarkly SDK onboarding: install dependency (or dual-SDK pair), configure env and secrets with consent, add init at entrypoint(s), verify compile. Nested under sdk-install; next is run.", diff --git a/skills/agentcontrol/aiconfig-agent-graphs/SKILL.md b/skills/agentcontrol/aiconfig-agent-graphs/SKILL.md new file mode 100644 index 0000000..8346943 --- /dev/null +++ b/skills/agentcontrol/aiconfig-agent-graphs/SKILL.md @@ -0,0 +1,15 @@ +--- +name: aiconfig-agent-graphs +description: "DEPRECATED redirect — this skill was renamed to agent-graphs. Do not use this skill; invoke agent-graphs instead. Kept only so old references to aiconfig-agent-graphs still point users to the new name." +license: Apache-2.0 +metadata: + version: 0.1.0 +--- + +# Deprecated — renamed to `agent-graphs` + +This skill has been renamed. **Do not follow any instructions here and do not perform the task from this skill.** + +Invoke the **`agent-graphs`** skill instead — it contains the current, complete instructions. + +If you arrived here from an old reference (documentation, a saved prompt, or automation), update it from `aiconfig-agent-graphs` to `agent-graphs`. diff --git a/skills/agentcontrol/aiconfig-ai-metrics/SKILL.md b/skills/agentcontrol/aiconfig-ai-metrics/SKILL.md new file mode 100644 index 0000000..f9911dc --- /dev/null +++ b/skills/agentcontrol/aiconfig-ai-metrics/SKILL.md @@ -0,0 +1,15 @@ +--- +name: aiconfig-ai-metrics +description: "DEPRECATED redirect — this skill was renamed to built-in-metrics. Do not use this skill; invoke built-in-metrics instead. Kept only so old references to aiconfig-ai-metrics still point users to the new name." +license: Apache-2.0 +metadata: + version: 0.1.0 +--- + +# Deprecated — renamed to `built-in-metrics` + +This skill has been renamed. **Do not follow any instructions here and do not perform the task from this skill.** + +Invoke the **`built-in-metrics`** skill instead — it contains the current, complete instructions. + +If you arrived here from an old reference (documentation, a saved prompt, or automation), update it from `aiconfig-ai-metrics` to `built-in-metrics`. diff --git a/skills/agentcontrol/aiconfig-create/SKILL.md b/skills/agentcontrol/aiconfig-create/SKILL.md new file mode 100644 index 0000000..8a5a2ba --- /dev/null +++ b/skills/agentcontrol/aiconfig-create/SKILL.md @@ -0,0 +1,15 @@ +--- +name: aiconfig-create +description: "DEPRECATED redirect — this skill was renamed to configs-create. Do not use this skill; invoke configs-create instead. Kept only so old references to aiconfig-create still point users to the new name." +license: Apache-2.0 +metadata: + version: 0.1.0 +--- + +# Deprecated — renamed to `configs-create` + +This skill has been renamed. **Do not follow any instructions here and do not perform the task from this skill.** + +Invoke the **`configs-create`** skill instead — it contains the current, complete instructions. + +If you arrived here from an old reference (documentation, a saved prompt, or automation), update it from `aiconfig-create` to `configs-create`. diff --git a/skills/agentcontrol/aiconfig-custom-metrics/SKILL.md b/skills/agentcontrol/aiconfig-custom-metrics/SKILL.md new file mode 100644 index 0000000..c13dd2a --- /dev/null +++ b/skills/agentcontrol/aiconfig-custom-metrics/SKILL.md @@ -0,0 +1,15 @@ +--- +name: aiconfig-custom-metrics +description: "DEPRECATED redirect — this skill was renamed to custom-metrics. Do not use this skill; invoke custom-metrics instead. Kept only so old references to aiconfig-custom-metrics still point users to the new name." +license: Apache-2.0 +metadata: + version: 0.1.0 +--- + +# Deprecated — renamed to `custom-metrics` + +This skill has been renamed. **Do not follow any instructions here and do not perform the task from this skill.** + +Invoke the **`custom-metrics`** skill instead — it contains the current, complete instructions. + +If you arrived here from an old reference (documentation, a saved prompt, or automation), update it from `aiconfig-custom-metrics` to `custom-metrics`. diff --git a/skills/agentcontrol/aiconfig-migrate/SKILL.md b/skills/agentcontrol/aiconfig-migrate/SKILL.md new file mode 100644 index 0000000..dbd8d56 --- /dev/null +++ b/skills/agentcontrol/aiconfig-migrate/SKILL.md @@ -0,0 +1,15 @@ +--- +name: aiconfig-migrate +description: "DEPRECATED redirect — this skill was renamed to migrate. Do not use this skill; invoke migrate instead. Kept only so old references to aiconfig-migrate still point users to the new name." +license: Apache-2.0 +metadata: + version: 0.1.0 +--- + +# Deprecated — renamed to `migrate` + +This skill has been renamed. **Do not follow any instructions here and do not perform the task from this skill.** + +Invoke the **`migrate`** skill instead — it contains the current, complete instructions. + +If you arrived here from an old reference (documentation, a saved prompt, or automation), update it from `aiconfig-migrate` to `migrate`. diff --git a/skills/agentcontrol/aiconfig-online-evals/SKILL.md b/skills/agentcontrol/aiconfig-online-evals/SKILL.md new file mode 100644 index 0000000..5adc06b --- /dev/null +++ b/skills/agentcontrol/aiconfig-online-evals/SKILL.md @@ -0,0 +1,15 @@ +--- +name: aiconfig-online-evals +description: "DEPRECATED redirect — this skill was renamed to online-evals. Do not use this skill; invoke online-evals instead. Kept only so old references to aiconfig-online-evals still point users to the new name." +license: Apache-2.0 +metadata: + version: 0.1.0 +--- + +# Deprecated — renamed to `online-evals` + +This skill has been renamed. **Do not follow any instructions here and do not perform the task from this skill.** + +Invoke the **`online-evals`** skill instead — it contains the current, complete instructions. + +If you arrived here from an old reference (documentation, a saved prompt, or automation), update it from `aiconfig-online-evals` to `online-evals`. diff --git a/skills/agentcontrol/aiconfig-projects/SKILL.md b/skills/agentcontrol/aiconfig-projects/SKILL.md new file mode 100644 index 0000000..f876ed5 --- /dev/null +++ b/skills/agentcontrol/aiconfig-projects/SKILL.md @@ -0,0 +1,15 @@ +--- +name: aiconfig-projects +description: "DEPRECATED redirect — this skill was renamed to projects. Do not use this skill; invoke projects instead. Kept only so old references to aiconfig-projects still point users to the new name." +license: Apache-2.0 +metadata: + version: 0.1.0 +--- + +# Deprecated — renamed to `projects` + +This skill has been renamed. **Do not follow any instructions here and do not perform the task from this skill.** + +Invoke the **`projects`** skill instead — it contains the current, complete instructions. + +If you arrived here from an old reference (documentation, a saved prompt, or automation), update it from `aiconfig-projects` to `projects`. diff --git a/skills/agentcontrol/aiconfig-snippets/SKILL.md b/skills/agentcontrol/aiconfig-snippets/SKILL.md new file mode 100644 index 0000000..a060fe5 --- /dev/null +++ b/skills/agentcontrol/aiconfig-snippets/SKILL.md @@ -0,0 +1,15 @@ +--- +name: aiconfig-snippets +description: "DEPRECATED redirect — this skill was renamed to snippets. Do not use this skill; invoke snippets instead. Kept only so old references to aiconfig-snippets still point users to the new name." +license: Apache-2.0 +metadata: + version: 0.1.0 +--- + +# Deprecated — renamed to `snippets` + +This skill has been renamed. **Do not follow any instructions here and do not perform the task from this skill.** + +Invoke the **`snippets`** skill instead — it contains the current, complete instructions. + +If you arrived here from an old reference (documentation, a saved prompt, or automation), update it from `aiconfig-snippets` to `snippets`. diff --git a/skills/agentcontrol/aiconfig-targeting/SKILL.md b/skills/agentcontrol/aiconfig-targeting/SKILL.md new file mode 100644 index 0000000..46632e8 --- /dev/null +++ b/skills/agentcontrol/aiconfig-targeting/SKILL.md @@ -0,0 +1,15 @@ +--- +name: aiconfig-targeting +description: "DEPRECATED redirect — this skill was renamed to configs-targeting. Do not use this skill; invoke configs-targeting instead. Kept only so old references to aiconfig-targeting still point users to the new name." +license: Apache-2.0 +metadata: + version: 0.1.0 +--- + +# Deprecated — renamed to `configs-targeting` + +This skill has been renamed. **Do not follow any instructions here and do not perform the task from this skill.** + +Invoke the **`configs-targeting`** skill instead — it contains the current, complete instructions. + +If you arrived here from an old reference (documentation, a saved prompt, or automation), update it from `aiconfig-targeting` to `configs-targeting`. diff --git a/skills/agentcontrol/aiconfig-tools/SKILL.md b/skills/agentcontrol/aiconfig-tools/SKILL.md new file mode 100644 index 0000000..e695187 --- /dev/null +++ b/skills/agentcontrol/aiconfig-tools/SKILL.md @@ -0,0 +1,15 @@ +--- +name: aiconfig-tools +description: "DEPRECATED redirect — this skill was renamed to tools. Do not use this skill; invoke tools instead. Kept only so old references to aiconfig-tools still point users to the new name." +license: Apache-2.0 +metadata: + version: 0.1.0 +--- + +# Deprecated — renamed to `tools` + +This skill has been renamed. **Do not follow any instructions here and do not perform the task from this skill.** + +Invoke the **`tools`** skill instead — it contains the current, complete instructions. + +If you arrived here from an old reference (documentation, a saved prompt, or automation), update it from `aiconfig-tools` to `tools`. diff --git a/skills/agentcontrol/aiconfig-update/SKILL.md b/skills/agentcontrol/aiconfig-update/SKILL.md new file mode 100644 index 0000000..84c2577 --- /dev/null +++ b/skills/agentcontrol/aiconfig-update/SKILL.md @@ -0,0 +1,15 @@ +--- +name: aiconfig-update +description: "DEPRECATED redirect — this skill was renamed to configs-update. Do not use this skill; invoke configs-update instead. Kept only so old references to aiconfig-update still point users to the new name." +license: Apache-2.0 +metadata: + version: 0.1.0 +--- + +# Deprecated — renamed to `configs-update` + +This skill has been renamed. **Do not follow any instructions here and do not perform the task from this skill.** + +Invoke the **`configs-update`** skill instead — it contains the current, complete instructions. + +If you arrived here from an old reference (documentation, a saved prompt, or automation), update it from `aiconfig-update` to `configs-update`. diff --git a/skills/agentcontrol/aiconfig-variations/SKILL.md b/skills/agentcontrol/aiconfig-variations/SKILL.md new file mode 100644 index 0000000..745b882 --- /dev/null +++ b/skills/agentcontrol/aiconfig-variations/SKILL.md @@ -0,0 +1,15 @@ +--- +name: aiconfig-variations +description: "DEPRECATED redirect — this skill was renamed to configs-variations. Do not use this skill; invoke configs-variations instead. Kept only so old references to aiconfig-variations still point users to the new name." +license: Apache-2.0 +metadata: + version: 0.1.0 +--- + +# Deprecated — renamed to `configs-variations` + +This skill has been renamed. **Do not follow any instructions here and do not perform the task from this skill.** + +Invoke the **`configs-variations`** skill instead — it contains the current, complete instructions. + +If you arrived here from an old reference (documentation, a saved prompt, or automation), update it from `aiconfig-variations` to `configs-variations`. From 5650b2637103a974a69475141593c089d63a6db2 Mon Sep 17 00:00:00 2001 From: Scarlett Attensil Date: Thu, 21 May 2026 15:42:29 -0700 Subject: [PATCH 10/10] Repoint eval suites at renamed agentcontrol skills The rename moved the config skills out of skills/ai-configs/ into skills/agentcontrol/ (configs-create, configs-update, tools, configs-variations), but the eval suites still referenced the old skills/ai-configs/aiconfig-*/SKILL.md paths, causing ENOENT crashes that failed the Skill Evals gate. Update _manifest.js (skillKey/skillDir/readme), each promptfooconfig.yaml (prompts path + skill_slug), and eval-scores.json keys to point at the renamed skills. --- eval-scores.json | 8 +++---- evals/aiconfig-create/promptfooconfig.yaml | 4 ++-- evals/aiconfig-tools/promptfooconfig.yaml | 4 ++-- evals/aiconfig-update/promptfooconfig.yaml | 4 ++-- .../aiconfig-variations/promptfooconfig.yaml | 4 ++-- evals/scripts/_manifest.js | 24 +++++++++---------- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/eval-scores.json b/eval-scores.json index e250665..aaaba1a 100644 --- a/eval-scores.json +++ b/eval-scores.json @@ -2,7 +2,7 @@ "schemaVersion": 1, "updatedAt": "2026-05-20T19:16:00.938Z", "skills": { - "ai-configs/aiconfig-create": { + "agentcontrol/configs-create": { "score": 100, "passed": 4, "total": 4, @@ -32,7 +32,7 @@ } ] }, - "ai-configs/aiconfig-update": { + "agentcontrol/configs-update": { "score": 80, "passed": 4, "total": 5, @@ -67,7 +67,7 @@ } ] }, - "ai-configs/aiconfig-tools": { + "agentcontrol/tools": { "score": 75, "passed": 3, "total": 4, @@ -97,7 +97,7 @@ } ] }, - "ai-configs/aiconfig-variations": { + "agentcontrol/configs-variations": { "score": 80, "passed": 4, "total": 5, diff --git a/evals/aiconfig-create/promptfooconfig.yaml b/evals/aiconfig-create/promptfooconfig.yaml index d42ae6b..09c05ae 100644 --- a/evals/aiconfig-create/promptfooconfig.yaml +++ b/evals/aiconfig-create/promptfooconfig.yaml @@ -10,13 +10,13 @@ description: "End-to-end evaluation of the aiconfig-create skill" prompts: - - file://../../skills/ai-configs/aiconfig-create/SKILL.md + - file://../../skills/agentcontrol/configs-create/SKILL.md providers: - id: file://../providers/claude-skill-agent-sdk.js label: claude-skill-agent-sdk config: - skill_slug: aiconfig-create + skill_slug: configs-create tests: # ------------------------------------------------------------------ diff --git a/evals/aiconfig-tools/promptfooconfig.yaml b/evals/aiconfig-tools/promptfooconfig.yaml index 168f389..3fba3c7 100644 --- a/evals/aiconfig-tools/promptfooconfig.yaml +++ b/evals/aiconfig-tools/promptfooconfig.yaml @@ -10,13 +10,13 @@ description: "End-to-end evaluation of the aiconfig-tools skill" prompts: - - file://../../skills/ai-configs/aiconfig-tools/SKILL.md + - file://../../skills/agentcontrol/tools/SKILL.md providers: - id: file://../providers/claude-skill-agent-sdk.js label: claude-skill-agent-sdk config: - skill_slug: aiconfig-tools + skill_slug: tools tests: # ------------------------------------------------------------------ diff --git a/evals/aiconfig-update/promptfooconfig.yaml b/evals/aiconfig-update/promptfooconfig.yaml index d5061cb..fcf4d1b 100644 --- a/evals/aiconfig-update/promptfooconfig.yaml +++ b/evals/aiconfig-update/promptfooconfig.yaml @@ -8,13 +8,13 @@ description: "End-to-end evaluation of the aiconfig-update skill" prompts: - - file://../../skills/ai-configs/aiconfig-update/SKILL.md + - file://../../skills/agentcontrol/configs-update/SKILL.md providers: - id: file://../providers/claude-skill-agent-sdk.js label: claude-skill-agent-sdk config: - skill_slug: aiconfig-update + skill_slug: configs-update tests: # ------------------------------------------------------------------ diff --git a/evals/aiconfig-variations/promptfooconfig.yaml b/evals/aiconfig-variations/promptfooconfig.yaml index e8723db..6edcc82 100644 --- a/evals/aiconfig-variations/promptfooconfig.yaml +++ b/evals/aiconfig-variations/promptfooconfig.yaml @@ -9,13 +9,13 @@ description: "End-to-end evaluation of the aiconfig-variations skill" prompts: - - file://../../skills/ai-configs/aiconfig-variations/SKILL.md + - file://../../skills/agentcontrol/configs-variations/SKILL.md providers: - id: file://../providers/claude-skill-agent-sdk.js label: claude-skill-agent-sdk config: - skill_slug: aiconfig-variations + skill_slug: configs-variations tests: # ------------------------------------------------------------------ diff --git a/evals/scripts/_manifest.js b/evals/scripts/_manifest.js index 358db9a..711d87f 100644 --- a/evals/scripts/_manifest.js +++ b/evals/scripts/_manifest.js @@ -15,27 +15,27 @@ const SUITES = [ { suite: "aiconfig-create", - skillKey: "ai-configs/aiconfig-create", - skillDir: "skills/ai-configs/aiconfig-create", - readme: "skills/ai-configs/aiconfig-create/README.md", + skillKey: "agentcontrol/configs-create", + skillDir: "skills/agentcontrol/configs-create", + readme: "skills/agentcontrol/configs-create/README.md", }, { suite: "aiconfig-update", - skillKey: "ai-configs/aiconfig-update", - skillDir: "skills/ai-configs/aiconfig-update", - readme: "skills/ai-configs/aiconfig-update/README.md", + skillKey: "agentcontrol/configs-update", + skillDir: "skills/agentcontrol/configs-update", + readme: "skills/agentcontrol/configs-update/README.md", }, { suite: "aiconfig-tools", - skillKey: "ai-configs/aiconfig-tools", - skillDir: "skills/ai-configs/aiconfig-tools", - readme: "skills/ai-configs/aiconfig-tools/README.md", + skillKey: "agentcontrol/tools", + skillDir: "skills/agentcontrol/tools", + readme: "skills/agentcontrol/tools/README.md", }, { suite: "aiconfig-variations", - skillKey: "ai-configs/aiconfig-variations", - skillDir: "skills/ai-configs/aiconfig-variations", - readme: "skills/ai-configs/aiconfig-variations/README.md", + skillKey: "agentcontrol/configs-variations", + skillDir: "skills/agentcontrol/configs-variations", + readme: "skills/agentcontrol/configs-variations/README.md", }, { suite: "launchdarkly-flag-create",