diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 9fd72ee..9d3418e 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "launchdarkly", "version": "1.0.0", - "description": "LaunchDarkly agent skills and MCP servers for feature flag management, AI configuration, and metrics", + "description": "LaunchDarkly agent skills and MCP servers for feature flag management, AgentControl, and metrics", "author": { "name": "LaunchDarkly", "email": "support@launchdarkly.com" diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 92b8724..aaed45e 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "launchdarkly-mcp", "version": "0.6.1", - "description": "Connect Codex to LaunchDarkly over MCP: AI Configs and feature management (FM) hosted endpoints, plus bundled guidance for using the tools.", + "description": "Connect Codex to LaunchDarkly over MCP: configs and feature management (FM) hosted endpoints, plus bundled guidance for using the tools.", "author": { "name": "LaunchDarkly", "url": "https://launchdarkly.com" @@ -20,15 +20,15 @@ "mcpServers": "./.mcp.json", "interface": { "displayName": "LaunchDarkly", - "shortDescription": "Feature flags, AI Configs, and environments via MCP.", - "longDescription": "Use LaunchDarkly’s hosted MCP servers for AI Configs and FM, or run the same surface locally with npx @launchdarkly/mcp-server (see repository README). Authenticate with your LaunchDarkly API access token when prompted.", + "shortDescription": "Feature flags, configs, and environments via MCP.", + "longDescription": "Use LaunchDarkly’s hosted MCP servers for configs and FM, or run the same surface locally with npx @launchdarkly/mcp-server (see repository README). Authenticate with your LaunchDarkly API access token when prompted.", "developerName": "LaunchDarkly", "category": "Developer tools", "capabilities": ["Read", "Write"], "websiteURL": "https://launchdarkly.com", "defaultPrompt": [ "List my LaunchDarkly feature flags for this project.", - "Show AI Config targeting for the config I care about." + "Show config targeting for the config I care about." ], "brandColor": "#3DD6F5" } diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json index 7d572df..af8d486 100644 --- a/.cursor-plugin/plugin.json +++ b/.cursor-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "launchdarkly", "version": "1.0.2", - "description": "LaunchDarkly agent skills and mcp server for feature flag management, AI configuration, and skill authoring", + "description": "LaunchDarkly agent skills and mcp server for feature flag management, AgentControl, and skill authoring", "author": { "name": "LaunchDarkly", "email": "support@launchdarkly.com" diff --git a/README.md b/README.md index 46229fe..781c29c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # LaunchDarkly Agent Skills -LaunchDarkly's public collection of AI agent skills and playbooks. These skills encode repeatable workflows for working with LaunchDarkly, so coding agents can execute common tasks safely and consistently. +LaunchDarkly's public collection of agent skills and playbooks. These skills encode repeatable workflows for working with LaunchDarkly, so coding agents can execute common tasks safely and consistently. ## What Is This Repo? @@ -18,19 +18,19 @@ Agent Skills are modular, text-based playbooks that teach an agent how to perfor | `feature-flags/launchdarkly-flag-cleanup` | Safely remove flags from code using LaunchDarkly as the source of truth | | `feature-flags/launchdarkly-guarded-rollout` | Configure guarded rollouts with progressive traffic, metric monitoring, and rollback | -### AI Configs +### AgentControl | Skill | Description | |-------|-------------| -| `agentcontrol/configs-create` | Create AI Configs with variations for agent or completion mode | -| `agentcontrol/migrate` | Migrate an app with hardcoded LLM prompts to AI Configs in five stages (extract, wrap, tools, tracking, evals) | -| `agentcontrol/configs-update` | Update and delete AI Configs, manage lifecycle | -| `agentcontrol/configs-variations` | Manage AI Config variations for A/B testing | +| `agentcontrol/configs-create` | Create configs with variations for agent or completion mode | +| `agentcontrol/migrate` | Migrate an app with hardcoded LLM prompts to AgentControl in five stages (extract, wrap, tools, tracking, evals) | +| `agentcontrol/configs-update` | Update and delete configs, manage lifecycle | +| `agentcontrol/configs-variations` | Manage config variations for A/B testing | | `agentcontrol/tools` | Create and attach tools for function calling | -| `agentcontrol/projects` | Create and manage projects to organize AI Configs | -| `agentcontrol/online-evals` | Attach LLM-as-a-judge evaluators to AI Configs | -| `agentcontrol/configs-targeting` | Configure targeting rules for AI Config rollouts | -| `agentcontrol/snippets` | Create and manage reusable prompt snippets across AI Configs | +| `agentcontrol/projects` | Create and manage projects to organize configs | +| `agentcontrol/online-evals` | Attach LLM-as-a-judge evaluators to configs | +| `agentcontrol/configs-targeting` | Configure targeting rules for config rollouts | +| `agentcontrol/snippets` | Create and manage reusable prompt snippets across configs | | `agentcontrol/agent-graphs` | Create and manage multi-agent graphs with routing and handoffs | ### Experiments diff --git a/eval-scores.json b/eval-scores.json index aaaba1a..68ed226 100644 --- a/eval-scores.json +++ b/eval-scores.json @@ -11,22 +11,22 @@ "lastRun": "2026-05-20T19:16:00.930Z", "perTest": [ { - "description": "Create an AI Config for my customer support bot. It uses LangGraph and needs per", + "description": "Create a config for my customer support bot. It uses LangGraph and needs per", "pass": true, "score": 1 }, { - "description": "I need an AI Config for our blog summarizer. We call the OpenAI API directly wit", + "description": "I need a config for our blog summarizer. We call the OpenAI API directly wit", "pass": true, "score": 1 }, { - "description": "Create a new AI Config for our analytics agent. Project key is \"data-platform\".\n", + "description": "Create a new config for our analytics agent. Project key is \"data-platform\".\n", "pass": true, "score": 1 }, { - "description": "Create an AI Config called \"email-assistant\" for composing emails. Project is \"c", + "description": "Create a config called \"email-assistant\" for composing emails. Project is \"c", "pass": true, "score": 0.8333333333333334 } @@ -46,12 +46,12 @@ "score": 0.95 }, { - "description": "We no longer need the content-writer AI Config. Please remove it from active use", + "description": "We no longer need the content-writer config. Please remove it from active use", "pass": true, "score": 1 }, { - "description": "Update the support-chatbot AI Config in project \"support-app\": change the name t", + "description": "Update the support-chatbot config in project \"support-app\": change the name t", "pass": true, "score": 1 }, @@ -76,12 +76,12 @@ "lastRun": "2026-05-20T19:16:00.936Z", "perTest": [ { - "description": "I need to give our support-chatbot AI Config the ability to search our knowledge", + "description": "I need to give our support-chatbot config the ability to search our knowledge", "pass": true, "score": 1 }, { - "description": "Create an AI tool called \"create-ticket\" that accepts title (required string), p", + "description": "Create an agent tool called \"create-ticket\" that accepts title (required string), p", "pass": true, "score": 1 }, diff --git a/evals/aiconfig-create/promptfooconfig.yaml b/evals/aiconfig-create/promptfooconfig.yaml index 09c05ae..c29ca4c 100644 --- a/evals/aiconfig-create/promptfooconfig.yaml +++ b/evals/aiconfig-create/promptfooconfig.yaml @@ -28,7 +28,7 @@ tests: - description: "Happy path: creates agent-mode config for a LangGraph chatbot using setup-ai-config" vars: user_request: > - Create an AI Config for my customer support bot. It uses LangGraph and + Create a config for my customer support bot. It uses LangGraph and needs persistent instructions. The project key is "acme-app". Use Claude Sonnet as the model. codebase_context: "" @@ -82,7 +82,7 @@ tests: - type: llm-rubric value: | - Evaluate whether the agent correctly created an AI Config in agent mode. + Evaluate whether the agent correctly created a config in agent mode. Score 1.0 if all of the following are met: 1. The agent chose agent mode (appropriate for LangGraph with persistent instructions). 2. The agent created a config with a meaningful key and name derived from the use case. @@ -100,7 +100,7 @@ tests: - description: "Completion mode: creates completion-mode config for direct OpenAI API use" vars: user_request: > - I need an AI Config for our blog summarizer. We call the OpenAI API + I need a config for our blog summarizer. We call the OpenAI API directly with a messages array (system + user). Project key is "blog-app", use gpt-4o. codebase_context: > @@ -151,7 +151,7 @@ tests: - description: "Exploration: browses existing configs before creating when context is sparse" vars: user_request: > - Create a new AI Config for our analytics agent. Project key is "data-platform". + Create a new config for our analytics agent. Project key is "data-platform". codebase_context: "" assert: - type: javascript @@ -184,7 +184,7 @@ tests: - description: "Post-creation: response mentions targeting step or aiconfig-targeting" vars: user_request: > - Create an AI Config called "email-assistant" for composing emails. + Create a config called "email-assistant" for composing emails. Project is "comms-app", use gpt-4o-mini in completion mode. codebase_context: "" assert: diff --git a/evals/aiconfig-tools/promptfooconfig.yaml b/evals/aiconfig-tools/promptfooconfig.yaml index 3fba3c7..90af29d 100644 --- a/evals/aiconfig-tools/promptfooconfig.yaml +++ b/evals/aiconfig-tools/promptfooconfig.yaml @@ -3,8 +3,8 @@ # Run with shared defaults: # promptfoo eval -c shared/defaults.yaml -c aiconfig-tools/promptfooconfig.yaml # -# The aiconfig-tools skill covers creating AI tool definitions and attaching -# them to AI Config variations. Key invariant: tools must be created with +# The aiconfig-tools skill covers creating agent tool definitions and attaching +# them to config variations. Key invariant: tools must be created with # raw JSON Schema format (not OpenAI function-calling wrapper), and must be # created before being attached. description: "End-to-end evaluation of the aiconfig-tools skill" @@ -25,7 +25,7 @@ tests: - description: "Creates a tool with correct raw JSON Schema and attaches it to a variation" vars: user_request: > - I need to give our support-chatbot AI Config the ability to search our + I need to give our support-chatbot config the ability to search our knowledge base. Create a tool called "search-knowledge-base" with a query parameter and a limit parameter, then attach it to the "default" variation. Project key is "support-app". @@ -82,7 +82,7 @@ tests: - description: "Uses raw JSON Schema for tool schema, not the OpenAI function wrapper" vars: user_request: > - Create an AI tool called "create-ticket" that accepts title (required + Create an agent tool called "create-ticket" that accepts title (required string), priority (enum: low, medium, high), and description (optional string). Attach it to the default variation of the support-chatbot config. Project key is "support-app". diff --git a/evals/aiconfig-update/promptfooconfig.yaml b/evals/aiconfig-update/promptfooconfig.yaml index fcf4d1b..0724fe8 100644 --- a/evals/aiconfig-update/promptfooconfig.yaml +++ b/evals/aiconfig-update/promptfooconfig.yaml @@ -76,7 +76,7 @@ tests: - description: "Archives a config instead of deleting when asked to remove it" vars: user_request: > - We no longer need the content-writer AI Config. Please remove it from + We no longer need the content-writer config. Please remove it from active use. Project key is "marketing". codebase_context: > The content-writer config has 3 variations. It's been inactive for two @@ -114,7 +114,7 @@ tests: - description: "Updates config metadata — name, description, tags — without touching variations" vars: user_request: > - Update the support-chatbot AI Config in project "support-app": + Update the support-chatbot config in project "support-app": change the name to "Customer Support Agent", add a description "Handles tier-1 support inquiries", and add tags "support" and "production". codebase_context: > diff --git a/evals/mocks/tool-responses.json b/evals/mocks/tool-responses.json index 75817de..1824785 100644 --- a/evals/mocks/tool-responses.json +++ b/evals/mocks/tool-responses.json @@ -247,7 +247,7 @@ "key": "support-chatbot", "name": "Support Chatbot", "mode": "agent", - "description": "AI-powered support agent for customer tickets", + "description": "Agent-powered support agent for customer tickets", "tags": [ "support", "production" @@ -284,7 +284,7 @@ "key": "{{configKey}}", "name": "{{configName}}", "mode": "agent", - "description": "AI Config", + "description": "config", "tags": [], "archived": false, "variations": [ diff --git a/evals/tools/definitions.json b/evals/tools/definitions.json index 32e69f2..bb33eab 100644 --- a/evals/tools/definitions.json +++ b/evals/tools/definitions.json @@ -100,7 +100,7 @@ }, { "name": "list-ai-configs", - "description": "Search and browse AI Configs in a project. Returns a paginated list with key, name, mode (agent or completion), tags, and variation count. Use query to search by name or key.", + "description": "Search and browse configs in a project. Returns a paginated list with key, name, mode (agent or completion), tags, and variation count. Use query to search by name or key.", "input_schema": { "type": "object", "properties": { @@ -114,19 +114,19 @@ }, { "name": "get-ai-config", - "description": "Get detailed configuration for a single AI Config including all its variations. Each variation includes its model, instructions or messages, parameters, and attached tools.", + "description": "Get detailed configuration for a single config including all its variations. Each variation includes its model, instructions or messages, parameters, and attached tools.", "input_schema": { "type": "object", "properties": { "projectKey": { "type": "string", "description": "The project key" }, - "configKey": { "type": "string", "description": "The AI Config key" } + "configKey": { "type": "string", "description": "The config key" } }, "required": ["projectKey", "configKey"] } }, { "name": "create-ai-config", - "description": "Create a new AI Config in a project. This creates the config shell: use create-ai-config-variation next to add a model, prompts, and parameters. Mode determines whether variations use 'instructions' (agent) or 'messages' (completion).", + "description": "Create a new config in a project. This creates the config shell: use create-ai-config-variation next to add a model, prompts, and parameters. Mode determines whether variations use 'instructions' (agent) or 'messages' (completion).", "input_schema": { "type": "object", "properties": { @@ -142,12 +142,12 @@ }, { "name": "update-ai-config", - "description": "Update an AI Config's metadata: name, description, tags, or archive status. Does NOT modify variations: use update-ai-config-variation for model, prompt, or parameter changes. Set archived: true to archive (reversible).", + "description": "Update a config's metadata: name, description, tags, or archive status. Does NOT modify variations: use update-ai-config-variation for model, prompt, or parameter changes. Set archived: true to archive (reversible).", "input_schema": { "type": "object", "properties": { "projectKey": { "type": "string", "description": "The project key" }, - "configKey": { "type": "string", "description": "The AI Config key" }, + "configKey": { "type": "string", "description": "The config key" }, "name": { "type": "string", "description": "New config name" }, "description": { "type": "string", "description": "New description" }, "tags": { "type": "array", "items": { "type": "string" }, "description": "Replacement tags" }, @@ -158,12 +158,12 @@ }, { "name": "delete-ai-config", - "description": "Permanently delete an AI Config. THIS IS IRREVERSIBLE. Requires confirm=true to execute. Prefer archiving (update-ai-config with archived: true) when possible.", + "description": "Permanently delete a config. THIS IS IRREVERSIBLE. Requires confirm=true to execute. Prefer archiving (update-ai-config with archived: true) when possible.", "input_schema": { "type": "object", "properties": { "projectKey": { "type": "string", "description": "The project key" }, - "configKey": { "type": "string", "description": "The AI Config key" }, + "configKey": { "type": "string", "description": "The config key" }, "confirm": { "type": "boolean", "description": "Must be true to execute deletion" } }, "required": ["projectKey", "configKey", "confirm"] @@ -171,7 +171,7 @@ }, { "name": "setup-ai-config", - "description": "Create an AI Config with its first variation in one step. This is the recommended way to set up a new AI Config: it creates the config, adds a variation with model and prompts, and verifies everything is configured correctly. Returns the full config detail with variation. To attach AI tools, pass the 'tools' array with {key, version} entries. Use create-ai-config + create-ai-config-variation separately only when you need more control.", + "description": "Create a config with its first variation in one step. This is the recommended way to set up a new config: it creates the config, adds a variation with model and prompts, and verifies everything is configured correctly. Returns the full config detail with variation. To attach agent tools, pass the 'tools' array with {key, version} entries. Use create-ai-config + create-ai-config-variation separately only when you need more control.", "input_schema": { "type": "object", "properties": { @@ -204,12 +204,12 @@ "items": { "type": "object", "properties": { - "key": { "type": "string", "description": "AI tool key" }, + "key": { "type": "string", "description": "agent tool key" }, "version": { "type": "number", "description": "Tool version" } }, "required": ["key", "version"] }, - "description": "AI tools to attach" + "description": "agent tools to attach" } }, "required": ["projectKey", "key", "name", "mode", "variationKey", "variationName", "modelConfigKey", "modelName"] @@ -217,24 +217,24 @@ }, { "name": "get-ai-config-health", - "description": "Health check for an AI Config. Detects common issues: missing models (NO MODEL in UI), missing prompts, orphaned tool references, and empty configs with no variations. Returns a health verdict (healthy, warning, unhealthy) with specific issues and per-variation summaries. Run before updating or experimenting with a config.", + "description": "Health check for a config. Detects common issues: missing models (NO MODEL in UI), missing prompts, orphaned tool references, and empty configs with no variations. Returns a health verdict (healthy, warning, unhealthy) with specific issues and per-variation summaries. Run before updating or experimenting with a config.", "input_schema": { "type": "object", "properties": { "projectKey": { "type": "string", "description": "The project key" }, - "configKey": { "type": "string", "description": "The AI Config key" } + "configKey": { "type": "string", "description": "The config key" } }, "required": ["projectKey", "configKey"] } }, { "name": "create-ai-config-variation", - "description": "Create a variation for an AI Config. A variation defines the model, prompts, parameters, and tools. modelConfigKey must be in Provider.model-id format (e.g. OpenAI.gpt-4o, Anthropic.claude-sonnet-4-5) for models to display correctly in the UI. Agent-mode configs use 'instructions' (a string); completion-mode configs use 'messages' (an array of {role, content} objects). To attach AI tools, pass the 'tools' array with {key, version} entries (create tools first with create-ai-tool).", + "description": "Create a variation for a config. A variation defines the model, prompts, parameters, and tools. modelConfigKey must be in Provider.model-id format (e.g. OpenAI.gpt-4o, Anthropic.claude-sonnet-4-5) for models to display correctly in the UI. Agent-mode configs use 'instructions' (a string); completion-mode configs use 'messages' (an array of {role, content} objects). To attach agent tools, pass the 'tools' array with {key, version} entries (create tools first with create-ai-tool).", "input_schema": { "type": "object", "properties": { "projectKey": { "type": "string", "description": "The project key" }, - "configKey": { "type": "string", "description": "The AI Config key" }, + "configKey": { "type": "string", "description": "The config key" }, "key": { "type": "string", "description": "Unique variation key" }, "name": { "type": "string", "description": "Human-readable variation name" }, "modelConfigKey": { "type": "string", "description": "Provider.model-id format (e.g. OpenAI.gpt-4o)" }, @@ -258,12 +258,12 @@ "items": { "type": "object", "properties": { - "key": { "type": "string", "description": "AI tool key" }, + "key": { "type": "string", "description": "agent tool key" }, "version": { "type": "number", "description": "Tool version" } }, "required": ["key", "version"] }, - "description": "AI tools to attach" + "description": "agent tools to attach" } }, "required": ["projectKey", "configKey", "key", "name", "modelConfigKey", "modelName"] @@ -271,12 +271,12 @@ }, { "name": "update-ai-config-variation", - "description": "Update an AI Config variation's model, instructions/messages, parameters, or attached tools. All fields are optional: only provided fields are updated. To attach tools, pass the 'tools' array with {key, version} entries (create tools first with create-ai-tool). Pass an empty tools array to detach all tools.", + "description": "Update a config variation's model, instructions/messages, parameters, or attached tools. All fields are optional: only provided fields are updated. To attach tools, pass the 'tools' array with {key, version} entries (create tools first with create-ai-tool). Pass an empty tools array to detach all tools.", "input_schema": { "type": "object", "properties": { "projectKey": { "type": "string", "description": "The project key" }, - "configKey": { "type": "string", "description": "The AI Config key" }, + "configKey": { "type": "string", "description": "The config key" }, "variationKey": { "type": "string", "description": "The variation key to update" }, "modelConfigKey": { "type": "string", "description": "New Provider.model-id format key" }, "modelName": { "type": "string", "description": "New model identifier" }, @@ -299,7 +299,7 @@ "items": { "type": "object", "properties": { - "key": { "type": "string", "description": "AI tool key" }, + "key": { "type": "string", "description": "agent tool key" }, "version": { "type": "number", "description": "Tool version" } }, "required": ["key", "version"] @@ -312,12 +312,12 @@ }, { "name": "delete-ai-config-variation", - "description": "Permanently delete an AI Config variation. THIS IS IRREVERSIBLE. Requires confirm=true to execute.", + "description": "Permanently delete a config variation. THIS IS IRREVERSIBLE. Requires confirm=true to execute.", "input_schema": { "type": "object", "properties": { "projectKey": { "type": "string", "description": "The project key" }, - "configKey": { "type": "string", "description": "The AI Config key" }, + "configKey": { "type": "string", "description": "The config key" }, "variationKey": { "type": "string", "description": "The variation key to delete" }, "confirm": { "type": "boolean", "description": "Must be true to execute deletion" } }, @@ -326,12 +326,12 @@ }, { "name": "clone-ai-config-variation", - "description": "Clone an existing AI Config variation with selective overrides. Reads the source variation, applies any provided overrides (model, instructions, messages, parameters, tools), and creates a new variation. Returns both the source and created variation so you can compare the diff. Use this for A/B experimentation: change one thing at a time while keeping everything else constant.", + "description": "Clone an existing config variation with selective overrides. Reads the source variation, applies any provided overrides (model, instructions, messages, parameters, tools), and creates a new variation. Returns both the source and created variation so you can compare the diff. Use this for A/B experimentation: change one thing at a time while keeping everything else constant.", "input_schema": { "type": "object", "properties": { "projectKey": { "type": "string", "description": "The project key" }, - "configKey": { "type": "string", "description": "The AI Config key" }, + "configKey": { "type": "string", "description": "The config key" }, "sourceVariationKey": { "type": "string", "description": "Key of the variation to clone from" }, "key": { "type": "string", "description": "Key for the new variation" }, "name": { "type": "string", "description": "Name for the new variation" }, @@ -356,7 +356,7 @@ "items": { "type": "object", "properties": { - "key": { "type": "string", "description": "AI tool key" }, + "key": { "type": "string", "description": "agent tool key" }, "version": { "type": "number", "description": "Tool version" } }, "required": ["key", "version"] @@ -369,7 +369,7 @@ }, { "name": "list-ai-tools", - "description": "List AI tool definitions in a project. Returns each tool's key, description, and schema. Tools are attached to AI Config variations to give models function-calling capabilities.", + "description": "List agent tool definitions in a project. Returns each tool's key, description, and schema. Tools are attached to config variations to give models function-calling capabilities.", "input_schema": { "type": "object", "properties": { @@ -381,19 +381,19 @@ }, { "name": "get-ai-tool", - "description": "Get a single AI tool definition including its full schema. Use to inspect a tool's parameters before attaching it to an AI Config variation.", + "description": "Get a single agent tool definition including its full schema. Use to inspect a tool's parameters before attaching it to a config variation.", "input_schema": { "type": "object", "properties": { "projectKey": { "type": "string", "description": "The project key" }, - "toolKey": { "type": "string", "description": "The AI tool key" } + "toolKey": { "type": "string", "description": "The agent tool key" } }, "required": ["projectKey", "toolKey"] } }, { "name": "create-ai-tool", - "description": "Create a new AI tool definition in a project. The schema should be a raw JSON Schema object with type, properties, and required fields (e.g. {\"type\": \"object\", \"properties\": {...}}). Do NOT use the OpenAI function calling wrapper format. After creation, attach the tool to a variation using update-ai-config-variation.", + "description": "Create a new agent tool definition in a project. The schema should be a raw JSON Schema object with type, properties, and required fields (e.g. {\"type\": \"object\", \"properties\": {...}}). Do NOT use the OpenAI function calling wrapper format. After creation, attach the tool to a variation using update-ai-config-variation.", "input_schema": { "type": "object", "properties": { diff --git a/skills.json b/skills.json index 62bdcd6..5cb960f 100644 --- a/skills.json +++ b/skills.json @@ -102,15 +102,15 @@ }, { "name": "built-in-metrics", - "description": "Instrument an existing codebase with LaunchDarkly AI Config tracking. Walks the four-tier ladder (managed runner \u2192 provider package \u2192 custom extractor + trackMetricsOf \u2192 raw manual) and picks the lowest-ceremony option that still captures duration, tokens, and success/error.", + "description": "Instrument an existing codebase with LaunchDarkly config tracking. Walks the four-tier ladder (managed runner \u2192 provider package \u2192 custom extractor + trackMetricsOf \u2192 raw manual) and picks the lowest-ceremony option that still captures duration, tokens, and success/error.", "path": "skills/agentcontrol/built-in-metrics", "version": "1.0.0-experimental", "license": "Apache-2.0", - "compatibility": "Requires the LaunchDarkly server-side AI SDK (`launchdarkly-server-sdk-ai>=0.20.0` for Python or `@launchdarkly/server-sdk-ai>=0.20.0` for Node) and an existing AI Config." + "compatibility": "Requires the LaunchDarkly server-side AI SDK (`launchdarkly-server-sdk-ai>=0.20.0` for Python or `@launchdarkly/server-sdk-ai>=0.20.0` for Node) and an existing config." }, { "name": "configs-create", - "description": "Create and configure AI Configs in LaunchDarkly. Helps you choose between agent vs completion mode, create the config, add variations with models and prompts, and verify the setup.", + "description": "Create and configure configs in LaunchDarkly. Helps you choose between agent vs completion mode, create the config, add variations with models and prompts, and verify the setup.", "path": "skills/agentcontrol/configs-create", "version": "1.0.0-experimental", "license": "Apache-2.0", @@ -118,14 +118,14 @@ }, { "name": "configs-targeting", - "description": "Configure AI Config targeting rules to control which variations serve to different users. Enable percentage rollouts, attribute-based rules, segment targeting, and guarded rollouts.", + "description": "Configure config targeting rules to control which variations serve to different users. Enable percentage rollouts, attribute-based rules, segment targeting, and guarded rollouts.", "path": "skills/agentcontrol/configs-targeting", "version": "0.1.0", "compatibility": "Requires LaunchDarkly API access token with ai-configs:write permission." }, { "name": "configs-update", - "description": "Update, archive, and delete LaunchDarkly AI Configs and their variations. Use when you need to modify config properties, change model parameters, update instructions or messages, archive unused configs, or permanently remove them.", + "description": "Update, archive, and delete LaunchDarkly configs and their variations. Use when you need to modify config properties, change model parameters, update instructions or messages, archive unused configs, or permanently remove them.", "path": "skills/agentcontrol/configs-update", "version": "1.0.0-experimental", "license": "Apache-2.0", @@ -133,7 +133,7 @@ }, { "name": "configs-variations", - "description": "Experiment with AI configurations by creating and managing variations. Helps you test different models, prompts, and parameters to find what works best through systematic experimentation.", + "description": "Experiment with configs by creating and managing variations. Helps you test different models, prompts, and parameters to find what works best through systematic experimentation.", "path": "skills/agentcontrol/configs-variations", "version": "1.0.0-experimental", "license": "Apache-2.0", @@ -141,7 +141,7 @@ }, { "name": "custom-metrics", - "description": "Create, track, retrieve, update, and delete custom business metrics for AI Configs. Covers full lifecycle: define metric kinds via API, emit events via SDK, and query results.", + "description": "Create, track, retrieve, update, and delete custom business metrics for configs. Covers full lifecycle: define metric kinds via API, emit events via SDK, and query results.", "path": "skills/agentcontrol/custom-metrics", "version": "1.0.0-experimental", "license": "Apache-2.0", @@ -288,7 +288,7 @@ }, { "name": "migrate", - "description": "Migrate an application with hardcoded LLM prompts to a full LaunchDarkly AI Configs implementation in five stages: audit the code, wrap the call, move the tools, add tracking, attach evaluators. Use when the user wants to externalize model/prompt configuration, move from direct provider calls (OpenAI, Anthropic, Bedrock, Gemini, Strands) to a managed AI Config, or stage a full hardcoded-to-LaunchDarkly migration.", + "description": "Migrate an application with hardcoded LLM prompts to a full LaunchDarkly AgentControl implementation in five stages: audit the code, wrap the call, move the tools, add tracking, attach evaluators. Use when the user wants to externalize model/prompt configuration, move from direct provider calls (OpenAI, Anthropic, Bedrock, Gemini, Strands) to a managed config, or stage a full hardcoded-to-LaunchDarkly migration.", "path": "skills/agentcontrol/migrate", "version": "0.1.0", "license": "Apache-2.0", @@ -314,7 +314,7 @@ }, { "name": "online-evals", - "description": "Attach judges to AI Config variations for automatic LLM-as-a-judge evaluation. Create custom judges, configure sampling rates, and monitor quality scores.", + "description": "Attach judges to config variations for automatic LLM-as-a-judge evaluation. Create custom judges, configure sampling rates, and monitor quality scores.", "path": "skills/agentcontrol/online-evals", "version": "0.1.0", "compatibility": "Requires LaunchDarkly API access token with ai-configs:write permission. SDK versions Python v0.20.0+ or Node.js v0.20.0+ for automatic metric recording and the consolidated `track_judge_result` / `trackJudgeResult` API." @@ -344,7 +344,7 @@ }, { "name": "snippets", - "description": "Create and manage prompt snippets \u2014 reusable text blocks referenced inside AI Config variation prompts. Keeps common instructions, personas, and guardrails consistent across multiple configs.", + "description": "Create and manage prompt snippets \u2014 reusable text blocks referenced inside config variation prompts. Keeps common instructions, personas, and guardrails consistent across multiple configs.", "path": "skills/agentcontrol/snippets", "version": "0.1.0", "license": "Apache-2.0", @@ -352,7 +352,7 @@ }, { "name": "tools", - "description": "Give your AI agents capabilities through tools (function calling). Helps you identify what your AI needs to do, create tool definitions, and attach them to AI Config variations.", + "description": "Give your agents capabilities through tools (function calling). Helps you identify what your agent needs to do, create tool definitions, and attach them to config variations.", "path": "skills/agentcontrol/tools", "version": "1.0.0-experimental", "license": "Apache-2.0", diff --git a/skills/agentcontrol/agent-graphs/SKILL.md b/skills/agentcontrol/agent-graphs/SKILL.md index b067e1c..3f49719 100644 --- a/skills/agentcontrol/agent-graphs/SKILL.md +++ b/skills/agentcontrol/agent-graphs/SKILL.md @@ -8,9 +8,9 @@ metadata: version: "0.1.0" --- -# AI Config Agent Graphs +# Config Agent Graphs -You're using a skill that will guide you through creating and managing agent graphs in LaunchDarkly. Your job is to design the graph topology, create it with the right edges and handoffs, and verify the routing between AI Config nodes. +You're using a skill that will guide you through creating and managing agent graphs in LaunchDarkly. Your job is to design the graph topology, create it with the right edges and handoffs, and verify the routing between config nodes. ## Prerequisites @@ -24,15 +24,15 @@ This skill requires the remotely hosted LaunchDarkly MCP server to be configured **Optional MCP tools:** - `update-agent-graph` -- modify edges, root config, or description - `delete-agent-graph` -- permanently remove a graph -- `get-ai-config` -- inspect individual AI Configs that serve as nodes -- `create-ai-config` -- create new AI Configs to use as graph nodes +- `get-ai-config` -- inspect individual configs that serve as nodes +- `create-ai-config` -- create new configs to use as graph nodes ## Core Concepts ### What Are Agent Graphs? An agent graph is a directed graph where: -- **Nodes** are AI Configs (each config is an agent with its own model, prompt, and tools) +- **Nodes** are configs (each config is an agent with its own model, prompt, and tools) - **Edges** define routing between configs (source -> target) - **Handoff data** on edges controls how context is passed between agents - **Root config** is the entry point — the first agent that receives user input @@ -55,8 +55,8 @@ An agent graph is a directed graph where: Each edge has: - `key` -- unique identifier for the edge -- `sourceConfig` -- the AI Config key that routes FROM -- `targetConfig` -- the AI Config key that routes TO +- `sourceConfig` -- the config key that routes FROM +- `targetConfig` -- the config key that routes TO - `handoff` (optional) -- data/instructions passed during the transition ## Core Principles @@ -73,16 +73,16 @@ Each edge has: Before creating anything: -1. Identify the agents (AI Configs) needed — each is a graph node +1. Identify the agents (configs) needed — each is a graph node 2. Map the routing: which agent hands off to which? 3. Define handoff data: what context does each edge carry? 4. Identify the root config: which agent receives initial input? 5. Check existing graphs with `list-agent-graphs` to avoid duplicates -6. Check existing AI Configs with `get-ai-config` to see what nodes already exist +6. Check existing configs with `get-ai-config` to see what nodes already exist ### Step 2: Ensure Nodes Exist -Each node in the graph must be an existing AI Config. If configs don't exist yet: +Each node in the graph must be an existing config. If configs don't exist yet: 1. Use `create-ai-config` to create each agent config 2. Set up variations with appropriate models and prompts for each agent's role 3. Verify each config exists with `get-ai-config` @@ -90,11 +90,11 @@ Each node in the graph must be an existing AI Config. If configs don't exist yet ### Step 3: Create the Graph Use `create-agent-graph` with: -- `projectKey` -- the project containing the AI Configs +- `projectKey` -- the project containing the configs - `key` -- unique identifier for the graph - `name` -- human-readable display name - `description` (optional) -- explain the graph's purpose -- `rootConfigKey` -- the entry-point AI Config key +- `rootConfigKey` -- the entry-point config key - `edges` -- array of connections between configs ```json @@ -144,7 +144,7 @@ Use `create-agent-graph` with: ## What NOT to Do -- Don't create a graph before the AI Config nodes exist +- Don't create a graph before the config nodes exist - Don't forget handoff data when agents need context from predecessors - Don't create overly complex graphs — start simple and add nodes as needed - Don't delete a graph without understanding if it's actively used in agent workflows diff --git a/skills/agentcontrol/built-in-metrics/SKILL.md b/skills/agentcontrol/built-in-metrics/SKILL.md index 4b93473..186f4ad 100644 --- a/skills/agentcontrol/built-in-metrics/SKILL.md +++ b/skills/agentcontrol/built-in-metrics/SKILL.md @@ -1,16 +1,16 @@ --- name: built-in-metrics -description: "Instrument an existing codebase with LaunchDarkly AI Config tracking. Walks the four-tier ladder (managed runner → provider package → custom extractor + trackMetricsOf → raw manual) and picks the lowest-ceremony option that still captures duration, tokens, and success/error." +description: "Instrument an existing codebase with LaunchDarkly config tracking. Walks the four-tier ladder (managed runner → provider package → custom extractor + trackMetricsOf → raw manual) and picks the lowest-ceremony option that still captures duration, tokens, and success/error." license: Apache-2.0 -compatibility: Requires the LaunchDarkly server-side AI SDK (`launchdarkly-server-sdk-ai>=0.20.0` for Python or `@launchdarkly/server-sdk-ai>=0.20.0` for Node) and an existing AI Config. +compatibility: Requires the LaunchDarkly server-side AI SDK (`launchdarkly-server-sdk-ai>=0.20.0` for Python or `@launchdarkly/server-sdk-ai>=0.20.0` for Node) and an existing config. metadata: author: launchdarkly version: "1.0.0-experimental" --- -# AI Metrics Instrumentation +# Agent Metrics Instrumentation -You're using a skill that wires LaunchDarkly AI metrics around an existing provider call. Your job is to audit what's already there, pick the right tier from the ladder below, and implement it with the **least ceremony that still captures the metrics the Monitoring tab needs** (duration, input/output tokens, success/error, plus TTFT when streaming). +You're using a skill that wires LaunchDarkly agent metrics around an existing provider call. Your job is to audit what's already there, pick the right tier from the ladder below, and implement it with the **least ceremony that still captures the metrics the Monitoring tab needs** (duration, input/output tokens, success/error, plus TTFT when streaming). The single most important thing to get right: **default to the highest tier that fits the shape of the call**. Going lower ("just write the manual tracker calls") looks flexible but costs you drift, missed metrics, and legacy patterns the SDKs have moved past. @@ -38,7 +38,7 @@ Before picking a tier, find the provider call and answer these questions: - [ ] **Provider?** OpenAI, Anthropic, Bedrock, Gemini, Azure, custom HTTP? → cross-reference with the package availability matrix below. - [ ] **Streaming?** If yes, you'll need TTFT tracking, which means Tier 4 for the TTFT part even if the rest is Tier 2. - [ ] **Language?** Python or Node? Provider-package coverage differs between them. -- [ ] **Already using an AI Config?** If not, route to `configs-create` first — tracking requires a tracker, which is obtained by calling `create_tracker()` / `createTracker()` on the config object returned by `completion_config()` / `completionConfig()` / `createModel()`. +- [ ] **Already using a config?** If not, route to `configs-create` first — tracking requires a tracker, which is obtained by calling `create_tracker()` / `createTracker()` on the config object returned by `completion_config()` / `completionConfig()` / `createModel()`. - [ ] **On the current SDK API?** If the call site uses `aiclient.config(...)` / `aiClient.config(...)` or constructs an `AIConfig(...)` / `LDAIConfig` default, it's on the pre-0.20 surface. Migrate it as part of this work before adding tracking: - `aiclient.config(...)` → `aiclient.completion_config(...)` for one-shot/chat or `aiclient.agent_config(...)` for agent mode (mirror the call signature). Node is the same with camelCase. - `AIConfig(...)` default → `AICompletionConfigDefault(...)` or `AIAgentConfigDefault(...)` (Node: `LDAICompletionConfigDefault` / `LDAIAgentConfigDefault`). `AIConfig` is the base class the SDK returns; it isn't a valid default-value constructor — the typed `*Default` variants are. @@ -77,7 +77,7 @@ Guardrails that apply to every tier: Confirm the Monitoring tab fills in: - [ ] Run one real request through the instrumented path. -- [ ] Open the AI Config in LaunchDarkly → **Monitoring** tab. Duration, token counts, and generation counts should appear within 1–2 minutes. +- [ ] Open the config in LaunchDarkly → **Monitoring** tab. Duration, token counts, and generation counts should appear within 1–2 minutes. - [ ] Force an error (bad API key, zero `max_tokens`, whatever) and confirm the error count increments. - [ ] If streaming: verify TTFT appears. If it doesn't, you probably wrapped the stream creation with `trackMetricsOf` but didn't add the manual `trackTimeToFirstToken` call — see [streaming-tracking.md](references/streaming-tracking.md). @@ -103,7 +103,7 @@ Obtain a tracker via the factory on the config object: `tracker = config.create_ ## Related skills -- `configs-create` — prerequisite if the app doesn't have an AI Config yet -- `custom-metrics` — business metrics (conversion, resolution, retention) layered on top of the AI metrics this skill captures +- `configs-create` — prerequisite if the app doesn't have a config yet +- `custom-metrics` — business metrics (conversion, resolution, retention) layered on top of the agent metrics this skill captures - `online-evals` — automatic quality scoring (LLM-as-judge) on sampled live requests; complementary to the metrics here -- `migrate` — Stage 4 of the hardcoded-to-AI-Configs migration delegates to this skill +- `migrate` — Stage 4 of the hardcoded-to-AgentControl migration delegates to this skill diff --git a/skills/agentcontrol/built-in-metrics/references/gemini-tracking.md b/skills/agentcontrol/built-in-metrics/references/gemini-tracking.md index 32c5bf9..4d3129a 100644 --- a/skills/agentcontrol/built-in-metrics/references/gemini-tracking.md +++ b/skills/agentcontrol/built-in-metrics/references/gemini-tracking.md @@ -52,7 +52,7 @@ def map_to_gemini_messages(ld_messages): return (" ".join(system_parts) or None), contents def gemini_config_kwargs(params): - """Map AI Config parameter names to google-genai's GenerateContentConfig. + """Map config parameter names to google-genai's GenerateContentConfig. LaunchDarkly stores max_tokens (snake_case, matching the LD UI); Gemini's Python SDK expects max_output_tokens. Drop `tools` — they go on GenerateContentConfig.tools directly; leaving them here would double-pass.""" @@ -123,7 +123,7 @@ function mapToGeminiMessages( }; } -// Map AI Config parameter names to @google/genai's GenerateContentConfig keys. +// Map config parameter names to @google/genai's GenerateContentConfig keys. // LaunchDarkly stores max_tokens (snake_case, matching the LD UI); @google/genai // expects maxOutputTokens. Drop `tools` — they go on GenerateContentConfig.tools // directly; leaving them here would double-pass. diff --git a/skills/agentcontrol/built-in-metrics/references/langchain-tracking.md b/skills/agentcontrol/built-in-metrics/references/langchain-tracking.md index b3a084e..5412c2f 100644 --- a/skills/agentcontrol/built-in-metrics/references/langchain-tracking.md +++ b/skills/agentcontrol/built-in-metrics/references/langchain-tracking.md @@ -9,7 +9,7 @@ Three helpers do the heavy lifting. Use them — skipping any silently drops val | Helper | Purpose | |---|---| -| `create_langchain_model(config)` (Python) / `createLangChainModel(config)` (Node, bare export) | Build a LangChain chat model from the AI Config. Forwards **all** variation parameters (temperature, max_tokens, top_p, and so on), picks the correct LangChain chat class based on `config.provider.name`, and handles provider-name mapping internally (for example, LaunchDarkly's `"gemini"` → LangChain's `"google_genai"`). | +| `create_langchain_model(config)` (Python) / `createLangChainModel(config)` (Node, bare export) | Build a LangChain chat model from the config. Forwards **all** variation parameters (temperature, max_tokens, top_p, and so on), picks the correct LangChain chat class based on `config.provider.name`, and handles provider-name mapping internally (for example, LaunchDarkly's `"gemini"` → LangChain's `"google_genai"`). | | `build_structured_tools(config, registry)` (Python, `ldai_langchain.langchain_helper`) | Read `config.model.parameters.tools` and wrap the matching entries in your `{name: callable}` registry as LangChain `StructuredTool` instances ready for `bind_tools`. This is the first-class replacement for hand-rolled `resolve_tools` / `TOOL_REGISTRY` / `ALL_TOOLS` patterns — it handles async callables via `coroutine=` and uses the LD tool key as the `StructuredTool.name`, so `ToolNode` lookup works without extra mapping. | | `get_ai_metrics_from_response` (Python top-level import) / `getAIMetricsFromResponse` (Node, bare export) | Extract token usage from a LangChain response. Pass as the extractor argument to `track_metrics_of` / `trackMetricsOf`. | | `LangChainRunnerFactory` (Node) | Managed-runner factory: `new LangChainRunnerFactory().createModel(aiConfig)` wires the chat model into a `ManagedModel` that handles tracking end-to-end (Tier 1). | @@ -65,7 +65,7 @@ Nothing in the tracker or provider packages reads `custom` — it's a pass-throu ## Tier 2 — LangChain (single model, not a graph) -The common case: a one-shot LangChain call (ChatOpenAI, ChatAnthropic, ChatGoogleGenerativeAI, ChatBedrockConverse, etc.) against an AI Config in completion mode. +The common case: a one-shot LangChain call (ChatOpenAI, ChatAnthropic, ChatGoogleGenerativeAI, ChatBedrockConverse, etc.) against a config in completion mode. **Python:** @@ -250,7 +250,7 @@ for (const msg of result.messages ?? []) { `get_ai_metrics_from_response` / `getAIMetricsFromResponse` is defined on a single LangChain `AIMessage`. A LangGraph run produces N messages (model turn, tool result, model turn, tool result, final). If you pass the whole `result` to the extractor, you miss most of the token usage. Iterating and summing is deliberate — it's the same pattern the LaunchDarkly LangGraph guide uses. -## Binding AI-Config-attached tools with `build_structured_tools` +## Binding config-attached tools with `build_structured_tools` If the variation has tools attached (via `/tools`), use `build_structured_tools` rather than hand-rolling a `TOOL_REGISTRY` / `resolve_tools` / `ALL_TOOLS` shape. The helper reads `ai_config.model.parameters.tools`, picks the matching entries from your `{name: callable}` registry, wraps them as LangChain `StructuredTool` instances, and preserves the LD tool key as the `StructuredTool.name` (so `ToolNode(...)` lookup works without a second mapping). diff --git a/skills/agentcontrol/built-in-metrics/references/metrics-api.md b/skills/agentcontrol/built-in-metrics/references/metrics-api.md index 60b591f..4fa2ddb 100644 --- a/skills/agentcontrol/built-in-metrics/references/metrics-api.md +++ b/skills/agentcontrol/built-in-metrics/references/metrics-api.md @@ -1,6 +1,6 @@ # Metrics API -Retrieve AI Config metrics via the LaunchDarkly REST API. +Retrieve config metrics via the LaunchDarkly REST API. ## Endpoint @@ -26,7 +26,7 @@ import time import os def get_ai_config_metrics(project_key: str, config_key: str, env: str = "production", hours: int = 24): - """Get AI Config metrics for the last N hours.""" + """Get config metrics for the last N hours.""" API_TOKEN = os.environ.get("LAUNCHDARKLY_API_TOKEN") now = int(time.time()) diff --git a/skills/agentcontrol/built-in-metrics/references/strands-tracking.md b/skills/agentcontrol/built-in-metrics/references/strands-tracking.md index 062a006..21ab514 100644 --- a/skills/agentcontrol/built-in-metrics/references/strands-tracking.md +++ b/skills/agentcontrol/built-in-metrics/references/strands-tracking.md @@ -79,7 +79,7 @@ Pick the style that matches the rest of the codebase — the two variants record ## Provider dispatch stays in your code -Strands model classes are provider-specific (`AnthropicModel`, `OpenAIModel`, `BedrockModel`). To serve more than one provider from a single AI Config key, dispatch on `agent_config.provider.name` before constructing the `Agent`. See [agent-mode-frameworks.md § Strands Agent](../../migrate/references/agent-mode-frameworks.md) for the `create_strands_model` dispatcher, including the rule that `parameters.tools` must be dropped before being passed into the Strands model class (tools flow through the `Agent` constructor, not through model params). +Strands model classes are provider-specific (`AnthropicModel`, `OpenAIModel`, `BedrockModel`). To serve more than one provider from a single config key, dispatch on `agent_config.provider.name` before constructing the `Agent`. See [agent-mode-frameworks.md § Strands Agent](../../migrate/references/agent-mode-frameworks.md) for the `create_strands_model` dispatcher, including the rule that `parameters.tools` must be dropped before being passed into the Strands model class (tools flow through the `Agent` constructor, not through model params). ## Always flush before exit diff --git a/skills/agentcontrol/configs-create/README.md b/skills/agentcontrol/configs-create/README.md index cf21ee7..1433dee 100644 --- a/skills/agentcontrol/configs-create/README.md +++ b/skills/agentcontrol/configs-create/README.md @@ -1,12 +1,12 @@ -# LaunchDarkly AI Config Create Skill +# LaunchDarkly Config Create Skill -An Agent Skill for creating AI Configs in LaunchDarkly. Guides choosing agent vs completion mode, creating the config and variations, and verifying the result. +An Agent Skill for creating configs in LaunchDarkly. Guides choosing agent vs completion mode, creating the config and variations, and verifying the result. ## Overview This skill teaches agents how to: - Understand the use case and choose agent vs completion mode -- Create AI Configs using MCP tools (`setup-ai-config` for one-step, or `create-ai-config` + `create-ai-config-variation` for more control) +- Create configs using MCP tools (`setup-ai-config` for one-step, or `create-ai-config` + `create-ai-config-variation` for more control) - Set up model configuration with the correct `modelConfigKey` format - Verify creation via the tool response or `get-ai-config` @@ -21,11 +21,11 @@ This skill requires the remotely hosted LaunchDarkly MCP server to be configured ## Usage ``` -Create an AI config for our customer support agent +Create a config for our customer support agent ``` ``` -Set up an AI config for content generation using Claude +Set up a config for content generation using Claude ``` ## Structure @@ -38,10 +38,10 @@ configs-create/ ## Related -- [AI Config Projects](../projects/): Create projects first -- [AI Config Tools](../tools/): Add tools after creating config -- [AI Config Variations](../configs-variations/): Add more variations for experimentation -- [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) +- [config Projects](../projects/): Create projects first +- [config Tools](../tools/): Add tools after creating config +- [config Variations](../configs-variations/): Add more variations for experimentation +- [LaunchDarkly AgentControl Docs](https://docs.launchdarkly.com/home/ai-configs) ## License diff --git a/skills/agentcontrol/configs-create/SKILL.md b/skills/agentcontrol/configs-create/SKILL.md index 5a50668..875cd7d 100644 --- a/skills/agentcontrol/configs-create/SKILL.md +++ b/skills/agentcontrol/configs-create/SKILL.md @@ -1,6 +1,6 @@ --- name: configs-create -description: "Create and configure AI Configs in LaunchDarkly. Helps you choose between agent vs completion mode, create the config, add variations with models and prompts, and verify the setup." +description: "Create and configure configs in LaunchDarkly. Helps you choose between agent vs completion mode, create the config, add variations with models and prompts, and verify the setup." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server metadata: @@ -8,11 +8,11 @@ metadata: version: "1.0.0-experimental" --- -# Create AI Config +# Create Config -You're using a skill that will guide you through creating an AI Config in LaunchDarkly. Your job is to understand the use case, choose the right mode, create the config and its variations, and verify everything is set up correctly. +You're using a skill that will guide you through creating a config in LaunchDarkly. Your job is to understand the use case, choose the right mode, create the config and its variations, and verify everything is set up correctly. -> **⚠️ This skill creates a config — it does not make it servable.** A freshly-created AI Config has its **fallthrough pointing at an auto-generated disabled variation**, not at the variation you just created. The SDK will return `ai_config.enabled=False` on every evaluation until you flip targeting on and point the fallthrough at your new variation. This is not a bug — it's the default state. **You must run `/configs-targeting` (or the equivalent REST / CLI call shown in Step 5) before verifying against the SDK**, or verification will look like the LD-served path is broken when it isn't. The single most common failure mode users hit with this skill is skipping the targeting step and spending time debugging `enabled=False` in their application code. +> **⚠️ This skill creates a config — it does not make it servable.** A freshly-created config has its **fallthrough pointing at an auto-generated disabled variation**, not at the variation you just created. The SDK will return `ai_config.enabled=False` on every evaluation until you flip targeting on and point the fallthrough at your new variation. This is not a bug — it's the default state. **You must run `/configs-targeting` (or the equivalent REST / CLI call shown in Step 5) before verifying against the SDK**, or verification will look like the LD-served path is broken when it isn't. The single most common failure mode users hit with this skill is skipping the targeting step and spending time debugging `enabled=False` in their application code. ## Prerequisites @@ -41,7 +41,7 @@ When the user provides enough context (use case, model, mode), proceed through t Before creating, identify what you're building: - **What framework?** LangGraph, LangChain, CrewAI, Strands, OpenAI SDK, Anthropic SDK, custom -- **What does the AI need?** Just text generation, or tools/function calling? +- **What does the agent need?** Just text generation, or tools/function calling? - **Agent or completion?** See the decision matrix below ### Step 2: Choose Agent vs Completion Mode @@ -127,7 +127,7 @@ If you used `setup-ai-config`, verification is automatic: the response includes 3. Instructions or messages are present 4. Parameters are set -**Use `get-ai-config` for the verification call — do not drop to raw `curl` + `jq`.** The MCP tool returns a typed object you can inspect directly. Hand-rolled `jq` filters against the REST response routinely break: the AI Configs detail endpoint returns the variation list under different keys depending on `expand`, and a filter like `.variations.items[]` will fail with `Cannot index array with string "items"` when the response shape is a bare array. If you must call the REST API, use `jq -e .` first to inspect the actual shape before drilling in. +**Use `get-ai-config` for the verification call — do not drop to raw `curl` + `jq`.** The MCP tool returns a typed object you can inspect directly. Hand-rolled `jq` filters against the REST response routinely break: the configs detail endpoint returns the variation list under different keys depending on `expand`, and a filter like `.variations.items[]` will fail with `Cannot index array with string "items"` when the response shape is a bare array. If you must call the REST API, use `jq -e .` first to inspect the actual shape before drilling in. **Report results:** - Config created with correct structure @@ -154,7 +154,7 @@ Print this checklist verbatim to the user after Step 4, then wait for confirmati > - Fallthrough variation: `{variationKey}` (the one this skill just created) > > Verify after targeting is flipped by: -> 1. Opening the AI Config in the LD UI, switching to the correct environment, and confirming "Default rule serves: `{variationName}`" is shown with targeting **On**. +> 1. Opening the config in the LD UI, switching to the correct environment, and confirming "Default rule serves: `{variationName}`" is shown with targeting **On**. > 2. Running a quick test: `ai_config = ai_client.{completion|agent}_config(...)` and asserting `ai_config.enabled is True`. #### Direct shortcut if the user wants to flip targeting without invoking the sibling skill @@ -184,7 +184,7 @@ ldcli resources ai-configs update-ai-config-targeting \ --data '{"instructions":[{"kind":"updateFallthroughVariationOrRollout","variationId":""}]}' ``` -Do not use `turnTargetingOn` — that semantic-patch instruction does **not** work for AI Configs. `updateFallthroughVariationOrRollout` is the only instruction that actually flips the fallthrough. +Do not use `turnTargetingOn` — that semantic-patch instruction does **not** work for configs. `updateFallthroughVariationOrRollout` is the only instruction that actually flips the fallthrough. ## modelConfigKey Format diff --git a/skills/agentcontrol/configs-targeting/README.md b/skills/agentcontrol/configs-targeting/README.md index 073b85c..da1b63f 100644 --- a/skills/agentcontrol/configs-targeting/README.md +++ b/skills/agentcontrol/configs-targeting/README.md @@ -1,11 +1,11 @@ -# LaunchDarkly AI Config Targeting Skill +# LaunchDarkly Config Targeting Skill -An Agent Skill for configuring AI Config targeting rules via the LaunchDarkly API. +An Agent Skill for configuring config targeting rules via the LaunchDarkly API. ## Overview This skill teaches agents how to: -- Turn targeting on/off for AI Configs +- Turn targeting on/off for configs - Add attribute-based targeting rules - Configure percentage rollouts for A/B testing - Set fallthrough (default) variations @@ -18,7 +18,7 @@ Copy `skills/agentcontrol/configs-targeting/` into your agent client's skills pa ## Prerequisites - LaunchDarkly API access token with `ai-configs:write` permission -- Existing AI Config with variations (use `configs-create` skill) +- Existing config with variations (use `configs-create` skill) - Understanding of contexts (see `context-basic` skill) ## Usage @@ -41,9 +41,9 @@ configs-targeting/ ## Related -- [AI Config Create](../configs-create/) - Create AI Configs first -- [AI Config Variations](../configs-variations/) - Create variations to target -- [AI Config Online Evals](../online-evals/) - Attach judges +- [config Create](../configs-create/) - Create configs first +- [config Variations](../configs-variations/) - Create variations to target +- [config Online Evals](../online-evals/) - Attach judges - [Targeting Docs](https://docs.launchdarkly.com/home/ai-configs/target) ## License diff --git a/skills/agentcontrol/configs-targeting/SKILL.md b/skills/agentcontrol/configs-targeting/SKILL.md index 7903507..3b1bb7c 100644 --- a/skills/agentcontrol/configs-targeting/SKILL.md +++ b/skills/agentcontrol/configs-targeting/SKILL.md @@ -1,22 +1,22 @@ --- name: configs-targeting -description: Configure AI Config targeting rules to control which variations serve to different users. Enable percentage rollouts, attribute-based rules, segment targeting, and guarded rollouts. +description: Configure config targeting rules to control which variations serve to different users. Enable percentage rollouts, attribute-based rules, segment targeting, and guarded rollouts. compatibility: Requires LaunchDarkly API access token with ai-configs:write permission. metadata: author: launchdarkly version: "0.1.0" --- -# AI Config Targeting +# Config Targeting -Configure targeting rules for AI Configs to control which variations serve to different contexts. Works the same for both completion and agent mode. +Configure targeting rules for configs to control which variations serve to different contexts. Works the same for both completion and agent mode. ## Prerequisites -- LaunchDarkly account with AI Configs enabled +- LaunchDarkly account with AgentControl enabled - API access token with write permissions - Project key and environment key -- Existing AI Config with variations (use `configs-create` skill) +- Existing config with variations (use `configs-create` skill) ## API Key Detection @@ -38,7 +38,7 @@ Targeting rules evaluate in this order (same as feature flags): ### Semantic Patch API -AI Config targeting uses semantic patch instructions: +config targeting uses semantic patch instructions: ``` PATCH /api/v2/projects/{projectKey}/ai-configs/{configKey}/targeting @@ -68,7 +68,7 @@ Response includes `variations` array with `_id` (UUID) for each variation. Edit the default rule to serve the variation you created. -> **Important:** The `turnTargetingOn` instruction does not work for AI Configs. Use `updateFallthroughVariationOrRollout` instead. +> **Important:** The `turnTargetingOn` instruction does not work for configs. Use `updateFallthroughVariationOrRollout` instead. ```bash # First, get variation IDs from Step 1 response @@ -159,7 +159,7 @@ import os from typing import Dict, List, Optional class AIConfigTargeting: - """Manager for AI Config targeting rules""" + """Manager for config targeting rules""" def __init__(self, api_token: str, project_key: str): self.api_token = api_token @@ -212,9 +212,9 @@ class AIConfigTargeting: def enable_config(self, config_key: str, environment: str, variation_key: str = "default") -> bool: """ - Enable an AI Config by setting fallthrough to an enabled variation. + Enable a config by setting fallthrough to an enabled variation. - Note: turnTargetingOn doesn't work for AI Configs. Instead, set the + Note: turnTargetingOn doesn't work for configs. Instead, set the fallthrough from the disabled variation (index 0) to an enabled one. """ variation_id = self.get_variation_id(config_key, variation_key) @@ -323,7 +323,7 @@ class AIConfigTargeting: ## Instruction Reference -> **Note:** `turnTargetingOn` and `turnTargetingOff` do not work for AI Configs. AI Configs have targeting enabled by default. To "enable" a config, set the fallthrough to an enabled variation using `updateFallthroughVariationOrRollout`. +> **Note:** `turnTargetingOn` and `turnTargetingOff` do not work for configs. Configs have targeting enabled by default. To "enable" a config, set the fallthrough to an enabled variation using `updateFallthroughVariationOrRollout`. ### Rules | Kind | Description | @@ -494,14 +494,14 @@ After configuring targeting: ## Related Skills -- `configs-create` - Create AI Configs with variations +- `configs-create` - Create configs with variations - `configs-variations` - Manage variations - `online-evals` - Attach judges - `segments` - Create segments for targeting ## References -- [Target with AI Configs](https://docs.launchdarkly.com/home/ai-configs/target) +- [Target with AgentControl](https://docs.launchdarkly.com/home/ai-configs/target) - [Targeting Rules](https://docs.launchdarkly.com/home/flags/target-rules) - [JSON Targeting](https://docs.launchdarkly.com/home/flags/json-targeting) - [Guarded Rollouts](https://docs.launchdarkly.com/home/releases/guarded-rollouts) diff --git a/skills/agentcontrol/configs-update/README.md b/skills/agentcontrol/configs-update/README.md index 4582294..5406ae0 100644 --- a/skills/agentcontrol/configs-update/README.md +++ b/skills/agentcontrol/configs-update/README.md @@ -1,6 +1,6 @@ -# LaunchDarkly AI Config Update Skill +# LaunchDarkly Config Update Skill -An Agent Skill for updating, archiving, and deleting AI Configs and their variations. +An Agent Skill for updating, archiving, and deleting configs and their variations. ## Overview @@ -43,9 +43,9 @@ configs-update/ ## Related -- [AI Config Create](../configs-create/): Create configs -- [AI Config Variations](../configs-variations/): Add or test variations -- [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) +- [config Create](../configs-create/): Create configs +- [config Variations](../configs-variations/): Add or test variations +- [LaunchDarkly AgentControl Docs](https://docs.launchdarkly.com/home/ai-configs) ## License diff --git a/skills/agentcontrol/configs-update/SKILL.md b/skills/agentcontrol/configs-update/SKILL.md index e05d2c1..0473518 100644 --- a/skills/agentcontrol/configs-update/SKILL.md +++ b/skills/agentcontrol/configs-update/SKILL.md @@ -1,6 +1,6 @@ --- name: configs-update -description: "Update, archive, and delete LaunchDarkly AI Configs and their variations. Use when you need to modify config properties, change model parameters, update instructions or messages, archive unused configs, or permanently remove them." +description: "Update, archive, and delete LaunchDarkly configs and their variations. Use when you need to modify config properties, change model parameters, update instructions or messages, archive unused configs, or permanently remove them." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server metadata: @@ -8,9 +8,9 @@ metadata: version: "1.0.0-experimental" --- -# AI Config Update & Lifecycle +# Config Update & Lifecycle -You're using a skill that will guide you through updating, archiving, and deleting AI Configs and their variations. Your job is to understand the current state of the config, make the changes, and verify the result. +You're using a skill that will guide you through updating, archiving, and deleting configs and their variations. Your job is to understand the current state of the config, make the changes, and verify the result. ## Prerequisites diff --git a/skills/agentcontrol/configs-variations/README.md b/skills/agentcontrol/configs-variations/README.md index 307ca2e..e523709 100644 --- a/skills/agentcontrol/configs-variations/README.md +++ b/skills/agentcontrol/configs-variations/README.md @@ -1,6 +1,6 @@ -# LaunchDarkly AI Config Variations Skill +# LaunchDarkly Config Variations Skill -An Agent Skill for creating and managing AI Config variations to experiment with different models, prompts, and parameters. +An Agent Skill for creating and managing config variations to experiment with different models, prompts, and parameters. ## Overview @@ -37,10 +37,10 @@ configs-variations/ ## Related -- [AI Config Create](../configs-create/): Create the config first -- [AI Config Update](../configs-update/): Modify existing variations -- [AI Config Tools](../tools/): Attach tools to variations -- [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) +- [config Create](../configs-create/): Create the config first +- [config Update](../configs-update/): Modify existing variations +- [config Tools](../tools/): Attach tools to variations +- [LaunchDarkly AgentControl Docs](https://docs.launchdarkly.com/home/ai-configs) ## License diff --git a/skills/agentcontrol/configs-variations/SKILL.md b/skills/agentcontrol/configs-variations/SKILL.md index 26c58e0..967fdc9 100644 --- a/skills/agentcontrol/configs-variations/SKILL.md +++ b/skills/agentcontrol/configs-variations/SKILL.md @@ -1,6 +1,6 @@ --- name: configs-variations -description: "Experiment with AI configurations by creating and managing variations. Helps you test different models, prompts, and parameters to find what works best through systematic experimentation." +description: "Experiment with configs by creating and managing variations. Helps you test different models, prompts, and parameters to find what works best through systematic experimentation." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server metadata: @@ -8,9 +8,9 @@ metadata: version: "1.0.0-experimental" --- -# AI Config Variations +# Config Variations -You're using a skill that will guide you through testing and optimizing AI configurations through variations. Your job is to design experiments, create variations, and systematically find what works best. +You're using a skill that will guide you through testing and optimizing configs through variations. Your job is to design experiments, create variations, and systematically find what works best. ## Prerequisites diff --git a/skills/agentcontrol/custom-metrics/SKILL.md b/skills/agentcontrol/custom-metrics/SKILL.md index cca2111..30e7f79 100644 --- a/skills/agentcontrol/custom-metrics/SKILL.md +++ b/skills/agentcontrol/custom-metrics/SKILL.md @@ -1,6 +1,6 @@ --- name: custom-metrics -description: "Create, track, retrieve, update, and delete custom business metrics for AI Configs. Covers full lifecycle: define metric kinds via API, emit events via SDK, and query results." +description: "Create, track, retrieve, update, and delete custom business metrics for configs. Covers full lifecycle: define metric kinds via API, emit events via SDK, and query results." license: Apache-2.0 compatibility: Requires the LaunchDarkly server SDK and a LaunchDarkly API token with the `writer` role for metric management. metadata: @@ -8,7 +8,7 @@ metadata: version: "1.0.0-experimental" --- -# Custom Metrics for AI Configs +# Custom Metrics for Configs Full lifecycle management of custom business metrics: create metric definitions via API, track events via SDK, retrieve metric data, and manage metrics programmatically. @@ -16,7 +16,7 @@ Full lifecycle management of custom business metrics: create metric definitions - LaunchDarkly SDK initialized (see `sdk`) - LaunchDarkly API token with `writer` role for metric management -- Understanding of built-in AI metrics (see `built-in-metrics`) +- Understanding of built-in agent metrics (see `built-in-metrics`) ## API Key Detection @@ -126,7 +126,7 @@ def create_metric( ``` **Metric Kinds:** -- `custom` - Track any event (most common for AI metrics) +- `custom` - Track any event (most common for agent metrics) - `pageview` - Track page views - `click` - Track click events @@ -210,7 +210,7 @@ def track_satisfaction(ld_client, user_id: str, score: float, feedback_type: str ) def track_revenue(ld_client, user_id: str, revenue: float, source: str): - """Track revenue generated after AI interaction.""" + """Track revenue generated after agent interaction.""" context = Context.builder(user_id).set("tier", "premium").build() if revenue > 0: @@ -370,10 +370,10 @@ ld_client = ldclient.get() create_metric( PROJECT_KEY, "ai.task.completion", - name="AI Task Completion Rate", + name="Agent Task Completion Rate", kind="custom", is_numeric=True, - description="Tracks successful AI task completions" + description="Tracks successful agent task completions" ) # 2. Track events @@ -387,7 +387,7 @@ ld_client.flush() metric = get_metric(PROJECT_KEY, "ai.task.completion") # 4. Update metric name -rename_metric(PROJECT_KEY, "ai.task.completion", "AI Task Success Rate") +rename_metric(PROJECT_KEY, "ai.task.completion", "Agent Task Success Rate") # 5. List all metrics list_metrics(PROJECT_KEY) @@ -487,13 +487,13 @@ class SessionMetricsTracker: Custom metrics appear in: - **Metrics** page in LaunchDarkly UI -- **Monitoring tab** of your AI Config +- **Monitoring tab** of your config - Via API using `get_metric()` or `list_metrics()` ## Related Skills - `sdk` - SDK setup -- `built-in-metrics` - Built-in AI metrics (tokens, duration, cost) +- `built-in-metrics` - Built-in agent metrics (tokens, duration, cost) - `online-evals` - Quality metrics via judges ## References diff --git a/skills/agentcontrol/migrate/README.md b/skills/agentcontrol/migrate/README.md index 268997b..c92bfc8 100644 --- a/skills/agentcontrol/migrate/README.md +++ b/skills/agentcontrol/migrate/README.md @@ -1,10 +1,10 @@ -# LaunchDarkly AI Config Migrate Skill +# LaunchDarkly Config Migrate Skill -An Agent Skill for migrating an application with hardcoded LLM prompts to a full LaunchDarkly AI Configs implementation in five stages: extract, wrap, tools, tracking, evals. +An Agent Skill for migrating an application with hardcoded LLM prompts to a full LaunchDarkly AgentControl implementation in five stages: extract, wrap, tools, tracking, evals. ## Overview -This skill orchestrates the full migration journey from hardcoded `openai.chat.completions.create(model="gpt-4o", ...)` (or equivalent in any provider SDK) to a managed AI Config with tools, tracking, and judges. It delegates each stage to a focused skill and covers the tracker wiring inline — since no existing skill owns `tracker.track_*` calls. +This skill orchestrates the full migration journey from hardcoded `openai.chat.completions.create(model="gpt-4o", ...)` (or equivalent in any provider SDK) to a managed config with tools, tracking, and judges. It delegates each stage to a focused skill and covers the tracker wiring inline — since no existing skill owns `tracker.track_*` calls. The five stages: @@ -27,15 +27,15 @@ Copy `skills/agentcontrol/migrate/` into your agent client's skills path. ## Usage ``` -Migrate our chat service from hardcoded OpenAI prompts to LaunchDarkly AI Configs +Migrate our chat service from hardcoded OpenAI prompts to LaunchDarkly AgentControl ``` ``` -Our LangGraph agent has its model and instructions baked in — walk me through wrapping it in an AI Config +Our LangGraph agent has its model and instructions baked in — walk me through wrapping it in a config ``` ``` -Wire up the AI tracker and attach accuracy + relevance judges to our existing config +Wire up the agent tracker and attach accuracy + relevance judges to our existing config ``` ## Structure @@ -55,12 +55,12 @@ migrate/ ## Related -- [AI Config Create](../configs-create/): Delegated to by Stage 2 (wrap) -- [AI Config Tools](../tools/): Delegated to by Stage 3 (tools) -- [AI Config Online Evals](../online-evals/): Delegated to by Stage 5 (evals) -- [AI Config Variations](../configs-variations/): Next step after migration for A/B testing -- [AI Config Targeting](../configs-targeting/): Next step after migration for rollout control -- [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) +- [config Create](../configs-create/): Delegated to by Stage 2 (wrap) +- [config Tools](../tools/): Delegated to by Stage 3 (tools) +- [config Online Evals](../online-evals/): Delegated to by Stage 5 (evals) +- [config Variations](../configs-variations/): Next step after migration for A/B testing +- [config Targeting](../configs-targeting/): Next step after migration for rollout control +- [LaunchDarkly AgentControl Docs](https://docs.launchdarkly.com/home/ai-configs) ## License diff --git a/skills/agentcontrol/migrate/SKILL.md b/skills/agentcontrol/migrate/SKILL.md index fd0667e..4ac898b 100644 --- a/skills/agentcontrol/migrate/SKILL.md +++ b/skills/agentcontrol/migrate/SKILL.md @@ -1,6 +1,6 @@ --- name: migrate -description: "Migrate an application with hardcoded LLM prompts to a full LaunchDarkly AI Configs implementation in five stages: audit the code, wrap the call, move the tools, add tracking, attach evaluators. Use when the user wants to externalize model/prompt configuration, move from direct provider calls (OpenAI, Anthropic, Bedrock, Gemini, Strands) to a managed AI Config, or stage a full hardcoded-to-LaunchDarkly migration." +description: "Migrate an application with hardcoded LLM prompts to a full LaunchDarkly AgentControl implementation in five stages: audit the code, wrap the call, move the tools, add tracking, attach evaluators. Use when the user wants to externalize model/prompt configuration, move from direct provider calls (OpenAI, Anthropic, Bedrock, Gemini, Strands) to a managed config, or stage a full hardcoded-to-LaunchDarkly migration." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server metadata: @@ -8,13 +8,13 @@ metadata: version: "0.1.0" --- -# Migrate to AI Configs +# Migrate to AgentControl -You're using a skill that will guide you through migrating an application from hardcoded LLM prompts to a full LaunchDarkly AI Configs implementation. Your job is to run the migration in **five stages**, stopping at each stage for the user to confirm: +You're using a skill that will guide you through migrating an application from hardcoded LLM prompts to a full LaunchDarkly AgentControl implementation. Your job is to run the migration in **five stages**, stopping at each stage for the user to confirm: 1. **Audit the code** — read-only scan that produces a structured list of everything hardcoded (prompt, model, parameters, tools, app-scoped knobs). -2. **Wrap the call** — install the SDK, create the AI Config in LaunchDarkly with a fallback that mirrors the hardcoded values, and rewrite the call site to fetch the config fresh on every request. -3. **Move the tools** — extract each tool's JSON schema, attach it to the AI Config, and swap every call site that references the old tool list. +2. **Wrap the call** — install the SDK, create the config in LaunchDarkly with a fallback that mirrors the hardcoded values, and rewrite the call site to fetch the config fresh on every request. +3. **Move the tools** — extract each tool's JSON schema, attach it to the config, and swap every call site that references the old tool list. 4. **Add tracking** — wire the per-request tracker (duration, tokens, success/error) around the provider call. 5. **Attach evaluators** — either offline evals via the Playground + Datasets, or online judges that score sampled traffic automatically. @@ -22,7 +22,7 @@ You're using a skill that will guide you through migrating an application from h > > 1. **Tracker in the wrong scope.** For an agent with a loop, mint `create_tracker()` once per user turn in a `setup_run` entry node — not inside `call_model`. Per-iteration factory calls produce N `runId`s and trip the at-most-once guards. See [agent-mode-frameworks.md § Custom `StateGraph`](references/agent-mode-frameworks.md). > 2. **`load_chat_model` wrapper reuse.** Templates like `langchain-ai/react-agent` ship a `load_chat_model(f"{provider}/{name}")` helper that wraps `init_chat_model(...)` and silently drops every variation parameter. **Delete it** (don't just avoid using it) and replace call sites with `create_langchain_model(ai_config)`. -> 3. **Fallthrough not flipped after `/configs-create`.** A freshly-created AI Config's fallthrough points at an auto-generated disabled variation, so the SDK returns `enabled=False` until `/configs-targeting` runs. Flip it before Stage 2 verification. +> 3. **Fallthrough not flipped after `/configs-create`.** A freshly-created config's fallthrough points at an auto-generated disabled variation, so the SDK returns `enabled=False` until `/configs-targeting` runs. Flip it before Stage 2 verification. ## Coverage — which shapes are well-trodden vs require extrapolation @@ -68,7 +68,7 @@ If a CHANGELOG entry post-dates this skill and changes an API you're about to us **Sibling skills the user runs at each stage:** - `projects` — pre-Stage 2, only if no project exists yet -- `configs-create` — Stage 2 (creates the AI Config and first variation) +- `configs-create` — Stage 2 (creates the config and first variation) - `tools` — Stage 3 (creates tool definitions and attaches them) - `configs-targeting` — between Stage 2 and Stage 4 (promotes the new variation to fallthrough so the SDK actually serves it) - `online-evals` — Stage 5 (attaches judges, creates custom judges) @@ -108,7 +108,7 @@ For each hardcoded target the audit finds, record: - File path and line range - Current value (model name, full prompt text, parameter dict) -- Target AI Config field (`model.name`, `model.parameters.temperature`, `messages[].content`, `instructions`) +- Target config field (`model.name`, `model.parameters.temperature`, `messages[].content`, `instructions`) - Whether the surrounding call uses function calling / tools (drives Stage 3) - Whether the surrounding call has retry logic (affects where Stage 4 tracker calls go) @@ -129,7 +129,7 @@ Hardcoded targets: Externalized prompt files: none (or e.g. "prompts/agents.yaml — CrewAI role/goal/backstory") Prompt-template registries: none (or e.g. langchain.hub.pull("rlm/rag-prompt") at app.py:14) Coverage totals: 3 hardcoded code targets · 0 externalized prompt files · 0 registry pulls -Proposed plan: single AI Config key `chat-assistant`, mirror fallback, Stage 3 (tools) skipped (no function calling), Stage 4 (tracking) inline, Stage 5 (evals) attach built-in accuracy judge. +Proposed plan: single config key `chat-assistant`, mirror fallback, Stage 3 (tools) skipped (no function calling), Stage 4 (tracking) inline, Stage 5 (evals) attach built-in accuracy judge. ``` **STOP.** Present this summary, state the coverage totals out loud (e.g. "I found **N** hardcoded code targets and **M** externalized prompt files — does that match what you expected?"), and wait for the user to reply with one of four explicit forms: @@ -182,7 +182,7 @@ This is the first stage that writes code. It has nine sub-steps. # that turns a config gap into a boot failure. import logging logging.getLogger(__name__).warning( - "LD_SDK_KEY not set; AI Configs will use fallback values only." + "LD_SDK_KEY not set; configs will use fallback values only." ) ldclient.set_config(Config("", offline=True)) @@ -198,7 +198,7 @@ This is the first stage that writes code. It has nine sub-steps. // key fails fast during waitForInitialization, and every agent_config / // completion_config call returns the fallback. Log a warning; do not throw. if (!process.env.LD_SDK_KEY) { - console.warn('LD_SDK_KEY not set; AI Configs will use fallback values only.'); + console.warn('LD_SDK_KEY not set; configs will use fallback values only.'); } const ldClient = init(process.env.LD_SDK_KEY ?? 'sdk-offline'); await ldClient.waitForInitialization({ timeout: 10 }).catch(() => { @@ -350,9 +350,9 @@ Delegate: **`tools`** (sub-step 2). ### Step 4: Instrument the tracker (Stage 4) -Delegate: **`built-in-metrics`** wires the per-request `tracker.track_*` calls (duration, tokens, success/error, feedback) around the provider call. Use **`custom-metrics`** alongside it if the app needs business metrics beyond the built-in AI ones. Note: do not confuse this with `launchdarkly-metric-instrument`, which is for `ldClient.track()` feature metrics — a different API. See [sdk-ai-tracker-patterns.md](references/sdk-ai-tracker-patterns.md) for the full per-method Python + Node matrix that the delegate skill draws on. +Delegate: **`built-in-metrics`** wires the per-request `tracker.track_*` calls (duration, tokens, success/error, feedback) around the provider call. Use **`custom-metrics`** alongside it if the app needs business metrics beyond the built-in agent ones. Note: do not confuse this with `launchdarkly-metric-instrument`, which is for `ldClient.track()` feature metrics — a different API. See [sdk-ai-tracker-patterns.md](references/sdk-ai-tracker-patterns.md) for the full per-method Python + Node matrix that the delegate skill draws on. -Hand off: print the AI Config key, variation key, provider, and whether the call is streaming, then tell the user: *"Run `/built-in-metrics` with these inputs, then come back here."* Do not auto-invoke. Return here for sub-step 5 (verify) once they're done. +Hand off: print the config key, variation key, provider, and whether the call is streaming, then tell the user: *"Run `/built-in-metrics` with these inputs, then come back here."* Do not auto-invoke. Return here for sub-step 5 (verify) once they're done. 1. **Create the tracker.** Obtain a per-execution tracker via the factory on the config returned in Stage 2: `tracker = config.create_tracker()` (Python) or `const tracker = aiConfig.createTracker();` (Node). Call the factory **once per user turn** and reuse the returned `tracker` for every tracking call in that turn — each call mints a fresh `runId` that tags every event emitted from the turn so they can be correlated via exported events or downstream queries. (The Monitoring tab aggregates today; run-level grouping is a downstream concern — but the `runId` is also what the SDK's at-most-once guards are keyed on, so minting a new one mid-turn breaks the guard semantics regardless of where the events end up.) @@ -422,7 +422,7 @@ Hand off: print the AI Config key, variation key, provider, and whether the call **Deferred feedback across processes.** If the thumbs-up UI fires in a different process than the one that produced the response, do **not** call `create_tracker()` again in the consumer — that mints a new `runId`. Persist the tracker's resumption token (`tracker.resumption_token` in Python, `tracker.resumptionToken` in Node) alongside the message, then rehydrate the tracker with `LDAIConfigTracker.from_resumption_token(...)` (Python) or `aiClient.createTracker(token, context)` (Node) in the feedback handler. -5. **Verify.** Hit the wrapped endpoint in staging, then open the AI Config in LaunchDarkly → Monitoring tab. Duration, token, and generation counts should appear within 1–2 minutes. If nothing shows up, walk the checklist in [sdk-ai-tracker-patterns.md](references/sdk-ai-tracker-patterns.md) under "Troubleshooting." +5. **Verify.** Hit the wrapped endpoint in staging, then open the config in LaunchDarkly → Monitoring tab. Duration, token, and generation counts should appear within 1–2 minutes. If nothing shows up, walk the checklist in [sdk-ai-tracker-patterns.md](references/sdk-ai-tracker-patterns.md) under "Troubleshooting." ### Step 5: Attach evaluations (Stage 5) @@ -430,7 +430,7 @@ Hand off: print the AI Config key, variation key, provider, and whether the call | Path | When to use | Supports agent mode? | |------|-------------|---------------------| - | **Offline eval** (recommended default for migration) | Pre-ship regression: run a fixed dataset through the new variation in the LD Playground and score against baseline. Best fit for migration because you want to prove the new AI Config behaves at least as well as the hardcoded version before shipping. | Yes — all modes | + | **Offline eval** (recommended default for migration) | Pre-ship regression: run a fixed dataset through the new variation in the LD Playground and score against baseline. Best fit for migration because you want to prove the new config behaves at least as well as the hardcoded version before shipping. | Yes — all modes | | **UI-attached auto judges** | Attach one or more judges to a variation in the LD UI; judges run on sampled live requests automatically. Zero code changes. | Completion mode only (the UI widget is completion-only today) | | **Programmatic direct-judge** | Call `ai_client.create_judge(...)` inside the request handler and `judge.evaluate(input, output)` on each call. Adds per-request cost and code complexity. Best for continuous live scoring of workflows where sampled auto-judges aren't enough. | Yes — all modes (the SDK handles both identically) | @@ -446,12 +446,12 @@ Hand off: print the AI Config key, variation key, provider, and whether the call Write this shape into the project's `datasets/README.md` (or equivalent) so the comparison pattern is reproducible after the migration ships. -3. **Hand off to `online-evals`** — only for UI-attached judges (completion mode) or to create custom judge AI Configs that will be referenced by the programmatic path. Tell the user: *"Run `/online-evals` with these inputs, then come back here."* Do not auto-invoke. Pass: - - The parent AI Config key and variation key +3. **Hand off to `online-evals`** — only for UI-attached judges (completion mode) or to create custom judge configs that will be referenced by the programmatic path. Tell the user: *"Run `/online-evals` with these inputs, then come back here."* Do not auto-invoke. Pass: + - The parent config key and variation key - A list of built-in judges (Accuracy, Relevance, Toxicity) or custom judge keys to create/attach - Target environment - The delegate handles creating custom judge AI Configs, attaching them via the variation PATCH endpoint, and setting fallthrough on each judge config. Offline eval does **not** go through this delegate — it's a Playground workflow, not an API write. + The delegate handles creating custom judge configs, attaching them via the variation PATCH endpoint, and setting fallthrough on each judge config. Offline eval does **not** go through this delegate — it's a Playground workflow, not an API write. 4. **For programmatic direct-judge: wire `create_judge` + `evaluate` + `track_judge_result`.** This is the only path at Stage 5 that writes code. The Python shape: @@ -459,7 +459,7 @@ Hand off: print the AI Config key, variation key, provider, and whether the call from ldai.client import AIJudgeConfigDefault judge = ai_client.create_judge( - judge_key, # judge AI Config key in LD + judge_key, # judge config key in LD ld_context, AIJudgeConfigDefault(enabled=False), # fallback: skip eval on SDK miss ) @@ -475,12 +475,12 @@ Hand off: print the AI Config key, variation key, provider, and whether the call ``` Four rules: - - **`create_judge` returns `Optional[Judge]`.** Always guard with `if judge and judge.enabled:` — it returns `None` if the judge AI Config is disabled for the context or the provider is missing. A direct `.evaluate()` on a `None` return will raise `AttributeError`. + - **`create_judge` returns `Optional[Judge]`.** Always guard with `if judge and judge.enabled:` — it returns `None` if the judge config is disabled for the context or the provider is missing. A direct `.evaluate()` on a `None` return will raise `AttributeError`. - **Pass `AIJudgeConfigDefault`**, not `AICompletionConfigDefault`. The `create_judge` `default` parameter is typed `Optional[AIJudgeConfigDefault]`; passing the completion type will not type-check and is a doc-level bug in some older examples. - **`sampling_rate` is a parameter on `evaluate()`**, not on `create_judge`. It defaults to `1.0` (evaluate every call). For live paths, pass something lower (0.1–0.25) to control cost. - **`evaluate()` returns a `JudgeResult`** (never `None`). Check `result.sampled` to know whether the evaluation actually ran, and call `track_judge_result(result)`. Node uses `trackJudgeResult(result)` and `LDJudgeResult` with the same `sampled` field. - **Ask the user which judge AI Config key to use.** LaunchDarkly ships three built-in judges — Accuracy, Relevance, Toxicity — but the actual AI Config **keys** for the built-ins are not canonical SDK constants and aren't documented. Have the user open **AI Configs > Library** in the LD UI and copy the key of the judge they want to reference, or create a custom judge AI Config via `configs-create` first. + **Ask the user which judge config key to use.** LaunchDarkly ships three built-in judges — Accuracy, Relevance, Toxicity — but the actual config **keys** for the built-ins are not canonical SDK constants and aren't documented. Have the user open **AgentControl > Library** in the LD UI and copy the key of the judge they want to reference, or create a custom judge config via `configs-create` first. 5. **Verify.** - **UI-attached auto judges:** trigger a request in staging, open the Monitoring tab → "Evaluator metrics" dropdown. Scores appear within 1–2 minutes at the configured sampling rate. @@ -526,7 +526,7 @@ These are ordered by how likely they are to show up as a first-run failure. The - **If the repo already contains a `load_chat_model(f"{provider}/{name}")` helper, delete it — don't just avoid using it.** This exact shape ships with `langchain-ai/react-agent` and is copied into dozens of derivative repos; look for `utils.load_chat_model`, `utils.build_model`, or any one-arg `init_chat_model` wrapper that splits a `"provider/model"` string. Re-using it is the first-run failure mode: every variation parameter (temperature, max_tokens, top_p, stop sequences) silently drops on the floor because `init_chat_model` only receives the name and provider. `create_langchain_model(ai_config)` is a one-for-one replacement that forwards the whole `model.parameters` dict. Replace every call site, then delete the wrapper file-side so the next reader can't reach for it. - **Same rule applies to hand-rolled `resolve_tools` / `TOOL_REGISTRY` / `ALL_TOOLS` helpers.** If the template already has a `resolve_tools(tool_keys)` or an `ALL_TOOLS` module-level list, import `build_structured_tools` from `ldai_langchain.langchain_helper` and delete the hand-rolled version. `build_structured_tools(ai_config, TOOL_REGISTRY_DICT)` reads `ai_config.model.parameters.tools` and wraps the matching callables as LangChain `StructuredTool`s with the LD tool key as the `StructuredTool.name` — so `ToolNode` lookup works without a second mapping. Don't leave both in the repo. - Don't put app-scoped knobs directly in `model.parameters`. `create_langchain_model` forwards every key in `parameters` to the provider SDK via `init_chat_model`, so a `max_search_results` / `retry_budget` / `feature_toggle` entry will crash the provider with an unexpected-keyword-argument error. The correct home is `model.custom`, which the provider helpers ignore and the app reads via `ai_config.model.get_custom("key")`. The MCP `update-ai-config-variation` tool does not currently expose top-level `custom`, so pick one of two paths: (a) PATCH the variation via the REST API to set `model.custom` directly, or (b) set it via MCP inside `parameters.custom` (as a nested dict) and use a defensive accessor that reads both locations. Full walk-through with code samples in [langchain-tracking.md § MCP caveat](../built-in-metrics/references/langchain-tracking.md). -- Don't re-encode tool schemas inside the fallback. When LaunchDarkly is unreachable the fallback should run without tools (or with whatever minimal provider-bound parameters the app needs to keep operating). Building a `_FALLBACK_TOOLS` array that duplicates the AI Config's tool schema re-introduces the hardcoded config the migration was supposed to move out of code. +- Don't re-encode tool schemas inside the fallback. When LaunchDarkly is unreachable the fallback should run without tools (or with whatever minimal provider-bound parameters the app needs to keep operating). Building a `_FALLBACK_TOOLS` array that duplicates the config's tool schema re-introduces the hardcoded config the migration was supposed to move out of code. - Don't import `LaunchDarklyCallbackHandler` from `ldai.langchain` — neither the class nor the dotted module path exists. The Python LangChain helper package is `ldai_langchain` (top-level module, underscore). Use `create_langchain_model(config)` + `track_metrics_of_async(get_ai_metrics_from_response, lambda: llm.ainvoke(messages))` as the canonical pattern. ### Stage / handoff discipline @@ -543,12 +543,12 @@ These are ordered by how likely they are to show up as a first-run failure. The - Don't wire evals before the tracker is in place. Judges score traffic; without Stage 4 traffic, there is nothing to judge. - Don't frame Stage 5 as "either UI or programmatic." There are **three** paths: offline eval (recommended default for migration), UI-attached auto judges (completion-mode only), and programmatic direct-judge. Offline eval is the one most people skip and usually the right starting point. - Don't pass `sampling_rate` to `create_judge` — it's a parameter on `Judge.evaluate()`, not `create_judge()`. -- Don't hardcode judge AI Config keys (`"accuracy-judge"`, `"relevance-judge"`, etc). The built-in keys are not canonical SDK constants; ask the user to look them up in **AI Configs > Library** in the LD UI. +- Don't hardcode judge config keys (`"accuracy-judge"`, `"relevance-judge"`, etc). The built-in keys are not canonical SDK constants; ask the user to look them up in **AgentControl > Library** in the LD UI. - Don't forget the `if judge and judge.enabled:` guard after `create_judge`. It returns `Optional[Judge]` and returns `None` when the judge config is disabled for the context. ### API surface gotchas -- Don't use `launchdarkly-metric-instrument` for Stage 4 (tracking). That skill is for `ldClient.track()` feature metrics, not AI `tracker.track_*` calls — they are different APIs. +- Don't use `launchdarkly-metric-instrument` for Stage 4 (tracking). That skill is for `ldClient.track()` feature metrics, not agent `tracker.track_*` calls — they are different APIs. - Don't use `track_request()` in Python — it does not exist in `launchdarkly-server-sdk-ai`. Use `track_metrics_of` with a provider-package or custom extractor, or drop to explicit `track_duration` + `track_tokens` + `track_success` / `track_error` if you're on the streaming path. - Don't pass `graph_key=...` to `tracker.track_*()` methods in Python — it is not an accepted argument. Trackers obtained inside a graph traversal are automatically configured with the correct graph key. @@ -560,7 +560,7 @@ These are ordered by how likely they are to show up as a first-run failure. The - `configs-variations` — add variations for A/B testing after migration is complete - `configs-targeting` — roll out new variations to users after migration is complete - `configs-update` — modify config properties as your app evolves -- `launchdarkly-metric-instrument` — for `ldClient.track()` feature metrics (NOT for AI tracker calls) +- `launchdarkly-metric-instrument` — for `ldClient.track()` feature metrics (NOT for agent tracker calls) ## References diff --git a/skills/agentcontrol/migrate/references/agent-graph-reference.md b/skills/agentcontrol/migrate/references/agent-graph-reference.md index 7c8d778..7e5512f 100644 --- a/skills/agentcontrol/migrate/references/agent-graph-reference.md +++ b/skills/agentcontrol/migrate/references/agent-graph-reference.md @@ -6,7 +6,7 @@ ## What an agent graph is -An **agent graph** is a directed graph where each node is its own AI Config (with its own instructions, model, parameters, and tools) and each edge carries routing metadata for handoffs. A supervisor node routes incoming requests to worker nodes based on the supervisor's output; worker nodes may themselves route to other workers or terminate. The graph lives in LaunchDarkly — both its topology and each node's config are managed as versioned resources and can be changed at runtime without redeploying. +An **agent graph** is a directed graph where each node is its own config (with its own instructions, model, parameters, and tools) and each edge carries routing metadata for handoffs. A supervisor node routes incoming requests to worker nodes based on the supervisor's output; worker nodes may themselves route to other workers or terminate. The graph lives in LaunchDarkly — both its topology and each node's config are managed as versioned resources and can be changed at runtime without redeploying. Why use it: @@ -21,7 +21,7 @@ The current Python API (verified against `launchdarkly-server-sdk-ai` main branc ```python def agent_graph(self, key: str, context: Context) -> AgentGraphDefinition: - """Retrieve an AI agent graph by key.""" + """Retrieve an agent graph by key.""" async def create_agent_graph( self, @@ -219,7 +219,7 @@ Do this in phases, not one big bang: 1. **Pick one worker node to migrate first.** Use the single-agent skill workflow on that worker in isolation — extract, wrap, tools, tracking, evals. Leave the rest of the multi-agent app hardcoded. 2. **Confirm the wrapped worker runs in production** for the traffic it serves today, with metrics flowing in the Monitoring tab. 3. **Migrate the supervisor** the same way — single-agent workflow — but keep its routing logic hardcoded initially (a big if/elif over the other workers). -4. **Create the AI Graph in LaunchDarkly** via the UI. Define nodes (one per worker + supervisor) and edges (with `handoff.route` metadata). +4. **Create the agent graph in LaunchDarkly** via the UI. Define nodes (one per worker + supervisor) and edges (with `handoff.route` metadata). 5. **Replace the hardcoded router** with the traversal pattern above. Call `ai_client.agent_graph(...)` instead of assembling the pipeline by hand. 6. **Verify the Monitoring tab** shows the graph-level metrics (`track_path`, `track_duration`, `track_total_tokens`, handoff success/failure counts) in addition to the per-node metrics. 7. **Only then** start moving routing decisions into LaunchDarkly edges and using targeting to change the graph topology per user segment. diff --git a/skills/agentcontrol/migrate/references/agent-mode-frameworks.md b/skills/agentcontrol/migrate/references/agent-mode-frameworks.md index cf892d0..027f3ae 100644 --- a/skills/agentcontrol/migrate/references/agent-mode-frameworks.md +++ b/skills/agentcontrol/migrate/references/agent-mode-frameworks.md @@ -1,6 +1,6 @@ # Agent-Mode Frameworks -How to wire an AI Config in **agent mode** into the frameworks that take a goal/instructions string: LangGraph, CrewAI, Strands, and custom ReAct loops. Also covers the **dynamic tool loading** pattern from the devrel-agents-tutorial — how to extract tool names from `config.tools` at runtime and instantiate the actual tool implementations without hardcoding. +How to wire a config in **agent mode** into the frameworks that take a goal/instructions string: LangGraph, CrewAI, Strands, and custom ReAct loops. Also covers the **dynamic tool loading** pattern from the devrel-agents-tutorial — how to extract tool names from `config.tools` at runtime and instantiate the actual tool implementations without hardcoding. ## When to pick agent mode @@ -171,26 +171,26 @@ def build_crew_agent(ai_client, user_id: str): return None # CrewAI expects role/goal/backstory — split the instructions or store them - # in the AI Config as three variables and pipe them in at runtime. + # in the Config as three variables and pipe them in at runtime. return Agent( role="Research Analyst", - goal="Produce a summary of recent AI Config adoption patterns.", + goal="Produce a summary of recent config adoption patterns.", backstory=config.instructions, llm=config.model.name, # CrewAI accepts a string or a LangChain model ) ``` **Pattern note:** CrewAI's `Agent` takes three separate fields. If you want to drive all three from LaunchDarkly, either: -- Use prompt **variables** on the AI Config (`{{role}}`, `{{goal}}`, `{{backstory}}`) and pass them as the `variables` argument to `agent_config(...)` +- Use prompt **variables** on the config (`{{role}}`, `{{goal}}`, `{{backstory}}`) and pass them as the `variables` argument to `agent_config(...)` - Or store a structured JSON blob in `instructions` and parse it in the app -Prompt variables are cleaner and keep the AI Config human-readable in the UI. +Prompt variables are cleaner and keep the config human-readable in the UI. ### Strands `Agent` Strands is a provider-agnostic, async-first agent SDK. The same `Agent` class runs against Anthropic, OpenAI, and Bedrock by swapping the `model` argument; tools are plain `@tool`-decorated Python functions passed through the constructor; and `SlidingWindowConversationManager` keeps short-term memory across `invoke_async` turns without external state. Agent-mode `instructions` maps directly to `Agent(system_prompt=...)`. -Strands does not ship a first-party LaunchDarkly provider package. To serve multiple providers from a single AI Config key, dispatch on `agent_config.provider.name` and construct the matching Strands model class. +Strands does not ship a first-party LaunchDarkly provider package. To serve multiple providers from a single config key, dispatch on `agent_config.provider.name` and construct the matching Strands model class. **Provider dispatcher.** Drop `parameters.tools` before passing params into the Strands model class — LaunchDarkly surfaces attached tools via a flat `parameters.tools` shape in the variation payload, but Strands receives tools via the `Agent` constructor. Passing `tools` through a second time via model `params` is an error. @@ -294,7 +294,7 @@ async def run_turn(ai_client, user_id: str, user_input: str): - The tracker is Tier 3: `tracker.track_duration_of(...)` + an explicit `track_tokens` call fed by `track_strands_metrics`. See [strands-tracking.md](../../built-in-metrics/references/strands-tracking.md) for the single-call `track_metrics_of_async` variant and the per-field breakdown of `accumulated_usage`. - Always `ldclient.get().flush()` before process exit in short-lived scripts — trailing events can otherwise be lost. -**TypeScript caveat.** The Strands TypeScript SDK ships `BedrockModel` and `OpenAIModel` only — it cannot run Anthropic-backed variations. If the app needs to serve both OpenAI and Anthropic from a single AI Config, use the Python SDK. +**TypeScript caveat.** The Strands TypeScript SDK ships `BedrockModel` and `OpenAIModel` only — it cannot run Anthropic-backed variations. If the app needs to serve both OpenAI and Anthropic from a single config, use the Python SDK. ### Custom `StateGraph` (bind_tools + ToolNode) @@ -337,7 +337,7 @@ builder.add_edge("tools", "call_model") graph = builder.compile() ``` -After — **run-scoped** architecture. The critical shape for the tracker factory is that **one user turn = one `runId` = one tracker**, not one LLM call = one tracker. A ReAct loop that calls `call_model` three times in a single turn must not mint three trackers, or billing and the Monitoring tab will treat the turn as three separate executions. The fix is to resolve the AI Config and mint the tracker once, in a dedicated entry node, and thread both through graph state for every subsequent node. +After — **run-scoped** architecture. The critical shape for the tracker factory is that **one user turn = one `runId` = one tracker**, not one LLM call = one tracker. A ReAct loop that calls `call_model` three times in a single turn must not mint three trackers, or billing and the Monitoring tab will treat the turn as three separate executions. The fix is to resolve the config and mint the tracker once, in a dedicated entry node, and thread both through graph state for every subsequent node. Three nodes, in order: @@ -363,7 +363,7 @@ def make_search(ai_config) -> Callable[..., Any]: # Registry of factories keyed by LD tool name. Each factory takes the # per-run ai_config and returns a ready-to-bind callable. This decouples -# the AI Config (tool metadata) from the app (implementations) and +# the Config (tool metadata) from the app (implementations) and # means tools never call get_agent_config() themselves. TOOL_FACTORIES: Dict[str, Callable[[Any], Callable[..., Any]]] = { "search": make_search, @@ -401,7 +401,7 @@ class State(TypedDict, total=False): async def setup_run(state: State, config: RunnableConfig) -> Dict[str, Any]: - """Runs once per user turn. Resolves the AI Config, mints the tracker, + """Runs once per user turn. Resolves the config, mints the tracker, builds the model and tools, and stashes everything on state. Node signature takes `config: RunnableConfig` rather than @@ -740,13 +740,13 @@ def run_turn(ai_client, user_id: str, user_input: str): raise ``` -The call site stays in your control; the AI Config just delivers `instructions`, `model.name`, `model.parameters`, and `tools`. Everything that's stable across the turn (model name, instructions, tool bindings, tracker) is hoisted out of the loop body — the loop itself only does message passing and tool dispatch. +The call site stays in your control; the config just delivers `instructions`, `model.name`, `model.parameters`, and `tools`. Everything that's stable across the turn (model name, instructions, tool bindings, tracker) is hoisted out of the loop body — the loop itself only does message passing and tool dispatch. **Do not** call `track_duration` / `track_tokens` / `track_success` inside the `for` body. The at-most-once guards will warn and drop the second-and-later calls on the same tracker, so per-step tracker calls will silently lose data. Accumulate inside the loop, emit once after. ## Dynamic tool loading — the "tools factory" pattern -The devrel-agents-tutorial uses a **dynamic tool factory** that reads tool names from `config.tools` and instantiates the actual tool implementations at runtime. This decouples the AI Config (which holds tool metadata) from the application (which holds the executable code). +The devrel-agents-tutorial uses a **dynamic tool factory** that reads tool names from `config.tools` and instantiates the actual tool implementations at runtime. This decouples the config (which holds tool metadata) from the application (which holds the executable code). ### The pattern @@ -754,7 +754,7 @@ The devrel-agents-tutorial uses a **dynamic tool factory** that reads tool names # tools_impl/dynamic_tool_factory.py — adapted from devrel-agents-tutorial def extract_tool_names(config) -> list[str]: - """Read the list of tool names from the AI Config.""" + """Read the list of tool names from the config.""" if not hasattr(config, "tools") or not config.tools: return [] return [tool.name if hasattr(tool, "name") else tool.get("name") for tool in config.tools] @@ -798,7 +798,7 @@ agent = create_agent( **What this gives you:** -- Toggle a tool on/off by editing the AI Config in LaunchDarkly — no redeploy needed to remove a tool from production +- Toggle a tool on/off by editing the config in LaunchDarkly — no redeploy needed to remove a tool from production - Roll out a new tool to 5% of users by editing targeting rules (combined with `configs-targeting`) - Keep the actual tool implementation code in the repo; only metadata lives in LaunchDarkly @@ -817,13 +817,13 @@ Do not hand-write the schema — LangChain already generated it from the functio ### Dynamic schemas from LaunchDarkly -The devrel tutorial also shows the reverse: reading a JSON schema **from** the AI Config and constructing a Pydantic model at runtime: +The devrel tutorial also shows the reverse: reading a JSON schema **from** the config and constructing a Pydantic model at runtime: ```python from pydantic import BaseModel, Field, create_model def _create_dynamic_tool_input(tool_config: dict) -> type[BaseModel]: - """Build a Pydantic input schema from an AI Config tool's parameters.""" + """Build a Pydantic input schema from a config tool's parameters.""" properties = tool_config.get("properties", {}) fields = {} for name, cfg in properties.items(): @@ -857,9 +857,9 @@ For the full graph pattern, read [agent-graph-reference.md](agent-graph-referenc ## Keep the provider call in the repo -One rule that applies across all three frameworks: the **provider SDK call** (OpenAI, Anthropic, Bedrock) stays in your code. The AI Config only changes the inputs to that call — model name, instructions, parameters, tool list. It does not replace the provider SDK. That means: +One rule that applies across all three frameworks: the **provider SDK call** (OpenAI, Anthropic, Bedrock) stays in your code. The config only changes the inputs to that call — model name, instructions, parameters, tool list. It does not replace the provider SDK. That means: - You keep full control of error handling, retries, timeouts, custom headers - You keep full control of streaming logic and backpressure - You keep full control of authentication (API keys, IAM roles, Bedrock Converse sessions) -- The AI Config is additive — removing it gets you back to the original hardcoded app, provided the fallback mirrors the old values +- The config is additive — removing it gets you back to the original hardcoded app, provided the fallback mirrors the old values diff --git a/skills/agentcontrol/migrate/references/fallback-defaults-pattern.md b/skills/agentcontrol/migrate/references/fallback-defaults-pattern.md index 4a02575..4538106 100644 --- a/skills/agentcontrol/migrate/references/fallback-defaults-pattern.md +++ b/skills/agentcontrol/migrate/references/fallback-defaults-pattern.md @@ -2,11 +2,11 @@ Every `completion_config` / `agent_config` call takes a fallback. The fallback runs when LaunchDarkly is unreachable, the SDK is disabled, or the returned config has `enabled=False`. **It must mirror the hardcoded values you removed** so behavior is unchanged if LaunchDarkly is unavailable. -This doc covers three patterns in order of sophistication. Pick the one that matches the size of the app and the number of AI Configs it uses. +This doc covers three patterns in order of sophistication. Pick the one that matches the size of the app and the number of configs it uses. ## Pattern 1: Inline fallback (start here) -One config key, one fallback constant. Best for apps with a handful of AI Configs or a single call site. +One config key, one fallback constant. Best for apps with a handful of configs or a single call site. ### Python — completion mode @@ -101,13 +101,13 @@ const agent = await aiClient.agentConfig('support-agent', context, SUPPORT_AGENT ### When to use inline -- Fewer than ~5 AI Configs in the app +- Fewer than ~5 configs in the app - Fallback values don't change often (or change in lockstep with code deploys) - You want every fallback visible in a single source file for easy review ## Pattern 2: File-backed defaults -A JSON/YAML file holds every config's fallback; a loader at startup builds the default objects. Best for apps with many AI Configs or where the fallback needs to be environment-specific. +A JSON/YAML file holds every config's fallback; a loader at startup builds the default objects. Best for apps with many configs or where the fallback needs to be environment-specific. ### File shape @@ -206,16 +206,16 @@ config = ai_client.completion_config( ### When to use file-backed -- 5+ AI Configs, each with its own fallback +- 5+ configs, each with its own fallback - Fallback values are derived from the current LaunchDarkly state (see Pattern 3 for generation) - You want a single commit to update all fallbacks at once - You want different default files per environment (dev vs staging vs prod) ## Pattern 3: Bootstrap-generated defaults -A script fetches the current state of every AI Config from LaunchDarkly and writes the file used by Pattern 2. Best for large apps where keeping fallbacks in sync by hand is a maintenance burden. +A script fetches the current state of every config from LaunchDarkly and writes the file used by Pattern 2. Best for large apps where keeping fallbacks in sync by hand is a maintenance burden. -The devrel-agents-tutorial has a `bootstrap/create_configs.py` script that does this end-to-end: it creates the AI Configs in LaunchDarkly from a YAML manifest, then writes the `.ai_config_defaults.json` file that the app loads at startup. Use it as prior art — do not reproduce it inline. +The devrel-agents-tutorial has a `bootstrap/create_configs.py` script that does this end-to-end: it creates the configs in LaunchDarkly from a YAML manifest, then writes the `.ai_config_defaults.json` file that the app loads at startup. Use it as prior art — do not reproduce it inline. ### Sketch @@ -250,11 +250,11 @@ def dump_defaults(api_token: str, project_key: str, environment: str) -> dict: - Run the script in CI on every merge to main, commit the updated file - Or run it on deploy to bake the file into the container image -- Or run it manually when you know an AI Config has changed meaningfully +- Or run it manually when you know a config has changed meaningfully ### When to use bootstrap-generated -- 20+ AI Configs +- 20+ configs - You want fallbacks to track production state automatically - You have CI capacity to run the generator and commit the file diff --git a/skills/agentcontrol/migrate/references/phase-1-analysis-checklist.md b/skills/agentcontrol/migrate/references/phase-1-analysis-checklist.md index da16f00..a182d4f 100644 --- a/skills/agentcontrol/migrate/references/phase-1-analysis-checklist.md +++ b/skills/agentcontrol/migrate/references/phase-1-analysis-checklist.md @@ -34,7 +34,7 @@ Grep the source tree for provider SDK imports so you know which one the app actu ### 3. Hardcoded model configs -Look for the three things that need to move into the AI Config: +Look for the three things that need to move into the config: 1. **Model name** — grep for string literals: - `"gpt-4o"`, `"gpt-4o-mini"`, `"gpt-4-turbo"`, `"o1"`, `"o1-mini"` @@ -51,7 +51,7 @@ For each hit, record the file path, line number, and current value. ### 4. External prompt files & registries -Prompts often live outside `.py` / `.ts` source. **Open every plausible config file and read it before declaring the audit complete** — code-only grep will miss prompts loaded from YAML, prompt-template registries, or framework-specific manifests, and the resulting AI Config will not cover the real prompt surface area. +Prompts often live outside `.py` / `.ts` source. **Open every plausible config file and read it before declaring the audit complete** — code-only grep will miss prompts loaded from YAML, prompt-template registries, or framework-specific manifests, and the resulting config will not cover the real prompt surface area. **File-extension scan targets** (run from repo root): @@ -72,14 +72,14 @@ Prompts often live outside `.py` / `.ts` source. **Open every plausible config f | Framework | Where prompts live | |-----------|---------------------| | **CrewAI** | `agents.yaml` and `tasks.yaml` carry `role` / `goal` / `backstory` / `description` / `expected_output` per agent or task | -| **LangChain** | `.prompt` files (Promptfile format); any `langchain.hub.pull("name/key")` call — pulled prompts are remote and must either be inlined into the AI Config or replaced with a `messages` array sourced from the hub at audit time | +| **LangChain** | `.prompt` files (Promptfile format); any `langchain.hub.pull("name/key")` call — pulled prompts are remote and must either be inlined into the config or replaced with a `messages` array sourced from the hub at audit time | | **LangSmith** | `client.pull_prompt(...)` calls referencing a remote prompt repo | | **Pydantic / Settings** | classes with `prompt_*` fields backed by env vars or YAML overlays (Hydra / OmegaConf / Dynaconf) | | **Helm / k8s ConfigMap** | prompts stored as values overrides — search `values*.yaml` and `templates/*configmap*.yaml` | **For each hit, record**: file path, line range, the key holding the prompt, the loader call site that reads it, and any template placeholder syntax (Mustache vs Jinja vs Python `.format()`). The placeholder rewrite in Stage 2 sub-step 5 needs to know the source syntax to convert it to Mustache. -If a fallback file is already in use (see [fallback-defaults-pattern.md](fallback-defaults-pattern.md) — JSON/YAML loaded at startup), distinguish it from prompts that flow into the provider call. The fallback path is intentional infrastructure; only the latter migrates into the AI Config. +If a fallback file is already in use (see [fallback-defaults-pattern.md](fallback-defaults-pattern.md) — JSON/YAML loaded at startup), distinguish it from prompts that flow into the provider call. The fallback path is intentional infrastructure; only the latter migrates into the config. ### 5. Template placeholders in prompts @@ -103,7 +103,7 @@ Configuration that governs *tool* or *app* behavior rather than *model* behavior - Environment variables read inside tool implementations - Constants declared in `tools.py` or a config module that a tool reads at call time -If a value changes agent behavior between variations — it belongs in the AI Config. Stage 2 sub-step 5 (fallback) puts these in `ModelConfig(custom={...})`, **not** `parameters` (which is forwarded to the provider SDK and will crash on unknown kwargs). Tools read them via `ai_config.model.get_custom("key")`. +If a value changes agent behavior between variations — it belongs in the config. Stage 2 sub-step 5 (fallback) puts these in `ModelConfig(custom={...})`, **not** `parameters` (which is forwarded to the provider SDK and will crash on unknown kwargs). Tools read them via `ai_config.model.get_custom("key")`. ### 7. Existing LaunchDarkly SDK usage @@ -151,7 +151,7 @@ Feeds into Stage 2 (install + wrap). Quoted from the `ai-configs-relaunch-guides | LangChain / LangGraph | `@launchdarkly/server-sdk-ai-langchain` | `createLangChainModel(config)` (forwards all variation parameters and handles provider-name mapping) + `getAIMetricsFromResponse` with `trackMetricsOf` | | Vercel AI SDK | `@launchdarkly/server-sdk-ai-vercel` | `getAIMetricsFromResponse` + `trackMetricsOf`, or `VercelRunnerFactory.createVercelModel(aiConfig)` for a managed runner | -Python currently ships helper packages for OpenAI (`ldai_openai`) and LangChain (`ldai_langchain`). The LangChain Python package exposes `create_langchain_model(config)` (builds a LangChain chat model from the AI Config, forwarding every variation parameter and mapping LD provider names to LangChain equivalents), `convert_messages_to_langchain`, and `get_ai_metrics_from_response` — the same package covers LangGraph. Use `create_langchain_model(config)` + `track_metrics_of_async(get_ai_metrics_from_response, lambda: llm.ainvoke(messages))` as the canonical single-call pattern. See [langchain-tracking.md](../../built-in-metrics/references/langchain-tracking.md) for both LangChain and LangGraph patterns and [sdk-ai-tracker-patterns.md](sdk-ai-tracker-patterns.md) for the full tracker-method matrix. +Python currently ships helper packages for OpenAI (`ldai_openai`) and LangChain (`ldai_langchain`). The LangChain Python package exposes `create_langchain_model(config)` (builds a LangChain chat model from the config, forwarding every variation parameter and mapping LD provider names to LangChain equivalents), `convert_messages_to_langchain`, and `get_ai_metrics_from_response` — the same package covers LangGraph. Use `create_langchain_model(config)` + `track_metrics_of_async(get_ai_metrics_from_response, lambda: llm.ainvoke(messages))` as the canonical single-call pattern. See [langchain-tracking.md](../../built-in-metrics/references/langchain-tracking.md) for both LangChain and LangGraph patterns and [sdk-ai-tracker-patterns.md](sdk-ai-tracker-patterns.md) for the full tracker-method matrix. ## Phase 1 output format @@ -182,7 +182,7 @@ Coverage totals: N hardcoded code targets · M externalized prompt f Proposed plan: Stage 1 (Audit): Read-only manifest of hardcoded targets; flag placeholders for Mustache rewrite and knobs for model.custom - Stage 2 (Wrap): Install SDK, create AI Config 'chat-assistant', inline fallback mirrors current values (Mustache syntax), rewrite the call site + Stage 2 (Wrap): Install SDK, create config 'chat-assistant', inline fallback mirrors current values (Mustache syntax), rewrite the call site Stage 3 (Tools): Skipped (no function calling) / Attach 2 tools via tools Stage 4 (Tracking): Inline tracker wiring (track_duration + track_tokens + track_success/error) — run-scoped tracker for agent loops Stage 5 (Evals): Attach built-in 'accuracy' judge at 0.25 sampling via online-evals diff --git a/skills/agentcontrol/migrate/references/sdk-ai-tracker-patterns.md b/skills/agentcontrol/migrate/references/sdk-ai-tracker-patterns.md index c5446f3..fd4d188 100644 --- a/skills/agentcontrol/migrate/references/sdk-ai-tracker-patterns.md +++ b/skills/agentcontrol/migrate/references/sdk-ai-tracker-patterns.md @@ -1,6 +1,6 @@ -# SDK AI Tracker Patterns +# SDK Agent Tracker Patterns -The main novel content of this skill — a per-method reference for the LaunchDarkly AI Config tracker in Python and Node side by side. **No existing skill covers this.** The `launchdarkly-metric-instrument` skill is for `ldClient.track()` feature metrics, which is a different API. +The main novel content of this skill — a per-method reference for the LaunchDarkly config tracker in Python and Node side by side. **No existing skill covers this.** The `launchdarkly-metric-instrument` skill is for `ldClient.track()` feature metrics, which is a different API. All method names and signatures below describe the current public surface of `launchdarkly-server-sdk-ai` (Python) and `@launchdarkly/server-sdk-ai` (Node). If a method is not listed, it does not exist — do not invent it. For per-release breaking changes and renames, consult the SDK CHANGELOGs: @@ -211,7 +211,7 @@ The full programmatic direct-judge pattern (Python): from ldai.client import AIJudgeConfigDefault judge = ai_client.create_judge( - judge_key, # judge AI Config key in LD + judge_key, # judge config key in LD ld_context, AIJudgeConfigDefault(enabled=False), # fallback: skip eval on SDK miss ) @@ -438,10 +438,10 @@ The same resumption token carries the `runId`, so feedback lands on the same run Run the checklist in order. Each step rules out one cause. 1. **SDK key** — is `LD_SDK_KEY` the server-side key (starts with `sdk-`), not the client-side key or the API key? -2. **Enabled check** — is `ai_config.enabled` / `aiConfig.enabled` `True`? A disabled config will not record traffic. Check the AI Config's targeting in LaunchDarkly and confirm the context matches a rule that serves an enabled variation. +2. **Enabled check** — is `ai_config.enabled` / `aiConfig.enabled` `True`? A disabled config will not record traffic. Check the config's targeting in LaunchDarkly and confirm the context matches a rule that serves an enabled variation. 3. **Any tracker call at all** — did `track_success` / `trackSuccess` fire? Without at least one generation-level call, the Monitoring tab has nothing to show. Log a one-liner next to the call to confirm it runs. -4. **Config key match** — is the string passed to `completion_config` / `completionConfig` exactly the same as the AI Config key in LaunchDarkly? Keys are case-sensitive. -5. **Mode match** — if the code calls `completion_config` but the AI Config in LaunchDarkly is in agent mode (or vice versa), the SDK call will error out. Check the mode in the UI. +4. **Config key match** — is the string passed to `completion_config` / `completionConfig` exactly the same as the config key in LaunchDarkly? Keys are case-sensitive. +5. **Mode match** — if the code calls `completion_config` but the config in LaunchDarkly is in agent mode (or vice versa), the SDK call will error out. Check the mode in the UI. 6. **Flush on shutdown** — on short-lived processes (tests, scripts), call `ld_client.flush()` before exit. Long-running servers flush automatically on an interval. 7. **Data delay** — the Monitoring tab updates within 1–2 minutes. If you just deployed, wait and retry before debugging further. 8. **SDK version** — confirm the installed `launchdarkly-server-sdk-ai` (Python) / `@launchdarkly/server-sdk-ai` (Node) version supports the API the code is calling. Methods like `create_tracker` / `createTracker`, `runId`-grouped metrics, `track_judge_result`, and `trackToolCall` / `trackToolCalls` (Node) were added in recent releases — see the SDK CHANGELOGs linked at the top of this file for the version they landed in. diff --git a/skills/agentcontrol/online-evals/README.md b/skills/agentcontrol/online-evals/README.md index 156955e..87e7cf1 100644 --- a/skills/agentcontrol/online-evals/README.md +++ b/skills/agentcontrol/online-evals/README.md @@ -1,12 +1,12 @@ -# LaunchDarkly AI Config Online Evaluations Skill +# LaunchDarkly Config Online Evaluations Skill -An Agent Skill for attaching judges to AI Config variations for automatic LLM-as-a-judge evaluation. +An Agent Skill for attaching judges to config variations for automatic LLM-as-a-judge evaluation. ## Overview This skill teaches agents how to: -- Create custom judge AI Configs with evaluation criteria -- Attach judges to AI Config variations via API +- Create custom judge configs with evaluation criteria +- Attach judges to config variations via API - Configure sampling rates for cost control - Monitor evaluation results in the dashboard @@ -17,7 +17,7 @@ Copy `skills/agentcontrol/online-evals/` into your agent client's skills path. ## Prerequisites - LaunchDarkly API access token with `ai-configs:write` permission -- Existing AI Config with variations (use `configs-create` skill) +- Existing config with variations (use `configs-create` skill) - For custom judges: understanding of LLM-as-a-judge methodology ## Usage @@ -40,9 +40,9 @@ online-evals/ ## Related -- [AI Config Create](../configs-create/) - Create AI Configs first -- [AI Config Targeting](../configs-targeting/) - Enable targeting on judges -- [AI Config Variations](../configs-variations/) - Manage variations +- [config Create](../configs-create/) - Create configs first +- [config Targeting](../configs-targeting/) - Enable targeting on judges +- [config Variations](../configs-variations/) - Manage variations - [Online Evaluations Docs](https://docs.launchdarkly.com/home/ai-configs/online-evaluations) ## License diff --git a/skills/agentcontrol/online-evals/SKILL.md b/skills/agentcontrol/online-evals/SKILL.md index 1c939a0..0a4c12e 100644 --- a/skills/agentcontrol/online-evals/SKILL.md +++ b/skills/agentcontrol/online-evals/SKILL.md @@ -1,21 +1,21 @@ --- name: online-evals -description: Attach judges to AI Config variations for automatic LLM-as-a-judge evaluation. Create custom judges, configure sampling rates, and monitor quality scores. +description: Attach judges to config variations for automatic LLM-as-a-judge evaluation. Create custom judges, configure sampling rates, and monitor quality scores. compatibility: Requires LaunchDarkly API access token with ai-configs:write permission. SDK versions Python v0.20.0+ or Node.js v0.20.0+ for automatic metric recording and the consolidated `track_judge_result` / `trackJudgeResult` API. metadata: author: launchdarkly version: "0.1.0" --- -# AI Config Online Evaluations +# Config Online Evaluations -Attach judges to AI Config variations for automatic quality scoring using LLM-as-a-judge methodology. Judges evaluate responses and return scores between 0.0 and 1.0. +Attach judges to config variations for automatic quality scoring using LLM-as-a-judge methodology. Judges evaluate responses and return scores between 0.0 and 1.0. ## Prerequisites -- LaunchDarkly account with AI Configs enabled +- LaunchDarkly account with AgentControl enabled - API access token with write permissions -- Existing AI Config with variations (use `configs-create` skill) +- Existing config with variations (use `configs-create` skill) - For automatic metric recording and the consolidated judge-result API: Python AI SDK v0.20.0+ or Node.js AI SDK v0.20.0+ ## API Key Detection @@ -28,7 +28,7 @@ Attach judges to AI Config variations for automatic quality scoring using LLM-as ### What Are Judges? -Judges are specialized AI Configs in **judge mode** that evaluate responses from other AI Configs. They use an LLM to score outputs and return structured results: +Judges are specialized configs in **judge mode** that evaluate responses from other configs. They use an LLM to score outputs and return structured results: ```json { @@ -49,7 +49,7 @@ LaunchDarkly provides three pre-configured judges: ### Completion Mode Only -Judges can only be attached to **completion mode** AI Configs in the UI. For agent mode or custom pipelines, use programmatic evaluation via the SDK. +Judges can only be attached to **completion mode** configs in the UI. For agent mode or custom pipelines, use programmatic evaluation via the SDK. ### Restrictions @@ -61,7 +61,7 @@ Judges can only be attached to **completion mode** AI Configs in the UI. For age ### Step 1: Create Custom Judges (Optional) -For domain-specific evaluation, create judge AI Configs: +For domain-specific evaluation, create judge configs: ```bash # Create judge config @@ -128,9 +128,9 @@ curl -X PATCH "https://app.launchdarkly.com/api/v2/projects/{projectKey}/ai-conf ### Step 3: Set Fallthrough on Judges -Each judge AI Config needs its fallthrough set to the enabled variation. AI Configs default to the "disabled" variation (index 0). +Each judge config needs its fallthrough set to the enabled variation. Configs default to the "disabled" variation (index 0). -> **Note:** `turnTargetingOn` does not work for AI Configs. Use `updateFallthroughVariationOrRollout` instead. +> **Note:** `turnTargetingOn` does not work for configs. Use `updateFallthroughVariationOrRollout` instead. ```bash # First get the variation ID for "Default" from GET targeting response @@ -155,7 +155,7 @@ import os from typing import Optional class AIConfigJudges: - """Manager for AI Config judge attachments""" + """Manager for config judge attachments""" def __init__(self, api_token: str, project_key: str): self.api_token = api_token @@ -173,7 +173,7 @@ class AIConfigJudges: Attach judges to a variation. Args: - config_key: AI Config key + config_key: config key variation_key: Variation key judges: List of {"judgeConfigKey": str, "samplingRate": float} """ @@ -193,7 +193,7 @@ class AIConfigJudges: system_prompt: str, model: str = "OpenAI.gpt-4o-mini", is_inverted: bool = False) -> dict: """ - Create a judge AI Config. + Create a judge config. Args: key: Judge config key @@ -237,7 +237,7 @@ class AIConfigJudges: """ Set fallthrough to enable a judge config. - Note: turnTargetingOn doesn't work for AI Configs. Instead, set the + Note: turnTargetingOn doesn't work for configs. Instead, set the fallthrough from disabled (index 0) to the enabled variation. """ # Get variation ID @@ -307,11 +307,11 @@ async def async_main(): default_value = AICompletionConfigDefault(enabled=False) - # create_model() initializes with judges from AI Config + # create_model() initializes with judges from Config model = await aiclient.create_model(ai_config_key, context, default_value, {}) if not model: - print(f"AI configuration not enabled for: {ai_config_key}") + print(f"agent configuration not enabled for: {ai_config_key}") return user_input = 'How can LaunchDarkly help me?' @@ -369,7 +369,7 @@ async def async_main(): judge = aiclient.create_judge(judge_key, context, judge_default_value) if not judge: - print(f"AI judge configuration not enabled for key: {judge_key}") + print(f"agent judge configuration not enabled for key: {judge_key}") return input_text = 'You are a helpful assistant. How can you help me?' @@ -382,7 +382,7 @@ async def async_main(): print("Judge evaluation was skipped (sample rate or configuration issue)") return - # Track the consolidated result on the AI Config tracker if needed: + # Track the consolidated result on the Config tracker if needed: # tracker = ai_config.create_tracker() # tracker.track_judge_result(judge_result) @@ -395,7 +395,7 @@ async def async_main(): ldclient.get().close() ``` -> **Note:** Direct evaluation does not automatically record metrics. Obtain a tracker via `ai_config.create_tracker()` / `aiConfig.createTracker()` and call `tracker.track_judge_result(result)` / `tracker.trackJudgeResult(result)` to record scores for the AI Config you're evaluating. +> **Note:** Direct evaluation does not automatically record metrics. Obtain a tracker via `ai_config.create_tracker()` / `aiConfig.createTracker()` and call `tracker.track_judge_result(result)` / `tracker.trackJudgeResult(result)` to record scores for the config you're evaluating. ## Sampling Rates @@ -405,7 +405,7 @@ You can adjust sampling rates at any time from the Judges section of a variation ## Viewing Results -1. Navigate to **AI Configs** > select your config +1. Navigate to **configs** > select your config 2. Click **Monitoring** tab 3. Select **Evaluator metrics** from dropdown 4. View scores by variation and time range @@ -437,7 +437,7 @@ After attaching judges: ## Related Skills -- `configs-create` - Create AI Configs and judges +- `configs-create` - Create configs and judges - `configs-targeting` - Configure targeting rules - `configs-variations` - Manage variations diff --git a/skills/agentcontrol/projects/README.md b/skills/agentcontrol/projects/README.md index 0decb32..2abbcf3 100644 --- a/skills/agentcontrol/projects/README.md +++ b/skills/agentcontrol/projects/README.md @@ -1,4 +1,4 @@ -# LaunchDarkly AI Config Projects Skill +# LaunchDarkly Config Projects Skill An Agent Skill for setting up LaunchDarkly project management in a codebase. Guides exploration of the stack, assessment of the right approach, and integration that fits the architecture. @@ -30,7 +30,7 @@ Examples: Once installed, the skill activates automatically when you ask about project setup: ``` -Set up a LaunchDarkly project for our AI configs +Set up a LaunchDarkly project for our configs ``` ``` @@ -61,7 +61,7 @@ projects/ ## Related -- [LaunchDarkly AI Configs](https://docs.launchdarkly.com/home/ai-configs): Create AI Configs after setting up projects +- [LaunchDarkly AgentControl](https://docs.launchdarkly.com/home/ai-configs): Create configs after setting up projects - [LaunchDarkly Docs](https://docs.launchdarkly.com) - [Agent Skills Specification](https://agentskills.io/specification) diff --git a/skills/agentcontrol/projects/SKILL.md b/skills/agentcontrol/projects/SKILL.md index ddb94af..acb28bf 100644 --- a/skills/agentcontrol/projects/SKILL.md +++ b/skills/agentcontrol/projects/SKILL.md @@ -37,7 +37,7 @@ See [Quick Start](references/quick-start.md) for API usage patterns. ## What Are Projects? Projects are LaunchDarkly's top-level organizational containers that hold: -- All your AI Configs +- All your configs - Feature flags and segments - Multiple environments (Production and Test created by default) @@ -180,16 +180,16 @@ Project keys must follow these rules: ### By Team ``` -platform-ai → Platform Team AI -customer-ai → Customer Success Team AI -internal-ai → Internal Tools Team AI +platform-ai → Platform Team Agent +customer-ai → Customer Success Team Agent +internal-ai → Internal Tools Team Agent ``` ### By Application/Service ``` -mobile-ai → Mobile App AI Configs -web-ai → Web App AI Configs -api-ai → API Service AI Configs +mobile-ai → Mobile App configs +web-ai → Web App configs +api-ai → API Service configs ``` ### By Region/Deployment @@ -221,15 +221,15 @@ ai-apac → Asia-Pacific Region After setting up projects: -1. **Create AI Configs** - Use the `configs-create` skill +1. **Create configs** - Use the `configs-create` skill 2. **Set up SDK Integration** - Use the `sdk` skill 3. **Configure Targeting** - Use the `configs-targeting` skill ## Related Skills -- `configs-create` - Create AI Configs in projects +- `configs-create` - Create configs in projects - `sdk` - Integrate SDK in your application -- `configs-targeting` - Configure AI Config targeting +- `configs-targeting` - Configure config targeting - `configs-variations` - Manage config variations ## References diff --git a/skills/agentcontrol/projects/references/admin-tooling.md b/skills/agentcontrol/projects/references/admin-tooling.md index ec5abb3..afe7acd 100644 --- a/skills/agentcontrol/projects/references/admin-tooling.md +++ b/skills/agentcontrol/projects/references/admin-tooling.md @@ -195,7 +195,7 @@ pip install click tabulate **Usage:** ```bash # Create project -python cli/ldprojects.py create "My AI" my-ai -t ai-configs -t production +python cli/ldprojects.py create "My Agent" my-ai -t ai-configs -t production # List projects python cli/ldprojects.py list @@ -207,7 +207,7 @@ python cli/ldprojects.py get-key my-ai python cli/ldprojects.py get-key my-ai --env test --show-key # Clone project -python cli/ldprojects.py clone template-ai new-ai "New AI Project" +python cli/ldprojects.py clone template-ai new-ai "New Agent Project" # Export keys python cli/ldprojects.py export-keys my-ai -o keys.json diff --git a/skills/agentcontrol/projects/references/go-setup.md b/skills/agentcontrol/projects/references/go-setup.md index 8b7df49..e7a6a10 100644 --- a/skills/agentcontrol/projects/references/go-setup.md +++ b/skills/agentcontrol/projects/references/go-setup.md @@ -198,7 +198,7 @@ func main() { pm := launchdarkly.NewProjectManager("") project, err := pm.CreateProject( - "Customer AI Service", + "Customer Agent Service", "customer-ai", []string{"ai-configs", "production"}, ) @@ -386,7 +386,7 @@ func main() { **Usage:** ```bash -go run cmd/ldprojects/main.go create -name "My AI" -key my-ai -tags ai-configs,production +go run cmd/ldprojects/main.go create -name "My Agent" -key my-ai -tags ai-configs,production go run cmd/ldprojects/main.go list go run cmd/ldprojects/main.go get-key -project my-ai -env production ``` diff --git a/skills/agentcontrol/projects/references/iac-automation.md b/skills/agentcontrol/projects/references/iac-automation.md index 13e547a..b584fb1 100644 --- a/skills/agentcontrol/projects/references/iac-automation.md +++ b/skills/agentcontrol/projects/references/iac-automation.md @@ -36,13 +36,13 @@ variable "launchdarkly_access_token" { resource "launchdarkly_project" "customer_ai" { key = "customer-ai" - name = "Customer AI Service" + name = "Customer Agent Service" tags = ["ai-configs", "production", "terraform"] } resource "launchdarkly_project" "platform_ai" { key = "platform-ai" - name = "Platform AI Service" + name = "Platform Agent Service" tags = ["ai-configs", "production", "terraform"] } @@ -283,10 +283,10 @@ Manage projects with Ansible: vars: launchdarkly_api_token: "{{ lookup('env', 'LAUNCHDARKLY_API_TOKEN') }}" projects: - - name: "Customer AI Service" + - name: "Customer Agent Service" key: "customer-ai" tags: ["ai-configs", "production"] - - name: "Platform AI Service" + - name: "Platform Agent Service" key: "platform-ai" tags: ["ai-configs", "production"] @@ -345,14 +345,14 @@ import pulumi_launchdarkly as launchdarkly customer_ai = launchdarkly.Project( "customer-ai", key="customer-ai", - name="Customer AI Service", + name="Customer Agent Service", tags=["ai-configs", "production", "pulumi"] ) platform_ai = launchdarkly.Project( "platform-ai", key="platform-ai", - name="Platform AI Service", + name="Platform Agent Service", tags=["ai-configs", "production", "pulumi"] ) @@ -371,13 +371,13 @@ import * as launchdarkly from "@pulumi/launchdarkly"; // Create projects const customerAi = new launchdarkly.Project("customer-ai", { key: "customer-ai", - name: "Customer AI Service", + name: "Customer Agent Service", tags: ["ai-configs", "production", "pulumi"], }); const platformAi = new launchdarkly.Project("platform-ai", { key: "platform-ai", - name: "Platform AI Service", + name: "Platform Agent Service", tags: ["ai-configs", "production", "pulumi"], }); @@ -435,7 +435,7 @@ metadata: name: customer-ai spec: key: customer-ai - name: Customer AI Service + name: Customer Agent Service tags: - ai-configs - production @@ -513,7 +513,7 @@ setup-env: **Usage:** ```bash -make create-project NAME="My AI" KEY=my-ai TAGS=ai-configs,production +make create-project NAME="My Agent" KEY=my-ai TAGS=ai-configs,production make list-projects make get-key PROJECT=my-ai ENV=production make setup-env PROJECT=my-ai diff --git a/skills/agentcontrol/projects/references/multi-language-setup.md b/skills/agentcontrol/projects/references/multi-language-setup.md index 4d86de2..db705e6 100644 --- a/skills/agentcontrol/projects/references/multi-language-setup.md +++ b/skills/agentcontrol/projects/references/multi-language-setup.md @@ -15,7 +15,7 @@ One project, different services consume from different environments or contexts. **When to use:** - All services are part of the same application - Want centralized config management -- Services share the same AI configs +- Services share the same configs ``` Project: "my-app" diff --git a/skills/agentcontrol/projects/references/nodejs-setup.md b/skills/agentcontrol/projects/references/nodejs-setup.md index d8def14..bbb2157 100644 --- a/skills/agentcontrol/projects/references/nodejs-setup.md +++ b/skills/agentcontrol/projects/references/nodejs-setup.md @@ -329,7 +329,7 @@ program.parse(); **Usage:** ```bash -npm run ld-projects create "My AI" my-ai -t ai-configs production +npm run ld-projects create "My Agent" my-ai -t ai-configs production npm run ld-projects list npm run ld-projects get-key my-ai --env production ``` diff --git a/skills/agentcontrol/projects/references/project-cloning.md b/skills/agentcontrol/projects/references/project-cloning.md index c9794be..64ebd09 100644 --- a/skills/agentcontrol/projects/references/project-cloning.md +++ b/skills/agentcontrol/projects/references/project-cloning.md @@ -117,9 +117,9 @@ clone_for_regions("customer-ai", ["us", "eu", "apac"]) ``` **Result:** -- `customer-ai-us` - Customer AI - US -- `customer-ai-eu` - Customer AI - EU -- `customer-ai-apac` - Customer AI - APAC +- `customer-ai-us` - Customer Agent - US +- `customer-ai-eu` - Customer Agent - EU +- `customer-ai-apac` - Customer Agent - APAC ## Multi-Tenant Cloning @@ -174,9 +174,9 @@ cloneForTenants('saas-ai', tenants); ``` **Result:** -- `saas-ai-acme` - SaaS AI - Acme Corp -- `saas-ai-globex` - SaaS AI - Globex Inc -- `saas-ai-initech` - SaaS AI - Initech +- `saas-ai-acme` - SaaS Agent - Acme Corp +- `saas-ai-globex` - SaaS Agent - Globex Inc +- `saas-ai-initech` - SaaS Agent - Initech ## Team-Based Cloning @@ -200,7 +200,7 @@ def clone_for_teams(template_project: str, teams: List[str]): for team in teams: new_key = f"{team}-ai" - new_name = f"{team.title()} Team AI" + new_name = f"{team.title()} Team Agent" print(f"Creating {new_key} for {team} team...") project = clone_project( @@ -240,8 +240,8 @@ def clone_from_csv(source_key: str, csv_file: str): CSV format: project_key,project_name,tags - mobile-ai-us,Mobile AI US,"mobile,us,production" - mobile-ai-eu,Mobile AI EU,"mobile,eu,production" + mobile-ai-us,Mobile Agent US,"mobile,us,production" + mobile-ai-eu,Mobile Agent EU,"mobile,eu,production" """ pm = ProjectManager() created_projects = [] @@ -267,10 +267,10 @@ clone_from_csv("ai-template", "projects.csv") **projects.csv:** ```csv project_key,project_name,tags -mobile-ai-us,Mobile AI US,"mobile,us,production" -mobile-ai-eu,Mobile AI EU,"mobile,eu,production" -web-ai-us,Web AI US,"web,us,production" -web-ai-eu,Web AI EU,"web,eu,production" +mobile-ai-us,Mobile Agent US,"mobile,us,production" +mobile-ai-eu,Mobile Agent EU,"mobile,eu,production" +web-ai-us,Web Agent US,"web,us,production" +web-ai-eu,Web Agent EU,"web,eu,production" ``` ## Automated SDK Key Management @@ -340,9 +340,9 @@ def clone_projects_parallel(clones: List[Dict[str, str]], max_workers: int = 5): # Usage clones = [ - {"source_key": "template", "new_key": "team-a-ai", "new_name": "Team A AI"}, - {"source_key": "template", "new_key": "team-b-ai", "new_name": "Team B AI"}, - {"source_key": "template", "new_key": "team-c-ai", "new_name": "Team C AI"}, + {"source_key": "template", "new_key": "team-a-ai", "new_name": "Team A Agent"}, + {"source_key": "template", "new_key": "team-b-ai", "new_name": "Team B Agent"}, + {"source_key": "template", "new_key": "team-c-ai", "new_name": "Team C Agent"}, ] clone_projects_parallel(clones) diff --git a/skills/agentcontrol/projects/references/python-setup.md b/skills/agentcontrol/projects/references/python-setup.md index 87b4ab9..350b3e7 100644 --- a/skills/agentcontrol/projects/references/python-setup.md +++ b/skills/agentcontrol/projects/references/python-setup.md @@ -106,7 +106,7 @@ pm = ProjectManager() # Create new project project = pm.create_project( - name="Customer Support AI", + name="Customer Support Agent", key="support-ai", tags=["ai-configs", "production"] ) @@ -245,7 +245,7 @@ if __name__ == "__main__": **Usage:** ```bash -python cli/ld_projects.py create "My AI" my-ai --tags ai-configs +python cli/ld_projects.py create "My Agent" my-ai --tags ai-configs python cli/ld_projects.py list python cli/ld_projects.py get-key my-ai --env production ``` diff --git a/skills/agentcontrol/projects/references/quick-start.md b/skills/agentcontrol/projects/references/quick-start.md index eb91dec..9f0468c 100644 --- a/skills/agentcontrol/projects/references/quick-start.md +++ b/skills/agentcontrol/projects/references/quick-start.md @@ -22,7 +22,7 @@ Content-Type: application/json **Minimal Payload:** ```json { - "name": "My AI Project", + "name": "My Agent Project", "key": "my-ai-project" } ``` @@ -30,7 +30,7 @@ Content-Type: application/json **Recommended Payload:** ```json { - "name": "Customer Support AI", + "name": "Customer Support Agent", "key": "support-ai", "tags": ["ai-configs", "production"] } @@ -41,7 +41,7 @@ Content-Type: application/json ### Success (201 Created) ```json { - "name": "Customer Support AI", + "name": "Customer Support Agent", "key": "support-ai", "environments": { "items": [ @@ -140,7 +140,7 @@ Before creating a new project, you may want to check what exists. { "items": [ { - "name": "Customer Support AI", + "name": "Customer Support Agent", "key": "support-ai", "tags": ["ai-configs"] } @@ -164,7 +164,7 @@ After creating your project: 1. **Save SDK keys** to your environment configuration 2. **Initialize LaunchDarkly SDK** in your application -3. **Create AI Configs** within the project +3. **Create configs** within the project 4. **Test the integration** in test environment first See [Environment Configuration](env-config.md) for saving SDK keys to your codebase. diff --git a/skills/agentcontrol/snippets/SKILL.md b/skills/agentcontrol/snippets/SKILL.md index ecea329..99e34ae 100644 --- a/skills/agentcontrol/snippets/SKILL.md +++ b/skills/agentcontrol/snippets/SKILL.md @@ -1,6 +1,6 @@ --- name: snippets -description: "Create and manage prompt snippets — reusable text blocks referenced inside AI Config variation prompts. Keeps common instructions, personas, and guardrails consistent across multiple configs." +description: "Create and manage prompt snippets — reusable text blocks referenced inside config variation prompts. Keeps common instructions, personas, and guardrails consistent across multiple configs." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server metadata: @@ -8,9 +8,9 @@ metadata: version: "0.1.0" --- -# AI Config Prompt Snippets +# Config Prompt Snippets -You're using a skill that will guide you through creating and managing prompt snippets in LaunchDarkly. Your job is to identify reusable text, create snippets, reference them in AI Config variations, and verify everything is wired correctly. +You're using a skill that will guide you through creating and managing prompt snippets in LaunchDarkly. Your job is to identify reusable text, create snippets, reference them in config variations, and verify everything is wired correctly. ## Prerequisites @@ -30,9 +30,9 @@ This skill requires the remotely hosted LaunchDarkly MCP server to be configured ### What Are Prompt Snippets? -Prompt snippets are named, versioned text blocks stored at the project level. They contain reusable pieces of prompt text — personas, guardrails, output format instructions, domain knowledge — that can be shared across multiple AI Config variations. +Prompt snippets are named, versioned text blocks stored at the project level. They contain reusable pieces of prompt text — personas, guardrails, output format instructions, domain knowledge — that can be shared across multiple config variations. -When a snippet is updated, a new version is created. AI Config variations that reference the snippet can pick up the latest version, keeping all your configs in sync. +When a snippet is updated, a new version is created. Config variations that reference the snippet can pick up the latest version, keeping all your configs in sync. ### When to Use Snippets @@ -64,7 +64,7 @@ When a snippet is updated, a new version is created. AI Config variations that r Before creating snippets, understand what's shared: -1. List existing AI Configs in the project using `get-ai-config` for each +1. List existing configs in the project using `get-ai-config` for each 2. Look for repeated text across variation prompts 3. Identify text that should stay consistent (guardrails, personas, formats) 4. Check existing snippets with `list-prompt-snippets` to avoid duplicates @@ -112,7 +112,7 @@ Use `update-prompt-snippet` to modify an existing snippet. Only pass the fields } ``` -Each update creates a new version. Existing AI Config variations referencing the snippet can pick up the new version. +Each update creates a new version. Existing config variations referencing the snippet can pick up the new version. ## Edge Cases diff --git a/skills/agentcontrol/tools/README.md b/skills/agentcontrol/tools/README.md index 19f427a..75e8575 100644 --- a/skills/agentcontrol/tools/README.md +++ b/skills/agentcontrol/tools/README.md @@ -1,13 +1,13 @@ -# LaunchDarkly AI Config Tools Skill +# LaunchDarkly Config Tools Skill -An Agent Skill for creating tools (function calling) and attaching them to AI Config variations. Guides identifying capabilities, creating tool schemas, and verifying attachment. +An Agent Skill for creating tools (function calling) and attaching them to config variations. Guides identifying capabilities, creating tool schemas, and verifying attachment. ## Overview This skill teaches agents how to: -- Identify what capabilities the AI needs +- Identify what capabilities the agent needs - Create tool definitions using the `create-ai-tool` MCP tool -- Attach tools to AI Config variations via `update-ai-config-variation` +- Attach tools to config variations via `update-ai-config-variation` - Verify tools are properly connected via `get-ai-config` ## Installation (Local) @@ -38,9 +38,9 @@ tools/ ## Related -- [AI Config Create](../configs-create/): Create the config before adding tools -- [AI Config Variations](../configs-variations/): Manage variations that tools attach to -- [LaunchDarkly AI Configs Docs](https://docs.launchdarkly.com/home/ai-configs) +- [config Create](../configs-create/): Create the config before adding tools +- [config Variations](../configs-variations/): Manage variations that tools attach to +- [LaunchDarkly AgentControl Docs](https://docs.launchdarkly.com/home/ai-configs) ## License diff --git a/skills/agentcontrol/tools/SKILL.md b/skills/agentcontrol/tools/SKILL.md index ba8d16b..2d9ccfc 100644 --- a/skills/agentcontrol/tools/SKILL.md +++ b/skills/agentcontrol/tools/SKILL.md @@ -1,6 +1,6 @@ --- name: tools -description: "Give your AI agents capabilities through tools (function calling). Helps you identify what your AI needs to do, create tool definitions, and attach them to AI Config variations." +description: "Give your agents capabilities through tools (function calling). Helps you identify what your agent needs to do, create tool definitions, and attach them to config variations." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server metadata: @@ -8,9 +8,9 @@ metadata: version: "1.0.0-experimental" --- -# AI Config Tools +# Config Tools -You're using a skill that will guide you through adding capabilities to your AI agents through tools (function calling). Your job is to identify what your AI needs to do, create tool definitions, attach them to variations, and verify they work. +You're using a skill that will guide you through adding capabilities to your agents through tools (function calling). Your job is to identify what your agent needs to do, create tool definitions, attach them to variations, and verify they work. ## Prerequisites @@ -18,7 +18,7 @@ This skill requires the remotely hosted LaunchDarkly MCP server to be configured **Required MCP tools:** - `create-ai-tool` -- create a new tool definition with a schema -- `update-ai-config-variation` -- attach tools to an AI Config variation +- `update-ai-config-variation` -- attach tools to a config variation - `get-ai-config` -- verify tools are attached to the variation **Optional MCP tools:** @@ -27,7 +27,7 @@ This skill requires the remotely hosted LaunchDarkly MCP server to be configured ## Core Principles -1. **Start with Capabilities**: Think about what your AI needs to do before creating tools +1. **Start with Capabilities**: Think about what your agent needs to do before creating tools 2. **Framework Matters**: LangGraph/CrewAI often auto-generate schemas; OpenAI SDK needs manual schemas 3. **Create Before Attach**: Tools must exist before you can attach them to variations 4. **Verify**: The agent fetches the config to confirm attachment @@ -37,7 +37,7 @@ This skill requires the remotely hosted LaunchDarkly MCP server to be configured ### Step 1: Identify Needed Capabilities -What should the AI be able to do? +What should the agent be able to do? - Query databases, call APIs, perform calculations, send notifications - Check what exists in the codebase (API clients, functions) - Consider framework: LangGraph/LangChain auto-generate schemas; direct SDK needs manual schemas diff --git a/skills/onboarding/SKILL.md b/skills/onboarding/SKILL.md index 1ce801b..b007c8f 100644 --- a/skills/onboarding/SKILL.md +++ b/skills/onboarding/SKILL.md @@ -275,8 +275,8 @@ Generate the repo summary per [Onboarding Summary](references/1.8-summary.md). W 2. **Configuration** -- env var names, how secrets are managed, bundler-specific conventions 3. **Where to Find Things** -- dashboard links with real project key substituted 4. **How Feature Flags Work** -- a language-specific code example showing flag evaluation in this project's stack (not a generic snippet -- use the same pattern the agent wired during Step 5) -5. **Next Steps / Advanced Capabilities** -- links to Percentage Rollouts, Targeting Rules, Experimentation, AI Configs, Guarded Rollouts, and Observability -6. **AI Agent Integration** -- MCP server setup for continued agent-driven flag management +5. **Next Steps / Advanced Capabilities** -- links to Percentage Rollouts, Targeting Rules, Experimentation, configs, Guarded Rollouts, and Observability +6. **Agent Integration** -- MCP server setup for continued agent-driven flag management This is **not** the same file as `LAUNCHDARKLY_ONBOARDING.md`. The onboarding log is a working checklist; `LAUNCHDARKLY.md` is the **permanent reference** for the team. diff --git a/skills/onboarding/mcp-configure/SKILL.md b/skills/onboarding/mcp-configure/SKILL.md index ab5450e..2e7fd4c 100644 --- a/skills/onboarding/mcp-configure/SKILL.md +++ b/skills/onboarding/mcp-configure/SKILL.md @@ -107,7 +107,7 @@ First, tell the user how to create a token if they don't already have one: Then ask how they want to add the token to the MCP config: **D4-LOCAL -- BLOCKING:** Call your structured question tool now. -- question: "The local MCP server needs an API access token to authenticate with LaunchDarkly. You can create one at app.launchdarkly.com/settings/authorization/tokens/new. Once you have the token, how would you like to add it to your MCP config? We recommend adding it yourself — there is a non-zero risk when an AI agent handles secrets, as tokens may persist in conversation history, logs, or model context." +- question: "The local MCP server needs an API access token to authenticate with LaunchDarkly. You can create one at app.launchdarkly.com/settings/authorization/tokens/new. Once you have the token, how would you like to add it to your MCP config? We recommend adding it yourself — there is a non-zero risk when an agent handles secrets, as tokens may persist in conversation history, logs, or model context." - options: - "I'll add the token to the config myself — just tell me which file and variable" - "I have the token ready — go ahead and help me wire up the config" diff --git a/skills/onboarding/references/1.8-summary.md b/skills/onboarding/references/1.8-summary.md index 0b5e85f..7fc8fce 100644 --- a/skills/onboarding/references/1.8-summary.md +++ b/skills/onboarding/references/1.8-summary.md @@ -70,20 +70,20 @@ Here are some things you can do now that LaunchDarkly is set up: - **[Percentage Rollouts](https://launchdarkly.com/docs/home/targeting-flags/rollouts)** — Gradually roll out features to a percentage of users. - **[Targeting Rules](https://launchdarkly.com/docs/home/targeting-flags/targeting-rules)** — Target specific users, segments, or contexts. - **[Experimentation](https://launchdarkly.com/docs/home/about-experimentation)** — Run A/B tests and measure the impact of flag variations. -- **[AI Configs](https://launchdarkly.com/docs/home/ai-configs)** — Manage AI model configurations and prompts with feature flags. +- **[configs](https://launchdarkly.com/docs/home/ai-configs)** — Manage AI model configurations and prompts with feature flags. - **[Guarded Rollouts](https://launchdarkly.com/docs/home/guarded-rollouts)** — Automatically roll back flag changes based on metric guardrails. - **[Observability](https://launchdarkly.com/docs/home/observability)** — Monitor flag evaluations and SDK performance with built-in telemetry. -### AI Agent Integration (MCP Server) +### Agent Integration (MCP Server) -Install the [LaunchDarkly MCP server](https://github.com/launchdarkly/mcp-server) to let your AI agent manage feature flags directly from your editor. With it, your agent can: +Install the [LaunchDarkly MCP server](https://github.com/launchdarkly/mcp-server) to let your agent manage feature flags directly from your editor. With it, your agent can: - **Create and manage flags** — Ask your agent to create a new feature flag, and it will handle the API calls for you. - **Toggle flags on/off** — Turn features on or off across environments without leaving your editor. - **Set up targeting rules** — Configure percentage rollouts, user targeting, and segment-based rules through natural language. - **Clean up stale flags** — Ask your agent to find temporary flags that are fully rolled out and ready to archive. - **Run experiments** — Set up A/B tests and monitor results through your agent. -- **Manage AI configs** — Update model configurations and prompts managed by LaunchDarkly. +- **Manage configs** — Update model configurations and prompts managed by LaunchDarkly. **Two setup options:**