diff --git a/data/Admin/Drop Tables.sql b/data/Admin/Drop Tables.sql index 1cd043b..dec992c 100644 --- a/data/Admin/Drop Tables.sql +++ b/data/Admin/Drop Tables.sql @@ -8,6 +8,14 @@ IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[__EFM DROP TABLE [dbo].[__EFMigrationsHistory] GO +IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[Chat].[IntentEmbeddings]') AND type in (N'U')) +DROP TABLE [Chat].[IntentEmbeddings] +GO + +IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[Chat].[ChatGovernance]') AND type in (N'U')) +DROP TABLE [Chat].[ChatGovernance] +GO + IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[Chat].[ChatMessages]') AND type in (N'U')) DROP TABLE [Chat].[ChatMessages] GO diff --git a/docs/product/features/feature-semantic-intent-classification.md b/docs/product/features/feature-semantic-intent-classification.md new file mode 100644 index 0000000..935e51b --- /dev/null +++ b/docs/product/features/feature-semantic-intent-classification.md @@ -0,0 +1,223 @@ +# Semantic Intent Classification + +## Overview + +Add embedding-based intent classification as a controlled second tier between deterministic rule matching and the general Microsoft Agent Framework (MAF) fallback. The feature improves recognition of novel user phrasing while preserving deterministic routing, typed tool boundaries, authorization, observability, and a disabled-by-default rollout. + +This feature applies to both template-derived products: + +- `crucible-web` +- `agent-framework-quick-start` + +The implementations share the same contracts and behavior. Their embedding initialization triggers differ because Crucible has a StateOfYour business-seeding workflow while the quick-start application owns its startup lifecycle. + +## Business Problem + +Exact phrase matching is fast and reliable but misses requests expressed in unfamiliar language. Those requests fall through to the slower and more expensive agent inference path, even when the intended tool route is already known. + +For example, a catalog may contain `list my pipelines`, while a user asks `could you show my pipeline work`. Semantic classification should recognize the same intent without requiring an ever-growing list of exact phrases. + +## Business Value + +- Improve tool-routing coverage for novel phrasing. +- Preserve the deterministic rule path for known phrases and parameters. +- Reduce unnecessary full-agent inference. +- Keep semantic behavior measurable and reversible. +- Share one architecture across products derived from the same template. + +## User Story + +As a chat user, I want natural variations of a supported request to reach the correct tool, so that I do not need to use one exact command phrase. + +## Product Behavior + +The routing order is: + +```text +User message + | + v +Rule classifier + | + +--> Match ------------------------------> Deterministic route + | + +--> Miss and semantic disabled ---------> MAF fallback + | + +--> Miss and semantic enabled + | + v + Generate query embedding + | + v + Search intent embedding store + | + +--> Valid non-parameterized intent -> Deterministic route + | + +--> No confident match ------------> MAF fallback +``` + +Semantic infrastructure failures must return no semantic match and allow the normal MAF fallback. They must not break the chat request. + +## Current Scope + +Semantic matching currently supports non-parameterized intents. These intents can be routed from an intent name alone, such as: + +- List actors. +- List pipelines. +- Show recent messages. +- List playbooks. + +Parameterized intents remain rule-based until a separate capture-resolution design is implemented. Examples include selecting a pipeline by ID, searching for a supplied query, or retrieving an entity by name. The follow-up-message chat cycle remains the supported way to collect missing parameters. + +## Shared Contracts + +The following contracts are shared by both products: + +- `Embedding` +- `EmbeddingSource` +- `EmbeddingMatch` +- `IEmbeddingGenerator` +- `IIntentEmbeddingStore` +- `IIntentClassifier.ClassifyAsync(...)` +- `IntentClassificationOptions` + +`IntentDefinition.Examples` is the canonical source. Stored embeddings are a regenerated index and must not become a second source of truth. + +## Configuration + +The API configuration exposes the following section in every environment file: + +```json +"IntentClassification": { + "EnableSemantic": false, + "SemanticThreshold": 0.75, + "TopKResults": 5 +} +``` + +The section is bound through `IOptions`. `EnableSemantic` must remain `false` until evaluation and rollout approval are complete. + +Azure embedding settings are bound through `AzureOpenAIOptions`, including `EmbeddingDeploymentName` with the default deployment name `embedding-fast`. + +## Shared Implementation + +### Rule classifier + +`RuleIntentClassifier` remains the first tier. It handles exact examples, parameterized captures, and follow-up examples. Its contract is asynchronous so the routing pipeline can call both rule and semantic implementations consistently. + +### Semantic classifier + +`SemanticIntentClassifier`: + +1. Rejects empty messages. +2. Generates a query vector through `IEmbeddingGenerator`. +3. Searches `IIntentEmbeddingStore` with the configured threshold and top-K value. +4. Resolves the returned intent name against the current `IntentCatalog`. +5. Skips parameterized intents that require captures. +6. Returns the first valid catalog match. +7. Logs the selected intent and similarity score. + +### Hybrid classifier + +`HybridIntentClassifier`: + +1. Runs the rule classifier first. +2. Returns the rule match immediately. +3. Checks `EnableSemantic` after a rule miss. +4. Runs semantic classification only when enabled. +5. Returns `null` on non-cancellation semantic failures so MAF can handle the request. + +### Storage + +`SqlIntentEmbeddingStore` stores example vectors in the `Chat.IntentEmbeddings` table and performs cosine-similarity search. The store is behind `IIntentEmbeddingStore` so a future vector backend can replace SQL without changing the classifier contract. + +## Project Implementations + +### crucible-web + +Crucible seeds embeddings through `Seed.StateOfYour`: + +1. `StateOfYourSqlServerRuntimeScenario` invokes the seeder during product-data setup. +2. The seeder exits without work when semantic classification is disabled. +3. When enabled, it reads the current `IntentCatalog`. +4. It batches each intent's examples through the configured Azure embedding deployment. +5. It upserts the generated vectors into the Crucible SQL database. +6. Non-cancellation failures are logged and do not disable rule-based execution. + +This path is intended for the StateOfYour SQL-backed business scenario and requires a real SQL connection and embedding deployment when enabled. + +### agent-framework-quick-start + +The quick-start application seeds embeddings through `IntentEmbeddingInitializationService`: + +1. The hosted service starts with the application lifecycle. +2. It exits immediately when semantic classification is disabled. +3. It checks `IIntentEmbeddingStore.IsReadyAsync()` before generating vectors. +4. It reads the current `IntentCatalog` and batches examples. +5. It upserts vectors into the configured SQL database. +6. Initialization failures are logged without failing application startup. + +This path is idempotent at startup through the store readiness check. + +## Data Requirements + +The SQL migration creates `Chat.IntentEmbeddings` with: + +- Intent name. +- Embedding source. +- Original source text. +- Serialized vector. +- Weight. +- Created and updated timestamps. + +The migration is generated but must be applied through the normal deployment process. Application startup must not apply migrations. + +## Security and Governance + +- Semantic classification must respect the same owner, tenant, and authorization boundaries as deterministic routing. +- Embedding vectors and source text are governed application data and must not contain secrets or credentials. +- Logs must record routing decisions and scores without recording sensitive message content unnecessarily. +- The feature switch provides an immediate rollback to deterministic-only behavior. +- Azure credentials must come from approved configuration or secret stores and must never be committed to source or test fixtures. + +## Testing Requirements + +Both repositories require: + +- Generator tests with deterministic HTTP responses. +- Store tests for upsert, replacement, search, threshold filtering, ordering, deletion, readiness, and cosine similarity. +- Classifier tests for empty input, semantic match, rule-first behavior, disabled behavior, and provider failure fallback. +- Reqnroll scenarios covering the disabled switch and an enabled semantic match using deterministic embeddings and in-memory storage. +- API configuration parsing tests or equivalent validation confirming the switch is present and disabled by default. + +Live Azure-backed seeding is a separate environment validation. It requires a configured `embedding-fast` deployment, SQL database, credentials, and an applied migration. + +## Rollout + +1. Deploy the infrastructure with semantic classification disabled. +2. Run the offline accuracy and latency evaluation. +3. Validate live seeding in an approved environment. +4. Tune the threshold and top-K settings. +5. Run the end-to-end regression suite. +6. Enable semantic classification only after accuracy, false-positive, latency, and cost targets are approved. +7. Monitor rule, semantic, and MAF fallback rates with rollback available. + +## Out Of Scope + +- Semantic extraction of captures for parameterized intents. +- Playbook evaluation metric embeddings. +- Azure AI Search migration. +- Automatic production enablement without evaluation. +- A second source of truth for intent definitions. +- Migration application during application startup. + +## Definition Of Done + +- [ ] Shared contracts and behavior are implemented in both products. +- [ ] Both seeding strategies are implemented and feature-flagged. +- [ ] Deterministic, semantic, fallback, and failure paths are tested. +- [ ] Live Azure and SQL validation is completed in an approved environment. +- [ ] Phase 8.3 accuracy, latency, cost, and false-positive targets are documented. +- [ ] Production enablement is approved and reversible. +- [ ] Governance documentation contains durable principles only. +- [ ] Tactical implementation details remain in this product feature document. diff --git a/src/Infrastructure.AgentFramework/ChatMessageRoutingService.cs b/src/Infrastructure.AgentFramework/ChatMessageRoutingService.cs index 4d12591..c1109d5 100644 --- a/src/Infrastructure.AgentFramework/ChatMessageRoutingService.cs +++ b/src/Infrastructure.AgentFramework/ChatMessageRoutingService.cs @@ -54,7 +54,7 @@ public async Task ResolveReplyAsync( .Where(x => x.Role.Equals("user", StringComparison.OrdinalIgnoreCase)) .Select(x => x.Content) .ToList(); - var match = _intentClassifier.Classify(message, priorUserMessages); + var match = await _intentClassifier.ClassifyAsync(message, priorUserMessages, cancellationToken); var deterministicReply = match is null ? null : await RouteAsync(chatSessionId, match, cancellationToken); if (!string.IsNullOrWhiteSpace(deterministicReply)) { @@ -130,6 +130,7 @@ private async Task> BuildChatHistoryAsync( { IntentNames.QueryChatSessionsList => QueryChatSessionsListAsync(cancellationToken), IntentNames.QueryChatMessagesList => QueryChatMessagesListAsync(cancellationToken), + IntentNames.QueryChatMessagesForSession => QueryChatMessagesForSessionAsync(Guid.Parse(match.Captures!["sessionId"]), cancellationToken), IntentNames.QueryActorById => QueryActorByIdAsync(Guid.Parse(match.Captures!["id"]), cancellationToken), IntentNames.QueryActorsByName => QueryActorsByNameAsync(match, cancellationToken), IntentNames.QueryActorsList => QueryActorsListAsync(cancellationToken), @@ -217,6 +218,29 @@ private async Task QueryActorsByNameAsync(IntentMatch match, Cancellatio actor.CreatedOn.ToString("u", CultureInfo.InvariantCulture)])); } + private async Task QueryChatMessagesForSessionAsync(Guid sessionId, CancellationToken cancellationToken) + { + var messages = await _sender.Send(new GetMyChatSessionMessagesQuery + { + ChatSessionId = sessionId + }, cancellationToken); + + var items = messages.ToList(); + if (items.Count == 0) + { + return "No messages were found for this chat session."; + } + + return MarkdownTableFormatter.Format( + ["#", "Chat Session Id", "Timestamp (UTC)", "Role", "Content"], + items.Select((message, index) => (IReadOnlyList)[ + (index + 1).ToString(CultureInfo.InvariantCulture), + $"`{message.ChatSessionId:D}`", + message.Timestamp.ToString("u", CultureInfo.InvariantCulture), + message.Role, + message.Content])); + } + private async Task QueryActorsListAsync(CancellationToken cancellationToken) { var actors = await _sender.Send(new Core.Application.Actors.GetOurActorsQuery(), cancellationToken); diff --git a/src/Infrastructure.AgentFramework/ConfigureServices.cs b/src/Infrastructure.AgentFramework/ConfigureServices.cs index 0c58cf6..29c0ee8 100644 --- a/src/Infrastructure.AgentFramework/ConfigureServices.cs +++ b/src/Infrastructure.AgentFramework/ConfigureServices.cs @@ -1,4 +1,5 @@ using Goodtocode.AgentFramework.Infrastructure.AgentFramework.Options; +using Goodtocode.AgentFramework.Infrastructure.AgentFramework.Embeddings; using Goodtocode.AgentFramework.Infrastructure.AgentFramework.Providers; using Goodtocode.AgentFramework.Infrastructure.AgentFramework.Execution; using Goodtocode.AgentFramework.Infrastructure.AgentFramework.Intents; @@ -70,12 +71,19 @@ public static IServiceCollection AddAgentFrameworkOpenAIServices(this IServiceCo services.AddOptions() .Bind(configuration.GetSection(AgentToolInstructionsOptions.SectionName)); + services.AddOptions() + .Bind(configuration.GetSection(IntentClassificationOptions.SectionName)) + .ValidateDataAnnotations() + .ValidateOnStart(); services.AddSingleton(); services.AddScoped(); services.AddSingleton(DefaultIntentCatalogFactory.Create()); - services.AddSingleton(); + services.AddSingleton(); + services.AddScoped(); + services.AddScoped(); + services.AddHostedService(); services.AddScoped(); services.AddScoped(); services.AddScoped(provider => provider.GetRequiredService()); diff --git a/src/Infrastructure.AgentFramework/Embeddings/AzureOpenAiEmbeddingGenerator.cs b/src/Infrastructure.AgentFramework/Embeddings/AzureOpenAiEmbeddingGenerator.cs new file mode 100644 index 0000000..4d3356b --- /dev/null +++ b/src/Infrastructure.AgentFramework/Embeddings/AzureOpenAiEmbeddingGenerator.cs @@ -0,0 +1,177 @@ +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Goodtocode.AgentFramework.Infrastructure.AgentFramework.Options; + +namespace Goodtocode.AgentFramework.Infrastructure.AgentFramework.Embeddings; + +/// +/// Implementation of IEmbeddingGenerator using Azure OpenAI (or Azure AI Foundry). +/// Generates 1536-dimensional vectors via text-embedding-3-small deployment. +/// Thread-safe; can be registered as scoped or singleton. +/// +public sealed class AzureOpenAiEmbeddingGenerator : IEmbeddingGenerator +{ + private readonly string _apiKey; + private readonly string _endpoint; + private readonly string _deploymentName; + private readonly ILogger _logger; + private readonly HttpClient _httpClient; + + /// + public int Dimension => 1536; // text-embedding-3-small standard dimension + + /// + /// Creates a new instance of the Azure OpenAI embedding generator. + /// Configuration is read from appsettings: + /// - AzureOpenAI:ApiKey (required) + /// - AzureOpenAI:Endpoint (required) + /// - AzureOpenAI:EmbeddingDeploymentName (optional, default: "embedding-fast") + /// + /// Configuration provider. + /// Structured logger. + /// If required configuration is missing. + public AzureOpenAiEmbeddingGenerator( + IOptions options, + ILogger logger, + HttpClient httpClient) + { + var config = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _apiKey = config.ApiKey; + _endpoint = config.Endpoint; + _deploymentName = config.EmbeddingDeploymentName; + if (string.IsNullOrWhiteSpace(_apiKey) || string.IsNullOrWhiteSpace(_endpoint)) + throw new InvalidOperationException("AzureOpenAI ApiKey and Endpoint are required for embedding generation."); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + } + + /// + public async Task GenerateAsync(string text, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(text)) + { + _logger.LogWarning("Attempted to generate embedding for empty text"); + throw new ArgumentException("Text cannot be null or whitespace", nameof(text)); + } + + var result = await GenerateBatchAsync(new[] { text }, cancellationToken); + if (!result.TryGetValue(text, out var vector)) + { + _logger.LogError("Embedding generation failed for text: {Text}", text); + throw new InvalidOperationException($"Failed to generate embedding for: {text}"); + } + + return vector; + } + + /// + public async Task> GenerateBatchAsync( + IEnumerable texts, + CancellationToken cancellationToken) + { + var textList = texts.ToList(); + if (textList.Count == 0) + { + _logger.LogDebug("Empty text batch provided to GenerateBatchAsync; returning empty dictionary"); + return new Dictionary(); + } + + try + { + _logger.LogInformation( + "Generating embeddings for {Count} texts via deployment {Deployment}", + textList.Count, + _deploymentName); + + // Build request + var requestBody = new + { + input = textList + }; + var jsonRequest = JsonSerializer.Serialize(requestBody); + var httpContent = new StringContent(jsonRequest, Encoding.UTF8, "application/json"); + + // Build URL + var url = $"{_endpoint.TrimEnd('/')}/openai/deployments/{_deploymentName}/embeddings?api-version=2024-02-15-preview"; + + // Create request with auth header + var request = new HttpRequestMessage(HttpMethod.Post, url) + { + Content = httpContent + }; + request.Headers.Add("api-key", _apiKey); + + // Call Azure OpenAI + var startTime = DateTime.UtcNow; + using var response = await _httpClient.SendAsync(request, cancellationToken); + var elapsed = DateTime.UtcNow - startTime; + + if (!response.IsSuccessStatusCode) + { + var errorContent = await response.Content.ReadAsStringAsync(cancellationToken); + _logger.LogError( + "Azure OpenAI embedding request failed: {StatusCode} {Reason}\nDetails: {ErrorContent}", + response.StatusCode, + response.ReasonPhrase, + errorContent); + throw new HttpRequestException( + $"Azure OpenAI embedding failed: {response.StatusCode} {response.ReasonPhrase}"); + } + + // Parse response + var responseContent = await response.Content.ReadAsStringAsync(cancellationToken); + using var jsonDoc = JsonDocument.Parse(responseContent); + var root = jsonDoc.RootElement; + + var result = new Dictionary(StringComparer.Ordinal); + + if (root.TryGetProperty("data", out var dataArray)) + { + var embeddingIndex = 0; + foreach (var item in dataArray.EnumerateArray()) + { + if (item.TryGetProperty("embedding", out var embeddingProp) && + item.TryGetProperty("index", out var indexProp)) + { + var index = indexProp.GetInt32(); + if (index >= 0 && index < textList.Count) + { + var vector = embeddingProp + .EnumerateArray() + .Select(e => e.GetSingle()) + .ToArray(); + + result[textList[index]] = vector; + embeddingIndex++; + } + } + } + } + + _logger.LogInformation( + "Successfully generated {Count} embeddings in {ElapsedMs}ms via {Deployment}", + result.Count, + elapsed.TotalMilliseconds, + _deploymentName); + + return result; + } + catch (OperationCanceledException) + { + _logger.LogWarning("Embedding generation was cancelled"); + throw; + } + catch (HttpRequestException ex) + { + _logger.LogError(ex, "HTTP error during embedding generation"); + throw; + } + catch (Exception ex) + { + _logger.LogError(ex, "Unexpected error during embedding generation"); + throw; + } + } +} diff --git a/src/Infrastructure.AgentFramework/Embeddings/Embedding.cs b/src/Infrastructure.AgentFramework/Embeddings/Embedding.cs new file mode 100644 index 0000000..04b6758 --- /dev/null +++ b/src/Infrastructure.AgentFramework/Embeddings/Embedding.cs @@ -0,0 +1,59 @@ +namespace Goodtocode.AgentFramework.Infrastructure.AgentFramework.Embeddings; + +/// +/// Immutable value type representing a single embedding vector and its metadata. +/// Used as the data transfer object between embedding generation, storage, and retrieval. +/// Not persisted directly; EF Core maps this to IntentEmbeddingEntity. +/// +public sealed record Embedding +{ + /// + /// Unique identifier for this embedding (primary key in storage). + /// Generated as a new Guid when creating an embedding for storage. + /// + public Guid Id { get; init; } + + /// + /// Associated intent name (e.g., "CreatePlaybook", "QueryPipelines"). + /// Used to group embeddings by intent and for filtering during search. + /// + public string IntentName { get; init; } = null!; + + /// + /// Categorization of the embedding source: Example, Description, or ToolMetadata. + /// Enables filtering and weighted scoring during similarity search. + /// + public EmbeddingSource Source { get; init; } + + /// + /// Original text from which the embedding was generated. + /// Preserved for reference, logging, and user-facing display of match context. + /// Example: "create a new playbook" or "Creates a playbook for evaluation". + /// + public string SourceText { get; init; } = null!; + + /// + /// Vector array (typically 1536-dim for text-embedding-3-small). + /// Serialized/deserialized by the backend store (SQL VECTOR, Azure Blob, etc.). + /// + public float[] Vector { get; init; } = null!; + + /// + /// Weight applied during similarity scoring (default 1.0). + /// Examples: 1.0 (highest), Descriptions: 0.8, ToolMetadata: 0.6. + /// Allows prioritizing certain embedding sources over others. + /// + public float Weight { get; init; } = 1.0f; + + /// + /// Timestamp when the embedding was created (UTC). + /// Used for auditing and determining embedding freshness. + /// + public DateTime CreatedAtUtc { get; init; } = DateTime.UtcNow; + + /// + /// Timestamp when the embedding was last updated (UTC). + /// Used for auditing and cache invalidation. + /// + public DateTime UpdatedAtUtc { get; init; } = DateTime.UtcNow; +} diff --git a/src/Infrastructure.AgentFramework/Embeddings/EmbeddingMatch.cs b/src/Infrastructure.AgentFramework/Embeddings/EmbeddingMatch.cs new file mode 100644 index 0000000..e15576c --- /dev/null +++ b/src/Infrastructure.AgentFramework/Embeddings/EmbeddingMatch.cs @@ -0,0 +1,35 @@ +namespace Goodtocode.AgentFramework.Infrastructure.AgentFramework.Embeddings; + +/// +/// Immutable value type representing a single search result from semantic similarity matching. +/// Returned by IIntentEmbeddingStore.SearchAsync to communicate which intent matched +/// and with what confidence level. +/// +public sealed record EmbeddingMatch +{ + /// + /// Intent name that matched the search query. + /// Used by the classifier to identify which intent definition to return. + /// + public string IntentName { get; init; } = null!; + + /// + /// Source of the matched embedding (Example, Description, ToolMetadata). + /// Provides context about the reliability/weight of the match. + /// + public EmbeddingSource Source { get; init; } + + /// + /// The original text associated with the matched embedding. + /// Useful for logging, debugging, and explaining to users why a match occurred. + /// Example: "create a new playbook" (the exact example that matched). + /// + public string SourceText { get; init; } = null!; + + /// + /// Cosine similarity score between the query vector and this embedding's vector. + /// Range: 0.0 (completely dissimilar) to 1.0 (identical). + /// Typically compared against a threshold (e.g., 0.75) to determine match acceptance. + /// + public float SimilarityScore { get; init; } +} diff --git a/src/Infrastructure.AgentFramework/Embeddings/EmbeddingSource.cs b/src/Infrastructure.AgentFramework/Embeddings/EmbeddingSource.cs new file mode 100644 index 0000000..83b2095 --- /dev/null +++ b/src/Infrastructure.AgentFramework/Embeddings/EmbeddingSource.cs @@ -0,0 +1,27 @@ +namespace Goodtocode.AgentFramework.Infrastructure.AgentFramework.Embeddings; + +/// +/// Categorizes the origin/type of an embedding vector. +/// Used to distinguish between user-authored examples, system descriptions, and derived metadata, +/// allowing weighted scoring and filtering during semantic search. +/// +public enum EmbeddingSource +{ + /// + /// Embedding generated from a user-written example phrasing (e.g., from IntentDefinition.Examples). + /// Weight: 1.0 (highest priority; users wrote these). + /// + Example = 0, + + /// + /// Embedding generated from a tool or intent description (e.g., from IntentDefinition.Description). + /// Weight: 0.8 (lower priority; tool authors wrote these; less user-centric). + /// + Description = 1, + + /// + /// Embedding generated from tool metadata, field names, or derived context. + /// Weight: 0.6 (lowest priority; system-generated; less reliable). + /// + ToolMetadata = 2 +} diff --git a/src/Infrastructure.AgentFramework/Embeddings/IEmbeddingGenerator.cs b/src/Infrastructure.AgentFramework/Embeddings/IEmbeddingGenerator.cs new file mode 100644 index 0000000..09a87ba --- /dev/null +++ b/src/Infrastructure.AgentFramework/Embeddings/IEmbeddingGenerator.cs @@ -0,0 +1,34 @@ +namespace Goodtocode.AgentFramework.Infrastructure.AgentFramework.Embeddings; + +/// +/// Abstracts embedding vector generation from a specific model/provider. +/// Implementations: Azure OpenAI, OpenAI public API, local models via Ollama, etc. +/// +public interface IEmbeddingGenerator +{ + /// + /// Generates an embedding vector for a single text string. + /// Prefer for multiple texts (more efficient). + /// + /// Text to embed (e.g., "create a new playbook"). + /// Cancellation token for async operations. + /// Float array of length . + Task GenerateAsync(string text, CancellationToken cancellationToken); + + /// + /// Generates embedding vectors for multiple texts in a single call. + /// More efficient than multiple calls; + /// API providers often allow batching to reduce round-trips. + /// + /// Enumerable of texts to embed. + /// Cancellation token for async operations. + /// Dictionary mapping input text → generated vector. Empty if inputs empty. + Task> GenerateBatchAsync(IEnumerable texts, CancellationToken cancellationToken); + + /// + /// Dimension of generated vectors. + /// Standard value for text-embedding-3-small: 1536. + /// Used for validation and storage allocation. + /// + int Dimension { get; } +} diff --git a/src/Infrastructure.AgentFramework/Embeddings/IIntentEmbeddingStore.cs b/src/Infrastructure.AgentFramework/Embeddings/IIntentEmbeddingStore.cs new file mode 100644 index 0000000..8506f3a --- /dev/null +++ b/src/Infrastructure.AgentFramework/Embeddings/IIntentEmbeddingStore.cs @@ -0,0 +1,56 @@ +namespace Goodtocode.AgentFramework.Infrastructure.AgentFramework.Embeddings; + +/// +/// Abstraction for intent embedding storage and semantic similarity search. +/// Backend-agnostic: implementations can use SQL Server VECTOR, Azure AI Search, Cosmos DB vCore, etc. +/// Enables future migration without changing routing logic. +/// +public interface IIntentEmbeddingStore +{ + /// + /// Stores a batch of embeddings for a specific intent. + /// Typically called during seeding (Seed.StateOfYour) or app startup (IHostedService). + /// Replaces any existing embeddings for the same intent (idempotent upsert). + /// + /// Intent name (e.g., "CreatePlaybook"). + /// Collection of embeddings to store. + /// Cancellation token. + Task UpsertIntentEmbeddingsAsync( + string intentName, + IEnumerable embeddings, + CancellationToken cancellationToken); + + /// + /// Semantic search: finds embeddings similar to the query vector. + /// Used by SemanticIntentClassifier to classify incoming user messages. + /// + /// Embedding of the user's message (generated by IEmbeddingGenerator). + /// Cancellation token. + /// Maximum number of results to return (default 5). + /// Minimum similarity score to accept (default 0.75). + /// + /// List of EmbeddingMatch results sorted by similarity (descending). + /// Empty if no matches above threshold. + /// + Task> SearchAsync( + float[] queryVector, + CancellationToken cancellationToken, + int topK = 5, + float similarityThreshold = 0.75f); + + /// + /// Deletes all embeddings for a specific intent. + /// Used during reseeding or when an intent is removed. + /// + /// Intent name to delete embeddings for. + /// Cancellation token. + Task DeleteIntentEmbeddingsAsync(string intentName, CancellationToken cancellationToken); + + /// + /// Health check: verifies embeddings are available and queryable. + /// Returns true if store is ready; false if empty or unavailable. + /// Used by IntentEmbeddingInitializationService (quick-start) to avoid regenerating. + /// + /// Cancellation token. + Task IsReadyAsync(CancellationToken cancellationToken); +} diff --git a/src/Infrastructure.AgentFramework/Embeddings/IntentEmbeddingInitializationService.cs b/src/Infrastructure.AgentFramework/Embeddings/IntentEmbeddingInitializationService.cs new file mode 100644 index 0000000..1c16ca9 --- /dev/null +++ b/src/Infrastructure.AgentFramework/Embeddings/IntentEmbeddingInitializationService.cs @@ -0,0 +1,81 @@ +using Goodtocode.AgentFramework.Infrastructure.AgentFramework.Intents; +using Goodtocode.AgentFramework.Infrastructure.AgentFramework.Options; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Goodtocode.AgentFramework.Infrastructure.AgentFramework.Embeddings; + +/// +/// Seeds intent example embeddings once at startup when semantic classification is enabled. +/// +public sealed class IntentEmbeddingInitializationService( + IServiceScopeFactory scopeFactory, + IOptions options, + ILogger logger) : IHostedService +{ + public async Task StartAsync(CancellationToken cancellationToken) + { + if (!options.Value.EnableSemantic) + { + logger.LogInformation("Semantic intent classification is disabled; skipping embedding initialization."); + return; + } + + try + { + await using var scope = scopeFactory.CreateAsyncScope(); + var store = scope.ServiceProvider.GetRequiredService(); + if (await store.IsReadyAsync(cancellationToken)) + { + logger.LogInformation("Intent embedding store is already initialized; skipping seeding."); + return; + } + + var catalog = scope.ServiceProvider.GetRequiredService(); + var generator = scope.ServiceProvider.GetRequiredService(); + + foreach (var intent in catalog.Intents) + { + var examples = intent.Examples + .Where(example => !string.IsNullOrWhiteSpace(example)) + .Distinct(StringComparer.Ordinal) + .ToArray(); + if (examples.Length == 0) + { + continue; + } + + var vectors = await generator.GenerateBatchAsync(examples, cancellationToken); + var embeddings = examples + .Where(vectors.ContainsKey) + .Select(example => new Embedding + { + Id = Guid.NewGuid(), + IntentName = intent.Name, + Source = EmbeddingSource.Example, + SourceText = example, + Vector = vectors[example], + Weight = 1f, + CreatedAtUtc = DateTime.UtcNow, + UpdatedAtUtc = DateTime.UtcNow + }); + + await store.UpsertIntentEmbeddingsAsync(intent.Name, embeddings, cancellationToken); + } + + logger.LogInformation("Intent embedding initialization completed for {IntentCount} intents.", catalog.Intents.Count); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception exception) + { + logger.LogError(exception, "Intent embedding initialization failed; continuing with rule-based classification."); + } + } + + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; +} diff --git a/src/Infrastructure.AgentFramework/Intents/DefaultIntentCatalogFactory.cs b/src/Infrastructure.AgentFramework/Intents/DefaultIntentCatalogFactory.cs index 343a83f..fbc3a44 100644 --- a/src/Infrastructure.AgentFramework/Intents/DefaultIntentCatalogFactory.cs +++ b/src/Infrastructure.AgentFramework/Intents/DefaultIntentCatalogFactory.cs @@ -75,6 +75,14 @@ public static class DefaultIntentCatalogFactory "show my message history" ]), + new IntentDefinition(IntentNames.QueryChatMessagesForSession, Examples: [], + Captures: + [ + new PhraseCapture("show messages for chat session ", "sessionId", CaptureKind.GuidDFormat), + new PhraseCapture("conversation history for chat session ", "sessionId", CaptureKind.GuidDFormat), + new PhraseCapture("messages in chat session ", "sessionId", CaptureKind.GuidDFormat) + ]), + // Level 4 deterministic routing for web search: guarantee the "search the web for [query]" // phrasing never falls through to the LLM's own tool-selection. The Capture extracts the // query part so it can be passed to the search tool. diff --git a/src/Infrastructure.AgentFramework/Intents/HybridIntentClassifier.cs b/src/Infrastructure.AgentFramework/Intents/HybridIntentClassifier.cs new file mode 100644 index 0000000..7d4b8b3 --- /dev/null +++ b/src/Infrastructure.AgentFramework/Intents/HybridIntentClassifier.cs @@ -0,0 +1,42 @@ +using Goodtocode.AgentFramework.Infrastructure.AgentFramework.Options; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Goodtocode.AgentFramework.Infrastructure.AgentFramework.Intents; + +/// +/// Runs deterministic classification first and semantic classification only when rules do not match. +/// +public sealed class HybridIntentClassifier( + RuleIntentClassifier ruleClassifier, + SemanticIntentClassifier semanticClassifier, + IOptions options, + ILogger logger) : IIntentClassifier +{ + public async Task ClassifyAsync( + string message, + IReadOnlyList? priorUserMessages = null, + CancellationToken cancellationToken = default) + { + var ruleMatch = await ruleClassifier.ClassifyAsync(message, priorUserMessages, cancellationToken); + if (ruleMatch is not null || !options.Value.EnableSemantic) + { + return ruleMatch; + } + + logger.LogDebug("No rule intent match; attempting semantic classification."); + try + { + return await semanticClassifier.ClassifyAsync(message, priorUserMessages, cancellationToken); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception exception) + { + logger.LogWarning(exception, "Semantic intent classification failed; falling back to the agent."); + return null; + } + } +} diff --git a/src/Infrastructure.AgentFramework/Intents/IIntentClassifier.cs b/src/Infrastructure.AgentFramework/Intents/IIntentClassifier.cs index 93c2915..49433df 100644 --- a/src/Infrastructure.AgentFramework/Intents/IIntentClassifier.cs +++ b/src/Infrastructure.AgentFramework/Intents/IIntentClassifier.cs @@ -10,5 +10,8 @@ namespace Goodtocode.AgentFramework.Infrastructure.AgentFramework.Intents; public interface IIntentClassifier { /// Attempts to classify against the registered and prior conversation context. - IntentMatch? Classify(string message, IReadOnlyList? priorUserMessages = null); + Task ClassifyAsync( + string message, + IReadOnlyList? priorUserMessages = null, + CancellationToken cancellationToken = default); } diff --git a/src/Infrastructure.AgentFramework/Intents/IntentNames.cs b/src/Infrastructure.AgentFramework/Intents/IntentNames.cs index 2d86826..7cdc912 100644 --- a/src/Infrastructure.AgentFramework/Intents/IntentNames.cs +++ b/src/Infrastructure.AgentFramework/Intents/IntentNames.cs @@ -9,6 +9,7 @@ public static class IntentNames { public const string QueryChatSessionsList = nameof(QueryChatSessionsList); public const string QueryChatMessagesList = nameof(QueryChatMessagesList); + public const string QueryChatMessagesForSession = nameof(QueryChatMessagesForSession); public const string QueryActorById = nameof(QueryActorById); public const string QueryActorsByName = nameof(QueryActorsByName); public const string QueryActorsList = nameof(QueryActorsList); diff --git a/src/Infrastructure.AgentFramework/Intents/RuleIntentClassifier.cs b/src/Infrastructure.AgentFramework/Intents/RuleIntentClassifier.cs index 55aa850..418b926 100644 --- a/src/Infrastructure.AgentFramework/Intents/RuleIntentClassifier.cs +++ b/src/Infrastructure.AgentFramework/Intents/RuleIntentClassifier.cs @@ -11,11 +11,14 @@ public sealed class RuleIntentClassifier(IntentCatalog catalog) : IIntentClassif { private readonly IntentCatalog _catalog = catalog; - public IntentMatch? Classify(string message, IReadOnlyList? priorUserMessages = null) + public Task ClassifyAsync( + string message, + IReadOnlyList? priorUserMessages = null, + CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(message)) { - return null; + return Task.FromResult(null); } foreach (var intent in _catalog.Intents) @@ -29,7 +32,7 @@ public sealed class RuleIntentClassifier(IntentCatalog catalog) : IIntentClassif { if (capture.TryMatch(message, out var value)) { - return new IntentMatch(intent, new Dictionary { [capture.CaptureName] = value }); + return Task.FromResult(new IntentMatch(intent, new Dictionary { [capture.CaptureName] = value })); } } } @@ -41,7 +44,7 @@ public sealed class RuleIntentClassifier(IntentCatalog catalog) : IIntentClassif { if (normalized.Contains(example, StringComparison.Ordinal)) { - return new IntentMatch(intent); + return Task.FromResult(new IntentMatch(intent)); } } } @@ -53,14 +56,14 @@ public sealed class RuleIntentClassifier(IntentCatalog catalog) : IIntentClassif { if (intent.FollowUpExamples?.Any(example => priorMessage.Trim().Equals(example, StringComparison.OrdinalIgnoreCase)) == true) { - return new IntentMatch(intent, new Dictionary + return Task.FromResult(new IntentMatch(intent, new Dictionary { ["followUp"] = message.Trim() - }); + })); } } } - return null; + return Task.FromResult(null); } } diff --git a/src/Infrastructure.AgentFramework/Intents/SemanticIntentClassifier.cs b/src/Infrastructure.AgentFramework/Intents/SemanticIntentClassifier.cs new file mode 100644 index 0000000..08cb661 --- /dev/null +++ b/src/Infrastructure.AgentFramework/Intents/SemanticIntentClassifier.cs @@ -0,0 +1,54 @@ +using Goodtocode.AgentFramework.Infrastructure.AgentFramework.Embeddings; +using Goodtocode.AgentFramework.Infrastructure.AgentFramework.Options; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Goodtocode.AgentFramework.Infrastructure.AgentFramework.Intents; + +/// +/// Classifies messages by comparing their embedding to seeded intent examples. +/// +public sealed class SemanticIntentClassifier( + IntentCatalog catalog, + IEmbeddingGenerator embeddingGenerator, + IIntentEmbeddingStore embeddingStore, + IOptions options, + ILogger logger) : IIntentClassifier +{ + public async Task ClassifyAsync( + string message, + IReadOnlyList? priorUserMessages = null, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(message)) + { + return null; + } + + var queryVector = await embeddingGenerator.GenerateAsync(message, cancellationToken); + var matches = await embeddingStore.SearchAsync( + queryVector, + cancellationToken, + options.Value.TopKResults, + options.Value.SemanticThreshold); + + foreach (var match in matches) + { + var intent = catalog.Intents.FirstOrDefault( + definition => definition.Name.Equals(match.IntentName, StringComparison.Ordinal)); + + if (intent is null || intent.Captures is { Count: > 0 }) + { + continue; + } + + logger.LogInformation( + "Semantic intent match resolved for {IntentName} with score {SimilarityScore}", + intent.Name, + match.SimilarityScore); + return new IntentMatch(intent); + } + + return null; + } +} diff --git a/src/Infrastructure.AgentFramework/Options/AzureOpenAIOptions.cs b/src/Infrastructure.AgentFramework/Options/AzureOpenAIOptions.cs index f9eb541..c0e9ae6 100644 --- a/src/Infrastructure.AgentFramework/Options/AzureOpenAIOptions.cs +++ b/src/Infrastructure.AgentFramework/Options/AzureOpenAIOptions.cs @@ -12,6 +12,9 @@ public sealed class AzureOpenAIOptions [Required] public string ChatDeploymentName { get; set; } = string.Empty; + /// Azure OpenAI deployment used to generate semantic intent embeddings. + public string EmbeddingDeploymentName { get; set; } = "embedding-fast"; + [Required] public string Endpoint { get; set; } = string.Empty; diff --git a/src/Infrastructure.AgentFramework/Options/IntentClassificationOptions.cs b/src/Infrastructure.AgentFramework/Options/IntentClassificationOptions.cs new file mode 100644 index 0000000..3d87f5c --- /dev/null +++ b/src/Infrastructure.AgentFramework/Options/IntentClassificationOptions.cs @@ -0,0 +1,22 @@ +using System.ComponentModel.DataAnnotations; + +namespace Goodtocode.AgentFramework.Infrastructure.AgentFramework.Options; + +/// +/// Controls semantic intent classification behavior. +/// +public sealed class IntentClassificationOptions +{ + public const string SectionName = "IntentClassification"; + + /// Whether semantic fallback is enabled. + public bool EnableSemantic { get; set; } + + /// Minimum weighted similarity required for a semantic match. + [Range(0, 1)] + public float SemanticThreshold { get; set; } = 0.75f; + + /// Maximum number of embedding matches requested from the store. + [Range(1, 100)] + public int TopKResults { get; set; } = 5; +} diff --git a/src/Infrastructure.SqlServer/ConfigureServices.cs b/src/Infrastructure.SqlServer/ConfigureServices.cs index 6ad9e0b..5b62ef6 100644 --- a/src/Infrastructure.SqlServer/ConfigureServices.cs +++ b/src/Infrastructure.SqlServer/ConfigureServices.cs @@ -1,4 +1,5 @@ using Goodtocode.AgentFramework.Core.Application.Abstractions; +using Goodtocode.AgentFramework.Infrastructure.AgentFramework.Embeddings; using Goodtocode.AgentFramework.Infrastructure.SqlServer.Persistence; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -16,6 +17,10 @@ public static IServiceCollection AddDbContextServices(this IServiceCollection se .UseLazyLoadingProxies()); services.AddScoped(); + services.AddHttpClient(); + services.AddScoped(provider => + provider.GetRequiredService()); + services.AddScoped(); return services; } diff --git a/src/Infrastructure.SqlServer/Embeddings/SqlIntentEmbeddingStore.cs b/src/Infrastructure.SqlServer/Embeddings/SqlIntentEmbeddingStore.cs new file mode 100644 index 0000000..ddea45f --- /dev/null +++ b/src/Infrastructure.SqlServer/Embeddings/SqlIntentEmbeddingStore.cs @@ -0,0 +1,271 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Goodtocode.AgentFramework.Infrastructure.AgentFramework.Embeddings; +using Goodtocode.AgentFramework.Infrastructure.SqlServer.Persistence; +using Goodtocode.AgentFramework.Infrastructure.SqlServer.Persistence.Entities; + +namespace Goodtocode.AgentFramework.Infrastructure.SqlServer.Embeddings; + +/// +/// SQL Server implementation of IIntentEmbeddingStore. +/// Stores embeddings in [Chat].[IntentEmbeddings] table and performs cosine similarity search. +/// For ~2500 embeddings, brute-force similarity computation is acceptable (<100ms). +/// +public sealed class SqlIntentEmbeddingStore : IIntentEmbeddingStore +{ + private readonly AgentFrameworkContext _context; + private readonly ILogger _logger; + + /// + /// Creates a new instance of the SQL Server embedding store. + /// + /// EF Core database context. + /// Structured logger. + public SqlIntentEmbeddingStore( + AgentFrameworkContext context, + ILogger logger) + { + _context = context ?? throw new ArgumentNullException(nameof(context)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + public async Task UpsertIntentEmbeddingsAsync( + string intentName, + IEnumerable embeddings, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(intentName)) + throw new ArgumentException("Intent name cannot be null or whitespace", nameof(intentName)); + + var embeddingsList = embeddings.ToList(); + if (embeddingsList.Count == 0) + { + _logger.LogWarning("No embeddings provided for intent {IntentName}", intentName); + return; + } + + try + { + // Delete existing embeddings for this intent + var existing = await _context.IntentEmbeddings + .Where(e => e.IntentName == intentName) + .ToListAsync(cancellationToken); + + if (existing.Count > 0) + { + _context.IntentEmbeddings.RemoveRange(existing); + _logger.LogInformation( + "Deleted {Count} existing embeddings for intent {IntentName}", + existing.Count, + intentName); + } + + // Insert new embeddings + var entities = embeddingsList.Select(e => new IntentEmbeddingEntity + { + Id = Guid.NewGuid(), + IntentName = intentName, + Source = (int)e.Source, + SourceText = e.SourceText, + Vector = e.Vector, + Weight = e.Weight, + CreatedAtUtc = DateTime.UtcNow, + UpdatedAtUtc = DateTime.UtcNow + }).ToList(); + + await _context.IntentEmbeddings.AddRangeAsync(entities, cancellationToken); + await _context.SaveChangesAsync(cancellationToken); + + _logger.LogInformation( + "Upserted {Count} embeddings for intent {IntentName}", + entities.Count, + intentName); + } + catch (OperationCanceledException) + { + _logger.LogWarning("Upsert operation was cancelled for intent {IntentName}", intentName); + throw; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error upserting embeddings for intent {IntentName}", intentName); + throw; + } + } + + /// + public async Task> SearchAsync( + float[] queryVector, + CancellationToken cancellationToken, + int topK = 5, + float similarityThreshold = 0.75f) + { + if (queryVector == null || queryVector.Length == 0) + throw new ArgumentException("Query vector cannot be null or empty", nameof(queryVector)); + + if (topK <= 0) + throw new ArgumentException("topK must be > 0", nameof(topK)); + + if (similarityThreshold < 0 || similarityThreshold > 1) + throw new ArgumentException("similarityThreshold must be between 0 and 1", nameof(similarityThreshold)); + + try + { + // Load all embeddings (brute-force acceptable for ~2500 embeddings) + var allEmbeddings = await _context.IntentEmbeddings + .AsNoTracking() + .ToListAsync(cancellationToken); + + if (allEmbeddings.Count == 0) + { + _logger.LogDebug("No embeddings found in store"); + return Array.Empty(); + } + + // Compute cosine similarity for each embedding + var matches = new List<(EmbeddingMatch match, float score)>(); + + foreach (var entity in allEmbeddings) + { + if (entity.Vector == null || entity.Vector.Length == 0) + continue; + + var similarity = CosineSimilarity(queryVector, entity.Vector); + + // Apply weight + var weightedScore = similarity * entity.Weight; + + if (weightedScore >= similarityThreshold) + { + var match = new EmbeddingMatch + { + IntentName = entity.IntentName, + Source = (EmbeddingSource)entity.Source, + SourceText = entity.SourceText, + SimilarityScore = weightedScore + }; + matches.Add((match, weightedScore)); + } + } + + // Sort by similarity descending and take top K + var results = matches + .OrderByDescending(m => m.score) + .Take(topK) + .Select(m => m.match) + .ToList(); + + _logger.LogInformation( + "Semantic search found {Count} matches (threshold={Threshold}, topK={TopK})", + results.Count, + similarityThreshold, + topK); + + return results; + } + catch (OperationCanceledException) + { + _logger.LogWarning("Search operation was cancelled"); + throw; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error during semantic search"); + throw; + } + } + + /// + public async Task DeleteIntentEmbeddingsAsync(string intentName, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(intentName)) + throw new ArgumentException("Intent name cannot be null or whitespace", nameof(intentName)); + + try + { + var existing = await _context.IntentEmbeddings + .Where(e => e.IntentName == intentName) + .ToListAsync(cancellationToken); + + _context.IntentEmbeddings.RemoveRange(existing); + await _context.SaveChangesAsync(cancellationToken); + var count = existing.Count; + + _logger.LogInformation("Deleted {Count} embeddings for intent {IntentName}", count, intentName); + } + catch (OperationCanceledException) + { + _logger.LogWarning("Delete operation was cancelled for intent {IntentName}", intentName); + throw; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error deleting embeddings for intent {IntentName}", intentName); + throw; + } + } + + /// + public async Task IsReadyAsync(CancellationToken cancellationToken) + { + try + { + var count = await _context.IntentEmbeddings + .AsNoTracking() + .CountAsync(cancellationToken); + + var isReady = count > 0; + _logger.LogInformation("Embedding store ready check: {Count} embeddings found", count); + return isReady; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error checking if embedding store is ready"); + return false; + } + } + + /// + /// Computes cosine similarity between two vectors. + /// Formula: (a · b) / (||a|| * ||b||) + /// Result range: -1.0 to 1.0 (typically 0.0 to 1.0 for text embeddings). + /// + /// First vector (typically query). + /// Second vector (typically stored embedding). + /// Cosine similarity score. + private static float CosineSimilarity(float[] a, float[] b) + { + if (a.Length != b.Length) + throw new ArgumentException("Vectors must have the same dimension", nameof(b)); + + if (a.Length == 0) + return 0f; + + // Compute dot product + double dotProduct = 0; + for (int i = 0; i < a.Length; i++) + { + dotProduct += a[i] * b[i]; + } + + // Compute magnitudes + double magnitudeA = 0; + double magnitudeB = 0; + for (int i = 0; i < a.Length; i++) + { + magnitudeA += a[i] * a[i]; + magnitudeB += b[i] * b[i]; + } + + magnitudeA = Math.Sqrt(magnitudeA); + magnitudeB = Math.Sqrt(magnitudeB); + + // Avoid division by zero + if (magnitudeA == 0 || magnitudeB == 0) + return 0f; + + // Return normalized dot product + return (float)(dotProduct / (magnitudeA * magnitudeB)); + } +} diff --git a/src/Infrastructure.SqlServer/Infrastructure.SqlServer.csproj b/src/Infrastructure.SqlServer/Infrastructure.SqlServer.csproj index 4d6c558..c078ef3 100644 --- a/src/Infrastructure.SqlServer/Infrastructure.SqlServer.csproj +++ b/src/Infrastructure.SqlServer/Infrastructure.SqlServer.csproj @@ -1,4 +1,4 @@ - + Goodtocode.AgentFramework.Infrastructure.SqlServer Goodtocode.AgentFramework.Infrastructure.SqlServer @@ -26,5 +26,6 @@ + \ No newline at end of file diff --git a/src/Infrastructure.SqlServer/Migrations/20260903055155_InitialCreate-AgentFrameworkContext.Designer.cs b/src/Infrastructure.SqlServer/Migrations/20260906234719_InitialCreate-AgentFrameworkContext.Designer.cs similarity index 84% rename from src/Infrastructure.SqlServer/Migrations/20260903055155_InitialCreate-AgentFrameworkContext.Designer.cs rename to src/Infrastructure.SqlServer/Migrations/20260906234719_InitialCreate-AgentFrameworkContext.Designer.cs index 5ccc6e8..ea22250 100644 --- a/src/Infrastructure.SqlServer/Migrations/20260903055155_InitialCreate-AgentFrameworkContext.Designer.cs +++ b/src/Infrastructure.SqlServer/Migrations/20260906234719_InitialCreate-AgentFrameworkContext.Designer.cs @@ -12,7 +12,7 @@ namespace Goodtocode.AgentFramework.Infrastructure.SqlServer.Migrations { [DbContext(typeof(AgentFrameworkContext))] - [Migration("20260903055155_InitialCreate-AgentFrameworkContext")] + [Migration("20260906234719_InitialCreate-AgentFrameworkContext")] partial class InitialCreateAgentFrameworkContext { /// @@ -338,6 +338,64 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.ToTable("ChatGovernance", "Chat"); }); + modelBuilder.Entity("Goodtocode.AgentFramework.Infrastructure.SqlServer.Persistence.Entities.IntentEmbeddingEntity", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier") + .HasColumnName("Id"); + + b.Property("CreatedAtUtc") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2") + .HasColumnName("CreatedAtUtc") + .HasDefaultValueSql("GETUTCDATE()"); + + b.Property("IntentName") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("nvarchar(256)") + .HasColumnName("IntentName") + .UseCollation("SQL_Latin1_General_CP1_CI_AS"); + + b.Property("Source") + .ValueGeneratedOnAdd() + .HasColumnType("int") + .HasDefaultValue(0) + .HasColumnName("Source"); + + b.Property("SourceText") + .IsRequired() + .HasColumnType("NVARCHAR(MAX)") + .HasColumnName("SourceText"); + + b.Property("UpdatedAtUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("datetime2") + .HasColumnName("UpdatedAtUtc") + .HasDefaultValueSql("GETUTCDATE()"); + + b.Property("Vector") + .IsRequired() + .HasColumnType("NVARCHAR(MAX)") + .HasColumnName("Vector"); + + b.Property("Weight") + .ValueGeneratedOnAdd() + .HasColumnType("REAL") + .HasDefaultValue(1f) + .HasColumnName("Weight"); + + b.HasKey("Id"); + + b.HasIndex("IntentName") + .HasDatabaseName("IX_IntentEmbeddings_IntentName"); + + b.HasIndex("IntentName", "Source") + .HasDatabaseName("IX_IntentEmbeddings_IntentName_Source"); + + b.ToTable("IntentEmbeddings", "Chat"); + }); + modelBuilder.Entity("Goodtocode.AgentFramework.Core.Domain.Chats.ChatMessageEntity", b => { b.HasOne("Goodtocode.AgentFramework.Core.Domain.Chats.ChatSessionEntity", "ChatSession") diff --git a/src/Infrastructure.SqlServer/Migrations/20260903055155_InitialCreate-AgentFrameworkContext.cs b/src/Infrastructure.SqlServer/Migrations/20260906234719_InitialCreate-AgentFrameworkContext.cs similarity index 86% rename from src/Infrastructure.SqlServer/Migrations/20260903055155_InitialCreate-AgentFrameworkContext.cs rename to src/Infrastructure.SqlServer/Migrations/20260906234719_InitialCreate-AgentFrameworkContext.cs index 8b442a4..d699f44 100644 --- a/src/Infrastructure.SqlServer/Migrations/20260903055155_InitialCreate-AgentFrameworkContext.cs +++ b/src/Infrastructure.SqlServer/Migrations/20260906234719_InitialCreate-AgentFrameworkContext.cs @@ -108,6 +108,25 @@ protected override void Up(MigrationBuilder migrationBuilder) .Annotation("SqlServer:Clustered", false); }); + migrationBuilder.CreateTable( + name: "IntentEmbeddings", + schema: "Chat", + columns: table => new + { + Id = table.Column(type: "uniqueidentifier", nullable: false), + IntentName = table.Column(type: "nvarchar(256)", maxLength: 256, nullable: false, collation: "SQL_Latin1_General_CP1_CI_AS"), + Source = table.Column(type: "int", nullable: false, defaultValue: 0), + SourceText = table.Column(type: "NVARCHAR(MAX)", nullable: false), + Vector = table.Column(type: "NVARCHAR(MAX)", nullable: false), + Weight = table.Column(type: "REAL", nullable: false, defaultValue: 1f), + CreatedAtUtc = table.Column(type: "datetime2", nullable: false, defaultValueSql: "GETUTCDATE()"), + UpdatedAtUtc = table.Column(type: "datetime2", nullable: false, defaultValueSql: "GETUTCDATE()") + }, + constraints: table => + { + table.PrimaryKey("PK_IntentEmbeddings", x => x.Id); + }); + migrationBuilder.CreateTable( name: "ChatMessages", schema: "Chat", @@ -191,6 +210,18 @@ protected override void Up(MigrationBuilder migrationBuilder) column: "Timestamp", unique: true) .Annotation("SqlServer:Clustered", true); + + migrationBuilder.CreateIndex( + name: "IX_IntentEmbeddings_IntentName", + schema: "Chat", + table: "IntentEmbeddings", + column: "IntentName"); + + migrationBuilder.CreateIndex( + name: "IX_IntentEmbeddings_IntentName_Source", + schema: "Chat", + table: "IntentEmbeddings", + columns: new[] { "IntentName", "Source" }); } /// @@ -208,6 +239,10 @@ protected override void Down(MigrationBuilder migrationBuilder) name: "ChatMessages", schema: "Chat"); + migrationBuilder.DropTable( + name: "IntentEmbeddings", + schema: "Chat"); + migrationBuilder.DropTable( name: "ChatSessions", schema: "Chat"); diff --git a/src/Infrastructure.SqlServer/Migrations/AgentFrameworkContextModelSnapshot.cs b/src/Infrastructure.SqlServer/Migrations/AgentFrameworkContextModelSnapshot.cs index ec0c137..067d27c 100644 --- a/src/Infrastructure.SqlServer/Migrations/AgentFrameworkContextModelSnapshot.cs +++ b/src/Infrastructure.SqlServer/Migrations/AgentFrameworkContextModelSnapshot.cs @@ -335,6 +335,64 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("ChatGovernance", "Chat"); }); + modelBuilder.Entity("Goodtocode.AgentFramework.Infrastructure.SqlServer.Persistence.Entities.IntentEmbeddingEntity", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier") + .HasColumnName("Id"); + + b.Property("CreatedAtUtc") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2") + .HasColumnName("CreatedAtUtc") + .HasDefaultValueSql("GETUTCDATE()"); + + b.Property("IntentName") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("nvarchar(256)") + .HasColumnName("IntentName") + .UseCollation("SQL_Latin1_General_CP1_CI_AS"); + + b.Property("Source") + .ValueGeneratedOnAdd() + .HasColumnType("int") + .HasDefaultValue(0) + .HasColumnName("Source"); + + b.Property("SourceText") + .IsRequired() + .HasColumnType("NVARCHAR(MAX)") + .HasColumnName("SourceText"); + + b.Property("UpdatedAtUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("datetime2") + .HasColumnName("UpdatedAtUtc") + .HasDefaultValueSql("GETUTCDATE()"); + + b.Property("Vector") + .IsRequired() + .HasColumnType("NVARCHAR(MAX)") + .HasColumnName("Vector"); + + b.Property("Weight") + .ValueGeneratedOnAdd() + .HasColumnType("REAL") + .HasDefaultValue(1f) + .HasColumnName("Weight"); + + b.HasKey("Id"); + + b.HasIndex("IntentName") + .HasDatabaseName("IX_IntentEmbeddings_IntentName"); + + b.HasIndex("IntentName", "Source") + .HasDatabaseName("IX_IntentEmbeddings_IntentName_Source"); + + b.ToTable("IntentEmbeddings", "Chat"); + }); + modelBuilder.Entity("Goodtocode.AgentFramework.Core.Domain.Chats.ChatMessageEntity", b => { b.HasOne("Goodtocode.AgentFramework.Core.Domain.Chats.ChatSessionEntity", "ChatSession") diff --git a/src/Infrastructure.SqlServer/Persistence/AgentFrameworkContext.cs b/src/Infrastructure.SqlServer/Persistence/AgentFrameworkContext.cs index 45d8e3b..f7bd8df 100644 --- a/src/Infrastructure.SqlServer/Persistence/AgentFrameworkContext.cs +++ b/src/Infrastructure.SqlServer/Persistence/AgentFrameworkContext.cs @@ -3,6 +3,7 @@ using Goodtocode.AgentFramework.Core.Domain.Actors; using Goodtocode.AgentFramework.Core.Domain.Chats; using Goodtocode.AgentFramework.Core.Domain.Governance; +using Goodtocode.AgentFramework.Infrastructure.SqlServer.Persistence.Entities; using Microsoft.EntityFrameworkCore.ChangeTracking; namespace Goodtocode.AgentFramework.Infrastructure.SqlServer.Persistence; @@ -15,6 +16,7 @@ public class AgentFrameworkContext : DbContext, IAgentFrameworkContext public DbSet ChatSessions => Set(); public DbSet Actors => Set(); public DbSet ChatGovernance => Set(); + public DbSet IntentEmbeddings => Set(); protected AgentFrameworkContext() { } diff --git a/src/Infrastructure.SqlServer/Persistence/Configurations/IntentEmbeddingConfiguration.cs b/src/Infrastructure.SqlServer/Persistence/Configurations/IntentEmbeddingConfiguration.cs new file mode 100644 index 0000000..00738ce --- /dev/null +++ b/src/Infrastructure.SqlServer/Persistence/Configurations/IntentEmbeddingConfiguration.cs @@ -0,0 +1,103 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.ChangeTracking; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using Goodtocode.AgentFramework.Infrastructure.SqlServer.Persistence.Entities; +using System.Text.Json; + +namespace Goodtocode.AgentFramework.Infrastructure.SqlServer.Persistence.Configurations; + +/// +/// EF Core entity configuration for . +/// Defines table mapping, constraints, indexes, and value converters. +/// +public sealed class IntentEmbeddingConfiguration : IEntityTypeConfiguration +{ + private static readonly ValueComparer VectorComparer = new( + (left, right) => left != null && right != null && left.SequenceEqual(right), + vector => vector.Aggregate(0, (hash, value) => HashCode.Combine(hash, value)), + vector => vector.ToArray()); + + private static string SerializeVector(float[] vector) + { + return JsonSerializer.Serialize(vector); + } + + private static float[] DeserializeVector(string json) + { + return JsonSerializer.Deserialize(json) ?? Array.Empty(); + } + + /// + /// Configures the IntentEmbeddingEntity for EF Core. + /// Table: [Chat].[IntentEmbeddings] + /// + public void Configure(EntityTypeBuilder builder) + { + // Table & Schema + builder.ToTable("IntentEmbeddings", "Chat"); + + // Primary Key + builder.HasKey(e => e.Id); + + // Properties + builder.Property(e => e.Id) + .ValueGeneratedNever(); + + builder.Property(e => e.IntentName) + .IsRequired() + .HasMaxLength(256) + .UseCollation("SQL_Latin1_General_CP1_CI_AS"); + + builder.Property(e => e.Source) + .IsRequired() + .HasDefaultValue(0); // Example = 0 + + builder.Property(e => e.SourceText) + .IsRequired() + .HasColumnType("NVARCHAR(MAX)"); + + builder.Property(e => e.Vector) + .IsRequired() + .Metadata.SetValueComparer(VectorComparer); + + builder.Property(e => e.Vector) + .HasConversion( + // To database: serialize float[] as JSON string + v => SerializeVector(v), + // From database: deserialize JSON string to float[] + v => DeserializeVector(v)) + .HasColumnType("NVARCHAR(MAX)"); + + builder.Property(e => e.Weight) + .IsRequired() + .HasDefaultValue(1.0f) + .HasColumnType("REAL"); + + builder.Property(e => e.CreatedAtUtc) + .IsRequired() + .ValueGeneratedOnAdd() + .HasDefaultValueSql("GETUTCDATE()"); + + builder.Property(e => e.UpdatedAtUtc) + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasDefaultValueSql("GETUTCDATE()"); + + // Indexes + builder.HasIndex(e => e.IntentName) + .HasDatabaseName("IX_IntentEmbeddings_IntentName"); + + builder.HasIndex(e => new { e.IntentName, e.Source }) + .HasDatabaseName("IX_IntentEmbeddings_IntentName_Source"); + + // Column mappings + builder.Property(e => e.Id).HasColumnName("Id"); + builder.Property(e => e.IntentName).HasColumnName("IntentName"); + builder.Property(e => e.Source).HasColumnName("Source"); + builder.Property(e => e.SourceText).HasColumnName("SourceText"); + builder.Property(e => e.Vector).HasColumnName("Vector"); + builder.Property(e => e.Weight).HasColumnName("Weight"); + builder.Property(e => e.CreatedAtUtc).HasColumnName("CreatedAtUtc"); + builder.Property(e => e.UpdatedAtUtc).HasColumnName("UpdatedAtUtc"); + } +} diff --git a/src/Infrastructure.SqlServer/Persistence/Entities/IntentEmbeddingEntity.cs b/src/Infrastructure.SqlServer/Persistence/Entities/IntentEmbeddingEntity.cs new file mode 100644 index 0000000..252aa7b --- /dev/null +++ b/src/Infrastructure.SqlServer/Persistence/Entities/IntentEmbeddingEntity.cs @@ -0,0 +1,66 @@ +namespace Goodtocode.AgentFramework.Infrastructure.SqlServer.Persistence.Entities; + +/// +/// EF Core entity representing a single intent embedding vector stored in SQL Server. +/// Maps to the [Chat].[IntentEmbeddings] table. +/// Immutable after creation (used for storage/retrieval only; mutations via SemanticIntentClassifier abstraction). +/// +public class IntentEmbeddingEntity +{ + /// + /// Creates an embedding persistence entity. + /// + public IntentEmbeddingEntity() + { + } + + /// + /// Unique identifier (primary key). + /// Generated as Guid.NewGuid() when creating an embedding for storage. + /// + public virtual Guid Id { get; set; } + + /// + /// Associated intent name (e.g., "CreatePlaybook"). + /// Used for grouping and filtering during search. + /// Foreign key concept, though no explicit constraint (loose coupling to IntentDefinition). + /// + public virtual string IntentName { get; set; } = null!; + + /// + /// Categorization of embedding source (Example=0, Description=1, ToolMetadata=2). + /// Stored as integer for efficiency; mapped from EmbeddingSource enum. + /// + public virtual int Source { get; set; } + + /// + /// Original text from which the embedding was generated. + /// Preserved for reference, logging, and user-facing display of match context. + /// + public virtual string SourceText { get; set; } = null!; + + /// + /// Embedding vector (1536-dim for text-embedding-3-small). + /// Stored as float array, serialized/deserialized by EF Core value converter. + /// + public virtual float[] Vector { get; set; } = null!; + + /// + /// Weight applied during similarity scoring (default 1.0). + /// Enables prioritizing certain embedding sources over others. + /// Stored as float in database for future SIMD scoring. + /// + public virtual float Weight { get; set; } = 1.0f; + + /// + /// Timestamp when embedding was created (UTC). + /// Set by EF Core on insert; never updated. + /// + public virtual DateTime CreatedAtUtc { get; set; } + + /// + /// Timestamp when embedding was last updated (UTC). + /// Updated by EF Core on upsert operations. + /// + public virtual DateTime UpdatedAtUtc { get; set; } +} diff --git a/src/Presentation.Api/ConfigureServices.cs b/src/Presentation.Api/ConfigureServices.cs index 8faacf5..07211a4 100644 --- a/src/Presentation.Api/ConfigureServices.cs +++ b/src/Presentation.Api/ConfigureServices.cs @@ -113,7 +113,7 @@ private static OpenApiInfo CreateVersionInfo(ApiVersionDescription description) { var info = new OpenApiInfo { - Title = $"GoodToCode Application API ({Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT")})", + Title = $"Quick-start for Microsoft Agent Framework API ({Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT")})", Version = description.ApiVersion.ToString(), Description = "An API to interact with this application", Contact = new OpenApiContact diff --git a/src/Presentation.Api/appsettings.Development.json b/src/Presentation.Api/appsettings.Development.json index 38f9644..5139310 100644 --- a/src/Presentation.Api/appsettings.Development.json +++ b/src/Presentation.Api/appsettings.Development.json @@ -33,7 +33,8 @@ "Endpoint": "http://localhost:11434/api" }, "AzureOpenAI": { - "ChatDeploymentName": "openai-fast", // aka. gpt-5.4-mini + "ChatDeploymentName": "openai-fast", + "EmbeddingDeploymentName": "embedding-fast", "Endpoint": "https://your-resource.openai.azure.com", "ApiKey": "" }, @@ -74,5 +75,10 @@ { "ToolName": "ActorsTool", "Order": 30, "Instructions": "Always call ActorsTool for actor or user-profile lookups instead of guessing identity details." }, { "ToolName": "WebSearchTool", "Order": 40, "Instructions": "Call WebSearchTool for current public-web information that is not available from chat sessions or actor data." } ] + }, + "IntentClassification": { + "EnableSemantic": false, + "SemanticThreshold": 0.75, + "TopKResults": 5 } } \ No newline at end of file diff --git a/src/Presentation.Api/appsettings.Production.json b/src/Presentation.Api/appsettings.Production.json index 8668b7a..a9bc0f1 100644 --- a/src/Presentation.Api/appsettings.Production.json +++ b/src/Presentation.Api/appsettings.Production.json @@ -33,7 +33,8 @@ "Endpoint": "http://localhost:11434/api" }, "AzureOpenAI": { - "ChatDeploymentName": "openai-chat", // aka. gpt-5.4 + "ChatDeploymentName": "openai-chat", + "EmbeddingDeploymentName": "embedding-fast", "Endpoint": "https://your-resource.openai.azure.com", "ApiKey": "" }, @@ -74,5 +75,10 @@ { "ToolName": "ActorsTool", "Order": 30, "Instructions": "Always call ActorsTool for actor or user-profile lookups instead of guessing identity details." }, { "ToolName": "WebSearchTool", "Order": 40, "Instructions": "Call WebSearchTool for current public-web information that is not available from chat sessions or actor data." } ] + }, + "IntentClassification": { + "EnableSemantic": false, + "SemanticThreshold": 0.75, + "TopKResults": 5 } } \ No newline at end of file diff --git a/src/Presentation.Api/appsettings.json b/src/Presentation.Api/appsettings.json index ebfb585..be38b56 100644 --- a/src/Presentation.Api/appsettings.json +++ b/src/Presentation.Api/appsettings.json @@ -29,5 +29,10 @@ "Instructions": "Call WebSearchTool for current public-web information that is not available from chat sessions or actor data." } ] + }, + "IntentClassification": { + "EnableSemantic": false, + "SemanticThreshold": 0.75, + "TopKResults": 5 } } diff --git a/src/Presentation.Api/appsettings.local.json b/src/Presentation.Api/appsettings.local.json index 4b1464e..d9421be 100644 --- a/src/Presentation.Api/appsettings.local.json +++ b/src/Presentation.Api/appsettings.local.json @@ -33,7 +33,8 @@ "Endpoint": "http://localhost:11434/api" }, "AzureOpenAI": { - "ChatDeploymentName": "openai-fast", // aka. gpt-5.4-mini + "ChatDeploymentName": "openai-fast", + "EmbeddingDeploymentName": "embedding-fast", "Endpoint": "https://your-resource.openai.azure.com/openai/v1", "ApiKey": "" }, @@ -74,5 +75,10 @@ { "ToolName": "ActorsTool", "Order": 30, "Instructions": "Always call ActorsTool for actor or user-profile lookups instead of guessing identity details." }, { "ToolName": "WebSearchTool", "Order": 40, "Instructions": "Call WebSearchTool for current public-web information that is not available from chat sessions or actor data." } ] + }, + "IntentClassification": { + "EnableSemantic": false, + "SemanticThreshold": 0.75, + "TopKResults": 5 } } \ No newline at end of file diff --git a/src/Presentation.Web/Features/HomePage.razor b/src/Presentation.Web/Features/HomePage.razor index 673884b..3668439 100644 --- a/src/Presentation.Web/Features/HomePage.razor +++ b/src/Presentation.Web/Features/HomePage.razor @@ -4,7 +4,7 @@ @attribute [AllowAnonymous] -GoodToCode Application +Quick-start for Microsoft Agent Framework @@ -13,7 +13,7 @@ - GoodToCode Application + Quick-start for Microsoft Agent Framework diff --git a/src/Presentation.Web/Shell/App.razor b/src/Presentation.Web/Shell/App.razor index ad876f1..a9a4e61 100644 --- a/src/Presentation.Web/Shell/App.razor +++ b/src/Presentation.Web/Shell/App.razor @@ -10,19 +10,19 @@ - + - - + + - - + + diff --git a/src/Presentation.Web/Shell/Layout/MainLayout.razor b/src/Presentation.Web/Shell/Layout/MainLayout.razor index b81d2e6..f1bee2a 100644 --- a/src/Presentation.Web/Shell/Layout/MainLayout.razor +++ b/src/Presentation.Web/Shell/Layout/MainLayout.razor @@ -19,7 +19,7 @@ GoodToCode - GoodToCode Application + Quick-start for Microsoft Agent Framework diff --git a/src/Tests.Integration/AgentFramework/AzureOpenAiEmbeddingGeneratorTests.cs b/src/Tests.Integration/AgentFramework/AzureOpenAiEmbeddingGeneratorTests.cs new file mode 100644 index 0000000..20e70ec --- /dev/null +++ b/src/Tests.Integration/AgentFramework/AzureOpenAiEmbeddingGeneratorTests.cs @@ -0,0 +1,104 @@ +using System.Net; +using System.Text; +using Goodtocode.AgentFramework.Infrastructure.AgentFramework.Embeddings; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Goodtocode.AgentFramework.Infrastructure.AgentFramework.Options; + +namespace Goodtocode.AgentFramework.Tests.Integration.AgentFramework; + +[TestClass] +public sealed class AzureOpenAiEmbeddingGeneratorTests +{ + [TestMethod] + public void DimensionReturns1536() + { + var generator = CreateGenerator(new QueueHandler()); + + Assert.AreEqual(1536, generator.Dimension); + } + + [TestMethod] + public async Task GenerateBatchAsyncReturnsEmptyForEmptyInput() + { + var generator = CreateGenerator(new QueueHandler()); + + var result = await generator.GenerateBatchAsync(Array.Empty(), CancellationToken.None); + + Assert.AreEqual(0, result.Count); + } + + [TestMethod] + public async Task GenerateAsyncReturnsVectorFromAzureResponse() + { + var generator = CreateGenerator(new QueueHandler(CreateResponse("[0.1, 0.2, 0.3]", 0))); + + var result = await generator.GenerateAsync("create a playbook", CancellationToken.None); + + CollectionAssert.AreEqual(new[] { 0.1f, 0.2f, 0.3f }, result); + } + + [TestMethod] + public async Task GenerateBatchAsyncMapsResponsesByInputIndex() + { + var handler = new QueueHandler(CreateBatchResponse()); + var generator = CreateGenerator(handler); + + var result = await generator.GenerateBatchAsync(new[] { "first", "second" }, CancellationToken.None); + + CollectionAssert.AreEqual(new[] { 0.1f }, result["first"]); + CollectionAssert.AreEqual(new[] { 0.2f }, result["second"]); + Assert.AreEqual(1, handler.SendCount); + } + + private static AzureOpenAiEmbeddingGenerator CreateGenerator(HttpMessageHandler handler) + { + return new AzureOpenAiEmbeddingGenerator( + Options.Create(new AzureOpenAIOptions + { + ApiKey = "test-key", + Endpoint = "https://example.test", + EmbeddingDeploymentName = "embedding-fast" + }), + NullLogger.Instance, + new HttpClient(handler)); + } + + private static HttpResponseMessage CreateResponse(string embedding, int index) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent( + $"{{\"data\":[{{\"index\":{index},\"embedding\":{embedding}}}]}}", + Encoding.UTF8, + "application/json") + }; + } + + private static HttpResponseMessage CreateBatchResponse() + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent( + "{\"data\":[{\"index\":0,\"embedding\":[0.1]},{\"index\":1,\"embedding\":[0.2]}]}", + Encoding.UTF8, + "application/json") + }; + } + + private sealed class QueueHandler(params HttpResponseMessage[] responses) : HttpMessageHandler + { + private readonly Queue responses = new(responses); + + public int SendCount { get; private set; } + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + SendCount++; + return Task.FromResult(responses.Dequeue()); + } + } +} diff --git a/src/Tests.Integration/AgentFramework/IntentClassifierTests.cs b/src/Tests.Integration/AgentFramework/IntentClassifierTests.cs new file mode 100644 index 0000000..64bc043 --- /dev/null +++ b/src/Tests.Integration/AgentFramework/IntentClassifierTests.cs @@ -0,0 +1,115 @@ +using Goodtocode.AgentFramework.Infrastructure.AgentFramework.Embeddings; +using Goodtocode.AgentFramework.Infrastructure.AgentFramework.Intents; +using Goodtocode.AgentFramework.Infrastructure.AgentFramework.Options; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; + +namespace Goodtocode.AgentFramework.Tests.Integration.AgentFramework; + +[TestClass] +public sealed class IntentClassifierTests +{ + [TestMethod] + public async Task SemanticClassifierReturnsCatalogIntentForMatchingEmbedding() + { + var classifier = CreateSemanticClassifier(new EmbeddingMatch + { + IntentName = "list-actors", + Source = EmbeddingSource.Example, + SourceText = "show actors", + SimilarityScore = 0.9f + }); + + var result = await classifier.ClassifyAsync("could you show me the people", cancellationToken: CancellationToken.None); + + Assert.IsNotNull(result); + Assert.AreEqual("list-actors", result.Intent.Name); + } + + [TestMethod] + public async Task HybridClassifierUsesRuleMatchBeforeSemanticFallback() + { + var catalog = new IntentCatalog([new IntentDefinition("list-actors", ["show actors"])]); + var rule = new RuleIntentClassifier(catalog); + var semantic = CreateSemanticClassifier(null); + var hybrid = new HybridIntentClassifier( + rule, + semantic, + Options.Create(new IntentClassificationOptions { EnableSemantic = true }), + NullLogger.Instance); + + var result = await hybrid.ClassifyAsync("show actors", cancellationToken: CancellationToken.None); + + Assert.IsNotNull(result); + Assert.AreEqual("list-actors", result.Intent.Name); + } + + [TestMethod] + public async Task HybridClassifierSkipsSemanticWhenDisabled() + { + var catalog = new IntentCatalog([new IntentDefinition("list-actors", ["show actors"])]); + var hybrid = new HybridIntentClassifier( + new RuleIntentClassifier(catalog), + CreateSemanticClassifier(new EmbeddingMatch + { + IntentName = "list-actors", + Source = EmbeddingSource.Example, + SourceText = "show actors", + SimilarityScore = 0.9f + }), + Options.Create(new IntentClassificationOptions { EnableSemantic = false }), + NullLogger.Instance); + + var result = await hybrid.ClassifyAsync("unmatched wording", cancellationToken: CancellationToken.None); + + Assert.IsNull(result); + } + + [TestMethod] + public async Task DefaultCatalogRoutesPerSessionMessagePromptsWithSessionCapture() + { + var sessionId = Guid.NewGuid(); + var classifier = new RuleIntentClassifier(DefaultIntentCatalogFactory.Create()); + + foreach (var prompt in new[] + { + $"Show messages for chat session {sessionId:D}", + $"Can you pull up the conversation history for chat session {sessionId:D}?" + }) + { + var result = await classifier.ClassifyAsync(prompt); + + Assert.IsNotNull(result); + Assert.AreEqual(IntentNames.QueryChatMessagesForSession, result!.Intent.Name); + Assert.AreEqual(sessionId.ToString("D"), result.Captures!["sessionId"]); + } + } + + private static SemanticIntentClassifier CreateSemanticClassifier(EmbeddingMatch? match) + { + var catalog = new IntentCatalog([new IntentDefinition("list-actors", ["show actors"])]); + return new SemanticIntentClassifier( + catalog, + new FakeEmbeddingGenerator(), + new FakeEmbeddingStore(match), + Options.Create(new IntentClassificationOptions { EnableSemantic = true }), + NullLogger.Instance); + } + + private sealed class FakeEmbeddingGenerator : IEmbeddingGenerator + { + public int Dimension => 2; + public Task GenerateAsync(string text, CancellationToken cancellationToken) => Task.FromResult(new[] { 1f, 0f }); + public Task> GenerateBatchAsync(IEnumerable texts, CancellationToken cancellationToken) => + Task.FromResult>(texts.ToDictionary(text => text, _ => new[] { 1f, 0f })); + } + + private sealed class FakeEmbeddingStore(EmbeddingMatch? match) : IIntentEmbeddingStore + { + public Task UpsertIntentEmbeddingsAsync(string intentName, IEnumerable embeddings, CancellationToken cancellationToken) => Task.CompletedTask; + public Task> SearchAsync(float[] queryVector, CancellationToken cancellationToken, int topK = 5, float similarityThreshold = 0.75f) => + Task.FromResult>(match is null ? [] : [match]); + public Task DeleteIntentEmbeddingsAsync(string intentName, CancellationToken cancellationToken) => Task.CompletedTask; + public Task IsReadyAsync(CancellationToken cancellationToken) => Task.FromResult(true); + } +} diff --git a/src/Tests.Integration/AgentFramework/SemanticIntentClassification.feature b/src/Tests.Integration/AgentFramework/SemanticIntentClassification.feature new file mode 100644 index 0000000..d1aa101 --- /dev/null +++ b/src/Tests.Integration/AgentFramework/SemanticIntentClassification.feature @@ -0,0 +1,16 @@ +@semanticIntentClassification +Feature: Semantic intent classification +As a chat routing system +I classify unfamiliar phrasing only when semantic classification is enabled + +Scenario Outline: Hybrid classification honors the semantic feature switch + Given the semantic classifier switch is "" + And the catalog contains intent "list-actors" with example "show actors" + And the semantic example "show actors" is seeded for intent "list-actors" + When I classify the message "" + Then the classified intent is "" + +Examples: + | enabled | message | expected | + | false | could you show me the people | none | + | true | could you show me the people | list-actors | diff --git a/src/Tests.Integration/AgentFramework/SemanticIntentClassification.feature.cs b/src/Tests.Integration/AgentFramework/SemanticIntentClassification.feature.cs new file mode 100644 index 0000000..cd17eee --- /dev/null +++ b/src/Tests.Integration/AgentFramework/SemanticIntentClassification.feature.cs @@ -0,0 +1,175 @@ +// ------------------------------------------------------------------------------ +// +// This code was generated by Reqnroll (https://reqnroll.net/). +// Reqnroll Version:3.0.0.0 +// Reqnroll Generator Version:3.0.0.0 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +// ------------------------------------------------------------------------------ +#region Designer generated code +#pragma warning disable +using Reqnroll; +namespace Goodtocode.AgentFramework.Tests.Integration.AgentFramework +{ + + + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Reqnroll", "3.0.0.0")] + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + [global::Microsoft.VisualStudio.TestTools.UnitTesting.TestClassAttribute()] + public partial class SemanticIntentClassificationFeature + { + + private global::Reqnroll.ITestRunner testRunner; + + private Microsoft.VisualStudio.TestTools.UnitTesting.TestContext _testContext; + + private static string[] featureTags = new string[] { + "semanticIntentClassification"}; + + private static global::Reqnroll.FeatureInfo featureInfo = new global::Reqnroll.FeatureInfo(new global::System.Globalization.CultureInfo("en-US"), "AgentFramework", "Semantic intent classification", "As a chat routing system\r\nI classify unfamiliar phrasing only when semantic class" + + "ification is enabled", global::Reqnroll.ProgrammingLanguage.CSharp, featureTags, InitializeCucumberMessages()); + +#line 1 "SemanticIntentClassification.feature" +#line hidden + + public virtual Microsoft.VisualStudio.TestTools.UnitTesting.TestContext TestContext + { + get + { + return this._testContext; + } + set + { + this._testContext = value; + } + } + + [global::Microsoft.VisualStudio.TestTools.UnitTesting.ClassInitializeAttribute()] + public static async global::System.Threading.Tasks.Task FeatureSetupAsync(Microsoft.VisualStudio.TestTools.UnitTesting.TestContext testContext) + { + } + + [global::Microsoft.VisualStudio.TestTools.UnitTesting.ClassCleanupAttribute()] + public static async global::System.Threading.Tasks.Task FeatureTearDownAsync() + { + await global::Reqnroll.TestRunnerManager.ReleaseFeatureAsync(featureInfo); + } + + [global::Microsoft.VisualStudio.TestTools.UnitTesting.TestInitializeAttribute()] + public async global::System.Threading.Tasks.Task TestInitializeAsync() + { + testRunner = global::Reqnroll.TestRunnerManager.GetTestRunnerForAssembly(featureHint: featureInfo); + try + { + if (((testRunner.FeatureContext != null) + && (testRunner.FeatureContext.FeatureInfo.Equals(featureInfo) == false))) + { + await testRunner.OnFeatureEndAsync(); + } + } + finally + { + if (((testRunner.FeatureContext != null) + && testRunner.FeatureContext.BeforeFeatureHookFailed)) + { + throw new global::Reqnroll.ReqnrollException("Scenario skipped because of previous before feature hook error"); + } + if ((testRunner.FeatureContext == null)) + { + await testRunner.OnFeatureStartAsync(featureInfo); + } + } + } + + [global::Microsoft.VisualStudio.TestTools.UnitTesting.TestCleanupAttribute()] + public async global::System.Threading.Tasks.Task TestTearDownAsync() + { + if ((testRunner == null)) + { + return; + } + try + { + await testRunner.OnScenarioEndAsync(); + } + finally + { + global::Reqnroll.TestRunnerManager.ReleaseTestRunner(testRunner); + testRunner = null; + } + } + + public void ScenarioInitialize(global::Reqnroll.ScenarioInfo scenarioInfo, global::Reqnroll.RuleInfo ruleInfo) + { + testRunner.OnScenarioInitialize(scenarioInfo, ruleInfo); + testRunner.ScenarioContext.ScenarioContainer.RegisterInstanceAs(_testContext); + } + + public async global::System.Threading.Tasks.Task ScenarioStartAsync() + { + await testRunner.OnScenarioStartAsync(); + } + + public async global::System.Threading.Tasks.Task ScenarioCleanupAsync() + { + await testRunner.CollectScenarioErrorsAsync(); + } + + private static global::Reqnroll.Formatters.RuntimeSupport.FeatureLevelCucumberMessages InitializeCucumberMessages() + { + return new global::Reqnroll.Formatters.RuntimeSupport.FeatureLevelCucumberMessages("AgentFramework/SemanticIntentClassification.feature.ndjson", 4); + } + + [global::Microsoft.VisualStudio.TestTools.UnitTesting.TestMethodAttribute(callerLineNumber: 6, DisplayName="Hybrid classification honors the semantic feature switch")] + [global::Microsoft.VisualStudio.TestTools.UnitTesting.DescriptionAttribute("Hybrid classification honors the semantic feature switch")] + [global::Microsoft.VisualStudio.TestTools.UnitTesting.TestPropertyAttribute("FeatureTitle", "Semantic intent classification")] + [global::Microsoft.VisualStudio.TestTools.UnitTesting.TestCategoryAttribute("semanticIntentClassification")] + [global::Microsoft.VisualStudio.TestTools.UnitTesting.DataRowAttribute("false", "could you show me the people", "none", "0", null, DisplayName="Hybrid classification honors the semantic feature switch(false,could you show me " + + "the people,none,0)")] + [global::Microsoft.VisualStudio.TestTools.UnitTesting.DataRowAttribute("true", "could you show me the people", "list-actors", "1", null, DisplayName="Hybrid classification honors the semantic feature switch(true,could you show me t" + + "he people,list-actors,1)")] + public async global::System.Threading.Tasks.Task HybridClassificationHonorsTheSemanticFeatureSwitch(string enabled, string message, string expected, string @__pickleIndex, string[] exampleTags) + { + string[] tagsOfScenario = exampleTags; + global::System.Collections.Specialized.OrderedDictionary argumentsOfScenario = new global::System.Collections.Specialized.OrderedDictionary(); + argumentsOfScenario.Add("enabled", enabled); + argumentsOfScenario.Add("message", message); + argumentsOfScenario.Add("expected", expected); + string pickleIndex = @__pickleIndex; + global::Reqnroll.ScenarioInfo scenarioInfo = new global::Reqnroll.ScenarioInfo("Hybrid classification honors the semantic feature switch", null, tagsOfScenario, argumentsOfScenario, featureTags, pickleIndex); + string[] tagsOfRule = ((string[])(null)); + global::Reqnroll.RuleInfo ruleInfo = null; +#line 6 +this.ScenarioInitialize(scenarioInfo, ruleInfo); +#line hidden + if ((global::Reqnroll.TagHelper.ContainsIgnoreTag(scenarioInfo.CombinedTags) || global::Reqnroll.TagHelper.ContainsIgnoreTag(featureTags))) + { + await testRunner.SkipScenarioAsync(); + } + else + { + await this.ScenarioStartAsync(); +#line 7 + await testRunner.GivenAsync(string.Format("the semantic classifier switch is \"{0}\"", enabled), ((string)(null)), ((global::Reqnroll.Table)(null)), "Given "); +#line hidden +#line 8 + await testRunner.AndAsync("the catalog contains intent \"list-actors\" with example \"show actors\"", ((string)(null)), ((global::Reqnroll.Table)(null)), "And "); +#line hidden +#line 9 + await testRunner.AndAsync("the semantic example \"show actors\" is seeded for intent \"list-actors\"", ((string)(null)), ((global::Reqnroll.Table)(null)), "And "); +#line hidden +#line 10 + await testRunner.WhenAsync(string.Format("I classify the message \"{0}\"", message), ((string)(null)), ((global::Reqnroll.Table)(null)), "When "); +#line hidden +#line 11 + await testRunner.ThenAsync(string.Format("the classified intent is \"{0}\"", expected), ((string)(null)), ((global::Reqnroll.Table)(null)), "Then "); +#line hidden + } + await this.ScenarioCleanupAsync(); + } + } +} +#pragma warning restore +#endregion diff --git a/src/Tests.Integration/AgentFramework/SemanticIntentClassificationStepDefinitions.cs b/src/Tests.Integration/AgentFramework/SemanticIntentClassificationStepDefinitions.cs new file mode 100644 index 0000000..fcfc741 --- /dev/null +++ b/src/Tests.Integration/AgentFramework/SemanticIntentClassificationStepDefinitions.cs @@ -0,0 +1,90 @@ +using Goodtocode.AgentFramework.Infrastructure.AgentFramework.Embeddings; +using Goodtocode.AgentFramework.Infrastructure.AgentFramework.Intents; +using Goodtocode.AgentFramework.Infrastructure.AgentFramework.Options; +using Goodtocode.AgentFramework.Infrastructure.SqlServer.Embeddings; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; + +namespace Goodtocode.AgentFramework.Tests.Integration.AgentFramework; + +[Binding] +[Scope(Tag = "semanticIntentClassification")] +public sealed class SemanticIntentClassificationStepDefinitions : TestBase +{ + private bool _enabled; + private IntentCatalog _catalog = new([]); + private HybridIntentClassifier? _classifier; + private IntentMatch? _result; + + [Given("the semantic classifier switch is \"(.*)\"")] + public void GivenTheSemanticClassifierSwitchIs(string enabled) + { + _enabled = bool.Parse(enabled); + } + + [Given("the catalog contains intent \"(.*)\" with example \"(.*)\"")] + public void GivenTheCatalogContainsIntentWithExample(string intentName, string example) + { + _catalog = new IntentCatalog([new IntentDefinition(intentName, [example])]); + } + + [Given("the semantic example \"(.*)\" is seeded for intent \"(.*)\"")] + public async Task GivenTheSemanticExampleIsSeededForIntent(string example, string intentName) + { + var store = new SqlIntentEmbeddingStore(context, NullLogger.Instance); + await store.UpsertIntentEmbeddingsAsync(intentName, [new Embedding + { + Id = Guid.NewGuid(), + IntentName = intentName, + Source = EmbeddingSource.Example, + SourceText = example, + Vector = [1f, 0f], + Weight = 1f, + CreatedAtUtc = DateTime.UtcNow, + UpdatedAtUtc = DateTime.UtcNow + }], CancellationToken.None); + + _classifier = new HybridIntentClassifier( + new RuleIntentClassifier(_catalog), + new SemanticIntentClassifier( + _catalog, + new FixedEmbeddingGenerator(), + store, + Options.Create(new IntentClassificationOptions + { + EnableSemantic = _enabled, + SemanticThreshold = 0.75f, + TopKResults = 5 + }), + NullLogger.Instance), + Options.Create(new IntentClassificationOptions { EnableSemantic = _enabled }), + NullLogger.Instance); + } + + [When("I classify the message \"(.*)\"")] + public async Task WhenIClassifyTheMessage(string message) + { + _result = await _classifier!.ClassifyAsync(message, cancellationToken: CancellationToken.None); + } + + [Then("the classified intent is \"(.*)\"")] + public void ThenTheClassifiedIntentIs(string expected) + { + if (expected.Equals("none", StringComparison.OrdinalIgnoreCase)) + { + Assert.IsNull(_result); + return; + } + + Assert.IsNotNull(_result); + Assert.AreEqual(expected, _result!.Intent.Name); + } + + private sealed class FixedEmbeddingGenerator : IEmbeddingGenerator + { + public int Dimension => 2; + public Task GenerateAsync(string text, CancellationToken cancellationToken) => Task.FromResult(new[] { 1f, 0f }); + public Task> GenerateBatchAsync(IEnumerable texts, CancellationToken cancellationToken) => + Task.FromResult>(texts.ToDictionary(text => text, _ => new[] { 1f, 0f })); + } +} diff --git a/src/Tests.Integration/Infrastructure/SqlIntentEmbeddingStoreTests.cs b/src/Tests.Integration/Infrastructure/SqlIntentEmbeddingStoreTests.cs new file mode 100644 index 0000000..f1974f3 --- /dev/null +++ b/src/Tests.Integration/Infrastructure/SqlIntentEmbeddingStoreTests.cs @@ -0,0 +1,83 @@ +using Goodtocode.AgentFramework.Core.Application.Common.Auth; +using Goodtocode.AgentFramework.Infrastructure.AgentFramework.Embeddings; +using Goodtocode.AgentFramework.Infrastructure.SqlServer.Embeddings; +using Goodtocode.AgentFramework.Infrastructure.SqlServer.Persistence; +using Goodtocode.AgentFramework.Infrastructure.SqlServer.Persistence.Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Goodtocode.AgentFramework.Tests.Integration.Infrastructure; + +[TestClass] +public sealed class SqlIntentEmbeddingStoreTests +{ + [TestMethod] + public async Task UpsertStoresAndReplacesEmbeddings() + { + await using var context = CreateContext(); + var store = CreateStore(context); + + await store.UpsertIntentEmbeddingsAsync("CreatePlaybook", CreateEmbeddings(), CancellationToken.None); + await store.UpsertIntentEmbeddingsAsync("CreatePlaybook", CreateEmbeddings().Take(1), CancellationToken.None); + + Assert.AreEqual(1, await context.IntentEmbeddings.CountAsync()); + } + + [TestMethod] + public async Task SearchReturnsWeightedTopMatch() + { + await using var context = CreateContext(); + var store = CreateStore(context); + await store.UpsertIntentEmbeddingsAsync("CreatePlaybook", CreateEmbeddings(), CancellationToken.None); + + var result = await store.SearchAsync(new[] { 1f, 0f }, CancellationToken.None, topK: 1, similarityThreshold: 0.5f); + + Assert.AreEqual(1, result.Count); + Assert.AreEqual("CreatePlaybook", result[0].IntentName); + Assert.AreEqual(1f, result[0].SimilarityScore, 0.001f); + } + + [TestMethod] + public async Task DeleteRemovesEmbeddingsAndReadinessReflectsStore() + { + await using var context = CreateContext(); + var store = CreateStore(context); + await store.UpsertIntentEmbeddingsAsync("CreatePlaybook", CreateEmbeddings().Take(1), CancellationToken.None); + + Assert.IsTrue(await store.IsReadyAsync(CancellationToken.None)); + await store.DeleteIntentEmbeddingsAsync("CreatePlaybook", CancellationToken.None); + + Assert.IsFalse(await store.IsReadyAsync(CancellationToken.None)); + } + + private static SqlIntentEmbeddingStore CreateStore(AgentFrameworkContext context) + => new(context, NullLogger.Instance); + + private static AgentFrameworkContext CreateContext() + { + var options = new DbContextOptionsBuilder() + .UseInMemoryDatabase(Guid.NewGuid().ToString()) + .Options; + return new AgentFrameworkContext(options, new TestRlsContext()); + } + + private static IEnumerable CreateEmbeddings() + { + yield return new Embedding + { + Id = Guid.NewGuid(), IntentName = "CreatePlaybook", Source = EmbeddingSource.Example, + SourceText = "create a playbook", Vector = new[] { 1f, 0f }, Weight = 1f + }; + yield return new Embedding + { + Id = Guid.NewGuid(), IntentName = "CreatePlaybook", Source = EmbeddingSource.Example, + SourceText = "make a playbook", Vector = new[] { 0f, 1f }, Weight = 1f + }; + } + + private sealed class TestRlsContext : IRlsContext + { + public Guid OwnerId { get; } = Guid.NewGuid(); + public Guid TenantId { get; } = Guid.NewGuid(); + } +}