diff --git a/samples/cs/GettingStarted/src/AudioTranscriptionExample/Program.cs b/samples/cs/GettingStarted/src/AudioTranscriptionExample/Program.cs index be1db5db..d47c206f 100644 --- a/samples/cs/GettingStarted/src/AudioTranscriptionExample/Program.cs +++ b/samples/cs/GettingStarted/src/AudioTranscriptionExample/Program.cs @@ -12,11 +12,8 @@ var mgr = FoundryLocalManager.Instance; -// Ensure that any Execution Provider (EP) downloads run and are completed. -// EP packages include dependencies and may be large. -// Download is only required again if a new version of the EP is released. -// For cross platform builds there is no dynamic EP download and this will return immediately. -await Utils.RunWithSpinner("Registering execution providers", mgr.EnsureEpsDownloadedAsync()); +// Download and register all execution providers. +await Utils.RunWithSpinner("Registering execution providers", mgr.DownloadAndRegisterEpsAsync()); // Get the model catalog @@ -48,6 +45,7 @@ await model.DownloadAsync(progress => // Get a chat client var audioClient = await model.GetAudioClientAsync(); +audioClient.Settings.Language = "en"; // Get a transcription with streaming outputs diff --git a/samples/cs/GettingStarted/src/FoundryLocalWebServer/Program.cs b/samples/cs/GettingStarted/src/FoundryLocalWebServer/Program.cs index f50ac1b0..91cf4442 100644 --- a/samples/cs/GettingStarted/src/FoundryLocalWebServer/Program.cs +++ b/samples/cs/GettingStarted/src/FoundryLocalWebServer/Program.cs @@ -18,11 +18,8 @@ var mgr = FoundryLocalManager.Instance; -// Ensure that any Execution Provider (EP) downloads run and are completed. -// EP packages include dependencies and may be large. -// Download is only required again if a new version of the EP is released. -// For cross platform builds there is no dynamic EP download and this will return immediately. -await Utils.RunWithSpinner("Registering execution providers", mgr.EnsureEpsDownloadedAsync()); +// Download and register all execution providers. +await Utils.RunWithSpinner("Registering execution providers", mgr.DownloadAndRegisterEpsAsync()); // Get the model catalog diff --git a/samples/cs/GettingStarted/src/HelloFoundryLocalSdk/Program.cs b/samples/cs/GettingStarted/src/HelloFoundryLocalSdk/Program.cs index 52efe410..65eb2b4b 100644 --- a/samples/cs/GettingStarted/src/HelloFoundryLocalSdk/Program.cs +++ b/samples/cs/GettingStarted/src/HelloFoundryLocalSdk/Program.cs @@ -15,11 +15,19 @@ var mgr = FoundryLocalManager.Instance; -// Ensure that any Execution Provider (EP) downloads run and are completed. +// Discover available execution providers and their registration status. +var eps = mgr.DiscoverEps(); +Console.WriteLine("Available execution providers:"); +foreach (var ep in eps) +{ + Console.WriteLine($" {ep.Name} (registered: {ep.IsRegistered})"); +} + +// Download and register all execution providers. // EP packages include dependencies and may be large. // Download is only required again if a new version of the EP is released. // For cross platform builds there is no dynamic EP download and this will return immediately. -await Utils.RunWithSpinner("Registering execution providers", mgr.EnsureEpsDownloadedAsync()); +await Utils.RunWithSpinner("Registering execution providers", mgr.DownloadAndRegisterEpsAsync()); // Get the model catalog diff --git a/samples/cs/GettingStarted/src/ModelManagementExample/Program.cs b/samples/cs/GettingStarted/src/ModelManagementExample/Program.cs index 2b6fe2e8..5b711fc5 100644 --- a/samples/cs/GettingStarted/src/ModelManagementExample/Program.cs +++ b/samples/cs/GettingStarted/src/ModelManagementExample/Program.cs @@ -16,11 +16,8 @@ var mgr = FoundryLocalManager.Instance; -// Ensure that any Execution Provider (EP) downloads run and are completed. -// EP packages include dependencies and may be large. -// Download is only required again if a new version of the EP is released. -// For cross platform builds there is no dynamic EP download and this will return immediately. -await Utils.RunWithSpinner("Registering execution providers", mgr.EnsureEpsDownloadedAsync()); +// Download and register all execution providers. +await Utils.RunWithSpinner("Registering execution providers", mgr.DownloadAndRegisterEpsAsync()); // Model catalog operations diff --git a/samples/cs/GettingStarted/src/ToolCallingFoundryLocalSdk/Program.cs b/samples/cs/GettingStarted/src/ToolCallingFoundryLocalSdk/Program.cs index 3cdf3d38..c63c0783 100644 --- a/samples/cs/GettingStarted/src/ToolCallingFoundryLocalSdk/Program.cs +++ b/samples/cs/GettingStarted/src/ToolCallingFoundryLocalSdk/Program.cs @@ -18,11 +18,8 @@ var mgr = FoundryLocalManager.Instance; -// Ensure that any Execution Provider (EP) downloads run and are completed. -// EP packages include dependencies and may be large. -// Download is only required again if a new version of the EP is released. -// For cross platform builds there is no dynamic EP download and this will return immediately. -await Utils.RunWithSpinner("Registering execution providers", mgr.EnsureEpsDownloadedAsync()); +// Download and register all execution providers. +await Utils.RunWithSpinner("Registering execution providers", mgr.DownloadAndRegisterEpsAsync()); // Get the model catalog diff --git a/samples/cs/GettingStarted/src/ToolCallingFoundryLocalWebServer/Program.cs b/samples/cs/GettingStarted/src/ToolCallingFoundryLocalWebServer/Program.cs index 6d6937fd..4909cfe7 100644 --- a/samples/cs/GettingStarted/src/ToolCallingFoundryLocalWebServer/Program.cs +++ b/samples/cs/GettingStarted/src/ToolCallingFoundryLocalWebServer/Program.cs @@ -20,11 +20,8 @@ var mgr = FoundryLocalManager.Instance; -// Ensure that any Execution Provider (EP) downloads run and are completed. -// EP packages include dependencies and may be large. -// Download is only required again if a new version of the EP is released. -// For cross platform builds there is no dynamic EP download and this will return immediately. -await Utils.RunWithSpinner("Registering execution providers", mgr.EnsureEpsDownloadedAsync()); +// Download and register all execution providers. +await Utils.RunWithSpinner("Registering execution providers", mgr.DownloadAndRegisterEpsAsync()); // Get the model catalog diff --git a/sdk/cs/README.md b/sdk/cs/README.md index f58e41e0..95ec1caa 100644 --- a/sdk/cs/README.md +++ b/sdk/cs/README.md @@ -48,7 +48,10 @@ dotnet build src/Microsoft.AI.Foundry.Local.csproj /p:UseWinML=true ### Triggering EP download -EP download can be time-consuming. Call `EnsureEpsDownloadedAsync` early (after initialization) to separate the download step from catalog access: +EP management is explicit via two methods: + +- **`DiscoverEps()`** — returns an array of `EpInfo` describing each available EP and whether it is already registered. +- **`DownloadAndRegisterEpsAsync(names?, ct?)`** — downloads and registers the specified EPs (or all available EPs if no names are given). This is a blocking call that returns an `EpDownloadResult`. ```csharp // Initialize the manager first (see Quick Start) @@ -56,13 +59,22 @@ await FoundryLocalManager.CreateAsync( new Configuration { AppName = "my-app" }, NullLogger.Instance); -await FoundryLocalManager.Instance.EnsureEpsDownloadedAsync(); +var mgr = FoundryLocalManager.Instance; -// Now catalog access won't trigger an EP download -var catalog = await FoundryLocalManager.Instance.GetCatalogAsync(); +// Discover what EPs are available +var eps = mgr.DiscoverEps(); +foreach (var ep in eps) + Console.WriteLine($"{ep.Name} — registered: {ep.IsRegistered}"); + +// Download and register all EPs +var result = await mgr.DownloadAndRegisterEpsAsync(); +Console.WriteLine($"Success: {result.Success}, Status: {result.Status}"); + +// Or download only specific EPs +var result2 = await mgr.DownloadAndRegisterEpsAsync(new[] { eps[0].Name }); ``` -If you skip this step, EPs are downloaded automatically the first time you access the catalog. Once cached, subsequent calls are fast. +Catalog access no longer blocks on EP downloads. Call `DownloadAndRegisterEpsAsync` explicitly when you need hardware-accelerated execution providers. ## Quick Start diff --git a/sdk/cs/docs/api/microsoft.ai.foundry.local.foundrylocalmanager.md b/sdk/cs/docs/api/microsoft.ai.foundry.local.foundrylocalmanager.md index 93f162b7..b5324e89 100644 --- a/sdk/cs/docs/api/microsoft.ai.foundry.local.foundrylocalmanager.md +++ b/sdk/cs/docs/api/microsoft.ai.foundry.local.foundrylocalmanager.md @@ -97,8 +97,8 @@ The model catalog. **Remarks:** The catalog is populated on first use. - If you are using a WinML build this will trigger a one-off execution provider download if not already done. - It is recommended to call [FoundryLocalManager.EnsureEpsDownloadedAsync(Nullable<CancellationToken>)](./microsoft.ai.foundry.local.foundrylocalmanager.md#ensureepsdownloadedasyncnullablecancellationtoken) first to separate out the two steps. + Catalog access no longer blocks on execution provider downloads. + Call [FoundryLocalManager.DownloadAndRegisterEpsAsync](./microsoft.ai.foundry.local.foundrylocalmanager.md#downloadandregisterepsasync) to explicitly download and register execution providers when hardware acceleration is needed. ### **StartWebServiceAsync(Nullable<CancellationToken>)** @@ -141,27 +141,39 @@ Optional cancellation token. [Task](https://docs.microsoft.com/en-us/dotnet/api/system.threading.tasks.task)
Task stopping the web service. -### **EnsureEpsDownloadedAsync(Nullable<CancellationToken>)** +### **DiscoverEps()** -Ensure execution providers are downloaded and registered. - Only relevant when using WinML. - - Execution provider download can be time consuming due to the size of the packages. - Once downloaded, EPs are not re-downloaded unless a new version is available, so this method will be fast - on subsequent calls. +Discover available execution providers and their registration status. ```csharp -public Task EnsureEpsDownloadedAsync(Nullable ct) +public EpInfo[] DiscoverEps() +``` + +#### Returns + +`EpInfo[]`
+An array of `EpInfo` objects, each with `Name` and `IsRegistered` properties. + +### **DownloadAndRegisterEpsAsync(IEnumerable<string>?, Nullable<CancellationToken>)** + +Download and register execution providers. This is a blocking call. + +```csharp +public Task DownloadAndRegisterEpsAsync(IEnumerable? names = null, CancellationToken? ct = null) ``` #### Parameters +`names` [IEnumerable<string>?](https://docs.microsoft.com/en-us/dotnet/api/system.collections.generic.ienumerable-1)
+Optional list of EP names to download. If null or empty, all available EPs are downloaded. + `ct` [Nullable<CancellationToken>](https://docs.microsoft.com/en-us/dotnet/api/system.nullable-1)
Optional cancellation token. #### Returns -[Task](https://docs.microsoft.com/en-us/dotnet/api/system.threading.tasks.task)
+[Task<EpDownloadResult>](https://docs.microsoft.com/en-us/dotnet/api/system.threading.tasks.task-1)
+An `EpDownloadResult` with `Success`, `Status`, `RegisteredEps`, and `FailedEps` properties. ### **Dispose()** diff --git a/sdk/cs/src/Catalog.cs b/sdk/cs/src/Catalog.cs index eb9ba0d7..6f8f5711 100644 --- a/sdk/cs/src/Catalog.cs +++ b/sdk/cs/src/Catalog.cs @@ -17,7 +17,6 @@ internal sealed class Catalog : ICatalog, IDisposable { private readonly Dictionary _modelAliasToModel = new(); private readonly Dictionary _modelIdToModelVariant = new(); - private DateTime _lastFetch; private readonly IModelLoadManager _modelLoadManager; private readonly ICoreInterop _coreInterop; @@ -32,8 +31,6 @@ private Catalog(IModelLoadManager modelLoadManager, ICoreInterop coreInterop, IL _coreInterop = coreInterop; _logger = logger; - _lastFetch = DateTime.MinValue; - CoreInteropRequest? input = null; var response = coreInterop.ExecuteCommand("get_catalog_name", input); if (response.Error != null) @@ -145,12 +142,6 @@ private async Task> GetLoadedModelsImplAsync(CancellationToke private async Task UpdateModels(CancellationToken? ct) { - // TODO: make this configurable - if (DateTime.Now - _lastFetch < TimeSpan.FromHours(6)) - { - return; - } - CoreInteropRequest? input = null; var result = await _coreInterop.ExecuteCommandAsync("get_model_list", input, ct).ConfigureAwait(false); @@ -189,8 +180,6 @@ private async Task UpdateModels(CancellationToken? ct) _modelIdToModelVariant[variant.Id] = variant; } - - _lastFetch = DateTime.Now; } public void Dispose() diff --git a/sdk/cs/src/Detail/JsonSerializationContext.cs b/sdk/cs/src/Detail/JsonSerializationContext.cs index 894f9454..4621a43c 100644 --- a/sdk/cs/src/Detail/JsonSerializationContext.cs +++ b/sdk/cs/src/Detail/JsonSerializationContext.cs @@ -24,6 +24,8 @@ namespace Microsoft.AI.Foundry.Local.Detail; [JsonSerializable(typeof(AudioCreateTranscriptionRequest))] [JsonSerializable(typeof(AudioCreateTranscriptionResponse))] [JsonSerializable(typeof(string[]))] // list loaded or cached models +[JsonSerializable(typeof(EpInfo[]))] +[JsonSerializable(typeof(EpDownloadResult))] [JsonSerializable(typeof(JsonElement))] [JsonSerializable(typeof(ResponseFormatExtended))] [JsonSerializable(typeof(ToolChoice))] diff --git a/sdk/cs/src/EpInfo.cs b/sdk/cs/src/EpInfo.cs new file mode 100644 index 00000000..d170ac0e --- /dev/null +++ b/sdk/cs/src/EpInfo.cs @@ -0,0 +1,45 @@ +// -------------------------------------------------------------------------------------------------------------------- +// +// Copyright (c) Microsoft. All rights reserved. +// +// -------------------------------------------------------------------------------------------------------------------- + +namespace Microsoft.AI.Foundry.Local; + +using System.Text.Json.Serialization; + +/// +/// Describes a discoverable execution provider bootstrapper. +/// +public record EpInfo +{ + /// The identifier of the bootstrapper/execution provider (e.g. "CUDAExecutionProvider"). + [JsonPropertyName("Name")] + public required string Name { get; init; } + + /// True if this EP has already been successfully downloaded and registered. + [JsonPropertyName("IsRegistered")] + public required bool IsRegistered { get; init; } +} + +/// +/// Result of an explicit EP download and registration operation. +/// +public record EpDownloadResult +{ + /// True if all requested EPs were successfully downloaded and registered. + [JsonPropertyName("Success")] + public required bool Success { get; init; } + + /// Human-readable status message. + [JsonPropertyName("Status")] + public required string Status { get; init; } + + /// Names of EPs that were successfully registered. + [JsonPropertyName("RegisteredEps")] + public required string[] RegisteredEps { get; init; } + + /// Names of EPs that failed to register. + [JsonPropertyName("FailedEps")] + public required string[] FailedEps { get; init; } +} diff --git a/sdk/cs/src/FoundryLocalManager.cs b/sdk/cs/src/FoundryLocalManager.cs index 639be3a2..5fe70e0a 100644 --- a/sdk/cs/src/FoundryLocalManager.cs +++ b/sdk/cs/src/FoundryLocalManager.cs @@ -97,9 +97,9 @@ public static async Task CreateAsync(Configuration configuration, ILogger logger /// Optional cancellation token. /// The model catalog. /// - /// The catalog is populated on first use. - /// If you are using a WinML build this will trigger a one-off execution provider download if not already done. - /// It is recommended to call first to separate out the two steps. + /// The catalog is populated on first use and returns models based on currently available execution providers. + /// To ensure all hardware-accelerated models are listed, call first to + /// register execution providers, then access the catalog. /// public async Task GetCatalogAsync(CancellationToken? ct = null) { @@ -135,19 +135,42 @@ await Utils.CallWithExceptionHandling(() => StopWebServiceImplAsync(ct), } /// - /// Ensure execution providers are downloaded and registered. - /// Only relevant when using WinML. - /// - /// Execution provider download can be time consuming due to the size of the packages. - /// Once downloaded, EPs are not re-downloaded unless a new version is available, so this method will be fast - /// on subsequent calls. + /// Discovers all available execution provider bootstrappers. + /// Returns metadata about each EP including whether it is already registered. /// + /// Array of EP bootstrapper info describing available EPs. + public EpInfo[] DiscoverEps() + { + var result = _coreInterop!.ExecuteCommand("discover_eps"); + if (result.Error != null) + { + throw new FoundryLocalException($"Error discovering execution providers: {result.Error}", _logger); + } + + return JsonSerializer.Deserialize(result.Data!, JsonSerializationContext.Default.EpInfoArray) + ?? Array.Empty(); + } + + /// + /// Downloads and registers execution providers. This is a blocking call that completes when all + /// requested EPs have been processed. + /// + /// + /// Optional subset of EP bootstrapper names to download (as returned by ). + /// If null or empty, all discoverable EPs are downloaded. + /// /// Optional cancellation token. - public async Task EnsureEpsDownloadedAsync(CancellationToken? ct = null) + /// Result describing which EPs succeeded and which failed. + /// + /// Catalog and model requests use whatever EPs are currently registered and do not block on EP downloads. + /// After downloading new EPs, re-fetch the model catalog to include models requiring the newly registered EPs. + /// + public async Task DownloadAndRegisterEpsAsync(IEnumerable? names = null, + CancellationToken? ct = null) { - await Utils.CallWithExceptionHandling(() => EnsureEpsDownloadedImplAsync(ct), - "Error ensuring execution providers downloaded.", _logger) - .ConfigureAwait(false); + return await Utils.CallWithExceptionHandling(() => DownloadAndRegisterEpsImplAsync(names, ct), + "Error downloading execution providers.", _logger) + .ConfigureAwait(false); } private FoundryLocalManager(Configuration configuration, ILogger logger) @@ -259,17 +282,32 @@ private async Task StopWebServiceImplAsync(CancellationToken? ct = null) Urls = null; } - private async Task EnsureEpsDownloadedImplAsync(CancellationToken? ct = null) + private async Task DownloadAndRegisterEpsImplAsync(IEnumerable? names = null, + CancellationToken? ct = null) { - using var disposable = await asyncLock.LockAsync().ConfigureAwait(false); CoreInteropRequest? input = null; - var result = await _coreInterop!.ExecuteCommandAsync("ensure_eps_downloaded", input, ct); + if (names != null) + { + var namesList = string.Join(",", names); + if (!string.IsNullOrEmpty(namesList)) + { + input = new CoreInteropRequest + { + Params = new Dictionary { { "Names", namesList } } + }; + } + } + + var result = await _coreInterop!.ExecuteCommandAsync("download_and_register_eps", input, ct).ConfigureAwait(false); if (result.Error != null) { - throw new FoundryLocalException($"Error ensuring execution providers downloaded: {result.Error}", _logger); + throw new FoundryLocalException($"Error downloading execution providers: {result.Error}", _logger); } + + return JsonSerializer.Deserialize(result.Data!, JsonSerializationContext.Default.EpDownloadResult) + ?? throw new FoundryLocalException("Failed to deserialize EP download result.", _logger); } protected virtual void Dispose(bool disposing) diff --git a/sdk/js/README.md b/sdk/js/README.md index 9b08f9ac..1490e778 100644 --- a/sdk/js/README.md +++ b/sdk/js/README.md @@ -34,6 +34,27 @@ When WinML is enabled: > **Note:** The `--winml` flag is only relevant on Windows. On macOS and Linux, the standard installation is used regardless of this flag. +### Explicit EP Management + +You can explicitly discover and download execution providers using the `discoverEps()` and `downloadAndRegisterEps()` methods: + +```typescript +// Discover available EPs and their status +const eps = manager.discoverEps(); +for (const ep of eps) { + console.log(`${ep.name} — registered: ${ep.isRegistered}`); +} + +// Download and register all available EPs +const result = manager.downloadAndRegisterEps(); +console.log(`Success: ${result.success}, Status: ${result.status}`); + +// Download only specific EPs +const result2 = manager.downloadAndRegisterEps([eps[0].name]); +``` + +Catalog access does not block on EP downloads. Call `downloadAndRegisterEps()` when you need hardware-accelerated execution providers. + ## Quick Start ```typescript diff --git a/sdk/js/docs/README.md b/sdk/js/docs/README.md index 5e50e636..0cb39e1b 100644 --- a/sdk/js/docs/README.md +++ b/sdk/js/docs/README.md @@ -153,6 +153,70 @@ object: string; *** +### EpDownloadResult + +Result of an explicit EP download and registration operation. + +#### Properties + +##### failedEps + +```ts +failedEps: string[]; +``` + +Names of EPs that failed to register. + +##### registeredEps + +```ts +registeredEps: string[]; +``` + +Names of EPs that were successfully registered. + +##### status + +```ts +status: string; +``` + +Human-readable status message. + +##### success + +```ts +success: boolean; +``` + +True if all requested EPs were successfully downloaded and registered. + +*** + +### EpInfo + +Describes a discoverable execution provider bootstrapper. + +#### Properties + +##### isRegistered + +```ts +isRegistered: boolean; +``` + +True if this EP has already been successfully downloaded and registered. + +##### name + +```ts +name: string; +``` + +The identifier of the bootstrapper/execution provider (e.g. "CUDAExecutionProvider"). + +*** + ### FoundryLocalConfig Configuration options for the Foundry Local SDK. diff --git a/sdk/js/docs/classes/FoundryLocalManager.md b/sdk/js/docs/classes/FoundryLocalManager.md index 63bb2dd1..a2a2507f 100644 --- a/sdk/js/docs/classes/FoundryLocalManager.md +++ b/sdk/js/docs/classes/FoundryLocalManager.md @@ -87,6 +87,44 @@ Error - If the web service is not running. *** +### discoverEps() + +```ts +discoverEps(): EpInfo[]; +``` + +Discovers available execution providers (EPs) and their registration status. + +#### Returns + +[`EpInfo`](../README.md#epinfo)[] + +An array of EpInfo describing each available EP. + +*** + +### downloadAndRegisterEps() + +```ts +downloadAndRegisterEps(names?): EpDownloadResult; +``` + +Downloads and registers execution providers. This is a blocking call. + +#### Parameters + +| Parameter | Type | Description | +| ------ | ------ | ------ | +| `names?` | `string`[] | Optional array of EP names to download. If omitted, all available EPs are downloaded. | + +#### Returns + +[`EpDownloadResult`](../README.md#epdownloadresult) + +An EpDownloadResult with the outcome of the operation. + +*** + ### startWebService() ```ts diff --git a/sdk/js/examples/chat-completion.ts b/sdk/js/examples/chat-completion.ts index a9e2d59a..07f1407d 100644 --- a/sdk/js/examples/chat-completion.ts +++ b/sdk/js/examples/chat-completion.ts @@ -18,6 +18,17 @@ async function main() { }); console.log('✓ SDK initialized successfully'); + const availableEps = manager.discoverEps(); + console.log(`\nAvailable execution providers: ${availableEps.map((ep) => ep.name).join(', ')}`); + + console.log('\nDownloading and registering execution providers...'); + const downloadResult = manager.downloadAndRegisterEps(); + if (downloadResult.success) { + console.log('✓ All execution providers registered successfully'); + } else { + console.log(`⚠️ Some execution providers failed to download and/or register: ${downloadResult.failedEps.join(', ')}`); + } + // Explore available models console.log('\nFetching available models...'); const catalog = manager.catalog; @@ -37,7 +48,7 @@ async function main() { console.log(` - ${cachedModel.alias}`); } - const modelAlias = 'MODEL_ALIAS'; // Replace with a valid model alias from the list above + const modelAlias = 'qwen2.5-0.5b'; // Load the model first console.log(`\nLoading model ${modelAlias}...`); diff --git a/sdk/js/src/catalog.ts b/sdk/js/src/catalog.ts index bf2ae5c9..234c6cba 100644 --- a/sdk/js/src/catalog.ts +++ b/sdk/js/src/catalog.ts @@ -15,7 +15,6 @@ export class Catalog { private _models: Model[] = []; private modelAliasToModel: Map = new Map(); private modelIdToModelVariant: Map = new Map(); - private lastFetch: number = 0; constructor(coreInterop: CoreInterop, modelLoadManager: ModelLoadManager) { this.coreInterop = coreInterop; @@ -32,11 +31,6 @@ export class Catalog { } private async updateModels(): Promise { - // TODO: make this configurable - if ((Date.now() - this.lastFetch) < 6 * 60 * 60 * 1000) { // 6 hours - return; - } - // Potential network call to fetch model list const modelListJson = this.coreInterop.executeCommand("get_model_list"); let modelsInfo: ModelInfo[] = []; @@ -64,8 +58,6 @@ export class Catalog { this.modelIdToModelVariant.set(variant.id, variant); } - - this.lastFetch = Date.now(); } /** diff --git a/sdk/js/src/foundryLocalManager.ts b/sdk/js/src/foundryLocalManager.ts index bc408f78..5cb2a24e 100644 --- a/sdk/js/src/foundryLocalManager.ts +++ b/sdk/js/src/foundryLocalManager.ts @@ -3,6 +3,7 @@ import { CoreInterop } from './detail/coreInterop.js'; import { ModelLoadManager } from './detail/modelLoadManager.js'; import { Catalog } from './catalog.js'; import { ResponsesClient } from './openai/responsesClient.js'; +import { EpInfo, EpDownloadResult } from './types.js'; /** * The main entry point for the Foundry Local SDK. @@ -94,6 +95,60 @@ export class FoundryLocalManager { return this._urls.length > 0; } + /** + * Discovers available execution providers (EPs) and their registration status. + * @returns An array of EpInfo describing each available EP. + */ + public discoverEps(): EpInfo[] { + const response = this.coreInterop.executeCommand("discover_eps"); + type RawEpInfo = { + Name: string; + IsRegistered: boolean; + }; + + try { + const raw = JSON.parse(response) as RawEpInfo[]; + return raw.map((ep) => ({ + name: ep.Name, + isRegistered: ep.IsRegistered + })); + } catch (error) { + throw new Error(`Failed to decode JSON response from discover_eps: ${error}. Response was: ${response}`); + } + } + + /** + * Downloads and registers execution providers. This is a blocking call. + * @param names - Optional array of EP names to download. If omitted, all available EPs are downloaded. + * @returns An EpDownloadResult with the outcome of the operation. + */ + public downloadAndRegisterEps(names?: string[]): EpDownloadResult { + const params: { Params?: { Names: string } } = {}; + if (names && names.length > 0) { + params.Params = { Names: names.join(",") }; + } + const response = this.coreInterop.executeCommand("download_and_register_eps", Object.keys(params).length > 0 ? params : undefined); + + type RawEpDownloadResult = { + Success: boolean; + Status: string; + RegisteredEps: string[]; + FailedEps: string[]; + }; + + try { + const raw = JSON.parse(response) as RawEpDownloadResult; + return { + success: raw.Success, + status: raw.Status, + registeredEps: raw.RegisteredEps, + failedEps: raw.FailedEps + }; + } catch (error) { + throw new Error(`Failed to decode JSON response from download_and_register_eps: ${error}. Response was: ${response}`); + } + } + /** * Creates a ResponsesClient for interacting with the Responses API. * The web service must be started first via `startWebService()`. diff --git a/sdk/js/src/types.ts b/sdk/js/src/types.ts index 40a9110b..521ae34b 100644 --- a/sdk/js/src/types.ts +++ b/sdk/js/src/types.ts @@ -67,6 +67,30 @@ export interface ToolChoice { name?: string; } +// ============================================================================ +// Execution Provider Types +// ============================================================================ + +/** Describes a discoverable execution provider bootstrapper. */ +export interface EpInfo { + /** The identifier of the bootstrapper/execution provider (e.g. "CUDAExecutionProvider"). */ + name: string; + /** True if this EP has already been successfully downloaded and registered. */ + isRegistered: boolean; +} + +/** Result of an explicit EP download and registration operation. */ +export interface EpDownloadResult { + /** True if all requested EPs were successfully downloaded and registered. */ + success: boolean; + /** Human-readable status message. */ + status: string; + /** Names of EPs that were successfully registered. */ + registeredEps: string[]; + /** Names of EPs that failed to register. */ + failedEps: string[]; +} + // ============================================================================ // Responses API Types // Aligned with OpenAI Responses API / OpenResponses spec and diff --git a/sdk/js/test/foundryLocalManager.test.ts b/sdk/js/test/foundryLocalManager.test.ts index 5ab40043..9c85ad3f 100644 --- a/sdk/js/test/foundryLocalManager.test.ts +++ b/sdk/js/test/foundryLocalManager.test.ts @@ -16,4 +16,64 @@ describe('Foundry Local Manager Tests', () => { // We don't assert the exact name as it might change, but we ensure it exists expect(catalog.name).to.be.a('string'); }); + + it('downloadAndRegisterEps should call command without params when names are omitted', function() { + const manager = getTestManager() as any; + const calls: unknown[][] = []; + const originalExecuteCommand = manager.coreInterop.executeCommand; + + manager.coreInterop.executeCommand = (...args: unknown[]) => { + calls.push(args); + return JSON.stringify({ + Success: true, + Status: 'All providers registered', + RegisteredEps: ['CUDAExecutionProvider'], + FailedEps: [] + }); + }; + + try { + const result = manager.downloadAndRegisterEps(); + expect(calls).to.deep.equal([['download_and_register_eps', undefined]]); + expect(result).to.deep.equal({ + success: true, + status: 'All providers registered', + registeredEps: ['CUDAExecutionProvider'], + failedEps: [] + }); + } finally { + manager.coreInterop.executeCommand = originalExecuteCommand; + } + }); + + it('downloadAndRegisterEps should send Names param when subset is provided', function() { + const manager = getTestManager() as any; + const calls: unknown[][] = []; + const originalExecuteCommand = manager.coreInterop.executeCommand; + + manager.coreInterop.executeCommand = (...args: unknown[]) => { + calls.push(args); + return JSON.stringify({ + Success: false, + Status: 'Some providers failed', + RegisteredEps: ['CUDAExecutionProvider'], + FailedEps: ['OpenVINOExecutionProvider'] + }); + }; + + try { + const result = manager.downloadAndRegisterEps(['CUDAExecutionProvider', 'OpenVINOExecutionProvider']); + expect(calls).to.deep.equal([ + ['download_and_register_eps', { Params: { Names: 'CUDAExecutionProvider,OpenVINOExecutionProvider' } }] + ]); + expect(result).to.deep.equal({ + success: false, + status: 'Some providers failed', + registeredEps: ['CUDAExecutionProvider'], + failedEps: ['OpenVINOExecutionProvider'] + }); + } finally { + manager.coreInterop.executeCommand = originalExecuteCommand; + } + }); }); diff --git a/sdk/python/README.md b/sdk/python/README.md index 7cc8b44c..3a6fd67b 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -18,7 +18,7 @@ Two package variants are published — choose the one that matches your target h | Variant | Package | Native backends | |---|---|---| -| Standard (cross-platform) | `foundry-local-sdk` | CPU / DirectML / CUDA | +| Standard (cross-platform) | `foundry-local-sdk` | CPU / WebGPU / CUDA | | WinML (Windows only) | `foundry-local-sdk-winml` | Windows ML + all standard backends | ```bash @@ -70,6 +70,26 @@ foundry-local-install --winml --verbose > **Note:** The standard and WinML native packages use different PyPI package names (`foundry-local-core` vs `foundry-local-core-winml`) so they can coexist in the same pip index, but they should not be installed in the same Python environment simultaneously. +## Explicit EP Management + +You can explicitly discover and download execution providers (EPs): + +```python +# Discover available EPs and registration status +eps = manager.discover_eps() +for ep in eps: + print(f"{ep.name} - registered: {ep.is_registered}") + +# Download and register all available EPs +result = manager.download_and_register_eps() +print(f"Success: {result.success}, Status: {result.status}") + +# Download only specific EPs +result2 = manager.download_and_register_eps([eps[0].name]) +``` + +Catalog access does not block on EP downloads. Call `download_and_register_eps()` when you need hardware-accelerated execution providers. + ## Quick Start ```python @@ -200,6 +220,8 @@ manager.stop_web_service() |---|---| | `Configuration` | SDK configuration (app name, cache dir, log level, web service settings) | | `FoundryLocalManager` | Singleton entry point – initialization, catalog access, web service | +| `EpInfo` | Discoverable execution provider info (`name`, `is_registered`) | +| `EpDownloadResult` | Result of EP download/registration (`success`, `status`, `registered_eps`, `failed_eps`) | | `Catalog` | Model discovery – listing, lookup by alias/ID, cached/loaded queries | | `Model` | Groups variants under one alias – select, load, unload, create clients | | `ModelVariant` | Specific model variant – download, cache, load/unload, create clients | diff --git a/sdk/python/examples/chat_completion.py b/sdk/python/examples/chat_completion.py index 60eefd5e..c0c58048 100644 --- a/sdk/python/examples/chat_completion.py +++ b/sdk/python/examples/chat_completion.py @@ -19,6 +19,15 @@ def main(): FoundryLocalManager.initialize(config) manager = FoundryLocalManager.instance + # Discover available EPs and register them explicitly when needed. + eps = manager.discover_eps() + print("Available execution providers:") + for ep in eps: + print(f" - {ep.name} (registered: {ep.is_registered})") + + ep_result = manager.download_and_register_eps() + print(f"EP registration success: {ep_result.success} ({ep_result.status})") + # 2. Print available models in the catalog and cache models = manager.catalog.list_models() print("Available models in catalog:") diff --git a/sdk/python/src/__init__.py b/sdk/python/src/__init__.py index 14534d19..fa416f5c 100644 --- a/sdk/python/src/__init__.py +++ b/sdk/python/src/__init__.py @@ -6,6 +6,7 @@ import sys from .configuration import Configuration +from .ep_types import EpDownloadResult, EpInfo from .foundry_local_manager import FoundryLocalManager from .version import __version__ @@ -20,4 +21,10 @@ _logger.addHandler(_sc) _logger.propagate = False -__all__ = ["Configuration", "FoundryLocalManager", "__version__"] +__all__ = [ + "Configuration", + "EpInfo", + "EpDownloadResult", + "FoundryLocalManager", + "__version__", +] diff --git a/sdk/python/src/catalog.py b/sdk/python/src/catalog.py index 767a9f08..96ef1b55 100644 --- a/sdk/python/src/catalog.py +++ b/sdk/python/src/catalog.py @@ -5,7 +5,6 @@ from __future__ import annotations -import datetime import logging import threading from typing import List, Optional @@ -25,7 +24,7 @@ class Catalog(): """Model catalog for discovering and querying available models. Provides methods to list models, look up by alias or ID, and query - cached or loaded models. The model list is refreshed every 6 hours. + cached or loaded models. The model list is refreshed on each query call. """ def __init__(self, model_load_manager: ModelLoadManager, core_interop: CoreInterop): @@ -42,7 +41,6 @@ def __init__(self, model_load_manager: ModelLoadManager, core_interop: CoreInter self._models: List[ModelInfo] = [] self._model_alias_to_model = {} self._model_id_to_model_variant = {} - self._last_fetch = datetime.datetime.min response = core_interop.execute_command("get_catalog_name") if response.error is not None: @@ -52,10 +50,6 @@ def __init__(self, model_load_manager: ModelLoadManager, core_interop: CoreInter def _update_models(self): with self._lock: - # refresh every 6 hours - if (datetime.datetime.now() - self._last_fetch) < datetime.timedelta(hours=6): - return - response = self._core_interop.execute_command("get_model_list") if response.error is not None: raise FoundryLocalException(f"Failed to get model list: {response.error}") @@ -80,7 +74,6 @@ def _update_models(self): self._model_id_to_model_variant[variant.id] = variant - self._last_fetch = datetime.datetime.now() self._models = models def list_models(self) -> List[Model]: diff --git a/sdk/python/src/ep_types.py b/sdk/python/src/ep_types.py new file mode 100644 index 00000000..c6bd6e13 --- /dev/null +++ b/sdk/python/src/ep_types.py @@ -0,0 +1,44 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class EpInfo: + """Metadata describing a discoverable execution provider (EP).""" + + name: str + is_registered: bool + + @staticmethod + def from_dict(data: dict) -> "EpInfo": + """Create ``EpInfo`` from core JSON payload (PascalCase fields).""" + return EpInfo( + name=data["Name"], + is_registered=data["IsRegistered"], + ) + + +@dataclass(frozen=True) +class EpDownloadResult: + """Result of an explicit EP download and registration operation.""" + + success: bool + status: str + registered_eps: list[str] + failed_eps: list[str] + + @staticmethod + def from_dict(data: dict) -> "EpDownloadResult": + """Create ``EpDownloadResult`` from core JSON payload (PascalCase fields).""" + return EpDownloadResult( + success=data["Success"], + status=data["Status"], + registered_eps=data["RegisteredEps"], + failed_eps=data["FailedEps"], + ) diff --git a/sdk/python/src/foundry_local_manager.py b/sdk/python/src/foundry_local_manager.py index 4486eaf1..9e6a5e14 100644 --- a/sdk/python/src/foundry_local_manager.py +++ b/sdk/python/src/foundry_local_manager.py @@ -9,10 +9,13 @@ import logging import threading +from typing import Optional + from .catalog import Catalog from .configuration import Configuration +from .ep_types import EpDownloadResult, EpInfo from .logging_helper import set_default_logger_severity -from .detail.core_interop import CoreInterop +from .detail.core_interop import CoreInterop, InteropRequest from .detail.model_load_manager import ModelLoadManager from .exception import FoundryLocalException @@ -71,17 +74,56 @@ def _initialize(self): self._model_load_manager = ModelLoadManager(self._core_interop, external_service_url) self.catalog = Catalog(self._model_load_manager, self._core_interop) - def ensure_eps_downloaded(self) -> None: - """Ensure execution providers are downloaded and registered (synchronous). - Only relevant when using WinML. + def discover_eps(self) -> list[EpInfo]: + """Discover available execution providers and their registration status. + + Returns: + List of ``EpInfo`` entries for all discoverable EPs. Raises: - FoundryLocalException: If execution provider download fails. + FoundryLocalException: If EP discovery fails or response JSON is invalid. """ - result = self._core_interop.execute_command("ensure_eps_downloaded") + response = self._core_interop.execute_command("discover_eps") + if response.error is not None: + raise FoundryLocalException(f"Error discovering execution providers: {response.error}") + + try: + payload = json.loads(response.data or "[]") + return [EpInfo.from_dict(item) for item in payload] + except Exception as e: + raise FoundryLocalException( + f"Failed to decode JSON response from discover_eps: {e}. Response was: {response.data}" + ) from e - if result.error is not None: - raise FoundryLocalException(f"Error ensuring execution providers downloaded: {result.error}") + def download_and_register_eps(self, names: Optional[list[str]] = None) -> EpDownloadResult: + """Download and register execution providers (blocking). + + Args: + names: Optional subset of EP names to download. If omitted or empty, + all discoverable EPs are downloaded. + + Returns: + ``EpDownloadResult`` describing operation status and per-EP outcomes. + + Raises: + FoundryLocalException: If the operation fails or response JSON is invalid. + """ + request = None + if names is not None and len(names) > 0: + request = InteropRequest(params={"Names": ",".join(names)}) + + response = self._core_interop.execute_command("download_and_register_eps", request) + if response.error is not None: + raise FoundryLocalException(f"Error downloading execution providers: {response.error}") + + try: + payload = json.loads(response.data or "{}") + return EpDownloadResult.from_dict(payload) + except Exception as e: + raise FoundryLocalException( + "Failed to decode JSON response from download_and_register_eps: " + f"{e}. Response was: {response.data}" + ) from e def start_web_service(self): """Start the optional web service. diff --git a/sdk/python/test/test_foundry_local_manager.py b/sdk/python/test/test_foundry_local_manager.py index b0a9c4e2..31528891 100644 --- a/sdk/python/test/test_foundry_local_manager.py +++ b/sdk/python/test/test_foundry_local_manager.py @@ -7,6 +7,22 @@ from __future__ import annotations +class _Response: + def __init__(self, data=None, error=None): + self.data = data + self.error = error + + +class _FakeCoreInterop: + def __init__(self, responses): + self._responses = responses + self.calls = [] + + def execute_command(self, command_name, command_input=None): + self.calls.append((command_name, command_input)) + return self._responses[command_name] + + class TestFoundryLocalManager: """Foundry Local Manager Tests.""" @@ -20,3 +36,48 @@ def test_should_return_catalog(self, manager): assert catalog is not None assert isinstance(catalog.name, str) assert len(catalog.name) > 0 + + def test_discover_eps_returns_ep_info(self, manager): + original_core = manager._core_interop + manager._core_interop = _FakeCoreInterop( + { + "discover_eps": _Response( + data='[{"Name":"CUDAExecutionProvider","IsRegistered":true}]', + error=None, + ) + } + ) + + try: + eps = manager.discover_eps() + finally: + manager._core_interop = original_core + + assert isinstance(eps, list) + assert len(eps) == 1 + assert eps[0].name == "CUDAExecutionProvider" + assert eps[0].is_registered is True + + def test_download_and_register_eps_returns_result(self, manager): + original_core = manager._core_interop + manager._core_interop = _FakeCoreInterop( + { + "download_and_register_eps": _Response( + data=( + '{"Success":true,"Status":"ok",' + '"RegisteredEps":["CUDAExecutionProvider"],"FailedEps":[]}' + ), + error=None, + ) + } + ) + + try: + result = manager.download_and_register_eps(["CUDAExecutionProvider"]) + finally: + manager._core_interop = original_core + + assert result.success is True + assert result.status == "ok" + assert result.registered_eps == ["CUDAExecutionProvider"] + assert result.failed_eps == [] diff --git a/sdk/rust/README.md b/sdk/rust/README.md index d76a7589..78a05ecf 100644 --- a/sdk/rust/README.md +++ b/sdk/rust/README.md @@ -60,6 +60,31 @@ foundry-local-sdk = { version = "0.1", features = ["winml"] } > **Note:** The `winml` feature is only relevant on Windows. On macOS and Linux, the standard build is used regardless. No code changes are needed — your application code stays the same. +### Explicit EP Management + +You can explicitly discover and download execution providers: + +```rust +use foundry_local_sdk::{FoundryLocalConfig, FoundryLocalManager}; + +let manager = FoundryLocalManager::create(FoundryLocalConfig::new("my_app"))?; + +// Discover available EPs and their status +let eps = manager.discover_eps()?; +for ep in &eps { + println!("{} — registered: {}", ep.name, ep.is_registered); +} + +// Download and register all available EPs +let result = manager.download_and_register_eps(None)?; +println!("Success: {}, Status: {}", result.success, result.status); + +// Download only specific EPs +let result = manager.download_and_register_eps(Some(&[eps[0].name.as_str()]))?; +``` + +Catalog access does not block on EP downloads. Call `download_and_register_eps` when you need hardware-accelerated execution providers. + ## Quick Start ```rust diff --git a/sdk/rust/src/catalog.rs b/sdk/rust/src/catalog.rs index 78485bff..b1ed10f4 100644 --- a/sdk/rust/src/catalog.rs +++ b/sdk/rust/src/catalog.rs @@ -1,9 +1,7 @@ //! Model catalog – discovers, caches, and looks up available models. use std::collections::HashMap; -use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; -use std::time::{Duration, Instant}; use crate::detail::core_interop::CoreInterop; use crate::detail::ModelLoadManager; @@ -12,35 +10,10 @@ use crate::model::Model; use crate::model_variant::ModelVariant; use crate::types::ModelInfo; -/// How long the catalog cache remains valid before a refresh. -const CACHE_TTL: Duration = Duration::from_secs(6 * 60 * 60); // 6 hours - -/// Shared flag allowing `ModelVariant` to signal that the catalog cache is -/// stale (e.g. after a download or removal). -#[derive(Clone, Debug)] -pub(crate) struct CacheInvalidator(Arc); - -impl CacheInvalidator { - fn new() -> Self { - Self(Arc::new(AtomicBool::new(false))) - } - - /// Mark the catalog cache as stale. - pub fn invalidate(&self) { - self.0.store(true, Ordering::Release); - } - - /// Check and clear the invalidation flag. - fn take(&self) -> bool { - self.0.swap(false, Ordering::AcqRel) - } -} - /// All mutable catalog data behind a single lock to prevent split-brain reads. struct CatalogState { models_by_alias: HashMap>, variants_by_id: HashMap>, - last_refresh: Option, } /// The model catalog provides discovery and lookup for all available models. @@ -51,7 +24,6 @@ pub struct Catalog { state: Mutex, /// Async gate ensuring only one refresh runs at a time. refresh_gate: tokio::sync::Mutex<()>, - invalidator: CacheInvalidator, } impl Catalog { @@ -63,7 +35,6 @@ impl Catalog { .execute_command("get_catalog_name", None) .unwrap_or_else(|_| "default".into()); - let invalidator = CacheInvalidator::new(); let catalog = Self { core, model_load_manager, @@ -71,10 +42,8 @@ impl Catalog { state: Mutex::new(CatalogState { models_by_alias: HashMap::new(), variants_by_id: HashMap::new(), - last_refresh: None, }), refresh_gate: tokio::sync::Mutex::new(()), - invalidator, }; // Perform initial synchronous refresh during construction. @@ -87,34 +56,11 @@ impl Catalog { &self.name } - /// Refresh the catalog from the native core if the cache has expired or - /// has been explicitly invalidated (e.g. after a download or removal). + /// Refresh the catalog from the native core. + /// The core handles its own caching, so this always fetches fresh data. pub async fn update_models(&self) -> Result<()> { - let invalidated = self.invalidator.take(); - - // Fast path: check under data lock (held briefly). - if !invalidated { - let s = self.lock_state()?; - if let Some(ts) = s.last_refresh { - if ts.elapsed() < CACHE_TTL { - return Ok(()); - } - } - } - - // Slow path: acquire refresh gate so only one thread refreshes. + // Acquire refresh gate so only one thread refreshes at a time. let _gate = self.refresh_gate.lock().await; - - // Re-check after acquiring the gate — another thread may have refreshed. - if !invalidated { - let s = self.lock_state()?; - if let Some(ts) = s.last_refresh { - if ts.elapsed() < CACHE_TTL { - return Ok(()); - } - } - } - self.force_refresh().await } @@ -220,7 +166,6 @@ impl Catalog { info, Arc::clone(&self.core), Arc::clone(&self.model_load_manager), - self.invalidator.clone(), ); let variant_arc = Arc::new(variant.clone()); id_map.insert(id, variant_arc); @@ -240,7 +185,6 @@ impl Catalog { let mut s = self.lock_state()?; s.models_by_alias = alias_map; s.variants_by_id = id_map; - s.last_refresh = Some(Instant::now()); Ok(()) } diff --git a/sdk/rust/src/foundry_local_manager.rs b/sdk/rust/src/foundry_local_manager.rs index f80a7176..a6306ec0 100644 --- a/sdk/rust/src/foundry_local_manager.rs +++ b/sdk/rust/src/foundry_local_manager.rs @@ -13,6 +13,7 @@ use crate::configuration::{Configuration, FoundryLocalConfig, Logger}; use crate::detail::core_interop::CoreInterop; use crate::detail::ModelLoadManager; use crate::error::{FoundryLocalError, Result}; +use crate::types::{EpInfo, EpDownloadResult}; /// Global singleton holder — only stores a successfully initialised manager. static INSTANCE: OnceLock = OnceLock::new(); @@ -133,4 +134,27 @@ impl FoundryLocalManager { .clear(); Ok(()) } + + /// Discover available execution providers and their registration status. + pub fn discover_eps(&self) -> Result> { + let raw = self.core.execute_command("discover_eps", None)?; + let eps: Vec = serde_json::from_str(&raw)?; + Ok(eps) + } + + /// Download and register execution providers. This is a blocking call. + /// + /// If `names` is `None` or empty, all available EPs are downloaded. + /// Otherwise only the named EPs are downloaded and registered. + pub fn download_and_register_eps(&self, names: Option<&[&str]>) -> Result { + let params = match names { + Some(n) if !n.is_empty() => { + Some(json!({ "Params": { "Names": n.join(",") } })) + } + _ => None, + }; + let raw = self.core.execute_command("download_and_register_eps", params.as_ref())?; + let result: EpDownloadResult = serde_json::from_str(&raw)?; + Ok(result) + } } diff --git a/sdk/rust/src/lib.rs b/sdk/rust/src/lib.rs index c6d6e6c4..2f3c6b71 100644 --- a/sdk/rust/src/lib.rs +++ b/sdk/rust/src/lib.rs @@ -20,8 +20,8 @@ pub use self::foundry_local_manager::FoundryLocalManager; pub use self::model::Model; pub use self::model_variant::ModelVariant; pub use self::types::{ - ChatResponseFormat, ChatToolChoice, DeviceType, ModelInfo, ModelSettings, Parameter, - PromptTemplate, Runtime, + ChatResponseFormat, ChatToolChoice, DeviceType, EpInfo, EpDownloadResult, + ModelInfo, ModelSettings, Parameter, PromptTemplate, Runtime, }; // Re-export OpenAI request types so callers can construct typed messages. diff --git a/sdk/rust/src/model_variant.rs b/sdk/rust/src/model_variant.rs index c4be6822..bd435365 100644 --- a/sdk/rust/src/model_variant.rs +++ b/sdk/rust/src/model_variant.rs @@ -6,7 +6,6 @@ use std::sync::Arc; use serde_json::json; -use crate::catalog::CacheInvalidator; use crate::detail::core_interop::CoreInterop; use crate::detail::ModelLoadManager; use crate::error::Result; @@ -21,7 +20,6 @@ pub struct ModelVariant { info: ModelInfo, core: Arc, model_load_manager: Arc, - cache_invalidator: CacheInvalidator, } impl fmt::Debug for ModelVariant { @@ -38,13 +36,11 @@ impl ModelVariant { info: ModelInfo, core: Arc, model_load_manager: Arc, - cache_invalidator: CacheInvalidator, ) -> Self { Self { info, core, model_load_manager, - cache_invalidator, } } @@ -106,7 +102,6 @@ impl ModelVariant { .await?; } } - self.cache_invalidator.invalidate(); Ok(()) } @@ -137,7 +132,6 @@ impl ModelVariant { .core .execute_command_async("remove_cached_model".into(), Some(params)) .await?; - self.cache_invalidator.invalidate(); Ok(result) } diff --git a/sdk/rust/src/types.rs b/sdk/rust/src/types.rs index bab2f9c8..28b37ed2 100644 --- a/sdk/rust/src/types.rs +++ b/sdk/rust/src/types.rs @@ -125,3 +125,27 @@ pub enum ChatToolChoice { /// Model must call the named function. Function(String), } + +/// Information about an available execution provider bootstrapper. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct EpInfo { + /// The name of the execution provider. + pub name: String, + /// Whether this EP is currently registered and ready for use. + pub is_registered: bool, +} + +/// Result of a download-and-register execution-provider operation. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct EpDownloadResult { + /// Whether all requested EPs were successfully registered. + pub success: bool, + /// Human-readable status message. + pub status: String, + /// Names of EPs that were successfully registered. + pub registered_eps: Vec, + /// Names of EPs that failed to register. + pub failed_eps: Vec, +} diff --git a/www/src/routes/models/service.ts b/www/src/routes/models/service.ts index de49a539..75e2901c 100644 --- a/www/src/routes/models/service.ts +++ b/www/src/routes/models/service.ts @@ -188,7 +188,6 @@ export class FoundryModelService { device: 'GPU', executionProviders: [ 'CUDAExecutionProvider', // NVIDIA CUDA - 'DmlExecutionProvider', // DirectML (Windows) 'TensorrtExecutionProvider', // NVIDIA TensorRT 'NvTensorRTRTXExecutionProvider', // NVIDIA TensorRT RTX (TRTRTX) 'WebGpuExecutionProvider', // WebGPU