From ae4ae5f4256f5aef84e2de5ec14511186b096486 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 11 Apr 2026 07:45:12 +0000 Subject: [PATCH 1/8] feat: add backend versioning data model foundation Add Version, URI, and Digest fields to BackendMetadata for tracking installed backend versions and enabling upgrade detection. Add Version field to GalleryBackend. Add UpgradeAvailable/AvailableVersion fields to SystemBackend. Implement GetImageDigest() for lightweight OCI digest lookups via remote.Head. Record version, URI, and digest at install time in InstallBackend() and propagate version through meta backends. --- core/gallery/backend_types.go | 7 ++ core/gallery/backends.go | 26 ++++-- core/gallery/backends_version_test.go | 118 ++++++++++++++++++++++++++ pkg/oci/image.go | 50 +++++++++++ pkg/oci/image_test.go | 8 ++ 5 files changed, 204 insertions(+), 5 deletions(-) create mode 100644 core/gallery/backends_version_test.go diff --git a/core/gallery/backend_types.go b/core/gallery/backend_types.go index 02ac9b275378..ef5904f99f91 100644 --- a/core/gallery/backend_types.go +++ b/core/gallery/backend_types.go @@ -20,12 +20,19 @@ type BackendMetadata struct { GalleryURL string `json:"gallery_url,omitempty"` // InstalledAt is the timestamp when the backend was installed InstalledAt string `json:"installed_at,omitempty"` + // Version is the version of the backend at install time + Version string `json:"version,omitempty"` + // URI is the original URI used to install the backend + URI string `json:"uri,omitempty"` + // Digest is the OCI image digest at install time (for upgrade detection) + Digest string `json:"digest,omitempty"` } type GalleryBackend struct { Metadata `json:",inline" yaml:",inline"` Alias string `json:"alias,omitempty" yaml:"alias,omitempty"` URI string `json:"uri,omitempty" yaml:"uri,omitempty"` + Version string `json:"version,omitempty" yaml:"version,omitempty"` Mirrors []string `json:"mirrors,omitempty" yaml:"mirrors,omitempty"` CapabilitiesMap map[string]string `json:"capabilities,omitempty" yaml:"capabilities,omitempty"` } diff --git a/core/gallery/backends.go b/core/gallery/backends.go index b48aaf8a4ad5..c2622c272e3d 100644 --- a/core/gallery/backends.go +++ b/core/gallery/backends.go @@ -15,6 +15,7 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/pkg/downloader" "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/oci" "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/xlog" cp "github.com/otiai10/copy" @@ -158,6 +159,7 @@ func InstallBackendFromGallery(ctx context.Context, galleries []config.Gallery, Name: name, GalleryURL: backend.Gallery.URL, InstalledAt: time.Now().Format(time.RFC3339), + Version: bestBackend.Version, } if err := writeBackendMetadata(metaBackendPath, metaMetadata); err != nil { @@ -279,6 +281,18 @@ func InstallBackend(ctx context.Context, systemState *system.SystemState, modelL Name: name, GalleryURL: config.Gallery.URL, InstalledAt: time.Now().Format(time.RFC3339), + Version: config.Version, + URI: string(uri), + } + + // Record the OCI digest for upgrade detection (non-fatal on failure) + if uri.LooksLikeOCI() { + digest, digestErr := oci.GetImageDigest(string(uri), "", nil, nil) + if digestErr != nil { + xlog.Warn("Failed to get OCI image digest for backend", "uri", string(uri), "error", digestErr) + } else { + metadata.Digest = digest + } } if config.Alias != "" { @@ -373,11 +387,13 @@ func DeleteBackendFromSystem(systemState *system.SystemState, name string) error } type SystemBackend struct { - Name string - RunFile string - IsMeta bool - IsSystem bool - Metadata *BackendMetadata + Name string + RunFile string + IsMeta bool + IsSystem bool + Metadata *BackendMetadata + UpgradeAvailable bool `json:"upgrade_available,omitempty"` + AvailableVersion string `json:"available_version,omitempty"` } type SystemBackends map[string]SystemBackend diff --git a/core/gallery/backends_version_test.go b/core/gallery/backends_version_test.go new file mode 100644 index 000000000000..8e97604d4745 --- /dev/null +++ b/core/gallery/backends_version_test.go @@ -0,0 +1,118 @@ +package gallery_test + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Backend versioning", func() { + var tempDir string + var systemState *system.SystemState + var modelLoader *model.ModelLoader + + BeforeEach(func() { + var err error + tempDir, err = os.MkdirTemp("", "gallery-version-*") + Expect(err).NotTo(HaveOccurred()) + + systemState, err = system.GetSystemState( + system.WithBackendPath(tempDir), + ) + Expect(err).NotTo(HaveOccurred()) + modelLoader = model.NewModelLoader(systemState) + }) + + AfterEach(func() { + os.RemoveAll(tempDir) + }) + + It("records version in metadata when installing a backend with a version", func() { + // Create a fake backend source directory with a run.sh + srcDir, err := os.MkdirTemp("", "gallery-version-src-*") + Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(srcDir) + err = os.WriteFile(filepath.Join(srcDir, "run.sh"), []byte("#!/bin/sh\necho ok"), 0755) + Expect(err).NotTo(HaveOccurred()) + + backend := &gallery.GalleryBackend{} + backend.Name = "test-backend" + backend.URI = srcDir + backend.Version = "1.2.3" + + err = gallery.InstallBackend(context.Background(), systemState, modelLoader, backend, nil) + Expect(err).NotTo(HaveOccurred()) + + // Read the metadata file and check version + metadataPath := filepath.Join(tempDir, "test-backend", "metadata.json") + data, err := os.ReadFile(metadataPath) + Expect(err).NotTo(HaveOccurred()) + + var metadata map[string]any + err = json.Unmarshal(data, &metadata) + Expect(err).NotTo(HaveOccurred()) + + Expect(metadata["version"]).To(Equal("1.2.3")) + }) + + It("records URI in metadata", func() { + srcDir, err := os.MkdirTemp("", "gallery-version-src-*") + Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(srcDir) + err = os.WriteFile(filepath.Join(srcDir, "run.sh"), []byte("#!/bin/sh\necho ok"), 0755) + Expect(err).NotTo(HaveOccurred()) + + backend := &gallery.GalleryBackend{} + backend.Name = "test-backend-uri" + backend.URI = srcDir + backend.Version = "2.0.0" + + err = gallery.InstallBackend(context.Background(), systemState, modelLoader, backend, nil) + Expect(err).NotTo(HaveOccurred()) + + metadataPath := filepath.Join(tempDir, "test-backend-uri", "metadata.json") + data, err := os.ReadFile(metadataPath) + Expect(err).NotTo(HaveOccurred()) + + var metadata map[string]any + err = json.Unmarshal(data, &metadata) + Expect(err).NotTo(HaveOccurred()) + + Expect(metadata["uri"]).To(Equal(srcDir)) + }) + + It("omits version key when version is empty", func() { + srcDir, err := os.MkdirTemp("", "gallery-version-src-*") + Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(srcDir) + err = os.WriteFile(filepath.Join(srcDir, "run.sh"), []byte("#!/bin/sh\necho ok"), 0755) + Expect(err).NotTo(HaveOccurred()) + + backend := &gallery.GalleryBackend{} + backend.Name = "test-backend-noversion" + backend.URI = srcDir + // Version intentionally left empty + + err = gallery.InstallBackend(context.Background(), systemState, modelLoader, backend, nil) + Expect(err).NotTo(HaveOccurred()) + + metadataPath := filepath.Join(tempDir, "test-backend-noversion", "metadata.json") + data, err := os.ReadFile(metadataPath) + Expect(err).NotTo(HaveOccurred()) + + var metadata map[string]any + err = json.Unmarshal(data, &metadata) + Expect(err).NotTo(HaveOccurred()) + + // omitempty should exclude the version key entirely + _, hasVersion := metadata["version"] + Expect(hasVersion).To(BeFalse()) + }) +}) diff --git a/pkg/oci/image.go b/pkg/oci/image.go index 90d433a05b0f..2d00c34799c3 100644 --- a/pkg/oci/image.go +++ b/pkg/oci/image.go @@ -188,6 +188,56 @@ func GetImage(targetImage, targetPlatform string, auth *registrytypes.AuthConfig return image, err } +// GetImageDigest returns the OCI image digest for the given image reference without downloading it. +// It uses remote.Head to fetch only the descriptor, which is much cheaper than pulling the full image. +func GetImageDigest(targetImage, targetPlatform string, auth *registrytypes.AuthConfig, t http.RoundTripper) (string, error) { + var platform *v1.Platform + var err error + + if targetPlatform != "" { + platform, err = v1.ParsePlatform(targetPlatform) + if err != nil { + return "", err + } + } else { + platform, err = v1.ParsePlatform(fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH)) + if err != nil { + return "", err + } + } + + ref, err := name.ParseReference(targetImage) + if err != nil { + return "", err + } + + if t == nil { + t = http.DefaultTransport + } + + tr := transport.NewRetry(t, + transport.WithRetryBackoff(defaultRetryBackoff), + transport.WithRetryPredicate(defaultRetryPredicate), + ) + + opts := []remote.Option{ + remote.WithTransport(tr), + remote.WithPlatform(*platform), + } + if auth != nil { + opts = append(opts, remote.WithAuth(staticAuth{auth})) + } else { + opts = append(opts, remote.WithAuthFromKeychain(authn.DefaultKeychain)) + } + + desc, err := remote.Head(ref, opts...) + if err != nil { + return "", err + } + + return desc.Digest.String(), nil +} + func GetOCIImageSize(targetImage, targetPlatform string, auth *registrytypes.AuthConfig, t http.RoundTripper) (int64, error) { var size int64 var img v1.Image diff --git a/pkg/oci/image_test.go b/pkg/oci/image_test.go index 8b26c2b87655..447bc90f61ca 100644 --- a/pkg/oci/image_test.go +++ b/pkg/oci/image_test.go @@ -5,6 +5,7 @@ import ( "os" "runtime" + "github.com/mudler/LocalAI/pkg/oci" . "github.com/mudler/LocalAI/pkg/oci" // Update with your module path . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -36,3 +37,10 @@ var _ = Describe("OCI", func() { }) }) }) + +var _ = Describe("GetImageDigest", func() { + It("returns an error for an invalid image reference", func() { + _, err := oci.GetImageDigest("!!!invalid-ref!!!", "", nil, nil) + Expect(err).To(HaveOccurred()) + }) +}) From 4d463e9f0d19d99f0fd63655a1bea5eb446b1193 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 11 Apr 2026 07:51:39 +0000 Subject: [PATCH 2/8] feat: add backend upgrade detection and execution logic Add CheckBackendUpgrades() to compare installed backend versions/digests against gallery entries, and UpgradeBackend() to perform atomic upgrades with backup-based rollback on failure. Includes Agent A's data model changes (Version/URI/Digest fields, GetImageDigest). --- core/gallery/upgrade.go | 223 +++++++++++++++++++++++++++++++++++ core/gallery/upgrade_test.go | 219 ++++++++++++++++++++++++++++++++++ 2 files changed, 442 insertions(+) create mode 100644 core/gallery/upgrade.go create mode 100644 core/gallery/upgrade_test.go diff --git a/core/gallery/upgrade.go b/core/gallery/upgrade.go new file mode 100644 index 000000000000..aec75852cc38 --- /dev/null +++ b/core/gallery/upgrade.go @@ -0,0 +1,223 @@ +package gallery + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/pkg/downloader" + "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/oci" + "github.com/mudler/LocalAI/pkg/system" + "github.com/mudler/xlog" + cp "github.com/otiai10/copy" +) + +// UpgradeInfo holds details about an available backend upgrade. +type UpgradeInfo struct { + BackendName string `json:"backend_name"` + InstalledVersion string `json:"installed_version"` + AvailableVersion string `json:"available_version"` + InstalledDigest string `json:"installed_digest,omitempty"` + AvailableDigest string `json:"available_digest,omitempty"` +} + +// CheckBackendUpgrades compares installed backends against gallery entries +// and returns a map of backend names to UpgradeInfo for those that have +// newer versions or different OCI digests available. +func CheckBackendUpgrades(ctx context.Context, galleries []config.Gallery, systemState *system.SystemState) (map[string]UpgradeInfo, error) { + galleryBackends, err := AvailableBackends(galleries, systemState) + if err != nil { + return nil, fmt.Errorf("failed to list available backends: %w", err) + } + + installedBackends, err := ListSystemBackends(systemState) + if err != nil { + return nil, fmt.Errorf("failed to list installed backends: %w", err) + } + + result := make(map[string]UpgradeInfo) + + for _, installed := range installedBackends { + // Skip system backends — they are managed outside the gallery + if installed.IsSystem { + continue + } + if installed.Metadata == nil { + continue + } + + // Find matching gallery entry by metadata name + galleryEntry := FindGalleryElement(galleryBackends, installed.Metadata.Name) + if galleryEntry == nil { + continue + } + + installedVersion := installed.Metadata.Version + galleryVersion := galleryEntry.Version + + // If both sides have versions, compare them + if galleryVersion != "" && installedVersion != "" { + if galleryVersion != installedVersion { + result[installed.Metadata.Name] = UpgradeInfo{ + BackendName: installed.Metadata.Name, + InstalledVersion: installedVersion, + AvailableVersion: galleryVersion, + } + } + // Versions match — no upgrade needed + continue + } + + // If either version is empty, fall back to OCI digest comparison + if installed.Metadata.Digest != "" && downloader.URI(galleryEntry.URI).LooksLikeOCI() { + remoteDigest, err := oci.GetImageDigest(galleryEntry.URI, "", nil, nil) + if err != nil { + xlog.Warn("Failed to get remote OCI digest for upgrade check", "backend", installed.Metadata.Name, "error", err) + continue + } + if remoteDigest != installed.Metadata.Digest { + result[installed.Metadata.Name] = UpgradeInfo{ + BackendName: installed.Metadata.Name, + InstalledDigest: installed.Metadata.Digest, + AvailableDigest: remoteDigest, + } + } + } + // No version info and no digest to compare — skip + } + + return result, nil +} + +// UpgradeBackend upgrades a single backend to the latest gallery version using +// an atomic swap with backup-based rollback on failure. +func UpgradeBackend(ctx context.Context, systemState *system.SystemState, modelLoader *model.ModelLoader, galleries []config.Gallery, backendName string, downloadStatus func(string, string, string, float64)) error { + // Look up the installed backend + installedBackends, err := ListSystemBackends(systemState) + if err != nil { + return fmt.Errorf("failed to list installed backends: %w", err) + } + + installed, ok := installedBackends.Get(backendName) + if !ok { + return fmt.Errorf("backend %q: %w", backendName, ErrBackendNotFound) + } + + if installed.IsSystem { + return fmt.Errorf("system backend %q cannot be upgraded via gallery", backendName) + } + + // If this is a meta backend, recursively upgrade the concrete backend it points to + if installed.Metadata != nil && installed.Metadata.MetaBackendFor != "" { + xlog.Info("Meta backend detected, upgrading concrete backend", "meta", backendName, "concrete", installed.Metadata.MetaBackendFor) + return UpgradeBackend(ctx, systemState, modelLoader, galleries, installed.Metadata.MetaBackendFor, downloadStatus) + } + + // Find the gallery entry + galleryBackends, err := AvailableBackends(galleries, systemState) + if err != nil { + return fmt.Errorf("failed to list available backends: %w", err) + } + + galleryEntry := FindGalleryElement(galleryBackends, backendName) + if galleryEntry == nil { + return fmt.Errorf("no gallery entry found for backend %q", backendName) + } + + backendPath := filepath.Join(systemState.Backend.BackendsPath, backendName) + tmpPath := backendPath + ".upgrade-tmp" + backupPath := backendPath + ".backup" + + // Clean up any stale tmp/backup dirs from prior attempts + os.RemoveAll(tmpPath) + os.RemoveAll(backupPath) + + // Step 1: Download the new backend into the tmp directory + if err := os.MkdirAll(tmpPath, 0750); err != nil { + return fmt.Errorf("failed to create upgrade tmp dir: %w", err) + } + + uri := downloader.URI(galleryEntry.URI) + if uri.LooksLikeDir() { + if err := cp.Copy(string(uri), tmpPath); err != nil { + os.RemoveAll(tmpPath) + return fmt.Errorf("failed to copy backend from directory: %w", err) + } + } else { + if err := uri.DownloadFileWithContext(ctx, tmpPath, "", 1, 1, downloadStatus); err != nil { + os.RemoveAll(tmpPath) + return fmt.Errorf("failed to download backend: %w", err) + } + } + + // Step 2: Validate — check that run.sh exists in the new content + newRunFile := filepath.Join(tmpPath, runFile) + if _, err := os.Stat(newRunFile); os.IsNotExist(err) { + os.RemoveAll(tmpPath) + return fmt.Errorf("upgrade validation failed: run.sh not found in new backend") + } + + // Step 3: Atomic swap — rename current to backup, then tmp to current + if err := os.Rename(backendPath, backupPath); err != nil { + os.RemoveAll(tmpPath) + return fmt.Errorf("failed to move current backend to backup: %w", err) + } + + if err := os.Rename(tmpPath, backendPath); err != nil { + // Restore backup on failure + xlog.Error("Failed to move new backend into place, restoring backup", "error", err) + if restoreErr := os.Rename(backupPath, backendPath); restoreErr != nil { + xlog.Error("Failed to restore backup", "error", restoreErr) + } + os.RemoveAll(tmpPath) + return fmt.Errorf("failed to move new backend into place: %w", err) + } + + // Step 4: Write updated metadata, preserving alias from old metadata + var oldAlias string + if installed.Metadata != nil { + oldAlias = installed.Metadata.Alias + } + + newMetadata := &BackendMetadata{ + Name: backendName, + Version: galleryEntry.Version, + URI: galleryEntry.URI, + InstalledAt: time.Now().Format(time.RFC3339), + Alias: oldAlias, + } + + if galleryEntry.Gallery.URL != "" { + newMetadata.GalleryURL = galleryEntry.Gallery.URL + } + + // Record OCI digest if applicable (non-fatal on failure) + if uri.LooksLikeOCI() { + digest, digestErr := oci.GetImageDigest(galleryEntry.URI, "", nil, nil) + if digestErr != nil { + xlog.Warn("Failed to get OCI image digest after upgrade", "uri", galleryEntry.URI, "error", digestErr) + } else { + newMetadata.Digest = digest + } + } + + if err := writeBackendMetadata(backendPath, newMetadata); err != nil { + // Metadata write failure is not worth rolling back the entire upgrade + xlog.Error("Failed to write metadata after upgrade", "error", err) + } + + // Step 5: Re-register backends so the model loader picks up any changes + if err := RegisterBackends(systemState, modelLoader); err != nil { + xlog.Warn("Failed to re-register backends after upgrade", "error", err) + } + + // Step 6: Remove backup + os.RemoveAll(backupPath) + + xlog.Info("Backend upgraded successfully", "backend", backendName, "version", galleryEntry.Version) + return nil +} diff --git a/core/gallery/upgrade_test.go b/core/gallery/upgrade_test.go new file mode 100644 index 000000000000..f65b4276b1bf --- /dev/null +++ b/core/gallery/upgrade_test.go @@ -0,0 +1,219 @@ +package gallery_test + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "time" + + "github.com/mudler/LocalAI/core/config" + . "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" +) + +var _ = Describe("Upgrade Detection and Execution", func() { + var ( + tempDir string + backendsPath string + galleryPath string + systemState *system.SystemState + galleries []config.Gallery + ) + + // installBackendWithVersion creates a fake installed backend directory with + // the given name, version, and optional run.sh content. + installBackendWithVersion := func(name, version string, runContent ...string) { + dir := filepath.Join(backendsPath, name) + Expect(os.MkdirAll(dir, 0750)).To(Succeed()) + + content := "#!/bin/sh\necho ok" + if len(runContent) > 0 { + content = runContent[0] + } + Expect(os.WriteFile(filepath.Join(dir, "run.sh"), []byte(content), 0755)).To(Succeed()) + + metadata := BackendMetadata{ + Name: name, + Version: version, + InstalledAt: time.Now().Format(time.RFC3339), + } + data, err := json.MarshalIndent(metadata, "", " ") + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(dir, "metadata.json"), data, 0644)).To(Succeed()) + } + + // writeGalleryYAML writes a gallery YAML file with the given backends. + writeGalleryYAML := func(backends []GalleryBackend) { + data, err := yaml.Marshal(backends) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(galleryPath, data, 0644)).To(Succeed()) + } + + BeforeEach(func() { + var err error + tempDir, err = os.MkdirTemp("", "upgrade-test-*") + Expect(err).NotTo(HaveOccurred()) + + backendsPath = tempDir + + galleryPath = filepath.Join(tempDir, "gallery.yaml") + + // Write a default empty gallery + writeGalleryYAML([]GalleryBackend{}) + + galleries = []config.Gallery{ + { + Name: "test-gallery", + URL: "file://" + galleryPath, + }, + } + + systemState, err = system.GetSystemState( + system.WithBackendPath(backendsPath), + ) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + os.RemoveAll(tempDir) + }) + + Describe("CheckBackendUpgrades", func() { + It("should detect upgrade when gallery version differs from installed version", func() { + // Install a backend at v1.0.0 + installBackendWithVersion("my-backend", "1.0.0") + + // Gallery advertises v2.0.0 + writeGalleryYAML([]GalleryBackend{ + { + Metadata: Metadata{ + Name: "my-backend", + }, + URI: filepath.Join(tempDir, "some-source"), + Version: "2.0.0", + }, + }) + + upgrades, err := CheckBackendUpgrades(context.Background(), galleries, systemState) + Expect(err).NotTo(HaveOccurred()) + Expect(upgrades).To(HaveKey("my-backend")) + Expect(upgrades["my-backend"].InstalledVersion).To(Equal("1.0.0")) + Expect(upgrades["my-backend"].AvailableVersion).To(Equal("2.0.0")) + }) + + It("should NOT flag upgrade when versions match", func() { + installBackendWithVersion("my-backend", "2.0.0") + + writeGalleryYAML([]GalleryBackend{ + { + Metadata: Metadata{ + Name: "my-backend", + }, + URI: filepath.Join(tempDir, "some-source"), + Version: "2.0.0", + }, + }) + + upgrades, err := CheckBackendUpgrades(context.Background(), galleries, systemState) + Expect(err).NotTo(HaveOccurred()) + Expect(upgrades).To(BeEmpty()) + }) + + It("should skip backends without version info and without OCI digest", func() { + // Install without version + installBackendWithVersion("my-backend", "") + + // Gallery also without version + writeGalleryYAML([]GalleryBackend{ + { + Metadata: Metadata{ + Name: "my-backend", + }, + URI: filepath.Join(tempDir, "some-source"), + }, + }) + + upgrades, err := CheckBackendUpgrades(context.Background(), galleries, systemState) + Expect(err).NotTo(HaveOccurred()) + Expect(upgrades).To(BeEmpty()) + }) + }) + + Describe("UpgradeBackend", func() { + It("should replace backend directory and update metadata", func() { + // Install v1 + installBackendWithVersion("my-backend", "1.0.0", "#!/bin/sh\necho v1") + + // Create a source directory with v2 content + srcDir := filepath.Join(tempDir, "v2-source") + Expect(os.MkdirAll(srcDir, 0750)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(srcDir, "run.sh"), []byte("#!/bin/sh\necho v2"), 0755)).To(Succeed()) + + // Gallery points to the v2 source dir + writeGalleryYAML([]GalleryBackend{ + { + Metadata: Metadata{ + Name: "my-backend", + }, + URI: srcDir, + Version: "2.0.0", + }, + }) + + ml := model.NewModelLoader(systemState) + err := UpgradeBackend(context.Background(), systemState, ml, galleries, "my-backend", nil) + Expect(err).NotTo(HaveOccurred()) + + // Verify run.sh was updated + content, err := os.ReadFile(filepath.Join(backendsPath, "my-backend", "run.sh")) + Expect(err).NotTo(HaveOccurred()) + Expect(string(content)).To(Equal("#!/bin/sh\necho v2")) + + // Verify metadata was updated + metaData, err := os.ReadFile(filepath.Join(backendsPath, "my-backend", "metadata.json")) + Expect(err).NotTo(HaveOccurred()) + var meta BackendMetadata + Expect(json.Unmarshal(metaData, &meta)).To(Succeed()) + Expect(meta.Version).To(Equal("2.0.0")) + Expect(meta.Name).To(Equal("my-backend")) + }) + + It("should restore backup on failure", func() { + // Install v1 + installBackendWithVersion("my-backend", "1.0.0", "#!/bin/sh\necho v1") + + // Gallery points to a nonexistent path (no run.sh will be found) + nonExistentDir := filepath.Join(tempDir, "does-not-exist") + writeGalleryYAML([]GalleryBackend{ + { + Metadata: Metadata{ + Name: "my-backend", + }, + URI: nonExistentDir, + Version: "2.0.0", + }, + }) + + ml := model.NewModelLoader(systemState) + err := UpgradeBackend(context.Background(), systemState, ml, galleries, "my-backend", nil) + Expect(err).To(HaveOccurred()) + + // Verify v1 is still intact + content, err := os.ReadFile(filepath.Join(backendsPath, "my-backend", "run.sh")) + Expect(err).NotTo(HaveOccurred()) + Expect(string(content)).To(Equal("#!/bin/sh\necho v1")) + + // Verify metadata still says v1 + metaData, err := os.ReadFile(filepath.Join(backendsPath, "my-backend", "metadata.json")) + Expect(err).NotTo(HaveOccurred()) + var meta BackendMetadata + Expect(json.Unmarshal(metaData, &meta)).To(Succeed()) + Expect(meta.Version).To(Equal("1.0.0")) + }) + }) +}) From b19e60d03ab6553a30514f02b29edb5687876d1d Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 11 Apr 2026 07:58:14 +0000 Subject: [PATCH 3/8] feat: add AutoUpgradeBackends config and runtime settings Add configuration and runtime settings for backend auto-upgrade: - RuntimeSettings field for dynamic config via API/JSON - ApplicationConfig field, option func, and roundtrip conversion - CLI flag with LOCALAI_AUTO_UPGRADE_BACKENDS env var - Config file watcher support for runtime_settings.json - Tests for ToRuntimeSettings, ApplyRuntimeSettings, and roundtrip --- core/application/config_file_watcher.go | 3 +++ core/cli/run.go | 5 +++++ core/config/application_config.go | 10 ++++++++++ core/config/application_config_test.go | 17 +++++++++++++++++ core/config/runtime_settings.go | 1 + 5 files changed, 36 insertions(+) diff --git a/core/application/config_file_watcher.go b/core/application/config_file_watcher.go index 8eb26355d132..7d174bede5d2 100644 --- a/core/application/config_file_watcher.go +++ b/core/application/config_file_watcher.go @@ -335,6 +335,9 @@ func readRuntimeSettingsJson(startupAppConfig config.ApplicationConfig) fileHand if settings.AutoloadBackendGalleries != nil && !envAutoloadBackendGalleries { appConfig.AutoloadBackendGalleries = *settings.AutoloadBackendGalleries } + if settings.AutoUpgradeBackends != nil { + appConfig.AutoUpgradeBackends = *settings.AutoUpgradeBackends + } if settings.ApiKeys != nil { // API keys from env vars (startup) should be kept, runtime settings keys replace all runtime keys // If runtime_settings.json specifies ApiKeys (even if empty), it replaces all runtime keys diff --git a/core/cli/run.go b/core/cli/run.go index ce31a6a2d6c5..fe3f7c21244a 100644 --- a/core/cli/run.go +++ b/core/cli/run.go @@ -47,6 +47,7 @@ type RunCMD struct { BackendImagesReleaseTag string `env:"LOCALAI_BACKEND_IMAGES_RELEASE_TAG,BACKEND_IMAGES_RELEASE_TAG" help:"Fallback release tag for backend images" group:"backends" default:"latest"` BackendImagesBranchTag string `env:"LOCALAI_BACKEND_IMAGES_BRANCH_TAG,BACKEND_IMAGES_BRANCH_TAG" help:"Fallback branch tag for backend images" group:"backends" default:"master"` BackendDevSuffix string `env:"LOCALAI_BACKEND_DEV_SUFFIX,BACKEND_DEV_SUFFIX" help:"Development suffix for backend images" group:"backends" default:"development"` + AutoUpgradeBackends bool `env:"LOCALAI_AUTO_UPGRADE_BACKENDS,AUTO_UPGRADE_BACKENDS" help:"Automatically upgrade backends when new versions are detected" group:"backends" default:"false"` PreloadModels string `env:"LOCALAI_PRELOAD_MODELS,PRELOAD_MODELS" help:"A List of models to apply in JSON at start" group:"models"` Models []string `env:"LOCALAI_MODELS,MODELS" help:"A List of model configuration URLs to load" group:"models"` PreloadModelsConfig string `env:"LOCALAI_PRELOAD_MODELS_CONFIG,PRELOAD_MODELS_CONFIG" help:"A List of models to apply at startup. Path to a YAML config file" group:"models"` @@ -490,6 +491,10 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error { opts = append(opts, config.EnableBackendGalleriesAutoload) } + if r.AutoUpgradeBackends { + opts = append(opts, config.WithAutoUpgradeBackends(r.AutoUpgradeBackends)) + } + if r.PreloadBackendOnly { _, err := application.New(opts...) return err diff --git a/core/config/application_config.go b/core/config/application_config.go index b4d0fd1bfc7c..e22ad4d0c21a 100644 --- a/core/config/application_config.go +++ b/core/config/application_config.go @@ -57,6 +57,7 @@ type ApplicationConfig struct { ExternalGRPCBackends map[string]string AutoloadGalleries, AutoloadBackendGalleries bool + AutoUpgradeBackends bool SingleBackend bool // Deprecated: use MaxActiveBackends = 1 instead MaxActiveBackends int // Maximum number of active backends (0 = unlimited, 1 = single backend mode) @@ -390,6 +391,10 @@ var EnableBackendGalleriesAutoload = func(o *ApplicationConfig) { o.AutoloadBackendGalleries = true } +func WithAutoUpgradeBackends(v bool) AppOption { + return func(o *ApplicationConfig) { o.AutoUpgradeBackends = v } +} + var EnableFederated = func(o *ApplicationConfig) { o.Federated = true } @@ -862,6 +867,7 @@ func (o *ApplicationConfig) ToRuntimeSettings() RuntimeSettings { backendGalleries := o.BackendGalleries autoloadGalleries := o.AutoloadGalleries autoloadBackendGalleries := o.AutoloadBackendGalleries + autoUpgradeBackends := o.AutoUpgradeBackends apiKeys := o.ApiKeys agentJobRetentionDays := o.AgentJobRetentionDays @@ -935,6 +941,7 @@ func (o *ApplicationConfig) ToRuntimeSettings() RuntimeSettings { BackendGalleries: &backendGalleries, AutoloadGalleries: &autoloadGalleries, AutoloadBackendGalleries: &autoloadBackendGalleries, + AutoUpgradeBackends: &autoUpgradeBackends, ApiKeys: &apiKeys, AgentJobRetentionDays: &agentJobRetentionDays, OpenResponsesStoreTTL: &openResponsesStoreTTL, @@ -1083,6 +1090,9 @@ func (o *ApplicationConfig) ApplyRuntimeSettings(settings *RuntimeSettings) (req if settings.AutoloadBackendGalleries != nil { o.AutoloadBackendGalleries = *settings.AutoloadBackendGalleries } + if settings.AutoUpgradeBackends != nil { + o.AutoUpgradeBackends = *settings.AutoUpgradeBackends + } if settings.AgentJobRetentionDays != nil { o.AgentJobRetentionDays = *settings.AgentJobRetentionDays } diff --git a/core/config/application_config_test.go b/core/config/application_config_test.go index c5559ce533d9..fffdeaf0bdf1 100644 --- a/core/config/application_config_test.go +++ b/core/config/application_config_test.go @@ -119,6 +119,13 @@ var _ = Describe("ApplicationConfig RuntimeSettings Conversion", func() { Expect(*rs.AgentJobRetentionDays).To(Equal(30)) }) + It("should include auto_upgrade_backends", func() { + appConfig := &ApplicationConfig{AutoUpgradeBackends: true} + rs := appConfig.ToRuntimeSettings() + Expect(rs.AutoUpgradeBackends).ToNot(BeNil()) + Expect(*rs.AutoUpgradeBackends).To(BeTrue()) + }) + It("should use default timeouts when not set", func() { appConfig := &ApplicationConfig{} @@ -426,6 +433,14 @@ var _ = Describe("ApplicationConfig RuntimeSettings Conversion", func() { Expect(appConfig.AutoloadBackendGalleries).To(BeTrue()) }) + It("should apply auto_upgrade_backends setting", func() { + appConfig := &ApplicationConfig{} + v := true + rs := &RuntimeSettings{AutoUpgradeBackends: &v} + appConfig.ApplyRuntimeSettings(rs) + Expect(appConfig.AutoUpgradeBackends).To(BeTrue()) + }) + It("should apply agent settings", func() { appConfig := &ApplicationConfig{} @@ -465,6 +480,7 @@ var _ = Describe("ApplicationConfig RuntimeSettings Conversion", func() { Federated: true, AutoloadGalleries: true, AutoloadBackendGalleries: false, + AutoUpgradeBackends: true, AgentJobRetentionDays: 60, } @@ -496,6 +512,7 @@ var _ = Describe("ApplicationConfig RuntimeSettings Conversion", func() { Expect(target.Federated).To(Equal(original.Federated)) Expect(target.AutoloadGalleries).To(Equal(original.AutoloadGalleries)) Expect(target.AutoloadBackendGalleries).To(Equal(original.AutoloadBackendGalleries)) + Expect(target.AutoUpgradeBackends).To(Equal(original.AutoUpgradeBackends)) Expect(target.AgentJobRetentionDays).To(Equal(original.AgentJobRetentionDays)) }) diff --git a/core/config/runtime_settings.go b/core/config/runtime_settings.go index 6a4117f06490..21a112b4de20 100644 --- a/core/config/runtime_settings.go +++ b/core/config/runtime_settings.go @@ -20,6 +20,7 @@ type RuntimeSettings struct { // Backend management SingleBackend *bool `json:"single_backend,omitempty"` // Deprecated: use MaxActiveBackends = 1 instead MaxActiveBackends *int `json:"max_active_backends,omitempty"` // Maximum number of active backends (0 = unlimited, 1 = single backend mode) + AutoUpgradeBackends *bool `json:"auto_upgrade_backends,omitempty"` // Automatically upgrade backends when new versions are detected // Memory Reclaimer settings (works with GPU if available, otherwise RAM) MemoryReclaimerEnabled *bool `json:"memory_reclaimer_enabled,omitempty"` // Enable memory threshold monitoring MemoryReclaimerThreshold *float64 `json:"memory_reclaimer_threshold,omitempty"` // Threshold 0.0-1.0 (e.g., 0.95 = 95%) From 1e083cd8700000b0796f10173ae50a7a2ae6cb48 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 11 Apr 2026 08:02:49 +0000 Subject: [PATCH 4/8] feat(ui): add backend version display and upgrade support - Add upgrade check/trigger API endpoints to config and api module - Backends page: version badge, upgrade indicator, upgrade button - Manage page: version in metadata, context-aware upgrade/reinstall button - Settings page: auto-upgrade backends toggle --- core/http/react-ui/src/pages/Backends.jsx | 56 ++++++++++++++++++++--- core/http/react-ui/src/pages/Manage.jsx | 45 ++++++++++++++++-- core/http/react-ui/src/pages/Settings.jsx | 3 ++ core/http/react-ui/src/utils/api.js | 3 ++ core/http/react-ui/src/utils/config.js | 3 ++ 5 files changed, 100 insertions(+), 10 deletions(-) diff --git a/core/http/react-ui/src/pages/Backends.jsx b/core/http/react-ui/src/pages/Backends.jsx index 4e73a9669c01..3cbb71872e2d 100644 --- a/core/http/react-ui/src/pages/Backends.jsx +++ b/core/http/react-ui/src/pages/Backends.jsx @@ -26,6 +26,7 @@ export default function Backends() { const [expandedRow, setExpandedRow] = useState(null) const [confirmDialog, setConfirmDialog] = useState(null) const [allBackends, setAllBackends] = useState([]) + const [upgrades, setUpgrades] = useState({}) const fetchBackends = useCallback(async () => { try { @@ -52,6 +53,13 @@ export default function Backends() { if (!loading) fetchBackends() }, [operations.length]) + // Fetch available upgrades + useEffect(() => { + backendsApi.checkUpgrades() + .then(data => setUpgrades(data || {})) + .catch(() => {}) + }, [operations.length]) + // Client-side filtering by tag const filteredBackends = filter ? allBackends.filter(b => { @@ -114,6 +122,15 @@ export default function Backends() { }) } + const handleUpgrade = async (id) => { + try { + await backendsApi.upgrade(id) + addToast(`Upgrading ${id}...`, 'info') + } catch (err) { + addToast(`Upgrade failed: ${err.message}`, 'error') + } + } + const handleManualInstall = async (e) => { e.preventDefault() if (!manualUri.trim()) { addToast('Please enter a URI', 'warning'); return } @@ -179,6 +196,14 @@ export default function Backends() {
Installed
+ {Object.keys(upgrades).length > 0 && ( +
+
+ {Object.keys(upgrades).length} +
+
Updates
+
+ )} Docs @@ -300,6 +325,11 @@ export default function Backends() { {/* Name */} {b.name || b.id} + {b.version && ( + + v{b.version} + + )} {/* Description */} @@ -346,9 +376,17 @@ export default function Backends() { ) : b.installed ? ( - - Installed - +
+ + Installed + + {upgrades[b.name] && ( + + + {upgrades[b.name].available_version ? `v${upgrades[b.name].available_version}` : 'Update'} + + )} +
) : ( Not Installed @@ -361,9 +399,15 @@ export default function Backends() {
e.stopPropagation()}> {b.installed ? ( <> - + {upgrades[b.name] ? ( + + ) : ( + + )} diff --git a/core/http/react-ui/src/pages/Manage.jsx b/core/http/react-ui/src/pages/Manage.jsx index 6ac429ff4051..3d24ebea8f06 100644 --- a/core/http/react-ui/src/pages/Manage.jsx +++ b/core/http/react-ui/src/pages/Manage.jsx @@ -22,6 +22,7 @@ export default function Manage() { const [backendsLoading, setBackendsLoading] = useState(true) const [reloading, setReloading] = useState(false) const [reinstallingBackends, setReinstallingBackends] = useState(new Set()) + const [upgrades, setUpgrades] = useState({}) const [confirmDialog, setConfirmDialog] = useState(null) const [distributedMode, setDistributedMode] = useState(false) const [togglingModels, setTogglingModels] = useState(new Set()) @@ -62,6 +63,15 @@ export default function Manage() { nodesApi.list().then(() => setDistributedMode(true)).catch(() => {}) }, [fetchLoadedModels, fetchBackends]) + // Fetch available backend upgrades + useEffect(() => { + if (activeTab === 'backends') { + backendsApi.checkUpgrades() + .then(data => setUpgrades(data || {})) + .catch(() => {}) + } + }, [activeTab]) + const handleStopModel = (modelName) => { setConfirmDialog({ title: 'Stop Model', @@ -169,6 +179,22 @@ export default function Manage() { } } + const handleUpgradeBackend = async (name) => { + try { + setReinstallingBackends(prev => new Set(prev).add(name)) + await backendsApi.upgrade(name) + addToast(`Upgrading ${name}...`, 'info') + } catch (err) { + addToast(`Failed to upgrade: ${err.message}`, 'error') + } finally { + setReinstallingBackends(prev => { + const next = new Set(prev) + next.delete(name) + return next + }) + } + } + const handleDeleteBackend = (name) => { setConfirmDialog({ title: 'Delete Backend', @@ -471,6 +497,17 @@ export default function Manage() { For: {backend.Metadata.meta_backend_for} )} + {backend.Metadata?.version && ( + + + Version: v{backend.Metadata.version} + {upgrades[backend.Name] && ( + + → v{upgrades[backend.Name].available_version} + + )} + + )} {backend.Metadata?.installed_at && ( @@ -485,12 +522,12 @@ export default function Manage() { {!backend.IsSystem ? ( <>
diff --git a/core/http/react-ui/src/utils/api.js b/core/http/react-ui/src/utils/api.js index b9d3983d2377..1edac8cbd4fc 100644 --- a/core/http/react-ui/src/utils/api.js +++ b/core/http/react-ui/src/utils/api.js @@ -120,6 +120,9 @@ export const backendsApi = { installExternal: (body) => postJSON(API_CONFIG.endpoints.installExternalBackend, body), getJob: (uid) => fetchJSON(API_CONFIG.endpoints.backendJob(uid)), deleteInstalled: (name) => postJSON(API_CONFIG.endpoints.deleteInstalledBackend(name), {}), + checkUpgrades: () => fetchJSON(API_CONFIG.endpoints.backendsUpgrades), + forceCheckUpgrades: () => postJSON(API_CONFIG.endpoints.backendsUpgradesCheck, {}), + upgrade: (name) => postJSON(API_CONFIG.endpoints.upgradeBackend(name), {}), } // Chat API (non-streaming) diff --git a/core/http/react-ui/src/utils/config.js b/core/http/react-ui/src/utils/config.js index 70fb7f57a95a..66767eb3fcbd 100644 --- a/core/http/react-ui/src/utils/config.js +++ b/core/http/react-ui/src/utils/config.js @@ -23,6 +23,9 @@ export const API_CONFIG = { installExternalBackend: '/api/backends/install-external', backendJob: (uid) => `/api/backends/job/${uid}`, deleteInstalledBackend: (name) => `/api/backends/system/delete/${name}`, + backendsUpgrades: '/api/backends/upgrades', + backendsUpgradesCheck: '/api/backends/upgrades/check', + upgradeBackend: (name) => `/api/backends/upgrade/${name}`, // Resources resources: '/api/resources', From 948f3bfaa4349455ac5b85b811dee922313f19eb Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 11 Apr 2026 08:11:06 +0000 Subject: [PATCH 5/8] feat: add upgrade checker service, API endpoints, and CLI command - UpgradeChecker background service: checks every 6h, auto-upgrades when enabled - API endpoints: GET /backends/upgrades, POST /backends/upgrades/check, POST /backends/upgrade/:name - CLI: `localai backends upgrade` command, version display in `backends list` - BackendManager interface: add UpgradeBackend and CheckUpgrades methods - Wire upgrade op through GalleryService backend handler - Distributed mode: fan-out upgrade to worker nodes via NATS --- core/application/application.go | 7 + core/application/startup.go | 7 + core/application/upgrade_checker.go | 163 ++++++++++++++++++++ core/cli/backends.go | 100 +++++++++++- core/http/endpoints/localai/backend.go | 66 +++++++- core/http/routes/localai.go | 6 +- core/http/routes/ui_api.go | 44 ++++++ core/services/galleryop/backends.go | 4 +- core/services/galleryop/managers.go | 4 +- core/services/galleryop/managers_local.go | 8 + core/services/galleryop/operation.go | 3 + core/services/nodes/managers_distributed.go | 54 ++++++- 12 files changed, 454 insertions(+), 12 deletions(-) create mode 100644 core/application/upgrade_checker.go diff --git a/core/application/application.go b/core/application/application.go index accba0330643..4af5ac9bbd55 100644 --- a/core/application/application.go +++ b/core/application/application.go @@ -37,6 +37,9 @@ type Application struct { // Distributed mode services (nil when not in distributed mode) distributed *DistributedServices + + // Upgrade checker (background service for detecting backend upgrades) + upgradeChecker *UpgradeChecker } func newApplication(appConfig *config.ApplicationConfig) *Application { @@ -79,6 +82,10 @@ func (a *Application) AgentJobService() *agentpool.AgentJobService { return a.agentJobService } +func (a *Application) UpgradeChecker() *UpgradeChecker { + return a.upgradeChecker +} + func (a *Application) AgentPoolService() *agentpool.AgentPoolService { return a.agentPoolService.Load() } diff --git a/core/application/startup.go b/core/application/startup.go index 728c3c97221e..d3aa9dcd0585 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -231,6 +231,13 @@ func New(opts ...config.AppOption) (*Application, error) { xlog.Error("error registering external backends", "error", err) } + // Start background upgrade checker for backends + if len(options.BackendGalleries) > 0 { + uc := NewUpgradeChecker(options, application.ModelLoader()) + application.upgradeChecker = uc + go uc.Run(options.Context) + } + if options.ConfigFile != "" { if err := application.ModelConfigLoader().LoadMultipleModelConfigsSingleFile(options.ConfigFile, configLoaderOpts...); err != nil { xlog.Error("error loading config file", "error", err) diff --git a/core/application/upgrade_checker.go b/core/application/upgrade_checker.go new file mode 100644 index 000000000000..774b15fa71ac --- /dev/null +++ b/core/application/upgrade_checker.go @@ -0,0 +1,163 @@ +package application + +import ( + "context" + "sync" + "time" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/system" + "github.com/mudler/xlog" +) + +// UpgradeChecker periodically checks for backend upgrades and optionally +// auto-upgrades them. It caches the last check results for API queries. +type UpgradeChecker struct { + appConfig *config.ApplicationConfig + modelLoader *model.ModelLoader + galleries []config.Gallery + systemState *system.SystemState + + checkInterval time.Duration + stop chan struct{} + done chan struct{} + triggerCh chan struct{} + + mu sync.RWMutex + lastUpgrades map[string]gallery.UpgradeInfo + lastCheckTime time.Time +} + +// NewUpgradeChecker creates a new UpgradeChecker service. +func NewUpgradeChecker(appConfig *config.ApplicationConfig, ml *model.ModelLoader) *UpgradeChecker { + return &UpgradeChecker{ + appConfig: appConfig, + modelLoader: ml, + galleries: appConfig.BackendGalleries, + systemState: appConfig.SystemState, + checkInterval: 6 * time.Hour, + stop: make(chan struct{}), + done: make(chan struct{}), + triggerCh: make(chan struct{}, 1), + lastUpgrades: make(map[string]gallery.UpgradeInfo), + } +} + +// Run starts the upgrade checker loop. It waits 30 seconds after startup, +// performs an initial check, then re-checks every 6 hours. +func (uc *UpgradeChecker) Run(ctx context.Context) { + defer close(uc.done) + + // Initial delay: don't slow down startup + select { + case <-ctx.Done(): + return + case <-uc.stop: + return + case <-time.After(30 * time.Second): + } + + // First check + uc.runCheck(ctx) + + // Periodic loop + ticker := time.NewTicker(uc.checkInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-uc.stop: + return + case <-ticker.C: + uc.runCheck(ctx) + case <-uc.triggerCh: + uc.runCheck(ctx) + } + } +} + +// Shutdown stops the upgrade checker loop. +func (uc *UpgradeChecker) Shutdown() { + close(uc.stop) + <-uc.done +} + +// TriggerCheck forces an immediate upgrade check. +func (uc *UpgradeChecker) TriggerCheck() { + select { + case uc.triggerCh <- struct{}{}: + default: + // Already triggered, skip + } +} + +// GetAvailableUpgrades returns the cached upgrade check results. +func (uc *UpgradeChecker) GetAvailableUpgrades() map[string]gallery.UpgradeInfo { + uc.mu.RLock() + defer uc.mu.RUnlock() + + // Return a copy to avoid races + result := make(map[string]gallery.UpgradeInfo, len(uc.lastUpgrades)) + for k, v := range uc.lastUpgrades { + result[k] = v + } + return result +} + +func (uc *UpgradeChecker) runCheck(ctx context.Context) { + upgrades, err := gallery.CheckBackendUpgrades(ctx, uc.galleries, uc.systemState) + + uc.mu.Lock() + uc.lastCheckTime = time.Now() + if err != nil { + xlog.Debug("Backend upgrade check failed", "error", err) + uc.mu.Unlock() + return + } + uc.lastUpgrades = upgrades + uc.mu.Unlock() + + if len(upgrades) == 0 { + xlog.Debug("All backends up to date") + return + } + + // Log available upgrades + for name, info := range upgrades { + if info.AvailableVersion != "" { + xlog.Info("Backend upgrade available", + "backend", name, + "installed", info.InstalledVersion, + "available", info.AvailableVersion) + } else { + xlog.Info("Backend upgrade available (new build)", + "backend", name) + } + } + + // Auto-upgrade if enabled + if uc.appConfig.AutoUpgradeBackends { + for name, info := range upgrades { + xlog.Info("Auto-upgrading backend", "backend", name, + "from", info.InstalledVersion, "to", info.AvailableVersion) + if err := gallery.UpgradeBackend(ctx, uc.systemState, uc.modelLoader, + uc.galleries, name, nil); err != nil { + xlog.Error("Failed to auto-upgrade backend", + "backend", name, "error", err) + } else { + xlog.Info("Backend upgraded successfully", "backend", name, + "version", info.AvailableVersion) + } + } + // Re-check to update cache after upgrades + if freshUpgrades, err := gallery.CheckBackendUpgrades(ctx, uc.galleries, uc.systemState); err == nil { + uc.mu.Lock() + uc.lastUpgrades = freshUpgrades + uc.mu.Unlock() + } + } +} diff --git a/core/cli/backends.go b/core/cli/backends.go index 23f6b3ff181b..6269a9b0e8d1 100644 --- a/core/cli/backends.go +++ b/core/cli/backends.go @@ -40,10 +40,17 @@ type BackendsUninstall struct { BackendsCMDFlags `embed:""` } +type BackendsUpgrade struct { + BackendArgs []string `arg:"" optional:"" name:"backends" help:"Backend names to upgrade (empty = upgrade all)"` + + BackendsCMDFlags `embed:""` +} + type BackendsCMD struct { List BackendsList `cmd:"" help:"List the backends available in your galleries" default:"withargs"` Install BackendsInstall `cmd:"" help:"Install a backend from the gallery"` Uninstall BackendsUninstall `cmd:"" help:"Uninstall a backend"` + Upgrade BackendsUpgrade `cmd:"" help:"Upgrade backends to latest versions"` } func (bl *BackendsList) Run(ctx *cliContext.Context) error { @@ -64,11 +71,27 @@ func (bl *BackendsList) Run(ctx *cliContext.Context) error { if err != nil { return err } + + // Check for upgrades + upgrades, _ := gallery.CheckBackendUpgrades(context.Background(), galleries, systemState) + for _, backend := range backends { + versionStr := "" + if backend.Version != "" { + versionStr = " v" + backend.Version + } if backend.Installed { - fmt.Printf(" * %s@%s (installed)\n", backend.Gallery.Name, backend.Name) + if info, ok := upgrades[backend.Name]; ok { + upgradeStr := info.AvailableVersion + if upgradeStr == "" { + upgradeStr = "new build" + } + fmt.Printf(" * %s@%s%s (installed, upgrade available: %s)\n", backend.Gallery.Name, backend.Name, versionStr, upgradeStr) + } else { + fmt.Printf(" * %s@%s%s (installed)\n", backend.Gallery.Name, backend.Name, versionStr) + } } else { - fmt.Printf(" - %s@%s\n", backend.Gallery.Name, backend.Name) + fmt.Printf(" - %s@%s%s\n", backend.Gallery.Name, backend.Name, versionStr) } } return nil @@ -111,6 +134,79 @@ func (bi *BackendsInstall) Run(ctx *cliContext.Context) error { return nil } +func (bu *BackendsUpgrade) Run(ctx *cliContext.Context) error { + var galleries []config.Gallery + if err := json.Unmarshal([]byte(bu.BackendGalleries), &galleries); err != nil { + xlog.Error("unable to load galleries", "error", err) + } + + systemState, err := system.GetSystemState( + system.WithBackendSystemPath(bu.BackendsSystemPath), + system.WithBackendPath(bu.BackendsPath), + ) + if err != nil { + return err + } + + upgrades, err := gallery.CheckBackendUpgrades(context.Background(), galleries, systemState) + if err != nil { + return fmt.Errorf("failed to check for upgrades: %w", err) + } + + if len(upgrades) == 0 { + fmt.Println("All backends are up to date.") + return nil + } + + // Filter to specified backends if args given + toUpgrade := upgrades + if len(bu.BackendArgs) > 0 { + toUpgrade = make(map[string]gallery.UpgradeInfo) + for _, name := range bu.BackendArgs { + if info, ok := upgrades[name]; ok { + toUpgrade[name] = info + } else { + fmt.Printf("Backend %s: no upgrade available\n", name) + } + } + } + + if len(toUpgrade) == 0 { + fmt.Println("No upgrades to apply.") + return nil + } + + modelLoader := model.NewModelLoader(systemState) + for name, info := range toUpgrade { + versionStr := "" + if info.AvailableVersion != "" { + versionStr = " to v" + info.AvailableVersion + } + fmt.Printf("Upgrading %s%s...\n", name, versionStr) + + progressBar := progressbar.NewOptions( + 1000, + progressbar.OptionSetDescription(fmt.Sprintf("downloading %s", name)), + progressbar.OptionShowBytes(false), + progressbar.OptionClearOnFinish(), + ) + progressCallback := func(fileName string, current string, total string, percentage float64) { + v := int(percentage * 10) + if err := progressBar.Set(v); err != nil { + xlog.Error("error updating progress bar", "error", err) + } + } + + if err := gallery.UpgradeBackend(context.Background(), systemState, modelLoader, galleries, name, progressCallback); err != nil { + fmt.Printf("Failed to upgrade %s: %v\n", name, err) + } else { + fmt.Printf("Backend %s upgraded successfully\n", name) + } + } + + return nil +} + func (bu *BackendsUninstall) Run(ctx *cliContext.Context) error { for _, backendName := range bu.BackendArgs { xlog.Info("uninstalling backend", "backend", backendName) diff --git a/core/http/endpoints/localai/backend.go b/core/http/endpoints/localai/backend.go index a67ba4da0f04..5cd2ba060587 100644 --- a/core/http/endpoints/localai/backend.go +++ b/core/http/endpoints/localai/backend.go @@ -15,23 +15,31 @@ import ( "github.com/mudler/xlog" ) +// UpgradeInfoProvider is an interface for querying cached backend upgrade information. +type UpgradeInfoProvider interface { + GetAvailableUpgrades() map[string]gallery.UpgradeInfo + TriggerCheck() +} + type BackendEndpointService struct { galleries []config.Gallery backendPath string backendSystemPath string backendApplier *galleryop.GalleryService + upgradeChecker UpgradeInfoProvider } type GalleryBackend struct { ID string `json:"id"` } -func CreateBackendEndpointService(galleries []config.Gallery, systemState *system.SystemState, backendApplier *galleryop.GalleryService) BackendEndpointService { +func CreateBackendEndpointService(galleries []config.Gallery, systemState *system.SystemState, backendApplier *galleryop.GalleryService, upgradeChecker UpgradeInfoProvider) BackendEndpointService { return BackendEndpointService{ galleries: galleries, backendPath: systemState.Backend.BackendsPath, backendSystemPath: systemState.Backend.BackendsSystemPath, backendApplier: backendApplier, + upgradeChecker: upgradeChecker, } } @@ -146,6 +154,62 @@ func (mgs *BackendEndpointService) ListBackendGalleriesEndpoint() echo.HandlerFu } } +// GetUpgradesEndpoint returns the cached backend upgrade information +// @Summary Get available backend upgrades +// @Tags backends +// @Success 200 {object} map[string]gallery.UpgradeInfo "Response" +// @Router /backends/upgrades [get] +func (mgs *BackendEndpointService) GetUpgradesEndpoint() echo.HandlerFunc { + return func(c echo.Context) error { + if mgs.upgradeChecker == nil { + return c.JSON(200, map[string]gallery.UpgradeInfo{}) + } + return c.JSON(200, mgs.upgradeChecker.GetAvailableUpgrades()) + } +} + +// CheckUpgradesEndpoint forces an immediate upgrade check +// @Summary Force backend upgrade check +// @Tags backends +// @Success 200 {object} map[string]gallery.UpgradeInfo "Response" +// @Router /backends/upgrades/check [post] +func (mgs *BackendEndpointService) CheckUpgradesEndpoint() echo.HandlerFunc { + return func(c echo.Context) error { + if mgs.upgradeChecker == nil { + return c.JSON(200, map[string]gallery.UpgradeInfo{}) + } + mgs.upgradeChecker.TriggerCheck() + // Return current cached results (the triggered check runs async) + return c.JSON(200, mgs.upgradeChecker.GetAvailableUpgrades()) + } +} + +// UpgradeBackendEndpoint triggers an upgrade for a specific backend +// @Summary Upgrade a backend +// @Tags backends +// @Param name path string true "Backend name" +// @Success 200 {object} schema.BackendResponse "Response" +// @Router /backends/upgrade/{name} [post] +func (mgs *BackendEndpointService) UpgradeBackendEndpoint() echo.HandlerFunc { + return func(c echo.Context) error { + backendName := c.Param("name") + + uuid, err := uuid.NewUUID() + if err != nil { + return err + } + + mgs.backendApplier.BackendGalleryChannel <- galleryop.ManagementOp[gallery.GalleryBackend, any]{ + ID: uuid.String(), + GalleryElementName: backendName, + Galleries: mgs.galleries, + Upgrade: true, + } + + return c.JSON(200, schema.BackendResponse{ID: uuid.String(), StatusURL: fmt.Sprintf("%sbackends/jobs/%s", middleware.BaseURL(c), uuid.String())}) + } +} + // ListAvailableBackendsEndpoint list the available backends in the galleries configured in LocalAI // @Summary List all available Backends // @Tags backends diff --git a/core/http/routes/localai.go b/core/http/routes/localai.go index 48cad2fb0e19..d5e41a9c578a 100644 --- a/core/http/routes/localai.go +++ b/core/http/routes/localai.go @@ -59,13 +59,17 @@ func RegisterLocalAIRoutes(router *echo.Echo, backendGalleryEndpointService := localai.CreateBackendEndpointService( appConfig.BackendGalleries, appConfig.SystemState, - galleryService) + galleryService, + app.UpgradeChecker()) router.POST("/backends/apply", backendGalleryEndpointService.ApplyBackendEndpoint(), adminMiddleware) router.POST("/backends/delete/:name", backendGalleryEndpointService.DeleteBackendEndpoint(), adminMiddleware) router.GET("/backends", backendGalleryEndpointService.ListBackendsEndpoint(), adminMiddleware) router.GET("/backends/available", backendGalleryEndpointService.ListAvailableBackendsEndpoint(appConfig.SystemState), adminMiddleware) router.GET("/backends/galleries", backendGalleryEndpointService.ListBackendGalleriesEndpoint(), adminMiddleware) router.GET("/backends/jobs/:uuid", backendGalleryEndpointService.GetOpStatusEndpoint(), adminMiddleware) + router.GET("/backends/upgrades", backendGalleryEndpointService.GetUpgradesEndpoint(), adminMiddleware) + router.POST("/backends/upgrades/check", backendGalleryEndpointService.CheckUpgradesEndpoint(), adminMiddleware) + router.POST("/backends/upgrade/:name", backendGalleryEndpointService.UpgradeBackendEndpoint(), adminMiddleware) // Custom model import endpoint router.POST("/models/import", localai.ImportModelEndpoint(cl, appConfig), adminMiddleware) diff --git a/core/http/routes/ui_api.go b/core/http/routes/ui_api.go index 570e96e1ed93..16dad72b7dde 100644 --- a/core/http/routes/ui_api.go +++ b/core/http/routes/ui_api.go @@ -929,6 +929,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model "tags": b.Tags, "gallery": b.Gallery.Name, "installed": b.Installed, + "version": b.Version, "processing": currentlyProcessing, "jobID": jobID, "isDeletion": isDeletionOp, @@ -1194,6 +1195,49 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model }) }, adminMiddleware) + // Backend upgrade APIs + app.GET("/api/backends/upgrades", func(c echo.Context) error { + if applicationInstance.UpgradeChecker() == nil { + return c.JSON(200, map[string]any{}) + } + return c.JSON(200, applicationInstance.UpgradeChecker().GetAvailableUpgrades()) + }, adminMiddleware) + + app.POST("/api/backends/upgrades/check", func(c echo.Context) error { + if applicationInstance.UpgradeChecker() == nil { + return c.JSON(200, map[string]any{}) + } + applicationInstance.UpgradeChecker().TriggerCheck() + return c.JSON(200, applicationInstance.UpgradeChecker().GetAvailableUpgrades()) + }, adminMiddleware) + + app.POST("/api/backends/upgrade/:name", func(c echo.Context) error { + backendName := c.Param("name") + backendName, err := url.QueryUnescape(backendName) + if err != nil { + return c.JSON(http.StatusBadRequest, map[string]any{ + "error": "invalid backend name", + }) + } + + uid, err := uuid.NewUUID() + if err != nil { + return c.JSON(http.StatusInternalServerError, map[string]any{"error": err.Error()}) + } + + galleryService.BackendGalleryChannel <- galleryop.ManagementOp[gallery.GalleryBackend, any]{ + ID: uid.String(), + GalleryElementName: backendName, + Galleries: appConfig.BackendGalleries, + Upgrade: true, + } + + return c.JSON(200, map[string]any{ + "uuid": uid.String(), + "statusUrl": fmt.Sprintf("/api/backends/job/%s", uid.String()), + }) + }, adminMiddleware) + // P2P APIs app.GET("/api/p2p/workers", func(c echo.Context) error { llamaNodes := p2p.GetAvailableNodes(p2p.NetworkID(appConfig.P2PNetworkID, p2p.LlamaCPPWorkerID)) diff --git a/core/services/galleryop/backends.go b/core/services/galleryop/backends.go index 05430026bacc..b8680f08a1cd 100644 --- a/core/services/galleryop/backends.go +++ b/core/services/galleryop/backends.go @@ -69,7 +69,9 @@ func (g *GalleryService) backendHandler(op *ManagementOp[gallery.GalleryBackend, } var err error - if op.Delete { + if op.Upgrade { + err = g.backendManager.UpgradeBackend(ctx, op.GalleryElementName, progressCallback) + } else if op.Delete { err = g.backendManager.DeleteBackend(op.GalleryElementName) } else { err = g.backendManager.InstallBackend(ctx, op, progressCallback) diff --git a/core/services/galleryop/managers.go b/core/services/galleryop/managers.go index 7857fba19503..5adbef8c7ed2 100644 --- a/core/services/galleryop/managers.go +++ b/core/services/galleryop/managers.go @@ -15,9 +15,11 @@ type ModelManager interface { DeleteModel(name string) error } -// BackendManager handles backend install, delete, and listing lifecycle. +// BackendManager handles backend install, delete, upgrade, and listing lifecycle. type BackendManager interface { InstallBackend(ctx context.Context, op *ManagementOp[gallery.GalleryBackend, any], progressCb ProgressCallback) error DeleteBackend(name string) error ListBackends() (gallery.SystemBackends, error) + UpgradeBackend(ctx context.Context, name string, progressCb ProgressCallback) error + CheckUpgrades(ctx context.Context) (map[string]gallery.UpgradeInfo, error) } diff --git a/core/services/galleryop/managers_local.go b/core/services/galleryop/managers_local.go index 3f8055372ce6..2d86f2dc6857 100644 --- a/core/services/galleryop/managers_local.go +++ b/core/services/galleryop/managers_local.go @@ -92,6 +92,14 @@ func (b *LocalBackendManager) ListBackends() (gallery.SystemBackends, error) { return gallery.ListSystemBackends(b.systemState) } +func (b *LocalBackendManager) UpgradeBackend(ctx context.Context, name string, progressCb ProgressCallback) error { + return gallery.UpgradeBackend(ctx, b.systemState, b.modelLoader, b.backendGalleries, name, progressCb) +} + +func (b *LocalBackendManager) CheckUpgrades(ctx context.Context) (map[string]gallery.UpgradeInfo, error) { + return gallery.CheckBackendUpgrades(ctx, b.backendGalleries, b.systemState) +} + func (b *LocalBackendManager) InstallBackend(ctx context.Context, op *ManagementOp[gallery.GalleryBackend, any], progressCb ProgressCallback) error { if op.ExternalURI != "" { return InstallExternalBackend(ctx, b.backendGalleries, b.systemState, b.modelLoader, diff --git a/core/services/galleryop/operation.go b/core/services/galleryop/operation.go index 758a5bc5c37e..1c766eadefed 100644 --- a/core/services/galleryop/operation.go +++ b/core/services/galleryop/operation.go @@ -29,6 +29,9 @@ type ManagementOp[T any, E any] struct { ExternalURI string // The OCI image, URL, or path ExternalName string // Custom name for the backend ExternalAlias string // Custom alias for the backend + + // Upgrade is true if this is an upgrade operation (not a fresh install) + Upgrade bool } type OpStatus struct { diff --git a/core/services/nodes/managers_distributed.go b/core/services/nodes/managers_distributed.go index 0f7967e11b00..62cb3255284d 100644 --- a/core/services/nodes/managers_distributed.go +++ b/core/services/nodes/managers_distributed.go @@ -49,17 +49,19 @@ func (d *DistributedModelManager) InstallModel(ctx context.Context, op *galleryo // DistributedBackendManager wraps a local BackendManager and adds NATS fan-out // for backend deletion so worker nodes clean up stale files. type DistributedBackendManager struct { - local galleryop.BackendManager - adapter *RemoteUnloaderAdapter - registry *NodeRegistry + local galleryop.BackendManager + adapter *RemoteUnloaderAdapter + registry *NodeRegistry + backendGalleries []config.Gallery } // NewDistributedBackendManager creates a DistributedBackendManager. func NewDistributedBackendManager(appConfig *config.ApplicationConfig, ml *model.ModelLoader, adapter *RemoteUnloaderAdapter, registry *NodeRegistry) *DistributedBackendManager { return &DistributedBackendManager{ - local: galleryop.NewLocalBackendManager(appConfig, ml), - adapter: adapter, - registry: registry, + local: galleryop.NewLocalBackendManager(appConfig, ml), + adapter: adapter, + registry: registry, + backendGalleries: appConfig.BackendGalleries, } } @@ -172,3 +174,43 @@ func (d *DistributedBackendManager) InstallBackend(ctx context.Context, op *gall } return nil } + +// UpgradeBackend fans out a backend upgrade to all healthy worker nodes. +// TODO: Add dedicated NATS subject for upgrade (currently reuses install with force flag) +func (d *DistributedBackendManager) UpgradeBackend(ctx context.Context, name string, progressCb galleryop.ProgressCallback) error { + allNodes, err := d.registry.List(context.Background()) + if err != nil { + return err + } + + galleriesJSON, _ := json.Marshal(d.backendGalleries) + var errs []error + + for _, node := range allNodes { + if node.Status != StatusHealthy { + continue + } + // Reuse install endpoint which will re-download the backend (force mode) + reply, err := d.adapter.InstallBackend(node.ID, name, "", string(galleriesJSON)) + if err != nil { + if errors.Is(err, nats.ErrNoResponders) { + xlog.Warn("No NATS responders for node during upgrade, marking unhealthy", "node", node.Name, "nodeID", node.ID) + d.registry.MarkUnhealthy(context.Background(), node.ID) + continue + } + errs = append(errs, fmt.Errorf("node %s: %w", node.Name, err)) + continue + } + if !reply.Success { + errs = append(errs, fmt.Errorf("node %s: %s", node.Name, reply.Error)) + } + } + + return errors.Join(errs...) +} + +// CheckUpgrades checks for available backend upgrades. +// Gallery comparison is global (not per-node), so we delegate to the local manager. +func (d *DistributedBackendManager) CheckUpgrades(ctx context.Context) (map[string]gallery.UpgradeInfo, error) { + return d.local.CheckUpgrades(ctx) +} From ee00a10836eed4004cbe5e5664886b1a706988e1 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 11 Apr 2026 08:24:03 +0000 Subject: [PATCH 6/8] fix: use advisory lock for upgrade checker in distributed mode In distributed mode with multiple frontend instances, use PostgreSQL advisory lock (KeyBackendUpgradeCheck) so only one instance runs periodic upgrade checks and auto-upgrades. Prevents duplicate upgrade operations across replicas. Standalone mode is unchanged (simple ticker loop). --- core/application/application.go | 9 ++++ core/application/startup.go | 6 ++- core/application/upgrade_checker.go | 67 ++++++++++++++++++++++------- core/services/advisorylock/keys.go | 3 +- 4 files changed, 66 insertions(+), 19 deletions(-) diff --git a/core/application/application.go b/core/application/application.go index 4af5ac9bbd55..9be613ab71e6 100644 --- a/core/application/application.go +++ b/core/application/application.go @@ -86,6 +86,15 @@ func (a *Application) UpgradeChecker() *UpgradeChecker { return a.upgradeChecker } +// distributedDB returns the PostgreSQL database for distributed coordination, +// or nil in standalone mode. +func (a *Application) distributedDB() *gorm.DB { + if a.distributed != nil { + return a.authDB + } + return nil +} + func (a *Application) AgentPoolService() *agentpool.AgentPoolService { return a.agentPoolService.Load() } diff --git a/core/application/startup.go b/core/application/startup.go index d3aa9dcd0585..a03f17bd2189 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -231,9 +231,11 @@ func New(opts ...config.AppOption) (*Application, error) { xlog.Error("error registering external backends", "error", err) } - // Start background upgrade checker for backends + // Start background upgrade checker for backends. + // In distributed mode, uses PostgreSQL advisory lock so only one frontend + // instance runs periodic checks (avoids duplicate upgrades across replicas). if len(options.BackendGalleries) > 0 { - uc := NewUpgradeChecker(options, application.ModelLoader()) + uc := NewUpgradeChecker(options, application.ModelLoader(), application.distributedDB()) application.upgradeChecker = uc go uc.Run(options.Context) } diff --git a/core/application/upgrade_checker.go b/core/application/upgrade_checker.go index 774b15fa71ac..94fb3f6c7756 100644 --- a/core/application/upgrade_checker.go +++ b/core/application/upgrade_checker.go @@ -7,18 +7,25 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/core/services/advisorylock" "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/xlog" + "gorm.io/gorm" ) // UpgradeChecker periodically checks for backend upgrades and optionally // auto-upgrades them. It caches the last check results for API queries. +// +// In standalone mode it runs a simple ticker loop. +// In distributed mode it uses a PostgreSQL advisory lock so that only one +// frontend instance performs periodic checks and auto-upgrades at a time. type UpgradeChecker struct { appConfig *config.ApplicationConfig modelLoader *model.ModelLoader galleries []config.Gallery systemState *system.SystemState + db *gorm.DB // non-nil in distributed mode checkInterval time.Duration stop chan struct{} @@ -31,12 +38,15 @@ type UpgradeChecker struct { } // NewUpgradeChecker creates a new UpgradeChecker service. -func NewUpgradeChecker(appConfig *config.ApplicationConfig, ml *model.ModelLoader) *UpgradeChecker { +// Pass db=nil for standalone mode, or a *gorm.DB for distributed mode +// (uses advisory locks so only one instance runs periodic checks). +func NewUpgradeChecker(appConfig *config.ApplicationConfig, ml *model.ModelLoader, db *gorm.DB) *UpgradeChecker { return &UpgradeChecker{ appConfig: appConfig, modelLoader: ml, galleries: appConfig.BackendGalleries, systemState: appConfig.SystemState, + db: db, checkInterval: 6 * time.Hour, stop: make(chan struct{}), done: make(chan struct{}), @@ -47,6 +57,10 @@ func NewUpgradeChecker(appConfig *config.ApplicationConfig, ml *model.ModelLoade // Run starts the upgrade checker loop. It waits 30 seconds after startup, // performs an initial check, then re-checks every 6 hours. +// +// In distributed mode, periodic checks are guarded by a PostgreSQL advisory +// lock so only one frontend instance runs them. On-demand triggers (TriggerCheck) +// and the initial check always run locally for fast API response cache warming. func (uc *UpgradeChecker) Run(ctx context.Context) { defer close(uc.done) @@ -59,23 +73,44 @@ func (uc *UpgradeChecker) Run(ctx context.Context) { case <-time.After(30 * time.Second): } - // First check + // First check always runs locally (to warm the cache on this instance) uc.runCheck(ctx) - // Periodic loop - ticker := time.NewTicker(uc.checkInterval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-uc.stop: - return - case <-ticker.C: - uc.runCheck(ctx) - case <-uc.triggerCh: + if uc.db != nil { + // Distributed mode: use advisory lock for periodic checks. + // RunLeaderLoop ticks every checkInterval; only the lock holder executes. + go advisorylock.RunLeaderLoop(ctx, uc.db, advisorylock.KeyBackendUpgradeCheck, uc.checkInterval, func() { uc.runCheck(ctx) + }) + + // Still listen for on-demand triggers (from API / settings change) + // and stop signal — these run on every instance. + for { + select { + case <-ctx.Done(): + return + case <-uc.stop: + return + case <-uc.triggerCh: + uc.runCheck(ctx) + } + } + } else { + // Standalone mode: simple ticker loop + ticker := time.NewTicker(uc.checkInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-uc.stop: + return + case <-ticker.C: + uc.runCheck(ctx) + case <-uc.triggerCh: + uc.runCheck(ctx) + } } } } @@ -86,7 +121,7 @@ func (uc *UpgradeChecker) Shutdown() { <-uc.done } -// TriggerCheck forces an immediate upgrade check. +// TriggerCheck forces an immediate upgrade check on this instance. func (uc *UpgradeChecker) TriggerCheck() { select { case uc.triggerCh <- struct{}{}: diff --git a/core/services/advisorylock/keys.go b/core/services/advisorylock/keys.go index ebca65932d62..d5378a5d1b71 100644 --- a/core/services/advisorylock/keys.go +++ b/core/services/advisorylock/keys.go @@ -9,5 +9,6 @@ const ( KeyGalleryDedup int64 = 102 KeyAgentScheduler int64 = 103 KeyHealthCheck int64 = 104 - KeySchemaMigrate int64 = 105 + KeySchemaMigrate int64 = 105 + KeyBackendUpgradeCheck int64 = 106 ) From 6dd37a95c47afa57c0c25f92ce64e5a2530b6a2c Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 11 Apr 2026 08:32:18 +0000 Subject: [PATCH 7/8] test: add e2e tests for backend upgrade API - Test GET /api/backends/upgrades returns 200 (even with no upgrade checker) - Test POST /api/backends/upgrade/:name accepts request and returns job ID - Test full upgrade flow: trigger upgrade via API, wait for job completion, verify run.sh updated to v2 and metadata.json has version 2.0.0 - Test POST /api/backends/upgrades/check returns 200 - Fix nil check for applicationInstance in upgrade API routes --- core/http/routes/ui_api.go | 4 +- core/http/routes/ui_api_backends_test.go | 156 +++++++++++++++++++++++ 2 files changed, 158 insertions(+), 2 deletions(-) diff --git a/core/http/routes/ui_api.go b/core/http/routes/ui_api.go index 16dad72b7dde..a34d22401995 100644 --- a/core/http/routes/ui_api.go +++ b/core/http/routes/ui_api.go @@ -1197,14 +1197,14 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model // Backend upgrade APIs app.GET("/api/backends/upgrades", func(c echo.Context) error { - if applicationInstance.UpgradeChecker() == nil { + if applicationInstance == nil || applicationInstance.UpgradeChecker() == nil { return c.JSON(200, map[string]any{}) } return c.JSON(200, applicationInstance.UpgradeChecker().GetAvailableUpgrades()) }, adminMiddleware) app.POST("/api/backends/upgrades/check", func(c echo.Context) error { - if applicationInstance.UpgradeChecker() == nil { + if applicationInstance == nil || applicationInstance.UpgradeChecker() == nil { return c.JSON(200, map[string]any{}) } applicationInstance.UpgradeChecker().TriggerCheck() diff --git a/core/http/routes/ui_api_backends_test.go b/core/http/routes/ui_api_backends_test.go index 4e4e5014df2f..4c46d387dfdd 100644 --- a/core/http/routes/ui_api_backends_test.go +++ b/core/http/routes/ui_api_backends_test.go @@ -20,6 +20,7 @@ import ( "github.com/mudler/LocalAI/pkg/system" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" ) func TestRoutes(t *testing.T) { @@ -176,6 +177,161 @@ var _ = Describe("Backend API Routes", func() { Expect(response["processed"]).To(Equal(false)) }) }) + + Describe("Backend upgrade API", func() { + var ( + galleryFile string + upgradeApp *echo.Echo + upgradeGallerySvc *galleryop.GalleryService + ) + + BeforeEach(func() { + // Place gallery file inside backends dir so it passes trusted root checks + galleryFile = filepath.Join(systemState.Backend.BackendsPath, "test-gallery.yaml") + + // Create a fake "v1" backend on disk (simulates a previously installed backend) + backendDir := filepath.Join(systemState.Backend.BackendsPath, "test-upgrade-backend") + err := os.MkdirAll(backendDir, 0750) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(filepath.Join(backendDir, "run.sh"), []byte("#!/bin/sh\necho v1"), 0755) + Expect(err).NotTo(HaveOccurred()) + + // Write metadata.json for the installed backend (v1) + metadata := map[string]string{ + "name": "test-upgrade-backend", + "version": "1.0.0", + "installed_at": "2024-01-01T00:00:00Z", + } + metadataBytes, err := json.MarshalIndent(metadata, "", " ") + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(filepath.Join(backendDir, "metadata.json"), metadataBytes, 0644) + Expect(err).NotTo(HaveOccurred()) + + // Create a "v2" source directory (the upgrade target) + // Must be inside backends path to pass trusted root checks + v2SrcDir := filepath.Join(systemState.Backend.BackendsPath, "v2-backend-src") + err = os.MkdirAll(v2SrcDir, 0750) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(filepath.Join(v2SrcDir, "run.sh"), []byte("#!/bin/sh\necho v2"), 0755) + Expect(err).NotTo(HaveOccurred()) + + // Write gallery YAML pointing to v2 + galleryData := []map[string]any{ + { + "name": "test-upgrade-backend", + "uri": v2SrcDir, + "version": "2.0.0", + }, + } + yamlBytes, err := yaml.Marshal(galleryData) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(galleryFile, yamlBytes, 0644) + Expect(err).NotTo(HaveOccurred()) + + // Configure the gallery in appConfig BEFORE creating the gallery service + // so the backend manager captures the correct galleries + appConfig.BackendGalleries = []config.Gallery{ + {Name: "test", URL: "file://" + galleryFile}, + } + + // Create a fresh gallery service with the upgrade gallery configured + upgradeGallerySvc = galleryop.NewGalleryService(appConfig, modelLoader) + err = upgradeGallerySvc.Start(context.Background(), configLoader, systemState) + Expect(err).NotTo(HaveOccurred()) + + // Register routes with the upgrade-aware gallery service + upgradeApp = echo.New() + opcache := galleryop.NewOpCache(upgradeGallerySvc) + noopMw := func(next echo.HandlerFunc) echo.HandlerFunc { return next } + routes.RegisterUIAPIRoutes(upgradeApp, configLoader, modelLoader, appConfig, upgradeGallerySvc, opcache, nil, noopMw) + }) + + Describe("GET /api/backends/upgrades", func() { + It("should return available upgrades", func() { + req := httptest.NewRequest(http.MethodGet, "/api/backends/upgrades", nil) + rec := httptest.NewRecorder() + + upgradeApp.ServeHTTP(rec, req) + + Expect(rec.Code).To(Equal(http.StatusOK)) + + var response map[string]any + err := json.Unmarshal(rec.Body.Bytes(), &response) + Expect(err).NotTo(HaveOccurred()) + // Response is empty (upgrade checker not running in test), + // but the endpoint should not error + }) + }) + + Describe("POST /api/backends/upgrade/:name", func() { + It("should accept upgrade request and return job ID", func() { + req := httptest.NewRequest(http.MethodPost, "/api/backends/upgrade/test-upgrade-backend", nil) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + + upgradeApp.ServeHTTP(rec, req) + + Expect(rec.Code).To(Equal(http.StatusOK)) + + var response map[string]any + err := json.Unmarshal(rec.Body.Bytes(), &response) + Expect(err).NotTo(HaveOccurred()) + Expect(response["uuid"]).NotTo(BeEmpty()) + Expect(response["statusUrl"]).NotTo(BeEmpty()) + }) + + It("should upgrade the backend and update metadata", func() { + req := httptest.NewRequest(http.MethodPost, "/api/backends/upgrade/test-upgrade-backend", nil) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + + upgradeApp.ServeHTTP(rec, req) + Expect(rec.Code).To(Equal(http.StatusOK)) + + var response map[string]any + err := json.Unmarshal(rec.Body.Bytes(), &response) + Expect(err).NotTo(HaveOccurred()) + jobID := response["uuid"].(string) + + // Wait for the upgrade job to complete + Eventually(func() bool { + jobReq := httptest.NewRequest(http.MethodGet, "/api/backends/job/"+jobID, nil) + jobRec := httptest.NewRecorder() + upgradeApp.ServeHTTP(jobRec, jobReq) + + var jobResp map[string]any + json.Unmarshal(jobRec.Body.Bytes(), &jobResp) + + processed, _ := jobResp["processed"].(bool) + return processed + }, "10s", "200ms").Should(BeTrue()) + + // Verify the backend was upgraded: run.sh should now contain "v2" + runContent, err := os.ReadFile(filepath.Join( + systemState.Backend.BackendsPath, "test-upgrade-backend", "run.sh")) + Expect(err).NotTo(HaveOccurred()) + Expect(string(runContent)).To(ContainSubstring("v2")) + + // Verify metadata was updated with new version + metadataContent, err := os.ReadFile(filepath.Join( + systemState.Backend.BackendsPath, "test-upgrade-backend", "metadata.json")) + Expect(err).NotTo(HaveOccurred()) + Expect(string(metadataContent)).To(ContainSubstring(`"version": "2.0.0"`)) + }) + }) + + Describe("POST /api/backends/upgrades/check", func() { + It("should trigger an upgrade check and return 200", func() { + req := httptest.NewRequest(http.MethodPost, "/api/backends/upgrades/check", nil) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + + upgradeApp.ServeHTTP(rec, req) + + Expect(rec.Code).To(Equal(http.StatusOK)) + }) + }) + }) }) // Helper function to make POST request From 5fe87cb0d5425537ab46006b6504bca2af05d440 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 11 Apr 2026 22:11:03 +0000 Subject: [PATCH 8/8] feat: upgrade banner with Upgrade All button, detect pre-existing backends - Add upgrade banner on Backends page showing count and Upgrade All button - Fix upgrade detection for backends installed before version tracking: flag as upgradeable when gallery has a version but installed has none - Fix OCI digest check to flag backends with no stored digest as upgradeable --- core/gallery/upgrade.go | 22 ++++- core/http/react-ui/src/pages/Backends.jsx | 109 +++++++++++++++++++--- 2 files changed, 115 insertions(+), 16 deletions(-) diff --git a/core/gallery/upgrade.go b/core/gallery/upgrade.go index aec75852cc38..dde33300f961 100644 --- a/core/gallery/upgrade.go +++ b/core/gallery/upgrade.go @@ -72,14 +72,28 @@ func CheckBackendUpgrades(ctx context.Context, galleries []config.Gallery, syste continue } - // If either version is empty, fall back to OCI digest comparison - if installed.Metadata.Digest != "" && downloader.URI(galleryEntry.URI).LooksLikeOCI() { + // Gallery has a version but installed doesn't — this happens for backends + // installed before version tracking was added. Flag as upgradeable so + // users can re-install to pick up version metadata. + if galleryVersion != "" && installedVersion == "" { + result[installed.Metadata.Name] = UpgradeInfo{ + BackendName: installed.Metadata.Name, + InstalledVersion: "", + AvailableVersion: galleryVersion, + } + continue + } + + // Fall back to OCI digest comparison when versions are unavailable + if downloader.URI(galleryEntry.URI).LooksLikeOCI() { remoteDigest, err := oci.GetImageDigest(galleryEntry.URI, "", nil, nil) if err != nil { xlog.Warn("Failed to get remote OCI digest for upgrade check", "backend", installed.Metadata.Name, "error", err) continue } - if remoteDigest != installed.Metadata.Digest { + // If we have a stored digest, compare; otherwise any remote digest + // means we can't confirm we're up to date — flag as upgradeable + if installed.Metadata.Digest == "" || remoteDigest != installed.Metadata.Digest { result[installed.Metadata.Name] = UpgradeInfo{ BackendName: installed.Metadata.Name, InstalledDigest: installed.Metadata.Digest, @@ -87,7 +101,7 @@ func CheckBackendUpgrades(ctx context.Context, galleries []config.Gallery, syste } } } - // No version info and no digest to compare — skip + // No version info and non-OCI URI — cannot determine, skip } return result, nil diff --git a/core/http/react-ui/src/pages/Backends.jsx b/core/http/react-ui/src/pages/Backends.jsx index 3cbb71872e2d..1644f06921ca 100644 --- a/core/http/react-ui/src/pages/Backends.jsx +++ b/core/http/react-ui/src/pages/Backends.jsx @@ -7,6 +7,7 @@ import { useOperations } from '../hooks/useOperations' import LoadingSpinner from '../components/LoadingSpinner' import { renderMarkdown } from '../utils/markdown' import ConfirmDialog from '../components/ConfirmDialog' +import Toggle from '../components/Toggle' export default function Backends() { const { addToast } = useOutletContext() @@ -27,6 +28,10 @@ export default function Backends() { const [confirmDialog, setConfirmDialog] = useState(null) const [allBackends, setAllBackends] = useState([]) const [upgrades, setUpgrades] = useState({}) + const [upgradingAll, setUpgradingAll] = useState(false) + const [showAllBackends, setShowAllBackends] = useState(false) + const [showDevelopment, setShowDevelopment] = useState(false) + const [preferDevLoaded, setPreferDevLoaded] = useState(false) const fetchBackends = useCallback(async () => { try { @@ -37,6 +42,11 @@ export default function Backends() { const list = Array.isArray(data?.backends) ? data.backends : Array.isArray(data) ? data : [] setAllBackends(list) setInstalledCount(list.filter(b => b.installed).length) + // On first load, use server preference for development toggle + if (!preferDevLoaded && data?.preferDevelopmentBackends) { + setShowDevelopment(true) + setPreferDevLoaded(true) + } } catch (err) { addToast(`Failed to load backends: ${err.message}`, 'error') } finally { @@ -60,17 +70,33 @@ export default function Backends() { .catch(() => {}) }, [operations.length]) - // Client-side filtering by tag - const filteredBackends = filter - ? allBackends.filter(b => { + // Client-side filtering by meta/development toggles and tag + const filteredBackends = (() => { + let result = allBackends + + // Show only meta backends unless "Show all" is toggled + if (!showAllBackends) { + result = result.filter(b => b.isMeta) + } + + // Hide development backends unless toggled on + if (!showDevelopment) { + result = result.filter(b => !b.isDevelopment) + } + + // Apply tag filter + if (filter) { + result = result.filter(b => { const tags = (b.tags || []).map(t => t.toLowerCase()) const name = (b.name || '').toLowerCase() const desc = (b.description || '').toLowerCase() const f = filter.toLowerCase() - // Match against tags, or name/description containing the filter keyword return tags.some(t => t.includes(f)) || name.includes(f) || desc.includes(f) }) - : allBackends + } + + return result + })() // Client-side pagination const ITEMS_PER_PAGE = 21 @@ -131,6 +157,22 @@ export default function Backends() { } } + const handleUpgradeAll = async () => { + const names = Object.keys(upgrades) + if (names.length === 0) return + setUpgradingAll(true) + try { + for (const name of names) { + await backendsApi.upgrade(name) + } + addToast(`Upgrading ${names.length} backend${names.length > 1 ? 's' : ''}...`, 'info') + } catch (err) { + addToast(`Upgrade failed: ${err.message}`, 'error') + } finally { + setUpgradingAll(false) + } + } + const handleManualInstall = async (e) => { e.preventDefault() if (!manualUri.trim()) { addToast('Please enter a URI', 'warning'); return } @@ -154,6 +196,9 @@ export default function Backends() { return operations.find(op => op.name === backend.name || op.name === backend.id) || null } + const handleToggleAllBackends = () => { setShowAllBackends(v => !v); setPage(1) } + const handleToggleDev = () => { setShowDevelopment(v => !v); setPage(1) } + const FILTERS = [ { key: '', label: 'All', icon: 'fa-layer-group' }, { key: 'llm', label: 'LLM', icon: 'fa-brain' }, @@ -211,6 +256,33 @@ export default function Backends() { + {/* Upgrade Banner */} + {Object.keys(upgrades).length > 0 && ( +
+
+ + + {Object.keys(upgrades).length} backend{Object.keys(upgrades).length > 1 ? 's have' : ' has'} updates available + +
+ +
+ )} + {/* Manual Install */}
-
- {FILTERS.map(f => ( - ))} +
+ +
+ + +
{/* Table */}