diff --git a/core/application/startup.go b/core/application/startup.go index b46af704618d..a047357a62fd 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -443,13 +443,20 @@ func New(opts ...config.AppOption) (*Application, error) { // Wire gallery generation counter into VRAM caches so they invalidate // when gallery data refreshes instead of using a fixed TTL. vram.SetGalleryGenerationFunc(gallery.GalleryGeneration) + if options.AutoloadGalleries { + if options.VRAMPersistentCache { + // Remote GGUF probes can transfer substantial metadata. Keep successful + // results across restarts so the startup warmer does not repeat that work. + vram.ConfigurePersistentCache(filepath.Join(options.SystemState.Model.ModelsPath, "..", "cache", "vram"), 24*time.Hour) + } - // Fill those caches ahead of the first visitor. An estimate for an entry - // nobody has asked about yet costs a remote probe of its weight files, and - // the model gallery asks for one per row, so without this the first page - // spends seconds filling in its own sizes while somebody watches it. - // Non-blocking, and bounded: see DefaultEstimateWarmConfig. - gallery.WarmEstimateCache(options.Context, options.Galleries, options.SystemState, gallery.EstimateWarmConfigFromEnv()) + // Fill those caches ahead of the first visitor. An estimate for an entry + // nobody has asked about yet costs a remote probe of its weight files, and + // the model gallery asks for one per row, so without this the first page + // spends seconds filling in its own sizes while somebody watches it. + // Non-blocking, and bounded: see DefaultEstimateWarmConfig. + gallery.WarmEstimateCache(options.Context, options.Galleries, options.SystemState, gallery.EstimateWarmConfigFromEnv()) + } if options.ConfigFile != "" { if err := application.ModelConfigLoader().LoadMultipleModelConfigsSingleFile(options.ConfigFile, configLoaderOpts...); err != nil { diff --git a/core/cli/run.go b/core/cli/run.go index 7d35fb693352..d11ed9a01426 100644 --- a/core/cli/run.go +++ b/core/cli/run.go @@ -53,6 +53,7 @@ type RunCMD struct { BackendGalleries string `env:"LOCALAI_BACKEND_GALLERIES,BACKEND_GALLERIES" help:"JSON list of backend galleries" group:"backends" default:"${backends}"` Galleries string `env:"LOCALAI_GALLERIES,GALLERIES" help:"JSON list of galleries" group:"models" default:"${galleries}"` AutoloadGalleries bool `env:"LOCALAI_AUTOLOAD_GALLERIES,AUTOLOAD_GALLERIES" group:"models" default:"true"` + VRAMPersistentCache bool `env:"LOCALAI_VRAM_PERSISTENT_CACHE,VRAM_PERSISTENT_CACHE" group:"models" default:"true" help:"Persist successful remote VRAM metadata probes across restarts"` AutoloadBackendGalleries bool `env:"LOCALAI_AUTOLOAD_BACKEND_GALLERIES,AUTOLOAD_BACKEND_GALLERIES" group:"backends" default:"true"` BackendImagesReleaseTag string `env:"LOCALAI_BACKEND_IMAGES_RELEASE_TAG,BACKEND_IMAGES_RELEASE_TAG" help:"Fallback release tag for backend images" group:"backends" default:"latest"` BackendImagesBranchTag string `env:"LOCALAI_BACKEND_IMAGES_BRANCH_TAG,BACKEND_IMAGES_BRANCH_TAG" help:"Fallback branch tag for backend images" group:"backends" default:"master"` @@ -300,6 +301,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error { config.WithF16(r.F16), config.WithStringGalleries(r.Galleries), config.WithBackendGalleries(r.BackendGalleries), + config.WithVRAMPersistentCache(r.VRAMPersistentCache), config.WithCors(r.CORS), config.WithCorsAllowOrigins(r.CORSAllowOrigins), config.WithDisableCSRF(r.DisableCSRF), diff --git a/core/config/application_config.go b/core/config/application_config.go index 009fc1ceae44..3778e1685962 100644 --- a/core/config/application_config.go +++ b/core/config/application_config.go @@ -124,6 +124,7 @@ type ApplicationConfig struct { ExternalGRPCBackends map[string]string AutoloadGalleries, AutoloadBackendGalleries bool + VRAMPersistentCache bool AutoUpgradeBackends bool PreferDevelopmentBackends bool @@ -281,6 +282,7 @@ func NewApplicationConfig(o ...AppOption) *ApplicationConfig { // toggle can still turn it off (a persisted false wins - see // loadRuntimeSettingsFromFile). EnableBackendLogging: true, + VRAMPersistentCache: true, ArtifactDownloadConcurrency: modelartifacts.DefaultDownloadConcurrency, AgentJobRetentionDays: 30, // Default: 30 days LRUEvictionMaxRetries: 30, // Default: 30 retries @@ -620,6 +622,10 @@ func WithAutoUpgradeBackends(v bool) AppOption { return func(o *ApplicationConfig) { o.AutoUpgradeBackends = v } } +func WithVRAMPersistentCache(v bool) AppOption { + return func(o *ApplicationConfig) { o.VRAMPersistentCache = v } +} + func WithRequireBackendIntegrity(v bool) AppOption { return func(o *ApplicationConfig) { o.RequireBackendIntegrity = v } } diff --git a/core/config/application_config_test.go b/core/config/application_config_test.go index 6860388e7efd..8d8a4b753cfb 100644 --- a/core/config/application_config_test.go +++ b/core/config/application_config_test.go @@ -1,6 +1,7 @@ package config import ( + "encoding/json" "time" . "github.com/onsi/ginkgo/v2" @@ -9,6 +10,15 @@ import ( var _ = Describe("ApplicationConfig RuntimeSettings Conversion", func() { Describe("ToRuntimeSettings", func() { + It("includes the persistent VRAM cache toggle", func() { + encoded, err := json.Marshal(NewApplicationConfig().ToRuntimeSettings()) + Expect(err).NotTo(HaveOccurred()) + + var settings map[string]any + Expect(json.Unmarshal(encoded, &settings)).To(Succeed()) + Expect(settings).To(HaveKeyWithValue("vram_persistent_cache", true)) + }) + It("should convert all fields correctly", func() { appConfig := &ApplicationConfig{ WatchDog: true, diff --git a/core/config/runtime_settings.go b/core/config/runtime_settings.go index 6e4381d8c97d..bd495104a478 100644 --- a/core/config/runtime_settings.go +++ b/core/config/runtime_settings.go @@ -59,6 +59,7 @@ type RuntimeSettings struct { BackendGalleries *[]Gallery `json:"backend_galleries,omitempty"` AutoloadGalleries *bool `json:"autoload_galleries,omitempty"` AutoloadBackendGalleries *bool `json:"autoload_backend_galleries,omitempty"` + VRAMPersistentCache *bool `json:"vram_persistent_cache,omitempty"` // API keys - No omitempty as we need to save empty arrays to clear keys ApiKeys *[]string `json:"api_keys"` diff --git a/core/config/runtime_settings_registry.go b/core/config/runtime_settings_registry.go index ce5774ab48c9..fda45e5bcdda 100644 --- a/core/config/runtime_settings_registry.go +++ b/core/config/runtime_settings_registry.go @@ -328,6 +328,10 @@ var runtimeSettingsFields = []fieldSpec{ func(s *RuntimeSettings) **bool { return &s.AutoloadBackendGalleries }, func(o *ApplicationConfig) bool { return o.AutoloadBackendGalleries }, func(o *ApplicationConfig, v bool) { o.AutoloadBackendGalleries = v }), + field("vram_persistent_cache", + func(s *RuntimeSettings) **bool { return &s.VRAMPersistentCache }, + func(o *ApplicationConfig) bool { return o.VRAMPersistentCache }, + func(o *ApplicationConfig, v bool) { o.VRAMPersistentCache = v }), // API keys: echoed for the UI, but the apply loops never touch them. // The settings endpoint and the file watcher own the env+runtime merge diff --git a/core/config/runtime_settings_startup.go b/core/config/runtime_settings_startup.go index 116d2a4f1478..9808c877b1a0 100644 --- a/core/config/runtime_settings_startup.go +++ b/core/config/runtime_settings_startup.go @@ -45,6 +45,7 @@ func DefaultRuntimeBaseline() *ApplicationConfig { o.BackendGalleries = mustGalleries(DefaultBackendGalleriesJSON) o.AutoloadGalleries = true o.AutoloadBackendGalleries = true + o.VRAMPersistentCache = true // core/cli/run.go injects WithMemoryReclaimer(enabled, threshold) // unconditionally, so the kong threshold default (0.95) reaches the // config even when the reclaimer flag is off - this overlay must match diff --git a/core/gallery/gallery.go b/core/gallery/gallery.go index 1d23dfbdb130..04752b3d1d2a 100644 --- a/core/gallery/gallery.go +++ b/core/gallery/gallery.go @@ -16,6 +16,7 @@ import ( "github.com/mudler/LocalAI/pkg/downloader" "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/utils" + "github.com/mudler/LocalAI/pkg/vram" "github.com/mudler/LocalAI/pkg/xsync" "github.com/mudler/xlog" @@ -457,6 +458,9 @@ func triggerGalleryRefresh(galleries []config.Gallery, systemState *system.Syste galleryGeneration.Add(1) } availableModelsMu.Unlock() + if changed { + vram.InvalidatePersistentCache() + } }() } diff --git a/core/http/endpoints/localai/settings.go b/core/http/endpoints/localai/settings.go index 15c6f6d925e8..606fe49c5711 100644 --- a/core/http/endpoints/localai/settings.go +++ b/core/http/endpoints/localai/settings.go @@ -4,6 +4,7 @@ import ( "encoding/json" "io" "net/http" + "path/filepath" "time" "github.com/labstack/echo/v4" @@ -12,6 +13,7 @@ import ( "github.com/mudler/LocalAI/core/http/endpoints/openresponses" "github.com/mudler/LocalAI/core/p2p" "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/pkg/vram" "github.com/mudler/LocalAI/pkg/vrambudget" "github.com/mudler/xlog" ) @@ -185,6 +187,13 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc { // Apply settings using centralized method watchdogChanged := appConfig.ApplyRuntimeSettings(&settings) + if settings.VRAMPersistentCache != nil || settings.AutoloadGalleries != nil { + if appConfig.VRAMPersistentCache && appConfig.AutoloadGalleries { + vram.ConfigurePersistentCache(filepath.Join(appConfig.SystemState.Model.ModelsPath, "..", "cache", "vram"), 24*time.Hour) + } else { + vram.DisablePersistentCache() + } + } // Handle API keys specially (merge with startup keys) if settings.ApiKeys != nil { diff --git a/core/http/react-ui/e2e/settings-backend-logging.spec.js b/core/http/react-ui/e2e/settings-backend-logging.spec.js index 7b5459d8973e..8b1d032563a9 100644 --- a/core/http/react-ui/e2e/settings-backend-logging.spec.js +++ b/core/http/react-ui/e2e/settings-backend-logging.spec.js @@ -18,6 +18,34 @@ test.describe('Settings - Backend Logging', () => { await expect(input).toHaveValue('4') }) + test('persistent VRAM cache can be toggled', async ({ page }) => { + const row = page.locator('.form-row', { hasText: 'Persist remote VRAM estimates' }) + await expect(row).toBeVisible() + + const checkbox = row.locator('input[type="checkbox"]') + const wasChecked = await checkbox.isChecked() + await checkbox.locator('..').click() + if (wasChecked) { + await expect(checkbox).not.toBeChecked() + } else { + await expect(checkbox).toBeChecked() + } + }) + + test('gallery startup loading and pre-warming can be toggled together', async ({ page }) => { + const row = page.locator('.form-row', { hasText: 'Load and pre-warm galleries on boot' }) + await expect(row).toBeVisible() + + const checkbox = row.locator('input[type="checkbox"]') + const wasChecked = await checkbox.isChecked() + await checkbox.locator('..').click() + if (wasChecked) { + await expect(checkbox).not.toBeChecked() + } else { + await expect(checkbox).toBeChecked() + } + }) + test('backend logging toggle can be toggled', async ({ page }) => { // Find the checkbox associated with backend logging const section = page.locator('div', { has: page.locator('text=Enable Backend Logging') }) diff --git a/core/http/react-ui/src/pages/Settings.jsx b/core/http/react-ui/src/pages/Settings.jsx index c1a95ad29ab3..df8990ff197d 100644 --- a/core/http/react-ui/src/pages/Settings.jsx +++ b/core/http/react-ui/src/pages/Settings.jsx @@ -482,12 +482,15 @@ export default function Settings() { Galleries
- + update('autoload_galleries', v)} /> update('autoload_backend_galleries', v)} /> + + update('vram_persistent_cache', v)} /> +