From 45527488985333b6afc2137ce4738b653788ded7 Mon Sep 17 00:00:00 2001
From: localai-org-maint-bot
<306269227+localai-org-maint-bot@users.noreply.github.com>
Date: Wed, 12 Aug 2026 22:23:15 +0000
Subject: [PATCH 1/4] fix(vram): persist remote probe metadata
The startup warmer repeated remote size and GGUF metadata probes after every restart because both caches lived only in memory. Store successful HTTP probes for 24 hours so frequent restarts reuse the prior results.
Bound the cache, reject invalid records, and purge it when gallery data changes. Local model files continue to bypass persistence.
Assisted-by: Codex:gpt-5
---
core/application/startup.go | 3 +
core/gallery/gallery.go | 4 +
docs/content/advanced/vram-management.md | 8 +-
pkg/vram/cache.go | 316 ++++++++++++++++++++++-
pkg/vram/cache_persistent_test.go | 228 ++++++++++++++++
5 files changed, 555 insertions(+), 4 deletions(-)
create mode 100644 pkg/vram/cache_persistent_test.go
diff --git a/core/application/startup.go b/core/application/startup.go
index b46af704618d..f811a216ff39 100644
--- a/core/application/startup.go
+++ b/core/application/startup.go
@@ -443,6 +443,9 @@ func New(opts ...config.AppOption) (*Application, error) {
// Wire gallery generation counter into VRAM caches so they invalidate
// when gallery data refreshes instead of using a fixed TTL.
vram.SetGalleryGenerationFunc(gallery.GalleryGeneration)
+ // Remote GGUF probes can transfer substantial metadata. Keep successful
+ // results across restarts so the startup warmer does not repeat that work.
+ vram.ConfigurePersistentCache(filepath.Join(options.SystemState.Model.ModelsPath, "..", "cache", "vram"), 24*time.Hour)
// Fill those caches ahead of the first visitor. An estimate for an entry
// nobody has asked about yet costs a remote probe of its weight files, and
diff --git a/core/gallery/gallery.go b/core/gallery/gallery.go
index 1d23dfbdb130..04752b3d1d2a 100644
--- a/core/gallery/gallery.go
+++ b/core/gallery/gallery.go
@@ -16,6 +16,7 @@ import (
"github.com/mudler/LocalAI/pkg/downloader"
"github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/LocalAI/pkg/utils"
+ "github.com/mudler/LocalAI/pkg/vram"
"github.com/mudler/LocalAI/pkg/xsync"
"github.com/mudler/xlog"
@@ -457,6 +458,9 @@ func triggerGalleryRefresh(galleries []config.Gallery, systemState *system.Syste
galleryGeneration.Add(1)
}
availableModelsMu.Unlock()
+ if changed {
+ vram.InvalidatePersistentCache()
+ }
}()
}
diff --git a/docs/content/advanced/vram-management.md b/docs/content/advanced/vram-management.md
index 229d39f9f4a9..8ed73065344a 100644
--- a/docs/content/advanced/vram-management.md
+++ b/docs/content/advanced/vram-management.md
@@ -460,8 +460,12 @@ context lengths, so you can see whether something will run before installing it.
Working that out means reading the metadata of a model's weight files, which for
a model you have not installed is a request to the host that serves them. It
takes a second or two the first time, and the gallery needs one per row. LocalAI
-caches the result, and warms that cache in the background at startup so the
-gallery reads instantly rather than filling in its own numbers while you watch.
+caches successful remote probes for 24 hours under the LocalAI data directory,
+and warms that cache in the background at startup so the gallery reads instantly
+rather than filling in its own numbers while you watch. The on-disk cache is
+reused after a restart, so frequent restarts do not download the same metadata
+again. Local model files are always inspected directly. The cache keeps at most
+4,096 entries and removes the oldest entries when it reaches that limit.
The same warm-up also describes each entry's **variants** - the alternative
builds of the same weights that the picker offers - because that costs the same
diff --git a/pkg/vram/cache.go b/pkg/vram/cache.go
index cbfaefed1b94..da7f90f8f5f2 100644
--- a/pkg/vram/cache.go
+++ b/pkg/vram/cache.go
@@ -2,9 +2,23 @@ package vram
import (
"context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "net/url"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
"sync"
+ "time"
)
+const persistentCacheEntryLimit = 4096
+const persistentCacheVersion = 1
+
+var defaultPersistentGuard = &persistentGenerationGuard{}
+
// galleryGenFunc returns the current gallery generation counter.
// When set, cache entries are invalidated when the generation changes.
// When nil (e.g., in tests or non-gallery contexts), entries never expire.
@@ -23,6 +37,135 @@ func currentGeneration() uint64 {
return 0
}
+// ConfigurePersistentCache replaces the process-wide estimator caches with
+// instances that reuse successful remote probes across server restarts.
+func ConfigurePersistentCache(dir string, ttl time.Duration) {
+ removeAbandonedPersistentTemps(dir)
+ prunePersistentEntries(dir, ttl, persistentCacheEntryLimit)
+ guard := &persistentGenerationGuard{dir: dir}
+ defaultPersistentGuard = guard
+ defaultCachedSizeResolver = newCachedSizeResolverWithGuard(defaultSizeResolver{}, dir, ttl, guard)
+ defaultCachedGGUFReader = newCachedGGUFReaderWithGuard(defaultGGUFReader{}, dir, ttl, guard)
+}
+
+// InvalidatePersistentCache removes remote probe results after the gallery
+// changes, including when no estimate is requested before the next restart.
+func InvalidatePersistentCache() {
+ defaultPersistentGuard.invalidate(currentGeneration())
+}
+
+func removeAbandonedPersistentTemps(dir string) {
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ return
+ }
+ for _, entry := range entries {
+ if entry.Type().IsRegular() && strings.HasPrefix(entry.Name(), ".vram-") {
+ _ = os.Remove(filepath.Join(dir, entry.Name()))
+ }
+ }
+}
+
+func removePersistentEntries(dir string) {
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ return
+ }
+ for _, entry := range entries {
+ name := entry.Name()
+ if entry.Type().IsRegular() && (strings.HasPrefix(name, "size-") || strings.HasPrefix(name, "gguf-")) {
+ _ = os.Remove(filepath.Join(dir, name))
+ }
+ }
+}
+
+type persistentGenerationGuard struct {
+ mu sync.Mutex
+ dir string
+ generation uint64
+ set bool
+}
+
+func (g *persistentGenerationGuard) invalidate(generation uint64) {
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ removePersistentEntries(g.dir)
+ g.generation = generation
+ g.set = true
+}
+
+func (g *persistentGenerationGuard) canRead(generation uint64) bool {
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ if !g.set {
+ g.generation = generation
+ g.set = true
+ return true
+ }
+ if g.generation == generation {
+ return true
+ }
+ removePersistentEntries(g.dir)
+ g.generation = generation
+ return false
+}
+
+func (g *persistentGenerationGuard) persist(generation uint64, write func()) {
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ if !g.set {
+ g.generation = generation
+ g.set = true
+ }
+ if g.generation == generation {
+ write()
+ }
+}
+
+func prunePersistentEntries(dir string, ttl time.Duration, limit int) {
+ if dir == "" || ttl <= 0 {
+ return
+ }
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ return
+ }
+ type cacheFile struct {
+ path string
+ modTime time.Time
+ }
+ files := make([]cacheFile, 0, len(entries))
+ for _, entry := range entries {
+ name := entry.Name()
+ if entry.Type().IsRegular() && (strings.HasPrefix(name, "size-") || strings.HasPrefix(name, "gguf-")) {
+ if info, err := entry.Info(); err == nil {
+ path := filepath.Join(dir, name)
+ if time.Since(info.ModTime()) > ttl {
+ _ = os.Remove(path)
+ continue
+ }
+ files = append(files, cacheFile{path: path, modTime: info.ModTime()})
+ }
+ }
+ }
+ if limit <= 0 || len(files) <= limit {
+ return
+ }
+ sort.Slice(files, func(i, j int) bool { return files[i].modTime.Before(files[j].modTime) })
+ for _, file := range files[:len(files)-limit] {
+ _ = os.Remove(file.path)
+ }
+}
+
+func persistentRemoteURI(uri string) bool {
+ parsed, err := url.Parse(uri)
+ if err != nil {
+ return false
+ }
+ scheme := strings.ToLower(parsed.Scheme)
+ return (scheme == "http" || scheme == "https") && parsed.Host != ""
+}
+
type sizeCacheEntry struct {
size int64
err error
@@ -33,6 +176,28 @@ type cachedSizeResolver struct {
underlying SizeResolver
mu sync.Mutex
cache map[string]sizeCacheEntry
+ diskDir string
+ diskTTL time.Duration
+ diskGuard *persistentGenerationGuard
+}
+
+type persistentSizeEntry struct {
+ Version int `json:"version"`
+ Size int64 `json:"size"`
+}
+
+func newCachedSizeResolver(underlying SizeResolver, diskDir string, diskTTL time.Duration) *cachedSizeResolver {
+ return newCachedSizeResolverWithGuard(underlying, diskDir, diskTTL, &persistentGenerationGuard{dir: diskDir})
+}
+
+func newCachedSizeResolverWithGuard(underlying SizeResolver, diskDir string, diskTTL time.Duration, guard *persistentGenerationGuard) *cachedSizeResolver {
+ return &cachedSizeResolver{
+ underlying: underlying,
+ cache: make(map[string]sizeCacheEntry),
+ diskDir: diskDir,
+ diskTTL: diskTTL,
+ diskGuard: guard,
+ }
}
func (c *cachedSizeResolver) ContentLength(ctx context.Context, uri string) (int64, error) {
@@ -43,13 +208,62 @@ func (c *cachedSizeResolver) ContentLength(ctx context.Context, uri string) (int
if ok && e.generation == gen {
return e.size, e.err
}
+ if persistentRemoteURI(uri) && c.canReadPersistent(gen) {
+ if size, ok := c.readPersistent(uri); ok {
+ c.mu.Lock()
+ c.cache[uri] = sizeCacheEntry{size: size, generation: gen}
+ c.mu.Unlock()
+ return size, nil
+ }
+ }
size, err := c.underlying.ContentLength(ctx, uri)
c.mu.Lock()
c.cache[uri] = sizeCacheEntry{size: size, err: err, generation: gen}
c.mu.Unlock()
+ if err == nil && persistentRemoteURI(uri) {
+ c.writePersistent(uri, size, gen)
+ }
return size, err
}
+func (c *cachedSizeResolver) canReadPersistent(generation uint64) bool {
+ return c.diskGuard.canRead(generation)
+}
+
+func (c *cachedSizeResolver) persistentPath(uri string) string {
+ digest := sha256.Sum256([]byte(uri))
+ return filepath.Join(c.diskDir, "size-"+hex.EncodeToString(digest[:])+".json")
+}
+
+func (c *cachedSizeResolver) readPersistent(uri string) (int64, bool) {
+ if c.diskDir == "" || c.diskTTL <= 0 {
+ return 0, false
+ }
+ path := c.persistentPath(uri)
+ info, err := os.Stat(path)
+ if err != nil || time.Since(info.ModTime()) > c.diskTTL {
+ return 0, false
+ }
+ data, err := os.ReadFile(path) // #nosec G304 -- path is a hash under the configured cache directory.
+ if err != nil {
+ return 0, false
+ }
+ var entry persistentSizeEntry
+ if json.Unmarshal(data, &entry) != nil || entry.Version != persistentCacheVersion || entry.Size <= 0 {
+ return 0, false
+ }
+ return entry.Size, true
+}
+
+func (c *cachedSizeResolver) writePersistent(uri string, size int64, generation uint64) {
+ if c.diskDir == "" || c.diskTTL <= 0 || os.MkdirAll(c.diskDir, 0o750) != nil {
+ return
+ }
+ c.diskGuard.persist(generation, func() {
+ writePersistentJSON(c.persistentPath(uri), persistentSizeEntry{Version: persistentCacheVersion, Size: size}, c.diskTTL)
+ })
+}
+
type ggufCacheEntry struct {
meta *GGUFMeta
err error
@@ -60,6 +274,28 @@ type cachedGGUFReader struct {
underlying GGUFMetadataReader
mu sync.Mutex
cache map[string]ggufCacheEntry
+ diskDir string
+ diskTTL time.Duration
+ diskGuard *persistentGenerationGuard
+}
+
+type persistentGGUFEntry struct {
+ Version int `json:"version"`
+ Meta *GGUFMeta `json:"meta"`
+}
+
+func newCachedGGUFReader(underlying GGUFMetadataReader, diskDir string, diskTTL time.Duration) *cachedGGUFReader {
+ return newCachedGGUFReaderWithGuard(underlying, diskDir, diskTTL, &persistentGenerationGuard{dir: diskDir})
+}
+
+func newCachedGGUFReaderWithGuard(underlying GGUFMetadataReader, diskDir string, diskTTL time.Duration, guard *persistentGenerationGuard) *cachedGGUFReader {
+ return &cachedGGUFReader{
+ underlying: underlying,
+ cache: make(map[string]ggufCacheEntry),
+ diskDir: diskDir,
+ diskTTL: diskTTL,
+ diskGuard: guard,
+ }
}
func (c *cachedGGUFReader) ReadMetadata(ctx context.Context, uri string) (*GGUFMeta, error) {
@@ -70,13 +306,89 @@ func (c *cachedGGUFReader) ReadMetadata(ctx context.Context, uri string) (*GGUFM
if ok && e.generation == gen {
return e.meta, e.err
}
+ if persistentRemoteURI(uri) && c.canReadPersistent(gen) {
+ if meta, ok := c.readPersistent(uri); ok {
+ c.mu.Lock()
+ c.cache[uri] = ggufCacheEntry{meta: meta, generation: gen}
+ c.mu.Unlock()
+ return meta, nil
+ }
+ }
meta, err := c.underlying.ReadMetadata(ctx, uri)
c.mu.Lock()
c.cache[uri] = ggufCacheEntry{meta: meta, err: err, generation: gen}
c.mu.Unlock()
+ if err == nil && meta != nil && persistentRemoteURI(uri) {
+ c.writePersistent(uri, meta, gen)
+ }
return meta, err
}
+func (c *cachedGGUFReader) canReadPersistent(generation uint64) bool {
+ return c.diskGuard.canRead(generation)
+}
+
+func (c *cachedGGUFReader) persistentPath(uri string) string {
+ digest := sha256.Sum256([]byte(uri))
+ return filepath.Join(c.diskDir, "gguf-"+hex.EncodeToString(digest[:])+".json")
+}
+
+func (c *cachedGGUFReader) readPersistent(uri string) (*GGUFMeta, bool) {
+ if c.diskDir == "" || c.diskTTL <= 0 {
+ return nil, false
+ }
+ path := c.persistentPath(uri)
+ info, err := os.Stat(path)
+ if err != nil || time.Since(info.ModTime()) > c.diskTTL {
+ return nil, false
+ }
+ data, err := os.ReadFile(path) // #nosec G304 -- path is a hash under the configured cache directory.
+ if err != nil {
+ return nil, false
+ }
+ var entry persistentGGUFEntry
+ if json.Unmarshal(data, &entry) != nil || entry.Version != persistentCacheVersion || !validPersistentGGUFMeta(entry.Meta) {
+ return nil, false
+ }
+ return entry.Meta, true
+}
+
+func (c *cachedGGUFReader) writePersistent(uri string, meta *GGUFMeta, generation uint64) {
+ if c.diskDir == "" || c.diskTTL <= 0 || os.MkdirAll(c.diskDir, 0o750) != nil {
+ return
+ }
+ c.diskGuard.persist(generation, func() {
+ writePersistentJSON(c.persistentPath(uri), persistentGGUFEntry{Version: persistentCacheVersion, Meta: meta}, c.diskTTL)
+ })
+}
+
+func validPersistentGGUFMeta(meta *GGUFMeta) bool {
+ return meta != nil && meta.BlockCount > 0 && meta.EmbeddingLength > 0 && meta.HeadCount > 0 && meta.HeadCountKV > 0
+}
+
+func writePersistentJSON(path string, value any, ttl time.Duration) {
+ data, err := json.Marshal(value)
+ if err != nil {
+ return
+ }
+ tmp, err := os.CreateTemp(filepath.Dir(path), ".vram-*.tmp")
+ if err != nil {
+ return
+ }
+ tmpPath := tmp.Name()
+ defer os.Remove(tmpPath)
+ if _, err = tmp.Write(data); err != nil {
+ _ = tmp.Close()
+ return
+ }
+ if err = tmp.Close(); err != nil {
+ return
+ }
+ if os.Rename(tmpPath, path) == nil {
+ prunePersistentEntries(filepath.Dir(path), ttl, persistentCacheEntryLimit)
+ }
+}
+
// DefaultCachedSizeResolver returns a cached SizeResolver using the default implementation.
// Entries are invalidated when the gallery generation changes.
func DefaultCachedSizeResolver() SizeResolver {
@@ -90,6 +402,6 @@ func DefaultCachedGGUFReader() GGUFMetadataReader {
}
var (
- defaultCachedSizeResolver = &cachedSizeResolver{underlying: defaultSizeResolver{}, cache: make(map[string]sizeCacheEntry)}
- defaultCachedGGUFReader = &cachedGGUFReader{underlying: defaultGGUFReader{}, cache: make(map[string]ggufCacheEntry)}
+ defaultCachedSizeResolver = newCachedSizeResolver(defaultSizeResolver{}, "", 0)
+ defaultCachedGGUFReader = newCachedGGUFReader(defaultGGUFReader{}, "", 0)
)
diff --git a/pkg/vram/cache_persistent_test.go b/pkg/vram/cache_persistent_test.go
new file mode 100644
index 000000000000..6b4fa45d5904
--- /dev/null
+++ b/pkg/vram/cache_persistent_test.go
@@ -0,0 +1,228 @@
+package vram
+
+import (
+ "context"
+ "errors"
+ "os"
+ "path/filepath"
+ "time"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+)
+
+type countingSizeResolver struct {
+ size int64
+ err error
+ calls int
+}
+
+type countingGGUFReader struct {
+ meta *GGUFMeta
+ err error
+ calls int
+}
+
+type blockingSizeResolver struct {
+ started chan struct{}
+ release chan struct{}
+}
+
+func (r *blockingSizeResolver) ContentLength(context.Context, string) (int64, error) {
+ close(r.started)
+ <-r.release
+ return 42, nil
+}
+
+func (r *countingGGUFReader) ReadMetadata(context.Context, string) (*GGUFMeta, error) {
+ r.calls++
+ return r.meta, r.err
+}
+
+func (r *countingSizeResolver) ContentLength(context.Context, string) (int64, error) {
+ r.calls++
+ return r.size, r.err
+}
+
+var _ = Describe("persistent VRAM metadata cache", func() {
+ AfterEach(func() {
+ ConfigurePersistentCache("", 0)
+ SetGalleryGenerationFunc(nil)
+ })
+
+ It("reuses a successful size probe after the in-memory cache is replaced", func() {
+ cacheDir := filepath.Join(GinkgoT().TempDir(), "vram")
+ firstSource := &countingSizeResolver{size: 42}
+ first := newCachedSizeResolver(firstSource, cacheDir, time.Hour)
+
+ size, err := first.ContentLength(context.Background(), "https://example.com/model.gguf")
+ Expect(err).NotTo(HaveOccurred())
+ Expect(size).To(Equal(int64(42)))
+ Expect(firstSource.calls).To(Equal(1))
+
+ secondSource := &countingSizeResolver{err: errors.New("unexpected remote probe")}
+ second := newCachedSizeResolver(secondSource, cacheDir, time.Hour)
+ size, err = second.ContentLength(context.Background(), "https://example.com/model.gguf")
+
+ Expect(err).NotTo(HaveOccurred())
+ Expect(size).To(Equal(int64(42)))
+ Expect(secondSource.calls).To(BeZero())
+ })
+
+ It("reuses successful GGUF metadata after the in-memory cache is replaced", func() {
+ cacheDir := filepath.Join(GinkgoT().TempDir(), "vram")
+ want := &GGUFMeta{BlockCount: 32, EmbeddingLength: 4096, HeadCount: 32, HeadCountKV: 8, MaximumContextLength: 131072}
+ firstSource := &countingGGUFReader{meta: want}
+ first := newCachedGGUFReader(firstSource, cacheDir, time.Hour)
+
+ meta, err := first.ReadMetadata(context.Background(), "https://example.com/model.gguf")
+ Expect(err).NotTo(HaveOccurred())
+ Expect(meta).To(Equal(want))
+ Expect(firstSource.calls).To(Equal(1))
+
+ secondSource := &countingGGUFReader{err: errors.New("unexpected remote probe")}
+ second := newCachedGGUFReader(secondSource, cacheDir, time.Hour)
+ meta, err = second.ReadMetadata(context.Background(), "https://example.com/model.gguf")
+
+ Expect(err).NotTo(HaveOccurred())
+ Expect(meta).To(Equal(want))
+ Expect(secondSource.calls).To(BeZero())
+ })
+
+ It("configures the default caches used by model estimates", func() {
+ cacheDir := filepath.Join(GinkgoT().TempDir(), "vram")
+ ConfigurePersistentCache(cacheDir, time.Hour)
+
+ Expect(defaultCachedSizeResolver.diskDir).To(Equal(cacheDir))
+ Expect(defaultCachedGGUFReader.diskDir).To(Equal(cacheDir))
+ Expect(defaultCachedSizeResolver.diskTTL).To(Equal(time.Hour))
+ Expect(defaultCachedGGUFReader.diskTTL).To(Equal(time.Hour))
+ Expect(defaultCachedSizeResolver.diskGuard).To(BeIdenticalTo(defaultCachedGGUFReader.diskGuard))
+ })
+
+ It("removes expired VRAM entries when the persistent cache is configured", func() {
+ cacheDir := GinkgoT().TempDir()
+ stale := filepath.Join(cacheDir, "size-stale.json")
+ abandoned := filepath.Join(cacheDir, ".vram-abandoned.tmp")
+ unrelated := filepath.Join(cacheDir, "keep.txt")
+ Expect(os.WriteFile(stale, []byte("{}"), 0o600)).To(Succeed())
+ Expect(os.WriteFile(abandoned, []byte("partial"), 0o600)).To(Succeed())
+ Expect(os.WriteFile(unrelated, []byte("keep"), 0o600)).To(Succeed())
+ old := time.Now().Add(-2 * time.Hour)
+ Expect(os.Chtimes(stale, old, old)).To(Succeed())
+
+ ConfigurePersistentCache(cacheDir, time.Hour)
+
+ Expect(stale).NotTo(BeAnExistingFile())
+ Expect(abandoned).NotTo(BeAnExistingFile())
+ Expect(unrelated).To(BeAnExistingFile())
+ })
+
+ It("does not reuse a persistent entry after the gallery generation changes", func() {
+ var generation uint64 = 1
+ SetGalleryGenerationFunc(func() uint64 { return generation })
+ cacheDir := GinkgoT().TempDir()
+ first := newCachedSizeResolver(&countingSizeResolver{size: 42}, cacheDir, time.Hour)
+ _, err := first.ContentLength(context.Background(), "https://example.com/model.gguf")
+ Expect(err).NotTo(HaveOccurred())
+
+ freshSource := &countingSizeResolver{size: 84}
+ second := newCachedSizeResolver(freshSource, cacheDir, time.Hour)
+ size, err := second.ContentLength(context.Background(), "https://example.com/model.gguf")
+ Expect(err).NotTo(HaveOccurred())
+ Expect(size).To(Equal(int64(42)))
+
+ generation = 2
+ size, err = second.ContentLength(context.Background(), "https://example.com/model.gguf")
+
+ Expect(err).NotTo(HaveOccurred())
+ Expect(size).To(Equal(int64(84)))
+ Expect(freshSource.calls).To(Equal(1))
+ })
+
+ It("does not persist probes for local model files", func() {
+ cacheDir := GinkgoT().TempDir()
+ first := newCachedSizeResolver(&countingSizeResolver{size: 42}, cacheDir, time.Hour)
+ _, err := first.ContentLength(context.Background(), "file:///models/model.gguf")
+ Expect(err).NotTo(HaveOccurred())
+
+ freshSource := &countingSizeResolver{size: 84}
+ second := newCachedSizeResolver(freshSource, cacheDir, time.Hour)
+ size, err := second.ContentLength(context.Background(), "file:///models/model.gguf")
+
+ Expect(err).NotTo(HaveOccurred())
+ Expect(size).To(Equal(int64(84)))
+ Expect(freshSource.calls).To(Equal(1))
+ })
+
+ It("falls back to the remote probe for an invalid persisted size", func() {
+ cacheDir := GinkgoT().TempDir()
+ resolver := newCachedSizeResolver(&countingSizeResolver{size: 42}, cacheDir, time.Hour)
+ Expect(os.WriteFile(resolver.persistentPath("https://example.com/model.gguf"), []byte(`{"size":-1}`), 0o600)).To(Succeed())
+
+ size, err := resolver.ContentLength(context.Background(), "https://example.com/model.gguf")
+
+ Expect(err).NotTo(HaveOccurred())
+ Expect(size).To(Equal(int64(42)))
+ })
+
+ It("falls back to the remote probe for empty persisted GGUF metadata", func() {
+ cacheDir := GinkgoT().TempDir()
+ want := &GGUFMeta{BlockCount: 32, EmbeddingLength: 4096, HeadCount: 32, HeadCountKV: 8}
+ reader := newCachedGGUFReader(&countingGGUFReader{meta: want}, cacheDir, time.Hour)
+ Expect(os.WriteFile(reader.persistentPath("https://example.com/model.gguf"), []byte(`{"meta":{}}`), 0o600)).To(Succeed())
+
+ meta, err := reader.ReadMetadata(context.Background(), "https://example.com/model.gguf")
+
+ Expect(err).NotTo(HaveOccurred())
+ Expect(meta).To(Equal(want))
+ })
+
+ It("keeps the persistent cache within its entry limit", func() {
+ cacheDir := GinkgoT().TempDir()
+ for _, name := range []string{"size-a.json", "size-b.json", "gguf-c.json"} {
+ Expect(os.WriteFile(filepath.Join(cacheDir, name), []byte("{}"), 0o600)).To(Succeed())
+ time.Sleep(time.Millisecond)
+ }
+
+ prunePersistentEntries(cacheDir, time.Hour, 2)
+
+ entries, err := os.ReadDir(cacheDir)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(entries).To(HaveLen(2))
+ Expect(filepath.Join(cacheDir, "size-a.json")).NotTo(BeAnExistingFile())
+ })
+
+ It("removes persistent entries when gallery data is invalidated", func() {
+ cacheDir := GinkgoT().TempDir()
+ ConfigurePersistentCache(cacheDir, time.Hour)
+ stale := filepath.Join(cacheDir, "size-stale.json")
+ Expect(os.WriteFile(stale, []byte(`{"version":1,"size":42}`), 0o600)).To(Succeed())
+
+ InvalidatePersistentCache()
+
+ Expect(stale).NotTo(BeAnExistingFile())
+ })
+
+ It("does not persist a probe that finishes after invalidation", func() {
+ var generation uint64 = 1
+ SetGalleryGenerationFunc(func() uint64 { return generation })
+ cacheDir := GinkgoT().TempDir()
+ guard := &persistentGenerationGuard{dir: cacheDir}
+ source := &blockingSizeResolver{started: make(chan struct{}), release: make(chan struct{})}
+ resolver := newCachedSizeResolverWithGuard(source, cacheDir, time.Hour, guard)
+ done := make(chan error, 1)
+ go func() {
+ _, err := resolver.ContentLength(context.Background(), "https://example.com/model.gguf")
+ done <- err
+ }()
+ Eventually(source.started).Should(BeClosed())
+
+ generation = 2
+ guard.invalidate(generation)
+ close(source.release)
+
+ Eventually(done).Should(Receive(BeNil()))
+ Expect(resolver.persistentPath("https://example.com/model.gguf")).NotTo(BeAnExistingFile())
+ })
+})
From db4f406726b952d53236db8a4284a9fb9ccf5e46 Mon Sep 17 00:00:00 2001
From: localai-org-maint-bot
<306269227+localai-org-maint-bot@users.noreply.github.com>
Date: Wed, 12 Aug 2026 23:02:42 +0000
Subject: [PATCH 2/4] fix(vram): check temporary file cleanup
The lint gate rejects the unchecked cleanup call in the persistent cache writer.
Assisted-by: Codex:gpt-5.6 [golangci-lint]
---
pkg/vram/cache.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pkg/vram/cache.go b/pkg/vram/cache.go
index da7f90f8f5f2..aac41a06e730 100644
--- a/pkg/vram/cache.go
+++ b/pkg/vram/cache.go
@@ -376,7 +376,7 @@ func writePersistentJSON(path string, value any, ttl time.Duration) {
return
}
tmpPath := tmp.Name()
- defer os.Remove(tmpPath)
+ defer func() { _ = os.Remove(tmpPath) }()
if _, err = tmp.Write(data); err != nil {
_ = tmp.Close()
return
From ad5ed470ffeab95083219b8294d382226317079c Mon Sep 17 00:00:00 2001
From: localai-org-maint-bot
<306269227+localai-org-maint-bot@users.noreply.github.com>
Date: Thu, 13 Aug 2026 21:07:10 +0000
Subject: [PATCH 3/4] fix(vram): make persistent cache optional
Remote metadata probes can transfer enough data that operators need
control over disk reuse and startup warming. Gallery autoload now gates
both behaviors, and the runtime setting applies changes immediately.
Assisted-by: Codex:gpt-5
---
core/application/startup.go | 24 +++++++++++--------
core/cli/run.go | 2 ++
core/config/application_config.go | 6 +++++
core/config/application_config_test.go | 10 ++++++++
core/config/runtime_settings.go | 1 +
core/config/runtime_settings_registry.go | 4 ++++
core/config/runtime_settings_startup.go | 1 +
core/http/endpoints/localai/settings.go | 9 +++++++
.../e2e/settings-backend-logging.spec.js | 14 +++++++++++
core/http/react-ui/src/pages/Settings.jsx | 3 +++
docs/content/advanced/vram-management.md | 4 ++++
docs/content/features/runtime-settings.md | 1 +
docs/content/reference/cli-reference.md | 1 +
pkg/vram/cache.go | 21 +++++++++++++++-
14 files changed, 90 insertions(+), 11 deletions(-)
diff --git a/core/application/startup.go b/core/application/startup.go
index f811a216ff39..a047357a62fd 100644
--- a/core/application/startup.go
+++ b/core/application/startup.go
@@ -443,16 +443,20 @@ func New(opts ...config.AppOption) (*Application, error) {
// Wire gallery generation counter into VRAM caches so they invalidate
// when gallery data refreshes instead of using a fixed TTL.
vram.SetGalleryGenerationFunc(gallery.GalleryGeneration)
- // Remote GGUF probes can transfer substantial metadata. Keep successful
- // results across restarts so the startup warmer does not repeat that work.
- vram.ConfigurePersistentCache(filepath.Join(options.SystemState.Model.ModelsPath, "..", "cache", "vram"), 24*time.Hour)
-
- // Fill those caches ahead of the first visitor. An estimate for an entry
- // nobody has asked about yet costs a remote probe of its weight files, and
- // the model gallery asks for one per row, so without this the first page
- // spends seconds filling in its own sizes while somebody watches it.
- // Non-blocking, and bounded: see DefaultEstimateWarmConfig.
- gallery.WarmEstimateCache(options.Context, options.Galleries, options.SystemState, gallery.EstimateWarmConfigFromEnv())
+ if options.AutoloadGalleries {
+ if options.VRAMPersistentCache {
+ // Remote GGUF probes can transfer substantial metadata. Keep successful
+ // results across restarts so the startup warmer does not repeat that work.
+ vram.ConfigurePersistentCache(filepath.Join(options.SystemState.Model.ModelsPath, "..", "cache", "vram"), 24*time.Hour)
+ }
+
+ // Fill those caches ahead of the first visitor. An estimate for an entry
+ // nobody has asked about yet costs a remote probe of its weight files, and
+ // the model gallery asks for one per row, so without this the first page
+ // spends seconds filling in its own sizes while somebody watches it.
+ // Non-blocking, and bounded: see DefaultEstimateWarmConfig.
+ gallery.WarmEstimateCache(options.Context, options.Galleries, options.SystemState, gallery.EstimateWarmConfigFromEnv())
+ }
if options.ConfigFile != "" {
if err := application.ModelConfigLoader().LoadMultipleModelConfigsSingleFile(options.ConfigFile, configLoaderOpts...); err != nil {
diff --git a/core/cli/run.go b/core/cli/run.go
index 7d35fb693352..d11ed9a01426 100644
--- a/core/cli/run.go
+++ b/core/cli/run.go
@@ -53,6 +53,7 @@ type RunCMD struct {
BackendGalleries string `env:"LOCALAI_BACKEND_GALLERIES,BACKEND_GALLERIES" help:"JSON list of backend galleries" group:"backends" default:"${backends}"`
Galleries string `env:"LOCALAI_GALLERIES,GALLERIES" help:"JSON list of galleries" group:"models" default:"${galleries}"`
AutoloadGalleries bool `env:"LOCALAI_AUTOLOAD_GALLERIES,AUTOLOAD_GALLERIES" group:"models" default:"true"`
+ VRAMPersistentCache bool `env:"LOCALAI_VRAM_PERSISTENT_CACHE,VRAM_PERSISTENT_CACHE" group:"models" default:"true" help:"Persist successful remote VRAM metadata probes across restarts"`
AutoloadBackendGalleries bool `env:"LOCALAI_AUTOLOAD_BACKEND_GALLERIES,AUTOLOAD_BACKEND_GALLERIES" group:"backends" default:"true"`
BackendImagesReleaseTag string `env:"LOCALAI_BACKEND_IMAGES_RELEASE_TAG,BACKEND_IMAGES_RELEASE_TAG" help:"Fallback release tag for backend images" group:"backends" default:"latest"`
BackendImagesBranchTag string `env:"LOCALAI_BACKEND_IMAGES_BRANCH_TAG,BACKEND_IMAGES_BRANCH_TAG" help:"Fallback branch tag for backend images" group:"backends" default:"master"`
@@ -300,6 +301,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
config.WithF16(r.F16),
config.WithStringGalleries(r.Galleries),
config.WithBackendGalleries(r.BackendGalleries),
+ config.WithVRAMPersistentCache(r.VRAMPersistentCache),
config.WithCors(r.CORS),
config.WithCorsAllowOrigins(r.CORSAllowOrigins),
config.WithDisableCSRF(r.DisableCSRF),
diff --git a/core/config/application_config.go b/core/config/application_config.go
index 009fc1ceae44..3778e1685962 100644
--- a/core/config/application_config.go
+++ b/core/config/application_config.go
@@ -124,6 +124,7 @@ type ApplicationConfig struct {
ExternalGRPCBackends map[string]string
AutoloadGalleries, AutoloadBackendGalleries bool
+ VRAMPersistentCache bool
AutoUpgradeBackends bool
PreferDevelopmentBackends bool
@@ -281,6 +282,7 @@ func NewApplicationConfig(o ...AppOption) *ApplicationConfig {
// toggle can still turn it off (a persisted false wins - see
// loadRuntimeSettingsFromFile).
EnableBackendLogging: true,
+ VRAMPersistentCache: true,
ArtifactDownloadConcurrency: modelartifacts.DefaultDownloadConcurrency,
AgentJobRetentionDays: 30, // Default: 30 days
LRUEvictionMaxRetries: 30, // Default: 30 retries
@@ -620,6 +622,10 @@ func WithAutoUpgradeBackends(v bool) AppOption {
return func(o *ApplicationConfig) { o.AutoUpgradeBackends = v }
}
+func WithVRAMPersistentCache(v bool) AppOption {
+ return func(o *ApplicationConfig) { o.VRAMPersistentCache = v }
+}
+
func WithRequireBackendIntegrity(v bool) AppOption {
return func(o *ApplicationConfig) { o.RequireBackendIntegrity = v }
}
diff --git a/core/config/application_config_test.go b/core/config/application_config_test.go
index 6860388e7efd..8d8a4b753cfb 100644
--- a/core/config/application_config_test.go
+++ b/core/config/application_config_test.go
@@ -1,6 +1,7 @@
package config
import (
+ "encoding/json"
"time"
. "github.com/onsi/ginkgo/v2"
@@ -9,6 +10,15 @@ import (
var _ = Describe("ApplicationConfig RuntimeSettings Conversion", func() {
Describe("ToRuntimeSettings", func() {
+ It("includes the persistent VRAM cache toggle", func() {
+ encoded, err := json.Marshal(NewApplicationConfig().ToRuntimeSettings())
+ Expect(err).NotTo(HaveOccurred())
+
+ var settings map[string]any
+ Expect(json.Unmarshal(encoded, &settings)).To(Succeed())
+ Expect(settings).To(HaveKeyWithValue("vram_persistent_cache", true))
+ })
+
It("should convert all fields correctly", func() {
appConfig := &ApplicationConfig{
WatchDog: true,
diff --git a/core/config/runtime_settings.go b/core/config/runtime_settings.go
index 6e4381d8c97d..bd495104a478 100644
--- a/core/config/runtime_settings.go
+++ b/core/config/runtime_settings.go
@@ -59,6 +59,7 @@ type RuntimeSettings struct {
BackendGalleries *[]Gallery `json:"backend_galleries,omitempty"`
AutoloadGalleries *bool `json:"autoload_galleries,omitempty"`
AutoloadBackendGalleries *bool `json:"autoload_backend_galleries,omitempty"`
+ VRAMPersistentCache *bool `json:"vram_persistent_cache,omitempty"`
// API keys - No omitempty as we need to save empty arrays to clear keys
ApiKeys *[]string `json:"api_keys"`
diff --git a/core/config/runtime_settings_registry.go b/core/config/runtime_settings_registry.go
index ce5774ab48c9..fda45e5bcdda 100644
--- a/core/config/runtime_settings_registry.go
+++ b/core/config/runtime_settings_registry.go
@@ -328,6 +328,10 @@ var runtimeSettingsFields = []fieldSpec{
func(s *RuntimeSettings) **bool { return &s.AutoloadBackendGalleries },
func(o *ApplicationConfig) bool { return o.AutoloadBackendGalleries },
func(o *ApplicationConfig, v bool) { o.AutoloadBackendGalleries = v }),
+ field("vram_persistent_cache",
+ func(s *RuntimeSettings) **bool { return &s.VRAMPersistentCache },
+ func(o *ApplicationConfig) bool { return o.VRAMPersistentCache },
+ func(o *ApplicationConfig, v bool) { o.VRAMPersistentCache = v }),
// API keys: echoed for the UI, but the apply loops never touch them.
// The settings endpoint and the file watcher own the env+runtime merge
diff --git a/core/config/runtime_settings_startup.go b/core/config/runtime_settings_startup.go
index 116d2a4f1478..9808c877b1a0 100644
--- a/core/config/runtime_settings_startup.go
+++ b/core/config/runtime_settings_startup.go
@@ -45,6 +45,7 @@ func DefaultRuntimeBaseline() *ApplicationConfig {
o.BackendGalleries = mustGalleries(DefaultBackendGalleriesJSON)
o.AutoloadGalleries = true
o.AutoloadBackendGalleries = true
+ o.VRAMPersistentCache = true
// core/cli/run.go injects WithMemoryReclaimer(enabled, threshold)
// unconditionally, so the kong threshold default (0.95) reaches the
// config even when the reclaimer flag is off - this overlay must match
diff --git a/core/http/endpoints/localai/settings.go b/core/http/endpoints/localai/settings.go
index 15c6f6d925e8..606fe49c5711 100644
--- a/core/http/endpoints/localai/settings.go
+++ b/core/http/endpoints/localai/settings.go
@@ -4,6 +4,7 @@ import (
"encoding/json"
"io"
"net/http"
+ "path/filepath"
"time"
"github.com/labstack/echo/v4"
@@ -12,6 +13,7 @@ import (
"github.com/mudler/LocalAI/core/http/endpoints/openresponses"
"github.com/mudler/LocalAI/core/p2p"
"github.com/mudler/LocalAI/core/schema"
+ "github.com/mudler/LocalAI/pkg/vram"
"github.com/mudler/LocalAI/pkg/vrambudget"
"github.com/mudler/xlog"
)
@@ -185,6 +187,13 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc {
// Apply settings using centralized method
watchdogChanged := appConfig.ApplyRuntimeSettings(&settings)
+ if settings.VRAMPersistentCache != nil || settings.AutoloadGalleries != nil {
+ if appConfig.VRAMPersistentCache && appConfig.AutoloadGalleries {
+ vram.ConfigurePersistentCache(filepath.Join(appConfig.SystemState.Model.ModelsPath, "..", "cache", "vram"), 24*time.Hour)
+ } else {
+ vram.DisablePersistentCache()
+ }
+ }
// Handle API keys specially (merge with startup keys)
if settings.ApiKeys != nil {
diff --git a/core/http/react-ui/e2e/settings-backend-logging.spec.js b/core/http/react-ui/e2e/settings-backend-logging.spec.js
index 7b5459d8973e..937836cbb760 100644
--- a/core/http/react-ui/e2e/settings-backend-logging.spec.js
+++ b/core/http/react-ui/e2e/settings-backend-logging.spec.js
@@ -18,6 +18,20 @@ test.describe('Settings - Backend Logging', () => {
await expect(input).toHaveValue('4')
})
+ test('persistent VRAM cache can be toggled', async ({ page }) => {
+ const row = page.locator('.form-row', { hasText: 'Persist remote VRAM estimates' })
+ await expect(row).toBeVisible()
+
+ const checkbox = row.locator('input[type="checkbox"]')
+ const wasChecked = await checkbox.isChecked()
+ await checkbox.locator('..').click()
+ if (wasChecked) {
+ await expect(checkbox).not.toBeChecked()
+ } else {
+ await expect(checkbox).toBeChecked()
+ }
+ })
+
test('backend logging toggle can be toggled', async ({ page }) => {
// Find the checkbox associated with backend logging
const section = page.locator('div', { has: page.locator('text=Enable Backend Logging') })
diff --git a/core/http/react-ui/src/pages/Settings.jsx b/core/http/react-ui/src/pages/Settings.jsx
index c1a95ad29ab3..c6d98fc11d24 100644
--- a/core/http/react-ui/src/pages/Settings.jsx
+++ b/core/http/react-ui/src/pages/Settings.jsx
@@ -488,6 +488,9 @@ export default function Settings() {
update('autoload_backend_galleries', v)} />
+
+ update('vram_persistent_cache', v)} />
+